diff --git a/Makefile b/Makefile index 08c1cae..85be191 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check description-drift-history iteration-ledger results-panel regression-history context-reports failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check description-drift-history iteration-ledger results-panel regression-history context-reports failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -17,6 +17,9 @@ route-confusion-check: description-optimization: $(PYTHON) scripts/run_description_optimization_suite.py +judge-blind-eval: + $(PYTHON) scripts/judge_blind_eval.py --description-file SKILL.md --cases evals/blind_holdout/trigger_cases.json --semantic-config evals/semantic_config.json + description-optimization-check: $(PYTHON) tests/verify_description_optimization.py diff --git a/README.md b/README.md index 13ac2dc..5398003 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Minimum commands: ```bash python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json python3 scripts/run_description_optimization_suite.py +python3 scripts/judge_blind_eval.py --description-file SKILL.md --cases evals/blind_holdout/trigger_cases.json --semantic-config evals/semantic_config.json python3 scripts/context_sizer.py . python3 scripts/resource_boundary_check.py . python3 scripts/governance_check.py . --require-manifest @@ -91,6 +92,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - packaging validation: `openai`, `claude`, and `generic` targets pass contract checks - description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately +- judge-backed blind eval: root, team frontend review, and governed incident command now pass an independent rubric judge on blind holdout prompts - packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected - failure library regressions: anti-pattern families pass automated checks - governance and resource-boundary checks are part of the default test path @@ -116,6 +118,15 @@ The design logic is simple: 5. Add quality gates only when they pay for themselves. 6. Export compatibility artifacts only for the clients you actually need. +## Method Doctrine + +The repository now treats method as a first-class asset instead of scattered guidance. + +- [Skill Engineering Method](references/skill-engineering-method.md) +- [Skill Archetypes](references/skill-archetypes.md) +- [Gate Selection](references/gate-selection.md) +- [Non-Skill Decision Tree](references/non-skill-decision-tree.md) + ## Why It Exists Most teams keep valuable operating knowledge scattered across chats, personal prompts, oral habits, and undocumented workflows. This project converts that hidden process knowledge into: @@ -164,6 +175,7 @@ Utility scripts that make the meta-skill operational: - `trigger_eval.py`: evaluates trigger descriptions with semantic intent concepts, explicit exclusions, and near-neighbor prompts - `run_eval_suite.py`: runs train/dev/holdout trigger suites, reports family-level regressions, and fails if aggregate regressions appear - `optimize_description.py`: generates candidate descriptions, scores them on dev, visible holdout, blind holdout, and adversarial holdout suites, then reports calibration and family health +- `judge_blind_eval.py`: applies an independent rubric judge to blind-holdout prompts so blind acceptance is not backed only by the main threshold scorer - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot @@ -196,6 +208,7 @@ Continuous integration entrypoint that runs the full local regression suite on p - The sample trigger report now covers a larger positive, negative, and near-neighbor set rather than a tiny demo set. - Train/dev/holdout trigger suites now separate iterative tuning from final verification. - Description optimization now uses dev for ranking, visible holdout for non-regression, blind holdout for acceptance, and adversarial holdout for harder route-collision checks without feeding the ranking loop. +- Judge-backed blind eval now adds a rubric-based second opinion for blind prompts, so blind acceptance is not decided by one scorer alone. - Description drift history now records adversarial calibration gaps and family coverage, so routing changes can be judged on confidence and family stability rather than raw error counts alone. - Route confusion is now tracked explicitly across the root meta-skill, frontend review skill, governed incident skill, and `no_route` cases, so route theft is visible instead of implicit. - Promotion policy now requires visible holdout, blind holdout, adversarial holdout, and route confusion to stay clean before a description should be considered promotable. diff --git a/SKILL.md b/SKILL.md index 9845cdc..cb32abe 100644 --- a/SKILL.md +++ b/SKILL.md @@ -12,37 +12,27 @@ Build reusable skill packages, not long prompts. ## Router Rules -- Route by the frontmatter `description` first; it is the main trigger surface. -- Keep `SKILL.md` as router plus execution skeleton only. -- Put long guidance in `references/`, deterministic logic in `scripts/`, static artifacts in `assets/`, and generated evidence in `reports/`. +- Route by frontmatter `description` first. +- Keep `SKILL.md` to routing plus the minimal execution skeleton. +- Push long guidance to `references/`, deterministic logic to `scripts/`, and evidence to `reports/`. - Use the lightest process that still makes the skill reliable. -- Package for reuse only when reuse is real. - -## Use Cases - -- create a new skill -- turn a workflow, runbook, or prompt set into a skill -- improve a skill's boundary, evals, or packaging -- design a team skill template or standard -- migrate a skill toward the open format ## Modes -- `Scaffold`: personal or exploratory; deliver `SKILL.md`, `agents/interface.yaml`, and only clearly needed folders. -- `Production`: reusable team skill; add focused `references/`, `scripts/`, and `evals/` only when they earn their keep. -- `Library`: important shared or meta skill; add trigger matrices, packaging guidance, and maintenance metadata. +- `Scaffold`: exploratory or personal. +- `Production`: team reuse with focused gates. +- `Library`: shared infrastructure or meta skill. -Detailed mode rules: [Operating Modes](references/operating-modes.md), [QA Ladder](references/qa-ladder.md), [Resource Boundary Spec](references/resource-boundaries.md). +Mode rules: [Operating Modes](references/operating-modes.md), [QA Ladder](references/qa-ladder.md), [Resource Boundary Spec](references/resource-boundaries.md), [Skill Engineering Method](references/skill-engineering-method.md). ## Compact Workflow -1. Capture the recurring job, outputs, trigger phrases, and risk. -2. Set one coherent boundary: one capability family, one trigger surface, one workflow. -3. Write the `description` early and test it before expanding the body on important skills. -4. Generate only the folders that earn their keep. Start from the basic template unless complexity is real. -5. Run the useful gates: `context_sizer.py`, `resource_boundary_check.py`, `governance_check.py`, `trigger_eval.py`, `optimize_description.py`, `cross_packager.py`. +1. Decide whether the request should become a skill, then choose the lightest archetype. +2. Capture the recurring job, outputs, trigger phrases, and exclusions. +3. Write the `description` early, then test route quality before expanding the package. +4. Add only the folders and gates that earn their keep: `trigger_eval.py`, `optimize_description.py`, `judge_blind_eval.py`, `resource_boundary_check.py`, `governance_check.py`, `cross_packager.py`. -Detailed playbooks: [Operating Modes](references/operating-modes.md) and [Trigger And Eval Playbook](references/eval-playbook.md). +Playbooks: [Skill Engineering Method](references/skill-engineering-method.md), [Skill Archetypes](references/skill-archetypes.md), [Gate Selection](references/gate-selection.md), [Non-Skill Decision Tree](references/non-skill-decision-tree.md), [Operating Modes](references/operating-modes.md), [Trigger And Eval Playbook](references/eval-playbook.md). ## Output Contract @@ -51,18 +41,16 @@ Unless the user asks otherwise, produce: 1. a working skill directory 2. a trigger-aware `SKILL.md` 3. aligned `agents/interface.yaml` -4. references, scripts, evals, reports, and `manifest.json` only when justified +4. optional `references/`, `scripts/`, `evals/`, `reports/`, and `manifest.json` only when justified 5. a short summary of boundary, exclusions, gates, and next steps ## Reference Map +- [Skill Engineering Method](references/skill-engineering-method.md) +- [Skill Archetypes](references/skill-archetypes.md) +- [Gate Selection](references/gate-selection.md) +- [Non-Skill Decision Tree](references/non-skill-decision-tree.md) - [Operating Modes](references/operating-modes.md) -- [QA Ladder](references/qa-ladder.md) - [Governance Model](references/governance.md) - [Resource Boundary Spec](references/resource-boundaries.md) -- [Skill Design Guidelines](references/skill_design_guidelines.md) -- [Client Compatibility](references/client-compatibility.md) -- [Comparative Analysis](references/comparative-analysis.md) -- [Meta-Skill Rubric](references/design-rubric.md) -- [Skill Template](references/skill-template.md) - [Trigger And Eval Playbook](references/eval-playbook.md) diff --git a/WORLD_CLASS_PLAN.md b/WORLD_CLASS_PLAN.md index 891c3e7..8e4127a 100644 --- a/WORLD_CLASS_PLAN.md +++ b/WORLD_CLASS_PLAN.md @@ -74,12 +74,14 @@ The current sprint is aimed at the four dimensions that most directly determine - route confusion matrix - contrastive route testing across sibling skills - mutation-based and adversarial route families +- judge-backed blind eval so blind acceptance does not depend on one scorer ### Method - promotion policy - explicit gate-selection doctrine - non-skill decision rules for when packaging should not happen +- formal skill engineering method and archetype taxonomy ### Execution diff --git a/evals/README.md b/evals/README.md index 17cc44b..ae82ddf 100644 --- a/evals/README.md +++ b/evals/README.md @@ -25,6 +25,7 @@ Use: python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt python3 scripts/run_eval_suite.py +python3 scripts/judge_blind_eval.py --description-file SKILL.md --cases evals/blind_holdout/trigger_cases.json --semantic-config evals/semantic_config.json python3 scripts/run_description_optimization_suite.py --history-snapshot-output evals/history/description_optimization/YYYY-MM-DD-adversarial-calibration-and-family-drift.json --snapshot-date YYYY-MM-DD --snapshot-id adversarial-calibration-and-family-drift --snapshot-label "Adversarial Calibration And Family Drift" python3 scripts/build_confusion_matrix.py --history-snapshot-output evals/history/YYYY-MM-DD-route-scorecard-foundation.json --snapshot-date YYYY-MM-DD --snapshot-id route-scorecard-foundation --snapshot-label "Route Scorecard Foundation" python3 scripts/render_eval_dashboard.py @@ -51,5 +52,6 @@ Regression scope now includes: - family-based reporting across workflow-to-skill, iterate-existing-skill, document-only, one-off, and future-outline cases - holdout verification - description optimization reports that compare baseline, current, and optimized route wording across dev, holdout, blind holdout, and adversarial holdout gates +- judge-backed blind-holdout verification that adds a rubric-based second opinion for blind prompts - calibration summaries that surface score gaps, threshold margins, and risk bands for each acceptance gate - family-level drift history that records which blind and adversarial families stay clean over time diff --git a/evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json b/evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json new file mode 100644 index 0000000..9cbe4c6 --- /dev/null +++ b/evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json @@ -0,0 +1,444 @@ +{ + "snapshot_id": "judge-backed-blind-eval", + "date": "2026-04-01", + "commit": "ce6cc74", + "label": "Judge-Backed Blind Eval", + "targets": [ + { + "name": "yao-meta-skill", + "winner_label": "Current", + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "winner_tokens": 65, + "current_tokens": 65, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.33, + "mean_positive_score": 0.672, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "blind_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.026, + "score_gap": 0.366, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.304, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "adversarial_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy", + "ok": true + }, + { + "name": "team-frontend-review", + "winner_label": "Current", + "winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "winner_tokens": 50, + "current_tokens": 50, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.088, + "mean_near_neighbor_score": 0.042, + "min_positive_score": 0.49, + "max_non_trigger_score": 0.211, + "score_gap": 0.279, + "threshold_margin": 0.01, + "positive_threshold_buffer": 0.01, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.226, + "min_positive_score": 0.504, + "max_non_trigger_score": 0.416, + "score_gap": 0.088, + "threshold_margin": 0.024, + "positive_threshold_buffer": 0.024, + "negative_threshold_buffer": 0.064, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.71, + "mean_non_trigger_score": 0.009, + "mean_near_neighbor_score": 0.018, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.035, + "score_gap": 0.509, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.305, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy", + "ok": true + }, + { + "name": "governed-incident-command", + "winner_label": "Current", + "winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "winner_tokens": 37, + "current_tokens": 37, + "winner_holdout_fp": 0, + "winner_holdout_fn": 1, + "current_holdout_fp": 0, + "current_holdout_fn": 1, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 2, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 1, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.367, + "mean_non_trigger_score": 0.019, + "mean_near_neighbor_score": 0.038, + "min_positive_score": 0.08, + "max_non_trigger_score": 0.058, + "score_gap": 0.022, + "threshold_margin": -0.4, + "positive_threshold_buffer": -0.4, + "negative_threshold_buffer": 0.422, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "overlap" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.899, + "mean_non_trigger_score": 0.168, + "mean_near_neighbor_score": 0.138, + "min_positive_score": 0.798, + "max_non_trigger_score": 0.386, + "score_gap": 0.412, + "threshold_margin": 0.094, + "positive_threshold_buffer": 0.318, + "negative_threshold_buffer": 0.094, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "healthy" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.827, + "mean_non_trigger_score": 0.027, + "mean_near_neighbor_score": 0.011, + "min_positive_score": 0.654, + "max_non_trigger_score": 0.056, + "score_gap": 0.598, + "threshold_margin": 0.284, + "positive_threshold_buffer": 0.314, + "negative_threshold_buffer": 0.284, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.69, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy", + "ok": true + } + ], + "notes": [ + "recorded family-level blind, judge-backed blind, and adversarial routing evidence", + "published calibration and drift history for description optimization" + ] +} diff --git a/evals/history/description_optimization/README.md b/evals/history/description_optimization/README.md index da74bad..bdb2a03 100644 --- a/evals/history/description_optimization/README.md +++ b/evals/history/description_optimization/README.md @@ -9,6 +9,7 @@ Each snapshot records: - per-target winner token counts - visible holdout errors - blind holdout errors when available +- judge-backed blind holdout errors and agreement when available - adversarial holdout errors when available - calibration summaries for blind and adversarial gates when available - family coverage summaries for blind and adversarial gates when available diff --git a/examples/governed-incident-command/optimization/reports/description_optimization.json b/examples/governed-incident-command/optimization/reports/description_optimization.json index 93bf03a..05d0090 100644 --- a/examples/governed-incident-command/optimization/reports/description_optimization.json +++ b/examples/governed-incident-command/optimization/reports/description_optimization.json @@ -3300,6 +3300,996 @@ ] } }, + "winner_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_packet_assembly": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_governed_workflow": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_one_off_update": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.", + "family": "blind_packet_assembly", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "strong_positive_concepts": [ + "severity_assessment", + "comms_and_owners" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "owner" + ], + "governed_workflow": [ + "governed" + ], + "incident_packet": [ + "governed incident packet" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.", + "family": "blind_governed_workflow", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment", + "comms_and_owners" + ], + "strong_positive_concepts": [ + "severity_assessment", + "comms_and_owners" + ], + "trigger_action_hits": [ + "run severity assessment", + "assemble incident communication" + ], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 5, + "negative_vote": 0, + "margin": 5, + "confidence": 0.96, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "incident communication" + ], + "reusable_packaging": [ + "reusable", + "workflow" + ], + "severity_assessment": [ + "severity assessment", + "severity" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Explain how severity levels are decided during incidents.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment" + ], + "strong_positive_concepts": [ + "severity_assessment" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain" + ], + "positive_vote": 1, + "negative_vote": 1, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Write a single incident update for customers from these notes.", + "family": "blind_one_off_update", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 0, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": {} + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Summarize the outage timeline, but do not package a governed workflow.", + "family": "blind_summary_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "governed_workflow" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "summarize" + ], + "positive_vote": 1, + "negative_vote": 1, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "governed_workflow": [ + "governed" + ], + "incident_inputs": [ + "timeline" + ], + "reusable_packaging": [ + "workflow", + "package" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Brainstorm how incident handling could improve next quarter without building the packet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "brainstorm" + ], + "positive_vote": 1, + "negative_vote": 3, + "margin": -2, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "brainstorm_only": [ + "brainstorm" + ] + } + } + } + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + } + }, + "current_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_packet_assembly": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_governed_workflow": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_one_off_update": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.", + "family": "blind_packet_assembly", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "strong_positive_concepts": [ + "severity_assessment", + "comms_and_owners" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "owner" + ], + "governed_workflow": [ + "governed" + ], + "incident_packet": [ + "governed incident packet" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.", + "family": "blind_governed_workflow", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment", + "comms_and_owners" + ], + "strong_positive_concepts": [ + "severity_assessment", + "comms_and_owners" + ], + "trigger_action_hits": [ + "run severity assessment", + "assemble incident communication" + ], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 5, + "negative_vote": 0, + "margin": 5, + "confidence": 0.96, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "incident communication" + ], + "reusable_packaging": [ + "reusable", + "workflow" + ], + "severity_assessment": [ + "severity assessment", + "severity" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Explain how severity levels are decided during incidents.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment" + ], + "strong_positive_concepts": [ + "severity_assessment" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain" + ], + "positive_vote": 1, + "negative_vote": 1, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Write a single incident update for customers from these notes.", + "family": "blind_one_off_update", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 0, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": {} + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Summarize the outage timeline, but do not package a governed workflow.", + "family": "blind_summary_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "governed_workflow" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "summarize" + ], + "positive_vote": 1, + "negative_vote": 1, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "governed_workflow": [ + "governed" + ], + "incident_inputs": [ + "timeline" + ], + "reusable_packaging": [ + "workflow", + "package" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Brainstorm how incident handling could improve next quarter without building the packet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "brainstorm" + ], + "positive_vote": 1, + "negative_vote": 3, + "margin": -2, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "brainstorm_only": [ + "brainstorm" + ] + } + } + } + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + } + }, + "baseline_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_packet_assembly": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_governed_workflow": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_one_off_update": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.", + "family": "blind_packet_assembly", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "incident_packet", + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "strong_positive_concepts": [ + "incident_packet", + "severity_assessment", + "comms_and_owners" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "owner" + ], + "governed_workflow": [ + "governed" + ], + "incident_packet": [ + "governed incident packet" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.", + "family": "blind_governed_workflow", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "reusable_packaging" + ], + "strong_positive_concepts": [ + "severity_assessment", + "comms_and_owners" + ], + "trigger_action_hits": [ + "run severity assessment", + "assemble incident communication" + ], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 6, + "negative_vote": 0, + "margin": 6, + "confidence": 1.0, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "incident communication" + ], + "reusable_packaging": [ + "reusable", + "workflow" + ], + "severity_assessment": [ + "severity assessment", + "severity" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Explain how severity levels are decided during incidents.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "severity_assessment" + ], + "strong_positive_concepts": [ + "severity_assessment" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain" + ], + "positive_vote": 1, + "negative_vote": 1, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Write a single incident update for customers from these notes.", + "family": "blind_one_off_update", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 0, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": {} + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Summarize the outage timeline, but do not package a governed workflow.", + "family": "blind_summary_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "incident_inputs", + "governed_workflow", + "reusable_packaging" + ], + "strong_positive_concepts": [ + "incident_inputs" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "summarize" + ], + "positive_vote": 3, + "negative_vote": 1, + "margin": 2, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "governed_workflow": [ + "governed" + ], + "incident_inputs": [ + "timeline" + ], + "reusable_packaging": [ + "workflow", + "package" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Brainstorm how incident handling could improve next quarter without building the packet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "brainstorm" + ], + "positive_vote": 1, + "negative_vote": 3, + "margin": -2, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "brainstorm_only": [ + "brainstorm" + ] + } + } + } + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.69, + "rubric_version": "blind-v1" + } + }, "winner_adversarial_holdout_report": { "threshold": 0.34, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", @@ -4588,6 +5578,30 @@ "baseline_recall": 0.5, "improved_recall": 1.0 }, + "winner_vs_current_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, "winner_vs_current_adversarial_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, @@ -4846,6 +5860,80 @@ } } }, + "judge_blind_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + } + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + } + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.69, + "rubric_version": "blind-v1" + } + }, + "winner_calibration": null, + "current_calibration": null, + "baseline_calibration": null, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + } + }, "adversarial_holdout_non_regression": { "winner": { "false_positives": 0, @@ -4958,8 +6046,11 @@ "current_holdout_total_errors": 1, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_total_errors": 0, + "current_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_total_errors": 0, "current_adversarial_holdout_total_errors": 0, + "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", "winner_adversarial_score_gap": 0.598, "candidate_count": 5, @@ -4967,6 +6058,7 @@ "baseline_dev_total_errors": 1, "baseline_holdout_total_errors": 2, "baseline_blind_holdout_total_errors": 1, + "baseline_judge_blind_holdout_total_errors": 0, "baseline_adversarial_holdout_total_errors": 0 } } diff --git a/examples/governed-incident-command/optimization/reports/description_optimization.md b/examples/governed-incident-command/optimization/reports/description_optimization.md index 6e98b8b..40bfcd1 100644 --- a/examples/governed-incident-command/optimization/reports/description_optimization.md +++ b/examples/governed-incident-command/optimization/reports/description_optimization.md @@ -26,6 +26,7 @@ Build governed incident command packets. Use when asked to standardize incident | --- | ---: | ---: | ---: | ---: | ---: | ---: | | Holdout | 0 | 1 | 0 | 1 | 0 | 2 | | Blind Holdout | 0 | 0 | 0 | 0 | 0 | 1 | +| Judge Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 | ## Calibration @@ -36,12 +37,19 @@ Build governed incident command packets. Use when asked to standardize incident | Blind Holdout | 0.412 | healthy | 0.167 | 0.412 | 0.065 | | Adversarial Holdout | 0.598 | healthy | 0.0 | 0.598 | 0.172 | +## Judge Blind Summary + +| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement | +| --- | ---: | ---: | ---: | ---: | +| Judge Blind Holdout | 1.0 | 0.657 | 1.0 | 1.0 | + ## Family Health | Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families | | --- | --- | --- | --- | --- | | Holdout | 5/6 | packet_assembly (1 errors) | 5/6 | 4/6 | | Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 5/6 | +| Judge Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 6/6 | | Adversarial Holdout | 6/6 | adversarial_single_update_collision (0 errors) | 6/6 | 6/6 | ## Selection Logic diff --git a/examples/team-frontend-review/optimization/reports/description_optimization.json b/examples/team-frontend-review/optimization/reports/description_optimization.json index 28c36da..bab7294 100644 --- a/examples/team-frontend-review/optimization/reports/description_optimization.json +++ b/examples/team-frontend-review/optimization/reports/description_optimization.json @@ -3942,6 +3942,1020 @@ ] } }, + "winner_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_pr_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_premerge": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_build_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_checklist_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_fix_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", + "family": "blind_pr_review", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "accessibility", + "ui_security", + "frontend_scope" + ], + "strong_positive_concepts": [ + "accessibility", + "ui_security" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 6, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "accessibility": [ + "a11y" + ], + "frontend_scope": [ + "react" + ], + "ui_security": [ + "unsafe rendering" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", + "family": "blind_premerge", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "review_frontend", + "accessibility", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "strong_positive_concepts": [ + "review_frontend", + "accessibility", + "state_coverage" + ], + "trigger_action_hits": [ + "run a pre-merge frontend review" + ], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 6, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 6, + "negative_vote": 0, + "margin": 6, + "confidence": 1.0, + "concept_evidence": { + "positive": { + "accessibility": [ + "accessibility" + ], + "frontend_scope": [ + "frontend", + "form" + ], + "review_frontend": [ + "frontend review", + "pre-merge frontend review" + ], + "state_coverage": [ + "missing states" + ], + "ux_regression": [ + "pre-merge" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Build a new React settings page and add error handling.", + "family": "blind_build_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "frontend_scope" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 0, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "react" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Explain why keyboard accessibility matters in forms.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "accessibility" + ], + "strong_positive_concepts": [ + "accessibility" + ], + "trigger_action_hits": [], + "input_hits": [ + "forms" + ], + "artifact_hits": [ + "forms" + ], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain" + ], + "positive_vote": 3, + "negative_vote": 1, + "margin": 2, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "accessibility": [ + "accessibility" + ] + }, + "negative": {} + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", + "family": "blind_checklist_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "review_frontend", + "frontend_scope" + ], + "strong_positive_concepts": [ + "review_frontend" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 0, + "margin": 2, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "frontend" + ], + "review_frontend": [ + "frontend review" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Fix the broken dropdown behavior in this component.", + "family": "blind_fix_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "frontend_scope" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 0, + "negative_vote": 0, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "component" + ] + }, + "negative": {} + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "current_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_pr_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_premerge": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_build_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_checklist_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_fix_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", + "family": "blind_pr_review", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "accessibility", + "ui_security", + "frontend_scope" + ], + "strong_positive_concepts": [ + "accessibility", + "ui_security" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 6, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "accessibility": [ + "a11y" + ], + "frontend_scope": [ + "react" + ], + "ui_security": [ + "unsafe rendering" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", + "family": "blind_premerge", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "review_frontend", + "accessibility", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "strong_positive_concepts": [ + "review_frontend", + "accessibility", + "state_coverage" + ], + "trigger_action_hits": [ + "run a pre-merge frontend review" + ], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 6, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 6, + "negative_vote": 0, + "margin": 6, + "confidence": 1.0, + "concept_evidence": { + "positive": { + "accessibility": [ + "accessibility" + ], + "frontend_scope": [ + "frontend", + "form" + ], + "review_frontend": [ + "frontend review", + "pre-merge frontend review" + ], + "state_coverage": [ + "missing states" + ], + "ux_regression": [ + "pre-merge" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Build a new React settings page and add error handling.", + "family": "blind_build_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "frontend_scope" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 0, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "react" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Explain why keyboard accessibility matters in forms.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "accessibility" + ], + "strong_positive_concepts": [ + "accessibility" + ], + "trigger_action_hits": [], + "input_hits": [ + "forms" + ], + "artifact_hits": [ + "forms" + ], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain" + ], + "positive_vote": 3, + "negative_vote": 1, + "margin": 2, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "accessibility": [ + "accessibility" + ] + }, + "negative": {} + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", + "family": "blind_checklist_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "review_frontend", + "frontend_scope" + ], + "strong_positive_concepts": [ + "review_frontend" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 0, + "margin": 2, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "frontend" + ], + "review_frontend": [ + "frontend review" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Fix the broken dropdown behavior in this component.", + "family": "blind_fix_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "frontend_scope" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 0, + "negative_vote": 0, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "component" + ] + }, + "negative": {} + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "baseline_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_pr_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_premerge": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_build_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_checklist_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_fix_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", + "family": "blind_pr_review", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "accessibility", + "ui_security", + "frontend_scope" + ], + "strong_positive_concepts": [ + "accessibility", + "ui_security" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 6, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "accessibility": [ + "a11y" + ], + "frontend_scope": [ + "react" + ], + "ui_security": [ + "unsafe rendering" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", + "family": "blind_premerge", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "review_frontend", + "accessibility", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "strong_positive_concepts": [ + "review_frontend", + "accessibility", + "state_coverage" + ], + "trigger_action_hits": [ + "run a pre-merge frontend review" + ], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 6, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 6, + "negative_vote": 0, + "margin": 6, + "confidence": 1.0, + "concept_evidence": { + "positive": { + "accessibility": [ + "accessibility" + ], + "frontend_scope": [ + "frontend", + "form" + ], + "review_frontend": [ + "frontend review", + "pre-merge frontend review" + ], + "state_coverage": [ + "missing states" + ], + "ux_regression": [ + "pre-merge" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Build a new React settings page and add error handling.", + "family": "blind_build_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "frontend_scope" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 0, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "react" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Explain why keyboard accessibility matters in forms.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "accessibility" + ], + "strong_positive_concepts": [ + "accessibility" + ], + "trigger_action_hits": [], + "input_hits": [ + "forms" + ], + "artifact_hits": [ + "forms" + ], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain" + ], + "positive_vote": 3, + "negative_vote": 1, + "margin": 2, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "accessibility": [ + "accessibility" + ] + }, + "negative": {} + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", + "family": "blind_checklist_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "review_frontend", + "frontend_scope" + ], + "strong_positive_concepts": [ + "review_frontend" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 0, + "margin": 2, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "frontend" + ], + "review_frontend": [ + "frontend review" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Fix the broken dropdown behavior in this component.", + "family": "blind_fix_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "frontend_scope" + ], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 0, + "negative_vote": 0, + "margin": 0, + "confidence": 0.5, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "component" + ] + }, + "negative": {} + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, "winner_adversarial_holdout_report": { "threshold": 0.34, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", @@ -5340,6 +6354,30 @@ "baseline_recall": 1.0, "improved_recall": 1.0 }, + "winner_vs_current_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, "winner_vs_current_adversarial_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, @@ -5569,6 +6607,80 @@ } } }, + "judge_blind_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "winner_calibration": null, + "current_calibration": null, + "baseline_calibration": null, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + } + }, "adversarial_holdout_non_regression": { "winner": { "false_positives": 0, @@ -5681,8 +6793,11 @@ "current_holdout_total_errors": 0, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_total_errors": 0, + "current_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_total_errors": 0, "current_adversarial_holdout_total_errors": 0, + "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", "winner_adversarial_score_gap": 0.509, "candidate_count": 5, @@ -5690,6 +6805,7 @@ "baseline_dev_total_errors": 4, "baseline_holdout_total_errors": 0, "baseline_blind_holdout_total_errors": 0, + "baseline_judge_blind_holdout_total_errors": 0, "baseline_adversarial_holdout_total_errors": 0 } } diff --git a/examples/team-frontend-review/optimization/reports/description_optimization.md b/examples/team-frontend-review/optimization/reports/description_optimization.md index 73d4d2a..0d005bc 100644 --- a/examples/team-frontend-review/optimization/reports/description_optimization.md +++ b/examples/team-frontend-review/optimization/reports/description_optimization.md @@ -26,6 +26,7 @@ Review frontend code for accessibility, UI security, missing states, and UX regr | --- | ---: | ---: | ---: | ---: | ---: | ---: | | Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | +| Judge Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 | ## Calibration @@ -36,12 +37,19 @@ Review frontend code for accessibility, UI security, missing states, and UX regr | Blind Holdout | 0.088 | tight | 0.333 | 0.088 | 0.082 | | Adversarial Holdout | 0.509 | healthy | 0.0 | 0.509 | 0.519 | +## Judge Blind Summary + +| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement | +| --- | ---: | ---: | ---: | ---: | +| Judge Blind Holdout | 1.0 | 0.703 | 1.0 | 1.0 | + ## Family Health | Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families | | --- | --- | --- | --- | --- | | Holdout | 6/6 | security_and_a11y (0 errors) | 6/6 | 6/6 | | Blind Holdout | 6/6 | blind_premerge (0 errors) | 6/6 | 6/6 | +| Judge Blind Holdout | 6/6 | blind_premerge (0 errors) | 6/6 | 6/6 | | Adversarial Holdout | 6/6 | adversarial_quality_gate_review (0 errors) | 6/6 | 6/6 | ## Selection Logic diff --git a/references/gate-selection.md b/references/gate-selection.md new file mode 100644 index 0000000..0200441 --- /dev/null +++ b/references/gate-selection.md @@ -0,0 +1,68 @@ +# Gate Selection + +Gate selection should track risk. More gates are not automatically better. + +## Minimum Gates By Risk + +| Risk Level | Typical Archetype | Required Gates | +| --- | --- | --- | +| low | Scaffold | `validate_skill.py`, `resource_boundary_check.py` | +| medium | Production | low-risk gates plus `trigger_eval.py` | +| high | Library | medium-risk gates plus `optimize_description.py`, route confusion, packaging validation | +| critical | Governed | high-risk gates plus governance scoring, regression history, promotion policy | + +## Trigger Gates + +Use trigger gates when: + +- a route can be confused with a nearby skill +- the workflow will be reused by multiple people +- the skill boundary must be defended over time + +Trigger gates, in order: + +1. `trigger_eval.py` +2. `optimize_description.py` +3. blind holdout +4. judge-backed blind holdout +5. adversarial holdout +6. route confusion + +## Packaging Gates + +Use packaging gates when: + +- output must be consumed by multiple clients +- metadata drift would break discoverability +- portability is part of the skill's value + +Packaging gates: + +- `validate_skill.py` +- `cross_packager.py` +- packager failure fixtures +- adapter snapshot checks + +## Governance Gates + +Use governance gates when: + +- the skill is shared infrastructure +- the skill has an owner and review cadence +- lifecycle state matters + +Governance gates: + +- `governance_check.py` +- regression history +- maturity scoring +- promotion policy + +## Context Gates + +Use context gates on every non-trivial skill. + +- `context_sizer.py` +- `resource_boundary_check.py` + +If rigor increases but initial load crosses the tier budget, move detail into `references/`, `scripts/`, or `reports/` before adding more checks. diff --git a/references/non-skill-decision-tree.md b/references/non-skill-decision-tree.md new file mode 100644 index 0000000..c50e8f2 --- /dev/null +++ b/references/non-skill-decision-tree.md @@ -0,0 +1,39 @@ +# Non-Skill Decision Tree + +Do not build a skill when the user really wants a direct answer or a lightweight artifact. + +## Use A Direct Answer Instead + +Prefer a normal response when the job is: + +- explain a concept +- summarize a document +- translate text +- brainstorm ideas without packaging +- answer a one-off question + +## Use A Document Instead + +Prefer a document or note when the output is: + +- wiki or knowledge-base content +- a reusable-looking document with no agent execution +- a policy memo +- a handoff note + +## Use A Script Instead + +Prefer a script without a full skill when: + +- the task is deterministic +- routing is not the hard part +- the user wants a utility, not a routed capability + +## Promote To A Skill Only When + +- the job is recurring +- discoverability matters +- the workflow needs a boundary +- the process benefits from reusable instructions or checks + +If there is doubt, start with `Scaffold` and upgrade only after reuse becomes real. diff --git a/references/skill-archetypes.md b/references/skill-archetypes.md new file mode 100644 index 0000000..cd8e6f5 --- /dev/null +++ b/references/skill-archetypes.md @@ -0,0 +1,86 @@ +# Skill Archetypes + +Use these archetypes to decide what kind of skill you are building before you decide how many files or gates to add. + +## Scaffold + +Purpose: + +- quick packaging for a real but still exploratory workflow + +Default assets: + +- `SKILL.md` +- `agents/interface.yaml` + +Use when: + +- reuse is plausible but not proven +- failure cost is low +- the workflow is still changing + +## Production + +Purpose: + +- compact skill for team reuse + +Default assets: + +- lean `SKILL.md` +- `agents/interface.yaml` +- selective `references/` +- selective `evals/` + +Use when: + +- route mistakes waste team time +- a checklist or focused script improves reliability + +## Library + +Purpose: + +- shared capability with visible evidence and portability expectations + +Default assets: + +- route evals +- packaging checks +- manifest metadata +- public reports + +Use when: + +- the skill will be reused across teams or clients +- the skill is likely to have near-neighbor route collisions + +## Governed + +Purpose: + +- high-trust skill with explicit ownership and review + +Default assets: + +- lifecycle metadata +- governance score +- review cadence +- regression history +- governed examples or policy references + +Use when: + +- the skill is operationally sensitive +- the skill influences incident, release, compliance, or organizational standards + +## Anti-Archetypes + +Do not force a request into a skill archetype when it is really: + +- a one-off answer +- a document +- a brainstorm +- an implementation task with no reusable process + +See [Non-Skill Decision Tree](non-skill-decision-tree.md). diff --git a/references/skill-engineering-method.md b/references/skill-engineering-method.md new file mode 100644 index 0000000..d326dfa --- /dev/null +++ b/references/skill-engineering-method.md @@ -0,0 +1,100 @@ +# Skill Engineering Method + +This doctrine defines the default method for turning messy workflow material into a reusable skill without bloating the entrypoint. + +## Core Loop + +1. Decide whether the request should become a skill at all. +2. Choose the smallest viable archetype. +3. Set one clear capability boundary. +4. Write and test the trigger description before expanding the body. +5. Add only the gates that match the risk. +6. Package and govern the skill only as far as real reuse demands. + +## Phase 1: Qualification + +Promote a request into a skill only when at least one of these is true: + +- the workflow will be reused +- the workflow is easy to route incorrectly +- deterministic scripts reduce repeated effort +- governance or portability matters + +Reject skill creation when the request is only: + +- explanation +- summary +- translation +- brainstorming +- documentation without agent execution +- a one-off answer with no reuse value + +See [Non-Skill Decision Tree](non-skill-decision-tree.md). + +## Phase 2: Archetype Selection + +Choose the lightest archetype that fits the job. + +- `Scaffold`: exploratory, personal, or short-lived +- `Production`: team-reused, quality-sensitive, but still compact +- `Library`: broad reuse, visible evidence, portability, and maintenance expectations +- `Governed`: organizationally sensitive or operationally critical; lifecycle and review are explicit + +See [Skill Archetypes](skill-archetypes.md). + +## Phase 3: Boundary Design + +Every skill should answer four questions clearly: + +- what recurring job does it own +- what outputs does it produce +- what near-neighbor requests should not route here +- what detail belongs outside `SKILL.md` + +Boundary work comes before polishing prose. + +## Phase 4: Trigger-First Authoring + +Author the frontmatter `description` before expanding the body. + +- start with the recurring job +- include the trigger actions that should route here +- include exclusions when confusion is plausible +- test the route before growing the file tree + +Trigger quality is improved through: + +- `trigger_eval.py` +- `optimize_description.py` +- blind holdout +- judge-backed blind holdout +- adversarial holdout +- route confusion + +## Phase 5: Gate Selection + +Add gates by risk, not by habit. + +- low-risk scaffolds: validate structure and context size +- production skills: trigger eval plus resource-boundary checks +- library skills: description optimization, route confusion, packaging checks +- governed skills: governance scoring, lifecycle metadata, regression history + +See [Gate Selection](gate-selection.md). + +## Phase 6: Promotion + +A candidate route or package is promotable only when: + +- visible holdout does not regress +- blind holdout does not regress +- judge-backed blind holdout does not regress +- adversarial holdout does not regress +- route confusion stays clean +- context and governance gates still pass + +See [Promotion Policy](../evals/promotion_policy.md). + +## Design Principle + +The method is only correct if rigor grows faster than context cost. If a new check or document makes the skill heavier without making it more reliable, remove or relocate it. diff --git a/reports/context_budget.json b/reports/context_budget.json index 5b50d7c..e9f286b 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -5,13 +5,13 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 898, - "other_text_tokens": 204894, - "estimated_initial_load_tokens": 989, - "estimated_total_text_tokens": 205792, - "relevant_file_count": 112, + "skill_body_tokens": 795, + "other_text_tokens": 225175, + "estimated_initial_load_tokens": 886, + "estimated_total_text_tokens": 225970, + "relevant_file_count": 119, "unused_resource_dirs": [], "quality_signal_points": 130, - "quality_density": 131.4 + "quality_density": 146.7 } } diff --git a/reports/context_budget.md b/reports/context_budget.md index 1007a39..afff72d 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | -| root | `.` | `production` | 1000 | 989 | 898 | 131.4 | - | ok | +| root | `.` | `production` | 1000 | 886 | 795 | 146.7 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 8ff0e0f..79721e2 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -6,9 +6,9 @@ "path": ".", "budget_tier": "production", "budget_limit": 1000, - "initial_tokens": 989, - "skill_body_tokens": 898, - "quality_density": 131.4, + "initial_tokens": 886, + "skill_body_tokens": 795, + "quality_density": 146.7, "unused_resource_dirs": [], "ok": true, "warnings": [] diff --git a/reports/description_drift_history.md b/reports/description_drift_history.md index 8e1aa01..33a037d 100644 --- a/reports/description_drift_history.md +++ b/reports/description_drift_history.md @@ -1,27 +1,33 @@ # Description Drift History -| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Drift Note | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | -| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | `Current` | 65 | 0 | 0 | - | - | - | - | - | - | initial description optimization suite without blind holdout | -| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | `Current` | 50 | 0 | 0 | - | - | - | - | - | - | compressed example description and stored first holdout report | -| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | `Current` | 37 | 0 | 1 | - | - | - | - | - | - | compressed governed description and reduced visible holdout misses | -| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 0 | 0.834 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy | -| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0.509 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy | -| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 0 | 0.598 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy | -| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | -| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | -| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 1 | +| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Judge Blind Errors | Judge Agreement | Adv FP | Adv FN | Adv Gap | Adv Risk | Drift Note | +| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | +| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | `Current` | 65 | 0 | 0 | - | - | - | - | - | - | - | - | initial description optimization suite without blind holdout | +| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | `Current` | 50 | 0 | 0 | - | - | - | - | - | - | - | - | compressed example description and stored first holdout report | +| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | `Current` | 37 | 0 | 1 | - | - | - | - | - | - | - | - | compressed governed description and reduced visible holdout misses | +| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | - | - | 0 | 0 | 0.834 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy | +| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | - | - | 0 | 0 | 0.509 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy | +| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | - | - | 0 | 0 | 0.598 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy | +| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | - | - | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | +| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | - | - | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | +| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | - | - | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 1 | +| 2026-04-01 | Judge-Backed Blind Eval | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 1.0 | 0 | 0 | 0.834 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy | +| 2026-04-01 | Judge-Backed Blind Eval | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 1.0 | 0 | 0 | 0.509 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy | +| 2026-04-01 | Judge-Backed Blind Eval | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 1.0 | 0 | 0 | 0.598 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy | ## Family Coverage -| Date | Label | Target | Blind Families | Adversarial Families | -| --- | --- | --- | --- | --- | -| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | - | - | -| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | - | - | -| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | - | - | -| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=adversarial_summary_translation_collision | -| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=adversarial_quality_gate_review | -| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=adversarial_single_update_collision | -| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | - | - | -| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | - | - | -| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | - | - | +| Date | Label | Target | Blind Families | Judge Blind Families | Adversarial Families | +| --- | --- | --- | --- | --- | --- | +| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | - | - | - | +| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | - | - | - | +| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | - | - | - | +| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | 6/6 clean; weakest=blind_operationalize | - | 6/6 clean; weakest=adversarial_summary_translation_collision | +| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | 6/6 clean; weakest=blind_premerge | - | 6/6 clean; weakest=adversarial_quality_gate_review | +| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | 6/6 clean; weakest=blind_summary_only | - | 6/6 clean; weakest=adversarial_single_update_collision | +| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | - | - | - | +| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | - | - | - | +| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | - | - | - | +| 2026-04-01 | Judge-Backed Blind Eval | `yao-meta-skill` | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=adversarial_summary_translation_collision | +| 2026-04-01 | Judge-Backed Blind Eval | `team-frontend-review` | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=adversarial_quality_gate_review | +| 2026-04-01 | Judge-Backed Blind Eval | `governed-incident-command` | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=adversarial_single_update_collision | diff --git a/reports/description_optimization.json b/reports/description_optimization.json index ffde081..23e170f 100644 --- a/reports/description_optimization.json +++ b/reports/description_optimization.json @@ -7449,6 +7449,1047 @@ ] } }, + "winner_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 0, + "margin": 3, + "confidence": 0.74, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow" + ], + "strong_positive_concepts": [ + "transform_workflow" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [ + "document_only" + ], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 1, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "prompts" + ], + "artifact_hits": [ + "prompts" + ], + "capability_overlap": 2, + "exclusive_negative_concepts": [ + "explain_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain", + "plain english" + ], + "positive_vote": 2, + "negative_vote": 3, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 2, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 2, + "margin": 1, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "current_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 0, + "margin": 3, + "confidence": 0.74, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow" + ], + "strong_positive_concepts": [ + "transform_workflow" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [ + "document_only" + ], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 1, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "prompts" + ], + "artifact_hits": [ + "prompts" + ], + "capability_overlap": 2, + "exclusive_negative_concepts": [ + "explain_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain", + "plain english" + ], + "positive_vote": 2, + "negative_vote": 3, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 2, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 2, + "margin": 1, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "baseline_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 0, + "margin": 3, + "confidence": 0.74, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow" + ], + "strong_positive_concepts": [ + "transform_workflow" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [ + "document_only" + ], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 1, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "prompts" + ], + "artifact_hits": [ + "prompts" + ], + "capability_overlap": 2, + "exclusive_negative_concepts": [ + "explain_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain", + "plain english" + ], + "positive_vote": 2, + "negative_vote": 3, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 2, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 2, + "margin": 1, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, "winner_adversarial_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", @@ -8936,6 +9977,30 @@ "baseline_recall": 1.0, "improved_recall": 1.0 }, + "winner_vs_current_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, "winner_vs_current_adversarial_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, @@ -9165,6 +10230,80 @@ } } }, + "judge_blind_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "winner_calibration": null, + "current_calibration": null, + "baseline_calibration": null, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + } + }, "adversarial_holdout_non_regression": { "winner": { "false_positives": 0, @@ -9277,8 +10416,11 @@ "current_holdout_total_errors": 0, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_total_errors": 0, + "current_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_total_errors": 0, "current_adversarial_holdout_total_errors": 0, + "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", "winner_adversarial_score_gap": 0.834, "candidate_count": 6, @@ -9286,6 +10428,7 @@ "baseline_dev_total_errors": 2, "baseline_holdout_total_errors": 0, "baseline_blind_holdout_total_errors": 0, + "baseline_judge_blind_holdout_total_errors": 0, "baseline_adversarial_holdout_total_errors": 0 } } diff --git a/reports/description_optimization.md b/reports/description_optimization.md index 911410f..3bb3678 100644 --- a/reports/description_optimization.md +++ b/reports/description_optimization.md @@ -27,6 +27,7 @@ Create, refactor, evaluate, and package agent skills from workflows, prompts, tr | --- | ---: | ---: | ---: | ---: | ---: | ---: | | Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | +| Judge Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 | ## Calibration @@ -37,12 +38,19 @@ Create, refactor, evaluate, and package agent skills from workflows, prompts, tr | Blind Holdout | 0.366 | watch | 0.333 | 0.366 | 0.369 | | Adversarial Holdout | 0.834 | healthy | 0.0 | 0.834 | 0.845 | +## Judge Blind Summary + +| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement | +| --- | ---: | ---: | ---: | ---: | +| Judge Blind Holdout | 1.0 | 0.66 | 1.0 | 1.0 | + ## Family Health | Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families | | --- | --- | --- | --- | --- | | Holdout | 12/12 | workflow_to_skill (0 errors) | 12/12 | 12/12 | | Blind Holdout | 6/6 | blind_operationalize (0 errors) | 6/6 | 6/6 | +| Judge Blind Holdout | 6/6 | blind_operationalize (0 errors) | 6/6 | 6/6 | | Adversarial Holdout | 6/6 | adversarial_summary_translation_collision (0 errors) | 6/6 | 6/6 | ## Selection Logic diff --git a/reports/description_optimization_suite.json b/reports/description_optimization_suite.json index effcbae..746276d 100644 --- a/reports/description_optimization_suite.json +++ b/reports/description_optimization_suite.json @@ -19,6 +19,13 @@ "baseline_blind_holdout_fp": 0, "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_fp": 0, "winner_adversarial_holdout_fn": 0, "current_adversarial_holdout_fp": 0, @@ -73,6 +80,23 @@ "risk_band": "healthy" } }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, "family_health": { "holdout": { "family_count": 12, @@ -94,6 +118,16 @@ "errors": 0 } }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, "adversarial_holdout": { "family_count": 6, "clean_family_count": 6, @@ -105,7 +139,7 @@ } } }, - "drift_note": "blind, adversarial, and calibration gates active", + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", "ok": true }, { @@ -127,6 +161,13 @@ "baseline_blind_holdout_fp": 0, "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_fp": 0, "winner_adversarial_holdout_fn": 0, "current_adversarial_holdout_fp": 0, @@ -181,6 +222,23 @@ "risk_band": "healthy" } }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, "family_health": { "holdout": { "family_count": 6, @@ -202,6 +260,16 @@ "errors": 0 } }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, "adversarial_holdout": { "family_count": 6, "clean_family_count": 6, @@ -213,7 +281,7 @@ } } }, - "drift_note": "blind, adversarial, and calibration gates active", + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", "ok": true }, { @@ -235,6 +303,13 @@ "baseline_blind_holdout_fp": 0, "baseline_blind_holdout_fn": 1, "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_fp": 0, "winner_adversarial_holdout_fn": 0, "current_adversarial_holdout_fp": 0, @@ -289,6 +364,23 @@ "risk_band": "healthy" } }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.69, + "rubric_version": "blind-v1" + } + }, "family_health": { "holdout": { "family_count": 6, @@ -316,6 +408,16 @@ "errors": 0 } }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, "adversarial_holdout": { "family_count": 6, "clean_family_count": 6, @@ -327,7 +429,7 @@ } } }, - "drift_note": "blind, adversarial, and calibration gates active", + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", "ok": true } ], diff --git a/reports/description_optimization_suite.md b/reports/description_optimization_suite.md index 53fdf31..a93a7ec 100644 --- a/reports/description_optimization_suite.md +++ b/reports/description_optimization_suite.md @@ -1,15 +1,15 @@ # Description Optimization Suite -| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Status | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | -| `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 0 | 0.834 | healthy | ok | -| `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0.509 | healthy | ok | -| `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 0 | 0.598 | healthy | ok | +| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Judge Blind Errors | Adv FP | Adv FN | Adv Gap | Adv Risk | Status | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | +| `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.834 | healthy | ok | +| `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.509 | healthy | ok | +| `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0.598 | healthy | ok | ## Family Coverage -| Target | Blind Families | Adversarial Families | -| --- | --- | --- | -| `yao-meta-skill` | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=adversarial_summary_translation_collision | -| `team-frontend-review` | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=adversarial_quality_gate_review | -| `governed-incident-command` | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=adversarial_single_update_collision | +| Target | Blind Families | Judge Blind Families | Adversarial Families | +| --- | --- | --- | --- | +| `yao-meta-skill` | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=adversarial_summary_translation_collision | +| `team-frontend-review` | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=adversarial_quality_gate_review | +| `governed-incident-command` | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=adversarial_single_update_collision | diff --git a/reports/iteration_ledger.md b/reports/iteration_ledger.md index cc263fd..113981f 100644 --- a/reports/iteration_ledger.md +++ b/reports/iteration_ledger.md @@ -12,17 +12,20 @@ ## Description Optimization Milestones -| Date | Label | Target | Blind Errors | Adversarial Errors | Adversarial Gap | Adversarial Risk | Drift Note | -| --- | --- | --- | ---: | ---: | ---: | --- | --- | -| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | - | - | - | - | initial description optimization suite without blind holdout | -| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | - | - | - | - | compressed example description and stored first holdout report | -| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | - | - | - | - | compressed governed description and reduced visible holdout misses | -| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | 0 | 0 | 0.834 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy | -| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | 0 | 0 | 0.509 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy | -| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | 0 | 0 | 0.598 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy | -| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | 0 | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | -| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | 0 | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | -| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | 0 | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 1 | +| Date | Label | Target | Blind Errors | Judge Blind Errors | Judge Agreement | Adversarial Errors | Adversarial Gap | Adversarial Risk | Drift Note | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- | +| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | - | - | - | - | - | - | initial description optimization suite without blind holdout | +| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | - | - | - | - | - | - | compressed example description and stored first holdout report | +| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | - | - | - | - | - | - | compressed governed description and reduced visible holdout misses | +| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | 0 | - | - | 0 | 0.834 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy | +| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | 0 | - | - | 0 | 0.509 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy | +| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | 0 | - | - | 0 | 0.598 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy | +| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | 0 | - | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | +| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | 0 | - | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | +| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | 0 | - | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 1 | +| 2026-04-01 | Judge-Backed Blind Eval | `yao-meta-skill` | 0 | 0 | 1.0 | 0 | 0.834 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy | +| 2026-04-01 | Judge-Backed Blind Eval | `team-frontend-review` | 0 | 0 | 1.0 | 0 | 0.509 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy | +| 2026-04-01 | Judge-Backed Blind Eval | `governed-incident-command` | 0 | 0 | 1.0 | 0 | 0.598 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy | ## Current Route Scorecard diff --git a/scripts/judge_blind_eval.py b/scripts/judge_blind_eval.py new file mode 100644 index 0000000..a3ff33d --- /dev/null +++ b/scripts/judge_blind_eval.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + +from trigger_eval import ( + collect_concept_hits, + desired_positive_concepts, + extract_description, + iter_case_items, + load_json, + load_semantic_config, + normalize, + phrase_present, + words, +) + + +DEFAULT_CONFIG_PATH = Path("evals/semantic_config.json") + + +def phrase_hits(text: str, phrases: list[str]) -> list[str]: + normalized = normalize(text) + return [phrase for phrase in phrases if phrase_present(normalized, phrase)] + + +def judge_prompt(description: str, prompt: str, config: dict) -> tuple[bool, dict]: + hints = config.get("optimizer_hints", {}) + generic_exclusion_phrases = [ + "explain", + "summarize", + "translate", + "brainstorm", + "teach me", + "plain english", + ] + desired = desired_positive_concepts(description, config) + positive_hits = collect_concept_hits(prompt, config["positive_concepts"]) + negative_hits = collect_concept_hits(prompt, config["negative_concepts"]) + + focused_positive = [name for name in desired if name in positive_hits] + strong_positive = [ + name + for name in focused_positive + if config["positive_concepts"][name]["weight"] >= 0.16 + ] + trigger_action_hits = phrase_hits(prompt, hints.get("trigger_actions", [])) + input_hits = phrase_hits(prompt, hints.get("inputs", [])) + artifact_hits = phrase_hits(prompt, hints.get("artifacts", [])) + exclusion_hits = phrase_hits(prompt, hints.get("exclusions", [])) + generic_exclusion_hits = phrase_hits(prompt, generic_exclusion_phrases) + + capability_words = words(hints.get("capability", "")) + prompt_words = words(prompt) + capability_overlap = len(capability_words & prompt_words) + + exclusive_negative = sorted( + name for name, hit in negative_hits.items() if hit.get("exclusive") + ) + nonexclusive_negative = sorted( + name for name, hit in negative_hits.items() if not hit.get("exclusive") + ) + + positive_vote = 0 + if trigger_action_hits: + positive_vote += 2 + if len(strong_positive) >= 2: + positive_vote += 2 + elif len(strong_positive) == 1: + positive_vote += 1 + if len(focused_positive) >= 3: + positive_vote += 1 + if input_hits or artifact_hits: + positive_vote += 1 + if capability_overlap >= 1: + positive_vote += 1 + + negative_vote = (2 * len(exclusive_negative)) + len(nonexclusive_negative) + if exclusion_hits: + negative_vote += 1 + if generic_exclusion_hits: + negative_vote += 1 + + trigger = positive_vote >= 3 and positive_vote > negative_vote + if exclusive_negative and positive_vote < 5: + trigger = False + if exclusion_hits and positive_vote < 4: + trigger = False + if generic_exclusion_hits and positive_vote < 4: + trigger = False + + margin = positive_vote - negative_vote + confidence = 0.5 + (0.08 * max(0, margin)) + if exclusive_negative: + confidence += 0.08 + if trigger_action_hits and len(strong_positive) >= 1: + confidence += 0.06 + confidence = max(0.0, min(1.0, confidence)) + + detail = { + "mode": "judge-rubric", + "focused_positive_concepts": focused_positive, + "strong_positive_concepts": strong_positive, + "trigger_action_hits": trigger_action_hits, + "input_hits": input_hits, + "artifact_hits": artifact_hits, + "capability_overlap": capability_overlap, + "exclusive_negative_concepts": exclusive_negative, + "nonexclusive_negative_concepts": nonexclusive_negative, + "exclusion_hits": exclusion_hits, + "generic_exclusion_hits": generic_exclusion_hits, + "positive_vote": positive_vote, + "negative_vote": negative_vote, + "margin": margin, + "confidence": round(confidence, 3), + "concept_evidence": { + "positive": { + name: positive_hits[name]["matched_phrases"] + for name in sorted(positive_hits) + }, + "negative": { + name: negative_hits[name]["matched_phrases"] + for name in sorted(negative_hits) + }, + }, + } + return trigger, detail + + +def classify_bucket(bucket: str) -> bool: + return bucket == "should_trigger" + + +def evaluate_judge(description: str, cases: dict, config: dict) -> dict: + results = {"should_trigger": [], "should_not_trigger": [], "near_neighbor": []} + fp = 0 + fn = 0 + bucket_stats = {} + family_stats: dict[str, dict] = {} + misfires = [] + confidence_total = 0.0 + confidence_count = 0 + + for bucket in ("should_trigger", "should_not_trigger", "near_neighbor"): + expected = classify_bucket(bucket) + items = iter_case_items(cases, bucket) + total = 0 + passed_count = 0 + for item in items: + prompt = item["text"] + family = item.get("family", "default") + predicted, detail = judge_prompt(description, prompt, config) + passed = predicted == expected + total += 1 + confidence_total += detail["confidence"] + confidence_count += 1 + if passed: + passed_count += 1 + if not passed and expected: + fn += 1 + if not passed and not expected: + fp += 1 + + record = { + "prompt": prompt, + "family": family, + "predicted_trigger": predicted, + "expected_trigger": expected, + "passed": passed, + "judge_detail": detail, + } + if abs(detail["margin"]) <= 1: + record["boundary_case"] = True + results[bucket].append(record) + + family_bucket = family_stats.setdefault( + family, + {"total": 0, "passed": 0, "false_positives": 0, "false_negatives": 0}, + ) + family_bucket["total"] += 1 + if passed: + family_bucket["passed"] += 1 + elif expected: + family_bucket["false_negatives"] += 1 + else: + family_bucket["false_positives"] += 1 + + if not passed: + misfires.append( + { + "bucket": bucket, + "family": family, + "prompt": prompt, + "reason": "false_negative" if expected else "false_positive", + "focused_positive_concepts": detail["focused_positive_concepts"], + "exclusive_negative_concepts": detail["exclusive_negative_concepts"], + "trigger_action_hits": detail["trigger_action_hits"], + "margin": detail["margin"], + } + ) + + bucket_stats[bucket] = { + "total": total, + "passed": passed_count, + "pass_rate": round(passed_count / total, 3) if total else None, + } + + for family, stats in family_stats.items(): + stats["pass_rate"] = round(stats["passed"] / stats["total"], 3) if stats["total"] else None + + tp = sum(1 for item in results["should_trigger"] if item["predicted_trigger"]) + precision = tp / (tp + fp) if (tp + fp) else None + recall = tp / (tp + fn) if (tp + fn) else None + + return { + "judge": "rubric-blind-v1", + "judge_explanation": ( + "The blind judge uses a rubric rather than the main threshold scorer. " + "It looks for trigger-action evidence, focused capability evidence, and " + "input or artifact evidence, then blocks on explicit exclusion and " + "exclusive negative signals. This acts as an independent second opinion " + "for blind-holdout prompts." + ), + "false_positives": fp, + "false_negatives": fn, + "precision": round(precision, 3) if precision is not None else None, + "recall": round(recall, 3) if recall is not None else None, + "bucket_stats": bucket_stats, + "family_stats": family_stats, + "misfires": misfires, + "results": results, + "judge_summary": { + "agreement_rate": round( + sum(bucket["passed"] for bucket in bucket_stats.values()) + / sum(bucket["total"] for bucket in bucket_stats.values()), + 3, + ) + if bucket_stats + else None, + "mean_confidence": round(confidence_total / confidence_count, 3) + if confidence_count + else None, + "rubric_version": "blind-v1", + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run a rubric-based blind trigger judge.") + parser.add_argument("--description", help="Description string to evaluate") + parser.add_argument("--description-file", help="Read description text from file") + parser.add_argument("--cases", required=True, help="JSON file with blind trigger cases") + parser.add_argument("--semantic-config", default=str(DEFAULT_CONFIG_PATH), help="Semantic config JSON") + args = parser.parse_args() + + description = args.description + if args.description_file: + description = extract_description(Path(args.description_file).read_text(encoding="utf-8")) + if not description: + raise SystemExit("Provide --description or --description-file") + + cases = load_json(Path(args.cases)) + config = load_semantic_config(Path(args.semantic_config)) + report = evaluate_judge(description, cases, config) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if report["false_positives"] > 0 or report["false_negatives"] > 0: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/optimize_description.py b/scripts/optimize_description.py index 4958ca1..560626e 100644 --- a/scripts/optimize_description.py +++ b/scripts/optimize_description.py @@ -4,6 +4,7 @@ import json from pathlib import Path from context_sizer import estimate_tokens +from judge_blind_eval import evaluate_judge from trigger_eval import ( compare_reports, evaluate, @@ -168,7 +169,7 @@ def summarize_candidate(candidate: dict, dev_report: dict, holdout_report: dict def summarize_gate_report(report: dict | None) -> dict | None: if not report: return None - return { + summary = { "false_positives": report["false_positives"], "false_negatives": report["false_negatives"], "precision": report["precision"], @@ -176,6 +177,9 @@ def summarize_gate_report(report: dict | None) -> dict | None: "near_neighbor_pass_rate": report["bucket_stats"]["near_neighbor"]["pass_rate"], "should_not_trigger_pass_rate": report["bucket_stats"]["should_not_trigger"]["pass_rate"], } + if report.get("judge_summary"): + summary["judge_summary"] = report["judge_summary"] + return summary def error_tuple(report: dict | None) -> tuple[int, int] | None: @@ -356,6 +360,13 @@ def optimize( baseline["description"], blind_holdout_cases, blind_holdout_threshold, config ) + judge_blind_reports = {} + if blind_holdout_cases: + judge_blind_reports["current"] = evaluate_judge(current["candidate"]["description"], blind_holdout_cases, config) + judge_blind_reports["winner"] = evaluate_judge(winner["candidate"]["description"], blind_holdout_cases, config) + if baseline: + judge_blind_reports["baseline"] = evaluate_judge(baseline["description"], blind_holdout_cases, config) + adversarial_reports = {} if adversarial_cases: adversarial_reports["current"] = evaluate( @@ -381,6 +392,9 @@ def optimize( "winner_blind_holdout_report": blind_reports.get("winner"), "current_blind_holdout_report": blind_reports.get("current"), "baseline_blind_holdout_report": blind_reports.get("baseline"), + "winner_judge_blind_holdout_report": judge_blind_reports.get("winner"), + "current_judge_blind_holdout_report": judge_blind_reports.get("current"), + "baseline_judge_blind_holdout_report": judge_blind_reports.get("baseline"), "winner_adversarial_holdout_report": adversarial_reports.get("winner"), "current_adversarial_holdout_report": adversarial_reports.get("current"), "baseline_adversarial_holdout_report": adversarial_reports.get("baseline"), @@ -411,6 +425,16 @@ def optimize( "winner_vs_baseline_blind_holdout": compare_reports(blind_reports["baseline"], blind_reports["winner"]) if blind_reports.get("baseline") and blind_reports.get("winner") else None, + "winner_vs_current_judge_blind_holdout": compare_reports( + judge_blind_reports["current"], judge_blind_reports["winner"] + ) + if judge_blind_reports.get("current") and judge_blind_reports.get("winner") + else None, + "winner_vs_baseline_judge_blind_holdout": compare_reports( + judge_blind_reports["baseline"], judge_blind_reports["winner"] + ) + if judge_blind_reports.get("baseline") and judge_blind_reports.get("winner") + else None, "winner_vs_current_adversarial_holdout": compare_reports( adversarial_reports["current"], adversarial_reports["winner"] ) @@ -436,6 +460,12 @@ def optimize( blind_reports.get("baseline"), blind_holdout_threshold if blind_holdout_cases else None, ), + "judge_blind_holdout_non_regression": build_gate_summary( + judge_blind_reports.get("winner"), + judge_blind_reports.get("current"), + judge_blind_reports.get("baseline"), + None, + ), "adversarial_holdout_non_regression": build_gate_summary( adversarial_reports.get("winner"), adversarial_reports.get("current"), @@ -464,12 +494,23 @@ def optimize( "current_blind_holdout_total_errors": sum(error_tuple(blind_reports.get("current"))) if blind_reports.get("current") else None, + "winner_judge_blind_holdout_total_errors": sum(error_tuple(judge_blind_reports.get("winner"))) + if judge_blind_reports.get("winner") + else None, + "current_judge_blind_holdout_total_errors": sum(error_tuple(judge_blind_reports.get("current"))) + if judge_blind_reports.get("current") + else None, "winner_adversarial_holdout_total_errors": sum(error_tuple(adversarial_reports.get("winner"))) if adversarial_reports.get("winner") else None, "current_adversarial_holdout_total_errors": sum(error_tuple(adversarial_reports.get("current"))) if adversarial_reports.get("current") else None, + "winner_judge_blind_agreement_rate": ( + report["acceptance_gates"]["judge_blind_holdout_non_regression"]["winner"].get("judge_summary", {}).get("agreement_rate") + if report["acceptance_gates"]["judge_blind_holdout_non_regression"]["winner"] + else None + ), "winner_adversarial_risk_band": report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"]["risk_band"] if report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"] else None, @@ -489,6 +530,9 @@ def optimize( report["summary"]["baseline_blind_holdout_total_errors"] = ( sum(error_tuple(blind_reports.get("baseline"))) if blind_reports.get("baseline") else None ) + report["summary"]["baseline_judge_blind_holdout_total_errors"] = ( + sum(error_tuple(judge_blind_reports.get("baseline"))) if judge_blind_reports.get("baseline") else None + ) report["summary"]["baseline_adversarial_holdout_total_errors"] = ( sum(error_tuple(adversarial_reports.get("baseline"))) if adversarial_reports.get("baseline") else None ) @@ -537,6 +581,7 @@ def render_markdown(report: dict, title: str) -> str: for gate_name, gate in ( ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), ): winner_gate = gate.get("winner") or {} @@ -560,6 +605,7 @@ def render_markdown(report: dict, title: str) -> str: for gate_name, gate in ( ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), ): winner_calibration = gate.get("winner_calibration") or {} @@ -571,6 +617,24 @@ def render_markdown(report: dict, title: str) -> str: f"| {gate_name} | {winner_calibration.get('score_gap', '-')} | {winner_calibration.get('risk_band', '-')} | {winner_calibration.get('boundary_case_rate', '-')} | {current_calibration.get('score_gap', '-')} | {baseline_calibration.get('score_gap', '-')} |" ) + lines.extend( + [ + "", + "## Judge Blind Summary", + "", + "| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement |", + "| --- | ---: | ---: | ---: | ---: |", + ] + ) + judge_gate = report["acceptance_gates"]["judge_blind_holdout_non_regression"] + judge_winner = (judge_gate.get("winner") or {}).get("judge_summary") or {} + judge_current = (judge_gate.get("current") or {}).get("judge_summary") or {} + judge_baseline = (judge_gate.get("baseline") or {}).get("judge_summary") or {} + if judge_winner or judge_current or judge_baseline: + lines.append( + f"| Judge Blind Holdout | {judge_winner.get('agreement_rate', '-')} | {judge_winner.get('mean_confidence', '-')} | {judge_current.get('agreement_rate', '-')} | {judge_baseline.get('agreement_rate', '-')} |" + ) + lines.extend( [ "", @@ -583,6 +647,7 @@ def render_markdown(report: dict, title: str) -> str: for gate_name, gate in ( ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), ): winner_health = gate.get("winner_family_health") or {} diff --git a/scripts/render_description_drift_history.py b/scripts/render_description_drift_history.py index adab4b7..88578d7 100644 --- a/scripts/render_description_drift_history.py +++ b/scripts/render_description_drift_history.py @@ -15,8 +15,8 @@ def render_markdown(snapshots: list[dict]) -> str: lines = [ "# Description Drift History", "", - "| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Drift Note |", - "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", + "| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Judge Blind Errors | Judge Agreement | Adv FP | Adv FN | Adv Gap | Adv Risk | Drift Note |", + "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", ] previous_by_target: dict[str, dict] = {} @@ -55,8 +55,10 @@ def render_markdown(snapshots: list[dict]) -> str: if not drift_note: drift_note = "initial snapshot" adversarial_calibration = (target.get("calibration", {}) or {}).get("adversarial_holdout") or {} + judge_blind = (target.get("judge_blind") or {}).get("winner") or {} + judge_errors = target.get("winner_judge_blind_holdout_total_errors") lines.append( - f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {display(target.get('winner_blind_holdout_fp'))} | {display(target.get('winner_blind_holdout_fn'))} | {display(target.get('winner_adversarial_holdout_fp'))} | {display(target.get('winner_adversarial_holdout_fn'))} | {display(adversarial_calibration.get('score_gap'))} | {display(adversarial_calibration.get('risk_band'))} | {drift_note} |" + f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {display(target.get('winner_blind_holdout_fp'))} | {display(target.get('winner_blind_holdout_fn'))} | {display(judge_errors)} | {display(judge_blind.get('agreement_rate'))} | {display(target.get('winner_adversarial_holdout_fp'))} | {display(target.get('winner_adversarial_holdout_fn'))} | {display(adversarial_calibration.get('score_gap'))} | {display(adversarial_calibration.get('risk_band'))} | {drift_note} |" ) previous_by_target[target["name"]] = target @@ -65,14 +67,14 @@ def render_markdown(snapshots: list[dict]) -> str: "", "## Family Coverage", "", - "| Date | Label | Target | Blind Families | Adversarial Families |", - "| --- | --- | --- | --- | --- |", + "| Date | Label | Target | Blind Families | Judge Blind Families | Adversarial Families |", + "| --- | --- | --- | --- | --- | --- |", ] ) for snapshot in snapshots: for target in snapshot.get("targets", []): lines.append( - f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | {family_display(target, 'blind_holdout')} | {family_display(target, 'adversarial_holdout')} |" + f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | {family_display(target, 'blind_holdout')} | {family_display(target, 'judge_blind_holdout')} | {family_display(target, 'adversarial_holdout')} |" ) return "\n".join(lines) + "\n" diff --git a/scripts/render_iteration_ledger.py b/scripts/render_iteration_ledger.py index b2af776..a85d394 100644 --- a/scripts/render_iteration_ledger.py +++ b/scripts/render_iteration_ledger.py @@ -38,17 +38,19 @@ def render_markdown(system_snapshots: list[dict], description_snapshots: list[di "", "## Description Optimization Milestones", "", - "| Date | Label | Target | Blind Errors | Adversarial Errors | Adversarial Gap | Adversarial Risk | Drift Note |", - "| --- | --- | --- | ---: | ---: | ---: | --- | --- |", + "| Date | Label | Target | Blind Errors | Judge Blind Errors | Judge Agreement | Adversarial Errors | Adversarial Gap | Adversarial Risk | Drift Note |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- |", ] ) for snapshot in description_snapshots: for target in snapshot.get("targets", []): blind_errors = target.get("winner_blind_holdout_total_errors") + judge_blind_errors = target.get("winner_judge_blind_holdout_total_errors") + judge_blind = (target.get("judge_blind") or {}).get("winner") or {} adversarial_errors = target.get("winner_adversarial_holdout_total_errors") adversarial_calibration = (target.get("calibration", {}) or {}).get("adversarial_holdout") or {} lines.append( - f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | {'-' if blind_errors is None else blind_errors} | {'-' if adversarial_errors is None else adversarial_errors} | {'-' if adversarial_calibration.get('score_gap') is None else adversarial_calibration.get('score_gap')} | {adversarial_calibration.get('risk_band', '-')} | {target.get('drift_note', '-')} |" + f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | {'-' if blind_errors is None else blind_errors} | {'-' if judge_blind_errors is None else judge_blind_errors} | {judge_blind.get('agreement_rate', '-')} | {'-' if adversarial_errors is None else adversarial_errors} | {'-' if adversarial_calibration.get('score_gap') is None else adversarial_calibration.get('score_gap')} | {adversarial_calibration.get('risk_band', '-')} | {target.get('drift_note', '-')} |" ) lines.extend( diff --git a/scripts/run_description_optimization_suite.py b/scripts/run_description_optimization_suite.py index 1dcdada..4bfbe0c 100644 --- a/scripts/run_description_optimization_suite.py +++ b/scripts/run_description_optimization_suite.py @@ -174,7 +174,7 @@ def build_history_snapshot(summary: dict, args: argparse.Namespace) -> dict: "label": args.snapshot_label, "targets": targets, "notes": [ - "recorded family-level blind and adversarial routing evidence", + "recorded family-level blind, judge-backed blind, and adversarial routing evidence", "published calibration and drift history for description optimization", ], } @@ -218,6 +218,15 @@ def main() -> None: blind_winner_fp, blind_winner_fn = report_errors(report["acceptance_gates"]["blind_holdout_non_regression"]["winner"]) blind_current_fp, blind_current_fn = report_errors(report["acceptance_gates"]["blind_holdout_non_regression"]["current"]) blind_baseline_fp, blind_baseline_fn = report_errors(report["acceptance_gates"]["blind_holdout_non_regression"]["baseline"]) + judge_blind_winner_fp, judge_blind_winner_fn = report_errors( + report["acceptance_gates"]["judge_blind_holdout_non_regression"]["winner"] + ) + judge_blind_current_fp, judge_blind_current_fn = report_errors( + report["acceptance_gates"]["judge_blind_holdout_non_regression"]["current"] + ) + judge_blind_baseline_fp, judge_blind_baseline_fn = report_errors( + report["acceptance_gates"]["judge_blind_holdout_non_regression"]["baseline"] + ) adversarial_winner_fp, adversarial_winner_fn = report_errors( report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner"] ) @@ -233,6 +242,8 @@ def main() -> None: and (winner_fp, winner_fn) <= (baseline_fp, baseline_fn) and (blind_winner_fp, blind_winner_fn) <= (blind_current_fp, blind_current_fn) and (blind_winner_fp, blind_winner_fn) <= (blind_baseline_fp, blind_baseline_fn) + and (judge_blind_winner_fp, judge_blind_winner_fn) <= (judge_blind_current_fp, judge_blind_current_fn) + and (judge_blind_winner_fp, judge_blind_winner_fn) <= (judge_blind_baseline_fp, judge_blind_baseline_fn) and (adversarial_winner_fp, adversarial_winner_fn) <= (adversarial_current_fp, adversarial_current_fn) and (adversarial_winner_fp, adversarial_winner_fn) <= (adversarial_baseline_fp, adversarial_baseline_fn) ) @@ -256,6 +267,13 @@ def main() -> None: "baseline_blind_holdout_fp": blind_baseline_fp, "baseline_blind_holdout_fn": blind_baseline_fn, "winner_blind_holdout_total_errors": blind_winner_fp + blind_winner_fn, + "winner_judge_blind_holdout_fp": judge_blind_winner_fp, + "winner_judge_blind_holdout_fn": judge_blind_winner_fn, + "current_judge_blind_holdout_fp": judge_blind_current_fp, + "current_judge_blind_holdout_fn": judge_blind_current_fn, + "baseline_judge_blind_holdout_fp": judge_blind_baseline_fp, + "baseline_judge_blind_holdout_fn": judge_blind_baseline_fn, + "winner_judge_blind_holdout_total_errors": judge_blind_winner_fp + judge_blind_winner_fn, "winner_adversarial_holdout_fp": adversarial_winner_fp, "winner_adversarial_holdout_fn": adversarial_winner_fn, "current_adversarial_holdout_fp": adversarial_current_fp, @@ -268,12 +286,18 @@ def main() -> None: "blind_holdout": report["acceptance_gates"]["blind_holdout_non_regression"]["winner_calibration"], "adversarial_holdout": report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"], }, + "judge_blind": { + "winner": (report["acceptance_gates"]["judge_blind_holdout_non_regression"]["winner"] or {}).get("judge_summary"), + "current": (report["acceptance_gates"]["judge_blind_holdout_non_regression"]["current"] or {}).get("judge_summary"), + "baseline": (report["acceptance_gates"]["judge_blind_holdout_non_regression"]["baseline"] or {}).get("judge_summary"), + }, "family_health": { "holdout": report["acceptance_gates"]["holdout_non_regression"]["winner_family_health"], "blind_holdout": report["acceptance_gates"]["blind_holdout_non_regression"]["winner_family_health"], + "judge_blind_holdout": report["acceptance_gates"]["judge_blind_holdout_non_regression"]["winner_family_health"], "adversarial_holdout": report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_family_health"], }, - "drift_note": "blind, adversarial, and calibration gates active", + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", "ok": target_ok, } ) @@ -285,25 +309,25 @@ def main() -> None: lines = [ "# Description Optimization Suite", "", - "| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Status |", - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", + "| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Judge Blind Errors | Adv FP | Adv FN | Adv Gap | Adv Risk | Status |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", ] for target in summary["targets"]: lines.append( - f"| `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {target['winner_blind_holdout_fp']} | {target['winner_blind_holdout_fn']} | {target['winner_adversarial_holdout_fp']} | {target['winner_adversarial_holdout_fn']} | {(target['calibration']['adversarial_holdout'] or {}).get('score_gap', '-')} | {(target['calibration']['adversarial_holdout'] or {}).get('risk_band', '-')} | {'ok' if target['ok'] else 'fail'} |" + f"| `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {target['winner_blind_holdout_fp']} | {target['winner_blind_holdout_fn']} | {target['winner_judge_blind_holdout_total_errors']} | {target['winner_adversarial_holdout_fp']} | {target['winner_adversarial_holdout_fn']} | {(target['calibration']['adversarial_holdout'] or {}).get('score_gap', '-')} | {(target['calibration']['adversarial_holdout'] or {}).get('risk_band', '-')} | {'ok' if target['ok'] else 'fail'} |" ) lines.extend( [ "", "## Family Coverage", "", - "| Target | Blind Families | Adversarial Families |", - "| --- | --- | --- |", + "| Target | Blind Families | Judge Blind Families | Adversarial Families |", + "| --- | --- | --- | --- |", ] ) for target in summary["targets"]: lines.append( - f"| `{target['name']}` | {family_gate_note(target, 'blind_holdout')} | {family_gate_note(target, 'adversarial_holdout')} |" + f"| `{target['name']}` | {family_gate_note(target, 'blind_holdout')} | {family_gate_note(target, 'judge_blind_holdout')} | {family_gate_note(target, 'adversarial_holdout')} |" ) (ROOT / "reports" / "description_optimization_suite.md").write_text("\n".join(lines) + "\n", encoding="utf-8") if args.history_snapshot_output: diff --git a/tests/verify_description_optimization.py b/tests/verify_description_optimization.py index 070326b..fe5c37a 100644 --- a/tests/verify_description_optimization.py +++ b/tests/verify_description_optimization.py @@ -86,6 +86,12 @@ def run_case(case: dict) -> dict: passed = passed and blind_winner is not None and total_errors(blind_winner) <= total_errors(blind_current) if blind_baseline: passed = passed and total_errors(blind_winner) <= total_errors(blind_baseline) + judge_blind_winner = payload["acceptance_gates"]["judge_blind_holdout_non_regression"]["winner"] + judge_blind_current = payload["acceptance_gates"]["judge_blind_holdout_non_regression"]["current"] + judge_blind_baseline = payload["acceptance_gates"]["judge_blind_holdout_non_regression"]["baseline"] + passed = passed and judge_blind_winner is not None and total_errors(judge_blind_winner) <= total_errors(judge_blind_current) + if judge_blind_baseline: + passed = passed and total_errors(judge_blind_winner) <= total_errors(judge_blind_baseline) adversarial_winner = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["winner"] adversarial_current = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["current"] adversarial_baseline = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["baseline"] @@ -94,9 +100,14 @@ def run_case(case: dict) -> dict: passed = passed and total_errors(adversarial_winner) <= total_errors(adversarial_baseline) winner_adversarial_calibration = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"] winner_adversarial_family = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_family_health"] + winner_judge_summary = (judge_blind_winner or {}).get("judge_summary") or {} + winner_judge_family = payload["acceptance_gates"]["judge_blind_holdout_non_regression"]["winner_family_health"] passed = passed and winner_adversarial_calibration is not None passed = passed and winner_adversarial_family is not None passed = passed and winner_adversarial_family.get("family_count", 0) >= 2 + passed = passed and winner_judge_summary.get("agreement_rate") is not None + passed = passed and winner_judge_family is not None + passed = passed and winner_judge_family.get("family_count", 0) >= 2 passed = passed and len(payload.get("candidates", [])) >= 3 return { "name": case["name"], @@ -108,6 +119,8 @@ def run_case(case: dict) -> dict: "current_tokens": current["estimated_tokens"], "baseline_tokens": baseline["estimated_tokens"] if baseline else None, "winner_blind_holdout_total_errors": total_errors(blind_winner) if blind_winner else None, + "winner_judge_blind_holdout_total_errors": total_errors(judge_blind_winner) if judge_blind_winner else None, + "winner_judge_blind_agreement_rate": winner_judge_summary.get("agreement_rate"), "winner_adversarial_holdout_total_errors": total_errors(adversarial_winner) if adversarial_winner else None, "winner_adversarial_risk_band": winner_adversarial_calibration.get("risk_band") if winner_adversarial_calibration else None, }