From 51a479d12ffe8b5a3abb68c504feeeb7bcc87b5e Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 31 Mar 2026 21:37:01 +0800 Subject: [PATCH] feat: expand family-based trigger regression suite --- Makefile | 7 +- README.md | 8 +- evals/README.md | 4 +- evals/dev/trigger_cases.json | 69 +- evals/holdout/trigger_cases.json | 37 +- evals/sample_trigger_report.json | 1622 ++++++++++++----- evals/semantic_config.json | 10 + evals/train/trigger_cases.json | 105 +- evals/trigger_cases.json | 211 ++- failures/README.md | 2 + .../document-export-vs-agent-skill/cases.json | 24 + failures/explain-not-package/cases.json | 24 + failures/future-outline-vs-build/cases.json | 19 + failures/one-off-vs-reusable/cases.json | 19 + scripts/run_eval_suite.py | 15 +- tests/verify_failure_regressions.py | 66 + 16 files changed, 1756 insertions(+), 486 deletions(-) create mode 100644 failures/document-export-vs-agent-skill/cases.json create mode 100644 failures/explain-not-package/cases.json create mode 100644 failures/future-outline-vs-build/cases.json create mode 100644 failures/one-off-vs-reusable/cases.json create mode 100644 tests/verify_failure_regressions.py diff --git a/Makefile b/Makefile index 33bbe36..3fd29ad 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite package-check package-failure-check snapshot-check validate lint test clean +.PHONY: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -8,6 +8,9 @@ eval: eval-suite: $(PYTHON) scripts/run_eval_suite.py +failure-regression-check: + $(PYTHON) tests/verify_failure_regressions.py + package-check: $(PYTHON) scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --output-dir dist --zip @@ -23,7 +26,7 @@ validate: lint: $(PYTHON) scripts/lint_skill.py . -test: eval eval-suite package-check package-failure-check snapshot-check validate lint +test: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint clean: rm -rf dist tests/tmp tests/tmp_snapshot diff --git a/README.md b/README.md index bf9bd9f..b79a5ab 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,10 @@ make test - `make test` passes locally - semantic trigger eval: `0` false positives, `0` false negatives on the current regression set -- eval suite: train / dev / holdout all pass +- eval suite: train / dev / holdout all pass, with family-based reporting - packaging validation: `openai`, `claude`, and `generic` targets pass contract checks - packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected +- failure library regressions: anti-pattern families pass automated checks ## What It Does @@ -111,7 +112,7 @@ Long-form material that should not bloat the main skill file. This includes desi Utility scripts that make the meta-skill operational: - `trigger_eval.py`: evaluates trigger descriptions with semantic intent concepts, explicit exclusions, and near-neighbor prompts -- `run_eval_suite.py`: runs train/dev/holdout trigger suites and fails if aggregate regressions appear +- `run_eval_suite.py`: runs train/dev/holdout trigger suites, reports family-level regressions, and fails if aggregate regressions appear - `context_sizer.py`: estimates context weight and warns when the initial load gets too large - `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain @@ -135,7 +136,7 @@ Continuous integration entrypoint that runs the full local regression suite on p - Train/dev/holdout trigger suites now separate iterative tuning from final verification. - Packaging validation now uses explicit contracts and YAML parsing, but it is still a lightweight local validation layer rather than a full platform integration suite. - `evals/failure-cases.md` captures known weak spots that should remain part of regression checks. -- `failures/` captures reusable anti-pattern writeups for routing, packaging, and authoring failures. +- `failures/` captures reusable anti-pattern writeups and machine-runnable failure cases for routing, packaging, and authoring failures. - `tests/verify_packager_failures.py` checks that invalid metadata, invalid YAML, and unsupported targets fail clearly. ### `templates/` @@ -207,6 +208,7 @@ This project is best for: - Examples: [examples/README.md](examples/README.md) - Evals: [evals/README.md](evals/README.md) - Failure library: [failures/README.md](failures/README.md) +- Failure regression check: [verify_failure_regressions.py](tests/verify_failure_regressions.py) - Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md) - Platform capability matrix: [references/platform-capability-matrix.md](references/platform-capability-matrix.md) - Failure fixtures: [tests/fixtures](tests/fixtures) diff --git a/evals/README.md b/evals/README.md index 1b784ea..a60cc36 100644 --- a/evals/README.md +++ b/evals/README.md @@ -4,7 +4,7 @@ This directory makes trigger quality and packaging quality more reproducible. Contents: -- `trigger_cases.json`: positive, negative, and near-neighbor prompts +- `trigger_cases.json`: full regression set with `family` labels - `train/`, `dev/`, `holdout/`: split trigger suites for iterative tuning and final verification - `semantic_config.json`: local semantic-intent concepts, exclusions, and weights - `baseline_description.txt`: intentionally weaker trigger description @@ -19,6 +19,7 @@ Use: python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt python3 scripts/run_eval_suite.py +python3 tests/verify_failure_regressions.py python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip python3 tests/verify_packager_failures.py ``` @@ -32,4 +33,5 @@ Regression scope now includes: - mixed-intent negatives - explicit "do not build a skill" negatives - semantic exclusion cases such as one-off, document-only, and future-outline prompts +- family-based reporting across workflow-to-skill, iterate-existing-skill, document-only, one-off, and future-outline cases - holdout verification diff --git a/evals/dev/trigger_cases.json b/evals/dev/trigger_cases.json index 2df8427..6dd5da6 100644 --- a/evals/dev/trigger_cases.json +++ b/evals/dev/trigger_cases.json @@ -10,18 +10,69 @@ "summarize" ], "should_trigger": [ - "Convert this operations checklist into a reusable skill.", - "Add trigger evals to this skill before sharing it with the team.", - "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks." + { + "text": "Convert this operations checklist into a reusable skill.", + "family": "workflow_to_skill" + }, + { + "text": "Add trigger evals to this skill before sharing it with the team.", + "family": "iterate_existing_skill" + }, + { + "text": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", + "family": "complex_multi_asset" + }, + { + "text": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", + "family": "complex_multi_asset" + }, + { + "text": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", + "family": "iterate_existing_skill" + } ], "should_not_trigger": [ - "Give me ideas for improving our process.", - "Review this note and tell me what it means in plain English.", - "Help me brainstorm several ways to improve packaging, but do not generate any skill files." + { + "text": "Give me ideas for improving our process.", + "family": "brainstorm_only" + }, + { + "text": "Review this note and tell me what it means in plain English.", + "family": "explain_only" + }, + { + "text": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", + "family": "brainstorm_only" + }, + { + "text": "Translate these notes into French and keep the structure exactly as-is.", + "family": "translate_only" + }, + { + "text": "Summarize this workflow and list the main points only.", + "family": "summary_only" + } ], "near_neighbor": [ - "Review this process note and explain it, no packaging needed.", - "Turn this into a checklist and template, but stop short of making a full skill.", - "Package this explanation as a document, not as an agent skill." + { + "text": "Review this process note and explain it, no packaging needed.", + "family": "explain_not_package" + }, + { + "text": "Turn this into a checklist and template, but stop short of making a full skill.", + "family": "partial_scaffold_not_full_skill" + }, + { + "text": "Package this explanation as a document, not as an agent skill.", + "family": "document_export_vs_agent_skill" + }, + { + "text": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build" + }, + { + "text": "Polish these notes into a reusable-looking document without turning them into an agent skill.", + "family": "document_export_vs_agent_skill" + } ] } diff --git a/evals/holdout/trigger_cases.json b/evals/holdout/trigger_cases.json index a21c77d..2c1f3bd 100644 --- a/evals/holdout/trigger_cases.json +++ b/evals/holdout/trigger_cases.json @@ -10,14 +10,41 @@ "summarize" ], "should_trigger": [ - "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", - "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution." + { + "text": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "family": "complex_multi_asset" + }, + { + "text": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", + "family": "iterate_existing_skill" + }, + { + "text": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", + "family": "workflow_to_skill" + } ], "should_not_trigger": [ - "I pasted a long process description below, but I only want a summary, not a reusable skill." + { + "text": "I pasted a long process description below, but I only want a summary, not a reusable skill.", + "family": "summary_only" + }, + { + "text": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_only" + } ], "near_neighbor": [ - "Help me shape an idea before we decide whether to build a skill.", - "Create an outline for a possible future skill, but do not build the skill yet." + { + "text": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build" + }, + { + "text": "Create an outline for a possible future skill, but do not build the skill yet.", + "family": "future_outline_vs_build" + }, + { + "text": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill" + } ] } diff --git a/evals/sample_trigger_report.json b/evals/sample_trigger_report.json index f8a87cb..f4b4515 100644 --- a/evals/sample_trigger_report.json +++ b/evals/sample_trigger_report.json @@ -7,25 +7,130 @@ "recall": 1.0, "bucket_stats": { "should_trigger": { - "total": 11, - "passed": 11, + "total": 16, + "passed": 16, "pass_rate": 1.0 }, "should_not_trigger": { - "total": 10, - "passed": 10, + "total": 15, + "passed": 15, "pass_rate": 1.0 }, "near_neighbor": { - "total": 10, - "passed": 10, + "total": 14, + "passed": 14, "pass_rate": 1.0 } }, "family_stats": { - "default": { - "total": 31, - "passed": 31, + "workflow_to_skill": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "package_for_team": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "meta_skill_creation": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "complex_multi_asset": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "one_off_vs_reusable": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_vs_build": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "partial_scaffold_not_full_skill": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_not_package": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 3, + "passed": 3, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 @@ -36,7 +141,7 @@ "should_trigger": [ { "prompt": "Create a skill from this repeated workflow.", - "family": "default", + "family": "workflow_to_skill", "score": 0.549, "predicted_trigger": true, "expected_trigger": true, @@ -76,53 +181,9 @@ } } }, - { - "prompt": "Improve this skill description and add evals.", - "family": "default", - "score": 0.353, - "predicted_trigger": true, - "expected_trigger": true, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [ - "eval_optimize", - "iterate_existing_skill" - ], - "extra_positive_concepts": [], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.291, - "support_score": 0, - "lexical_support": 0.571, - "negative_penalty": 0, - "coverage_boost": 0.04, - "concept_evidence": { - "positive": { - "eval_optimize": [ - "evals" - ], - "iterate_existing_skill": [ - "this skill", - "skill description" - ] - }, - "negative": {} - } - }, - "boundary_case": true - }, { "prompt": "Turn this runbook into a reusable agent skill.", - "family": "default", + "family": "workflow_to_skill", "score": 0.729, "predicted_trigger": true, "expected_trigger": true, @@ -166,9 +227,146 @@ } } }, + { + "prompt": "Convert this process note into a reusable skill package for the team.", + "family": "workflow_to_skill", + "score": 0.766, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0, + "lexical_support": 0.667, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "reusable", + "package" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Improve this skill description and add evals.", + "family": "iterate_existing_skill", + "score": 0.353, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.291, + "support_score": 0, + "lexical_support": 0.571, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals" + ], + "iterate_existing_skill": [ + "this skill", + "skill description" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Refactor this prompt into a proper skill package.", + "family": "iterate_existing_skill", + "score": 0.532, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": {} + } + } + }, { "prompt": "Package this skill for team reuse.", - "family": "default", + "family": "package_for_team", "score": 0.448, "predicted_trigger": true, "expected_trigger": true, @@ -216,9 +414,112 @@ } } }, + { + "prompt": "Create a meta-skill for packaging internal workflows.", + "family": "meta_skill_creation", + "score": 0.591, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.527, + "support_score": 0, + "lexical_support": 0.571, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "meta skill", + "meta-skill" + ], + "reuse_package": [ + "packaging", + "internal workflows" + ], + "team_operationalize": [ + "internal" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Standardize this recurring workflow as a shareable skill package with references.", + "family": "package_for_team", + "score": 0.749, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0.06, + "lexical_support": 0.273, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "multi_asset": [ + "references" + ], + "reuse_package": [ + "package" + ], + "team_operationalize": [ + "standardize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, { "prompt": "Convert this operations checklist into a reusable skill.", - "family": "default", + "family": "workflow_to_skill", "score": 0.415, "predicted_trigger": true, "expected_trigger": true, @@ -259,51 +560,9 @@ } } }, - { - "prompt": "Refactor this prompt into a proper skill package.", - "family": "default", - "score": 0.532, - "predicted_trigger": true, - "expected_trigger": true, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [ - "build_skill", - "reuse_package" - ], - "extra_positive_concepts": [], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, - "negative_penalty": 0, - "coverage_boost": 0.04, - "concept_evidence": { - "positive": { - "build_skill": [ - "skill package" - ], - "reuse_package": [ - "package" - ] - }, - "negative": {} - } - } - }, { "prompt": "Add trigger evals to this skill before sharing it with the team.", - "family": "default", + "family": "iterate_existing_skill", "score": 0.388, "predicted_trigger": true, "expected_trigger": true, @@ -352,57 +611,9 @@ }, "boundary_case": true }, - { - "prompt": "Create a meta-skill for packaging internal workflows.", - "family": "default", - "score": 0.591, - "predicted_trigger": true, - "expected_trigger": true, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [ - "build_skill", - "reuse_package", - "team_operationalize" - ], - "extra_positive_concepts": [], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.527, - "support_score": 0, - "lexical_support": 0.571, - "negative_penalty": 0, - "coverage_boost": 0.06, - "concept_evidence": { - "positive": { - "build_skill": [ - "meta skill", - "meta-skill" - ], - "reuse_package": [ - "packaging", - "internal workflows" - ], - "team_operationalize": [ - "internal" - ] - }, - "negative": {} - } - } - }, { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", - "family": "default", + "family": "complex_multi_asset", "score": 0.843, "predicted_trigger": true, "expected_trigger": true, @@ -456,9 +667,108 @@ } } }, + { + "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", + "family": "complex_multi_asset", + "score": 0.418, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.357, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "production-ready" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", + "family": "iterate_existing_skill", + "score": 0.563, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.429, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "distribution" + ], + "team_operationalize": [ + "team", + "distribution" + ] + }, + "negative": {} + } + } + }, { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", - "family": "default", + "family": "complex_multi_asset", "score": 0.468, "predicted_trigger": true, "expected_trigger": true, @@ -511,7 +821,7 @@ }, { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", - "family": "default", + "family": "iterate_existing_skill", "score": 0.729, "predicted_trigger": true, "expected_trigger": true, @@ -565,12 +875,58 @@ "negative": {} } } + }, + { + "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", + "family": "workflow_to_skill", + "score": 0.563, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.429, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": {} + } + } } ], "should_not_trigger": [ { "prompt": "Explain what a workflow is.", - "family": "default", + "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -614,88 +970,9 @@ } } }, - { - "prompt": "Write a product headline for this landing page.", - "family": "default", - "score": 0.0, - "predicted_trigger": false, - "expected_trigger": false, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [], - "extra_positive_concepts": [], - "matched_negative_concepts": [ - "document_only" - ], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.0, - "support_score": 0, - "lexical_support": 0.25, - "negative_penalty": 0.18, - "coverage_boost": 0.0, - "concept_evidence": { - "positive": {}, - "negative": { - "document_only": [ - "headline", - "landing page" - ] - } - } - } - }, - { - "prompt": "Summarize this random note.", - "family": "default", - "score": 0.0, - "predicted_trigger": false, - "expected_trigger": false, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [], - "extra_positive_concepts": [], - "matched_negative_concepts": [ - "summarize_only" - ], - "exclusive_negative_concepts": [ - "summarize_only" - ], - "semantic_coverage": 0.0, - "support_score": 0, - "lexical_support": 0.0, - "negative_penalty": 0.3, - "coverage_boost": 0.0, - "concept_evidence": { - "positive": {}, - "negative": { - "summarize_only": [ - "summarize" - ] - } - } - } - }, { "prompt": "Just explain what a skill is.", - "family": "default", + "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -735,8 +1012,8 @@ } }, { - "prompt": "Draft a blog title for this article.", - "family": "default", + "prompt": "Summarize this random note.", + "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -754,20 +1031,21 @@ "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ - "document_only" + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" ], - "exclusive_negative_concepts": [], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.286, - "negative_penalty": 0.18, + "lexical_support": 0.0, + "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { - "document_only": [ - "article", - "blog title" + "summarize_only": [ + "summarize" ] } } @@ -775,7 +1053,7 @@ }, { "prompt": "Translate this README into Japanese.", - "family": "default", + "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -818,9 +1096,164 @@ } } }, + { + "prompt": "Write a product headline for this landing page.", + "family": "document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "headline", + "landing page" + ] + } + } + } + }, + { + "prompt": "Draft a blog title for this article.", + "family": "document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "article", + "blog title" + ] + } + } + } + }, + { + "prompt": "Rewrite this paragraph more clearly, but do not package anything.", + "family": "document_only", + "score": 0.175, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.1, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "package" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Explain the difference between a runbook and a workflow.", + "family": "explain_only", + "score": 0.187, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow", + "runbook" + ] + }, + "negative": {} + } + } + }, { "prompt": "Give me ideas for improving our process.", - "family": "default", + "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -864,9 +1297,194 @@ } } }, + { + "prompt": "Review this note and tell me what it means in plain English.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "tell me what it means", + "plain english" + ] + } + } + } + }, + { + "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0.06, + "lexical_support": 0.2, + "negative_penalty": 0.66, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm", + "improve packaging" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Translate these notes into French and keep the structure exactly as-is.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "translate_only" + ], + "exclusive_negative_concepts": [ + "translate_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.273, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "translate_only": [ + "translate", + "into french" + ] + } + } + } + }, + { + "prompt": "Summarize this workflow and list the main points only.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.111, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "summarize_only": [ + "summarize" + ] + } + } + } + }, { "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", - "family": "default", + "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -921,49 +1539,8 @@ } }, { - "prompt": "Review this note and tell me what it means in plain English.", - "family": "default", - "score": 0.0, - "predicted_trigger": false, - "expected_trigger": false, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [], - "extra_positive_concepts": [], - "matched_negative_concepts": [ - "explain_only" - ], - "exclusive_negative_concepts": [ - "explain_only" - ], - "semantic_coverage": 0.0, - "support_score": 0, - "lexical_support": 0.083, - "negative_penalty": 0.26, - "coverage_boost": 0.0, - "concept_evidence": { - "positive": {}, - "negative": { - "explain_only": [ - "tell me what it means", - "plain english" - ] - } - } - } - }, - { - "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", - "family": "default", + "prompt": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -979,40 +1556,39 @@ "team_operationalize" ], "matched_desired_concepts": [ - "reuse_package" - ], - "extra_positive_concepts": [ - "multi_asset" + "reuse_package", + "transform_workflow" ], + "extra_positive_concepts": [], "matched_negative_concepts": [ - "brainstorm_only", + "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ - "brainstorm_only", + "explain_only", "no_build_directive" ], - "semantic_coverage": 0.182, - "support_score": 0.06, - "lexical_support": 0.2, - "negative_penalty": 0.66, - "coverage_boost": 0.0, + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.091, + "negative_penalty": 0.68, + "coverage_boost": 0.04, "concept_evidence": { "positive": { - "multi_asset": [ - "files" - ], "reuse_package": [ - "packaging" + "package" + ], + "transform_workflow": [ + "checklist" ] }, "negative": { - "brainstorm_only": [ - "brainstorm", - "improve packaging" + "explain_only": [ + "explain this", + "in simpler language" ], "no_build_directive": [ - "do not generate any skill files" + "do not package it" ] } } @@ -1022,7 +1598,7 @@ "near_neighbor": [ { "prompt": "Create a one-off prompt for this task.", - "family": "default", + "family": "one_off_vs_reusable", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1060,9 +1636,60 @@ } } }, + { + "prompt": "Write a custom answer for this request without creating a skill package.", + "family": "one_off_vs_reusable", + "score": 0.131, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "one_off_request" + ], + "exclusive_negative_concepts": [ + "one_off_request" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.364, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "one_off_request": [ + "custom answer", + "for this request" + ] + } + } + } + }, { "prompt": "Help me brainstorm process ideas without building a skill.", - "family": "default", + "family": "brainstorm_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1114,7 +1741,7 @@ }, { "prompt": "Improve this README but do not turn it into a skill.", - "family": "default", + "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1158,7 +1785,7 @@ }, { "prompt": "Make a checklist for this task, but not a reusable skill.", - "family": "default", + "family": "partial_scaffold_not_full_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1206,9 +1833,60 @@ } } }, + { + "prompt": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.231, + "negative_penalty": 0.42, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "no_build_directive": [ + "plain note", + "keep it as a plain note" + ] + } + } + } + }, { "prompt": "Review this process note and explain it, no packaging needed.", - "family": "default", + "family": "explain_not_package", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1261,111 +1939,9 @@ } } }, - { - "prompt": "Help me shape an idea before we decide whether to build a skill.", - "family": "default", - "score": 0.0, - "predicted_trigger": false, - "expected_trigger": false, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [ - "build_skill" - ], - "extra_positive_concepts": [], - "matched_negative_concepts": [ - "brainstorm_only", - "future_outline" - ], - "exclusive_negative_concepts": [ - "brainstorm_only", - "future_outline" - ], - "semantic_coverage": 0.309, - "support_score": 0, - "lexical_support": 0.308, - "negative_penalty": 0.46, - "coverage_boost": 0.0, - "concept_evidence": { - "positive": { - "build_skill": [ - "build a skill" - ] - }, - "negative": { - "brainstorm_only": [ - "shape an idea" - ], - "future_outline": [ - "before we decide whether to build" - ] - } - } - } - }, - { - "prompt": "Write a custom answer for this request without creating a skill package.", - "family": "default", - "score": 0.131, - "predicted_trigger": false, - "expected_trigger": false, - "passed": true, - "score_detail": { - "mode": "semantic-intent", - "desired_positive_concepts": [ - "build_skill", - "transform_workflow", - "reuse_package", - "eval_optimize", - "iterate_existing_skill", - "team_operationalize" - ], - "matched_desired_concepts": [ - "build_skill", - "reuse_package" - ], - "extra_positive_concepts": [], - "matched_negative_concepts": [ - "one_off_request" - ], - "exclusive_negative_concepts": [ - "one_off_request" - ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.364, - "negative_penalty": 0.24, - "coverage_boost": 0.04, - "concept_evidence": { - "positive": { - "build_skill": [ - "skill package" - ], - "reuse_package": [ - "package" - ] - }, - "negative": { - "one_off_request": [ - "custom answer", - "for this request" - ] - } - } - } - }, { "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", - "family": "default", + "family": "partial_scaffold_not_full_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1411,7 +1987,7 @@ }, { "prompt": "Package this explanation as a document, not as an agent skill.", - "family": "default", + "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1463,9 +2039,165 @@ } } }, + { + "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "iterate_existing_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.327, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.42, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "iterate_existing_skill": [ + "this skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.462, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "document_only": [ + "document", + "reusable-looking document" + ], + "no_build_directive": [ + "without turning them into an agent skill" + ] + } + } + } + }, + { + "prompt": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "semantic_coverage": 0.309, + "support_score": 0, + "lexical_support": 0.308, + "negative_penalty": 0.46, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ] + }, + "negative": { + "brainstorm_only": [ + "shape an idea" + ], + "future_outline": [ + "before we decide whether to build" + ] + } + } + } + }, { "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", - "family": "default", + "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, @@ -1509,6 +2241,62 @@ } } } + }, + { + "prompt": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.538, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document", + "document package", + "wiki" + ], + "no_build_directive": [ + "not an agent skill" + ] + } + } + } } ] }, @@ -1521,7 +2309,7 @@ "false_negative_delta": -2, "baseline_precision": 1.0, "improved_precision": 1.0, - "baseline_recall": 0.818, + "baseline_recall": 0.875, "improved_recall": 1.0 } } diff --git a/evals/semantic_config.json b/evals/semantic_config.json index 47cce4c..f698c0a 100644 --- a/evals/semantic_config.json +++ b/evals/semantic_config.json @@ -108,6 +108,8 @@ "explain what", "just explain", "explain it", + "explain this", + "in simpler language", "tell me what it means", "plain english" ] @@ -162,9 +164,14 @@ "do not generate any skill files", "not a reusable skill", "not as an agent skill", + "not an agent skill", + "without turning them into an agent skill", "do not turn it into a skill", + "do not package it", "stop short of making a full skill", "not a skill yet", + "plain note", + "keep it as a plain note", "no packaging needed" ] }, @@ -174,6 +181,9 @@ "phrases": [ "readme", "document", + "document package", + "reusable-looking document", + "wiki", "article", "blog title", "headline", diff --git a/evals/train/trigger_cases.json b/evals/train/trigger_cases.json index 1e8095f..a439d64 100644 --- a/evals/train/trigger_cases.json +++ b/evals/train/trigger_cases.json @@ -10,26 +10,97 @@ "summarize" ], "should_trigger": [ - "Create a skill from this repeated workflow.", - "Improve this skill description and add evals.", - "Turn this runbook into a reusable agent skill.", - "Package this skill for team reuse.", - "Refactor this prompt into a proper skill package.", - "Create a meta-skill for packaging internal workflows." + { + "text": "Create a skill from this repeated workflow.", + "family": "workflow_to_skill" + }, + { + "text": "Turn this runbook into a reusable agent skill.", + "family": "workflow_to_skill" + }, + { + "text": "Convert this process note into a reusable skill package for the team.", + "family": "workflow_to_skill" + }, + { + "text": "Improve this skill description and add evals.", + "family": "iterate_existing_skill" + }, + { + "text": "Refactor this prompt into a proper skill package.", + "family": "iterate_existing_skill" + }, + { + "text": "Package this skill for team reuse.", + "family": "package_for_team" + }, + { + "text": "Create a meta-skill for packaging internal workflows.", + "family": "meta_skill_creation" + }, + { + "text": "Standardize this recurring workflow as a shareable skill package with references.", + "family": "package_for_team" + } ], "should_not_trigger": [ - "Explain what a workflow is.", - "Write a product headline for this landing page.", - "Summarize this random note.", - "Just explain what a skill is.", - "Draft a blog title for this article.", - "Translate this README into Japanese." + { + "text": "Explain what a workflow is.", + "family": "explain_only" + }, + { + "text": "Just explain what a skill is.", + "family": "explain_only" + }, + { + "text": "Summarize this random note.", + "family": "summary_only" + }, + { + "text": "Translate this README into Japanese.", + "family": "translate_only" + }, + { + "text": "Write a product headline for this landing page.", + "family": "document_only" + }, + { + "text": "Draft a blog title for this article.", + "family": "document_only" + }, + { + "text": "Rewrite this paragraph more clearly, but do not package anything.", + "family": "document_only" + }, + { + "text": "Explain the difference between a runbook and a workflow.", + "family": "explain_only" + } ], "near_neighbor": [ - "Create a one-off prompt for this task.", - "Help me brainstorm process ideas without building a skill.", - "Improve this README but do not turn it into a skill.", - "Make a checklist for this task, but not a reusable skill.", - "Write a custom answer for this request without creating a skill package." + { + "text": "Create a one-off prompt for this task.", + "family": "one_off_vs_reusable" + }, + { + "text": "Write a custom answer for this request without creating a skill package.", + "family": "one_off_vs_reusable" + }, + { + "text": "Help me brainstorm process ideas without building a skill.", + "family": "brainstorm_vs_build" + }, + { + "text": "Improve this README but do not turn it into a skill.", + "family": "document_export_vs_agent_skill" + }, + { + "text": "Make a checklist for this task, but not a reusable skill.", + "family": "partial_scaffold_not_full_skill" + }, + { + "text": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "family": "partial_scaffold_not_full_skill" + } ] } diff --git a/evals/trigger_cases.json b/evals/trigger_cases.json index d41a273..a40816d 100644 --- a/evals/trigger_cases.json +++ b/evals/trigger_cases.json @@ -10,40 +10,189 @@ "summarize" ], "should_trigger": [ - "Create a skill from this repeated workflow.", - "Improve this skill description and add evals.", - "Turn this runbook into a reusable agent skill.", - "Package this skill for team reuse.", - "Convert this operations checklist into a reusable skill.", - "Refactor this prompt into a proper skill package.", - "Add trigger evals to this skill before sharing it with the team.", - "Create a meta-skill for packaging internal workflows.", - "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", - "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", - "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution." + { + "text": "Create a skill from this repeated workflow.", + "family": "workflow_to_skill" + }, + { + "text": "Turn this runbook into a reusable agent skill.", + "family": "workflow_to_skill" + }, + { + "text": "Convert this process note into a reusable skill package for the team.", + "family": "workflow_to_skill" + }, + { + "text": "Improve this skill description and add evals.", + "family": "iterate_existing_skill" + }, + { + "text": "Refactor this prompt into a proper skill package.", + "family": "iterate_existing_skill" + }, + { + "text": "Package this skill for team reuse.", + "family": "package_for_team" + }, + { + "text": "Create a meta-skill for packaging internal workflows.", + "family": "meta_skill_creation" + }, + { + "text": "Standardize this recurring workflow as a shareable skill package with references.", + "family": "package_for_team" + }, + { + "text": "Convert this operations checklist into a reusable skill.", + "family": "workflow_to_skill" + }, + { + "text": "Add trigger evals to this skill before sharing it with the team.", + "family": "iterate_existing_skill" + }, + { + "text": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", + "family": "complex_multi_asset" + }, + { + "text": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", + "family": "complex_multi_asset" + }, + { + "text": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", + "family": "iterate_existing_skill" + }, + { + "text": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "family": "complex_multi_asset" + }, + { + "text": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", + "family": "iterate_existing_skill" + }, + { + "text": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", + "family": "workflow_to_skill" + } ], "should_not_trigger": [ - "Explain what a workflow is.", - "Write a product headline for this landing page.", - "Summarize this random note.", - "Just explain what a skill is.", - "Draft a blog title for this article.", - "Translate this README into Japanese.", - "Give me ideas for improving our process.", - "I pasted a long process description below, but I only want a summary, not a reusable skill.", - "Review this note and tell me what it means in plain English.", - "Help me brainstorm several ways to improve packaging, but do not generate any skill files." + { + "text": "Explain what a workflow is.", + "family": "explain_only" + }, + { + "text": "Just explain what a skill is.", + "family": "explain_only" + }, + { + "text": "Summarize this random note.", + "family": "summary_only" + }, + { + "text": "Translate this README into Japanese.", + "family": "translate_only" + }, + { + "text": "Write a product headline for this landing page.", + "family": "document_only" + }, + { + "text": "Draft a blog title for this article.", + "family": "document_only" + }, + { + "text": "Rewrite this paragraph more clearly, but do not package anything.", + "family": "document_only" + }, + { + "text": "Explain the difference between a runbook and a workflow.", + "family": "explain_only" + }, + { + "text": "Give me ideas for improving our process.", + "family": "brainstorm_only" + }, + { + "text": "Review this note and tell me what it means in plain English.", + "family": "explain_only" + }, + { + "text": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", + "family": "brainstorm_only" + }, + { + "text": "Translate these notes into French and keep the structure exactly as-is.", + "family": "translate_only" + }, + { + "text": "Summarize this workflow and list the main points only.", + "family": "summary_only" + }, + { + "text": "I pasted a long process description below, but I only want a summary, not a reusable skill.", + "family": "summary_only" + }, + { + "text": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_only" + } ], "near_neighbor": [ - "Create a one-off prompt for this task.", - "Help me brainstorm process ideas without building a skill.", - "Improve this README but do not turn it into a skill.", - "Make a checklist for this task, but not a reusable skill.", - "Review this process note and explain it, no packaging needed.", - "Help me shape an idea before we decide whether to build a skill.", - "Write a custom answer for this request without creating a skill package.", - "Turn this into a checklist and template, but stop short of making a full skill.", - "Package this explanation as a document, not as an agent skill.", - "Create an outline for a possible future skill, but do not build the skill yet." + { + "text": "Create a one-off prompt for this task.", + "family": "one_off_vs_reusable" + }, + { + "text": "Write a custom answer for this request without creating a skill package.", + "family": "one_off_vs_reusable" + }, + { + "text": "Help me brainstorm process ideas without building a skill.", + "family": "brainstorm_vs_build" + }, + { + "text": "Improve this README but do not turn it into a skill.", + "family": "document_export_vs_agent_skill" + }, + { + "text": "Make a checklist for this task, but not a reusable skill.", + "family": "partial_scaffold_not_full_skill" + }, + { + "text": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "family": "partial_scaffold_not_full_skill" + }, + { + "text": "Review this process note and explain it, no packaging needed.", + "family": "explain_not_package" + }, + { + "text": "Turn this into a checklist and template, but stop short of making a full skill.", + "family": "partial_scaffold_not_full_skill" + }, + { + "text": "Package this explanation as a document, not as an agent skill.", + "family": "document_export_vs_agent_skill" + }, + { + "text": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build" + }, + { + "text": "Polish these notes into a reusable-looking document without turning them into an agent skill.", + "family": "document_export_vs_agent_skill" + }, + { + "text": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build" + }, + { + "text": "Create an outline for a possible future skill, but do not build the skill yet.", + "family": "future_outline_vs_build" + }, + { + "text": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill" + } ] } diff --git a/failures/README.md b/failures/README.md index 1fa386a..6284313 100644 --- a/failures/README.md +++ b/failures/README.md @@ -14,3 +14,5 @@ Current case library: - `explain-not-package` - `document-export-vs-agent-skill` - `future-outline-vs-build` + +Each family now includes a `cases.json` file so the anti-pattern can run in automated regression instead of living only as prose. diff --git a/failures/document-export-vs-agent-skill/cases.json b/failures/document-export-vs-agent-skill/cases.json new file mode 100644 index 0000000..2906459 --- /dev/null +++ b/failures/document-export-vs-agent-skill/cases.json @@ -0,0 +1,24 @@ +{ + "recommended_threshold": 0.33, + "should_trigger": [], + "should_not_trigger": [ + { + "text": "Rewrite this paragraph more clearly, but do not package anything.", + "family": "document_only" + } + ], + "near_neighbor": [ + { + "text": "Improve this README but do not turn it into a skill.", + "family": "document_export_vs_agent_skill" + }, + { + "text": "Package this explanation as a document, not as an agent skill.", + "family": "document_export_vs_agent_skill" + }, + { + "text": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill" + } + ] +} diff --git a/failures/explain-not-package/cases.json b/failures/explain-not-package/cases.json new file mode 100644 index 0000000..c08be99 --- /dev/null +++ b/failures/explain-not-package/cases.json @@ -0,0 +1,24 @@ +{ + "recommended_threshold": 0.33, + "should_trigger": [], + "should_not_trigger": [ + { + "text": "Explain what a workflow is.", + "family": "explain_only" + }, + { + "text": "Review this note and tell me what it means in plain English.", + "family": "explain_only" + } + ], + "near_neighbor": [ + { + "text": "Review this process note and explain it, no packaging needed.", + "family": "explain_not_package" + }, + { + "text": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_not_package" + } + ] +} diff --git a/failures/future-outline-vs-build/cases.json b/failures/future-outline-vs-build/cases.json new file mode 100644 index 0000000..185ef86 --- /dev/null +++ b/failures/future-outline-vs-build/cases.json @@ -0,0 +1,19 @@ +{ + "recommended_threshold": 0.33, + "should_trigger": [], + "should_not_trigger": [], + "near_neighbor": [ + { + "text": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build" + }, + { + "text": "Create an outline for a possible future skill, but do not build the skill yet.", + "family": "future_outline_vs_build" + }, + { + "text": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build" + } + ] +} diff --git a/failures/one-off-vs-reusable/cases.json b/failures/one-off-vs-reusable/cases.json new file mode 100644 index 0000000..6d9056b --- /dev/null +++ b/failures/one-off-vs-reusable/cases.json @@ -0,0 +1,19 @@ +{ + "recommended_threshold": 0.33, + "should_trigger": [], + "should_not_trigger": [], + "near_neighbor": [ + { + "text": "Create a one-off prompt for this task.", + "family": "one_off_vs_reusable" + }, + { + "text": "Write a custom answer for this request without creating a skill package.", + "family": "one_off_vs_reusable" + }, + { + "text": "Give me a single-use prompt for today's task, not a reusable skill.", + "family": "one_off_vs_reusable" + } + ] +} diff --git a/scripts/run_eval_suite.py b/scripts/run_eval_suite.py index daf873c..9076631 100644 --- a/scripts/run_eval_suite.py +++ b/scripts/run_eval_suite.py @@ -42,6 +42,7 @@ def main() -> None: suites = {} aggregate = {"false_positives": 0, "false_negatives": 0, "precision": [], "recall": []} + family_summary: dict[str, dict] = {} for name in ("train", "dev", "holdout"): report = run_case(script, description, baseline, root / name / "trigger_cases.json") suites[name] = report @@ -51,6 +52,18 @@ def main() -> None: aggregate["precision"].append(report["precision"]) if report["recall"] is not None: aggregate["recall"].append(report["recall"]) + for family, stats in report.get("family_stats", {}).items(): + slot = family_summary.setdefault( + family, + {"total": 0, "passed": 0, "false_positives": 0, "false_negatives": 0}, + ) + slot["total"] += stats["total"] + slot["passed"] += stats["passed"] + slot["false_positives"] += stats["false_positives"] + slot["false_negatives"] += stats["false_negatives"] + + for family, stats in family_summary.items(): + stats["pass_rate"] = round(stats["passed"] / stats["total"], 3) if stats["total"] else None summary = { "suite_count": len(suites), @@ -59,7 +72,7 @@ def main() -> None: "average_precision": round(sum(aggregate["precision"]) / len(aggregate["precision"]), 3) if aggregate["precision"] else None, "average_recall": round(sum(aggregate["recall"]) / len(aggregate["recall"]), 3) if aggregate["recall"] else None, } - output = {"summary": summary, "suites": suites} + output = {"summary": summary, "family_summary": family_summary, "suites": suites} print(json.dumps(output, ensure_ascii=False, indent=2)) if summary["false_positives"] > 0 or summary["false_negatives"] > 0: raise SystemExit(2) diff --git a/tests/verify_failure_regressions.py b/tests/verify_failure_regressions.py new file mode 100644 index 0000000..71080a9 --- /dev/null +++ b/tests/verify_failure_regressions.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +FAILURES_DIR = ROOT / "failures" +TRIGGER_EVAL = ROOT / "scripts" / "trigger_eval.py" +DESCRIPTION = ROOT / "evals" / "improved_description.txt" +BASELINE = ROOT / "evals" / "baseline_description.txt" +SEMANTIC_CONFIG = ROOT / "evals" / "semantic_config.json" + + +def run_case(case_file: Path) -> dict: + proc = subprocess.run( + [ + sys.executable, + str(TRIGGER_EVAL), + "--description-file", + str(DESCRIPTION), + "--baseline-description-file", + str(BASELINE), + "--cases", + str(case_file), + "--semantic-config", + str(SEMANTIC_CONFIG), + ], + capture_output=True, + text=True, + ) + payload = json.loads(proc.stdout) + payload["returncode"] = proc.returncode + payload["case_file"] = str(case_file.relative_to(ROOT)) + return payload + + +def main() -> None: + case_files = sorted(FAILURES_DIR.glob("*/cases.json")) + reports = [run_case(path) for path in case_files] + failures = [] + for report in reports: + if report["returncode"] != 0: + failures.append( + { + "case_file": report["case_file"], + "false_positives": report["false_positives"], + "false_negatives": report["false_negatives"], + "misfires": report["misfires"], + } + ) + + output = { + "ok": not failures, + "case_count": len(reports), + "families": [report["case_file"] for report in reports], + "failures": failures, + } + print(json.dumps(output, ensure_ascii=False, indent=2)) + if failures: + raise SystemExit(2) + + +if __name__ == "__main__": + main()