diff --git a/Makefile b/Makefile index 3fd29ad3..98791693 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint test clean +.PHONY: eval eval-suite results-panel failure-regression-check package-check package-failure-check snapshot-check validate lint test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -8,6 +8,9 @@ eval: eval-suite: $(PYTHON) scripts/run_eval_suite.py +results-panel: + $(PYTHON) scripts/render_eval_dashboard.py + failure-regression-check: $(PYTHON) tests/verify_failure_regressions.py diff --git a/README.md b/README.md index b79a5aba..0173f374 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,46 @@ make test ## Results -- `make test` passes locally -- semantic trigger eval: `0` false positives, `0` false negatives on the current regression set -- eval suite: train / dev / holdout all pass, with family-based reporting +The homepage panel below is generated from the current eval suite so the family-level outcome is visible without opening raw JSON. + + +- regression corpus: `66` prompts across `21` families +- aggregate result: `0` false positives, `0` false negatives, average precision `1.0`, average recall `1.0` +- suite status: + +| Suite | Cases | FP | FN | Precision | Recall | +| --- | ---: | ---: | ---: | ---: | ---: | +| train | 31 | 0 | 0 | 1.0 | 1.0 | +| dev | 22 | 0 | 0 | 1.0 | 1.0 | +| holdout | 13 | 0 | 0 | 1.0 | 1.0 | + +| Family | Cases | Pass Rate | +| --- | ---: | ---: | +| `brainstorm_only` | 2 | 1.0 | +| `brainstorm_vs_build` | 1 | 1.0 | +| `complex_multi_asset` | 3 | 1.0 | +| `document_export_vs_agent_skill` | 4 | 1.0 | +| `document_only` | 3 | 1.0 | +| `explain_not_package` | 1 | 1.0 | +| `explain_only` | 5 | 1.0 | +| `future_outline_vs_build` | 4 | 1.0 | +| `iterate_existing_skill` | 5 | 1.0 | +| `long_context_document_only` | 3 | 1.0 | +| `long_context_near_neighbor` | 3 | 1.0 | +| `long_context_summary_only` | 2 | 1.0 | +| `long_context_trigger` | 4 | 1.0 | +| `meta_skill_creation` | 1 | 1.0 | +| `one_off_vs_reusable` | 2 | 1.0 | +| `package_for_team` | 2 | 1.0 | +| `paraphrase_trigger` | 5 | 1.0 | +| `partial_scaffold_not_full_skill` | 4 | 1.0 | +| `summary_only` | 3 | 1.0 | +| `translate_only` | 4 | 1.0 | +| `workflow_to_skill` | 5 | 1.0 | + +Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md) + + - packaging validation: `openai`, `claude`, and `generic` targets pass contract checks - packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected - failure library regressions: anti-pattern families pass automated checks diff --git a/evals/README.md b/evals/README.md index a60cc360..829f9340 100644 --- a/evals/README.md +++ b/evals/README.md @@ -12,6 +12,7 @@ Contents: - `sample_trigger_report.json`: example comparison output using the current recommended threshold - `failure-cases.md`: current weak spots and regression targets - `packaging_expectations.json`: required packaging behaviors for supported targets +- `../reports/`: generated suite JSON plus the homepage-visible family summary panel source Use: @@ -19,6 +20,7 @@ Use: python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt python3 scripts/run_eval_suite.py +python3 scripts/render_eval_dashboard.py python3 tests/verify_failure_regressions.py python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip python3 tests/verify_packager_failures.py @@ -33,5 +35,7 @@ Regression scope now includes: - mixed-intent negatives - explicit "do not build a skill" negatives - semantic exclusion cases such as one-off, document-only, and future-outline prompts +- paraphrase families that avoid the original wording while preserving the same trigger intent +- long-context contamination cases where build intent or no-build intent appears after unrelated setup text - family-based reporting across workflow-to-skill, iterate-existing-skill, document-only, one-off, and future-outline cases - holdout verification diff --git a/evals/dev/trigger_cases.json b/evals/dev/trigger_cases.json index 6dd5da64..e64aacae 100644 --- a/evals/dev/trigger_cases.json +++ b/evals/dev/trigger_cases.json @@ -29,6 +29,18 @@ { "text": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill" + }, + { + "text": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", + "family": "paraphrase_trigger" + }, + { + "text": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", + "family": "paraphrase_trigger" + }, + { + "text": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", + "family": "long_context_trigger" } ], "should_not_trigger": [ @@ -51,6 +63,14 @@ { "text": "Summarize this workflow and list the main points only.", "family": "summary_only" + }, + { + "text": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", + "family": "long_context_document_only" + }, + { + "text": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", + "family": "long_context_summary_only" } ], "near_neighbor": [ @@ -73,6 +93,14 @@ { "text": "Polish these notes into a reusable-looking document without turning them into an agent skill.", "family": "document_export_vs_agent_skill" + }, + { + "text": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", + "family": "long_context_near_neighbor" + }, + { + "text": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", + "family": "future_outline_vs_build" } ] } diff --git a/evals/holdout/trigger_cases.json b/evals/holdout/trigger_cases.json index 2c1f3bdd..8ad1ea6f 100644 --- a/evals/holdout/trigger_cases.json +++ b/evals/holdout/trigger_cases.json @@ -21,6 +21,14 @@ { "text": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill" + }, + { + "text": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", + "family": "paraphrase_trigger" + }, + { + "text": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", + "family": "long_context_trigger" } ], "should_not_trigger": [ @@ -31,6 +39,14 @@ { "text": "Explain this release checklist in simpler language; do not package it.", "family": "explain_only" + }, + { + "text": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", + "family": "long_context_document_only" + }, + { + "text": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", + "family": "translate_only" } ], "near_neighbor": [ @@ -45,6 +61,10 @@ { "text": "Turn this into a document package for the wiki, not an agent skill.", "family": "document_export_vs_agent_skill" + }, + { + "text": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", + "family": "long_context_near_neighbor" } ] } diff --git a/evals/sample_trigger_report.json b/evals/sample_trigger_report.json index f4b4515f..776e426a 100644 --- a/evals/sample_trigger_report.json +++ b/evals/sample_trigger_report.json @@ -7,18 +7,18 @@ "recall": 1.0, "bucket_stats": { "should_trigger": { - "total": 16, - "passed": 16, + "total": 25, + "passed": 25, "pass_rate": 1.0 }, "should_not_trigger": { - "total": 15, - "passed": 15, + "total": 22, + "passed": 22, "pass_rate": 1.0 }, "near_neighbor": { - "total": 14, - "passed": 14, + "total": 19, + "passed": 19, "pass_rate": 1.0 } }, @@ -58,6 +58,20 @@ "false_negatives": 0, "pass_rate": 1.0 }, + "paraphrase_trigger": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, "explain_only": { "total": 5, "passed": 5, @@ -73,8 +87,8 @@ "pass_rate": 1.0 }, "translate_only": { - "total": 2, - "passed": 2, + "total": 4, + "passed": 4, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 @@ -93,6 +107,20 @@ "false_negatives": 0, "pass_rate": 1.0 }, + "long_context_summary_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, "one_off_vs_reusable": { "total": 2, "passed": 2, @@ -115,8 +143,8 @@ "pass_rate": 1.0 }, "partial_scaffold_not_full_skill": { - "total": 3, - "passed": 3, + "total": 4, + "passed": 4, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 @@ -129,6 +157,13 @@ "pass_rate": 1.0 }, "future_outline_vs_build": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { "total": 3, "passed": 3, "false_positives": 0, @@ -708,7 +743,8 @@ "production-ready" ], "transform_workflow": [ - "workflow" + "workflow", + "workflow fragments" ] }, "negative": {} @@ -912,7 +948,8 @@ "validation" ], "reuse_package": [ - "reusable" + "reusable", + "library entry" ], "transform_workflow": [ "process" @@ -921,6 +958,510 @@ "negative": {} } } + }, + { + "prompt": "Codify this recurring support escalation workflow into a maintained skill for the ops library.", + "family": "paraphrase_trigger", + "score": 0.397, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "ops library", + "maintained" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Operationalize this workflow as a reusable skill with validation notes and team library metadata.", + "family": "paraphrase_trigger", + "score": 0.88, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.429, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "operationalize this workflow as a reusable skill" + ], + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "team library", + "operationalize" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", + "family": "paraphrase_trigger", + "score": 0.471, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.4, + "support_score": 0, + "lexical_support": 0.538, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "reuse_package": [ + "team library", + "maintained" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", + "family": "paraphrase_trigger", + "score": 0.868, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.267, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this workflow" + ], + "eval_optimize": [ + "validate", + "route boundary" + ], + "reuse_package": [ + "package", + "distribution", + "shared capability" + ], + "team_operationalize": [ + "shared", + "distribution" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", + "family": "paraphrase_trigger", + "score": 0.858, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0.06, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "packaging checks" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "quarterly release routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below are release notes, rough notes, and a transcript from the last handoff. Ignore the noise and turn this runbook into a reusable agent skill with references and scripts so the team can reuse it every quarter.", + "family": "long_context_trigger", + "score": 0.758, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0.06, + "lexical_support": 0.387, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable", + "reuse" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "runbook", + "rough notes", + "transcript", + "release notes" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "We have a standard operating procedure, checklist, and prompt history scattered across docs. Formalize this process into a reusable capability, add validation, and package it for team reuse.", + "family": "long_context_trigger", + "score": 0.88, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.423, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "formalize this process into a reusable capability" + ], + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ], + "transform_workflow": [ + "process", + "checklist", + "prompt history", + "standard operating procedure" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", + "family": "long_context_trigger", + "score": 0.891, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0.06, + "lexical_support": 0.37, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill", + "skill package" + ], + "eval_optimize": [ + "validation" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ], + "transform_workflow": [ + "rough notes", + "transcript", + "onboarding flow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", + "family": "long_context_trigger", + "score": 0.744, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "turn this playbook into an agent capability package" + ], + "reuse_package": [ + "package", + "team library" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "prompt history", + "transcript", + "release notes" + ] + }, + "negative": {} + } + } } ], "should_not_trigger": [ @@ -1593,6 +2134,386 @@ } } } + }, + { + "prompt": "Below is a long SOP, but I only want a summary for leadership and a recap of the main points. Do not build anything reusable.", + "family": "long_context_summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.19, + "negative_penalty": 0.72, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "no_build_directive": [ + "do not build" + ], + "summarize_only": [ + "summary", + "only want a summary", + "recap" + ] + } + } + } + }, + { + "prompt": "Keep this as documentation only for the knowledge base and rewrite it as a reference document.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "document", + "reference document", + "knowledge base" + ] + } + } + } + }, + { + "prompt": "Translate this long onboarding flow into French and keep it as documentation only with no skill package.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.235, + "negative_penalty": 0.77, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "onboarding flow" + ] + }, + "negative": { + "no_build_directive": [ + "no skill package", + "keep it as documentation only" + ], + "translate_only": [ + "translate", + "into french" + ] + } + } + } + }, + { + "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.182, + "negative_penalty": 0.68, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "explain_only": [ + "explain it", + "in simpler language" + ], + "no_build_directive": [ + "do not convert it into a reusable capability", + "keep it as documentation only" + ] + } + } + } + }, + { + "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", + "family": "long_context_summary_only", + "score": 0.257, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.353, + "negative_penalty": 0.3, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "summarize_only": [ + "summary", + "recap" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.222, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document" + ], + "no_build_directive": [ + "not for agent execution" + ] + } + } + } + }, + { + "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.77, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "no_build_directive": [ + "no skill package", + "keep it as documentation only" + ], + "translate_only": [ + "translate", + "into russian" + ] + } + } + } } ], "near_neighbor": [ @@ -2062,15 +2983,17 @@ ], "extra_positive_concepts": [], "matched_negative_concepts": [ + "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ + "future_outline", "no_build_directive" ], "semantic_coverage": 0.327, "support_score": 0, "lexical_support": 0.286, - "negative_penalty": 0.42, + "negative_penalty": 0.64, "coverage_boost": 0.04, "concept_evidence": { "positive": { @@ -2082,6 +3005,9 @@ ] }, "negative": { + "future_outline": [ + "future-ready outline" + ], "no_build_directive": [ "do not build" ] @@ -2297,19 +3223,296 @@ } } } + }, + { + "prompt": "We may convert this workflow later, but for now keep it as documentation only and produce a reference document.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.158, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "reference document" + ], + "no_build_directive": [ + "keep it as documentation only" + ] + } + } + } + }, + { + "prompt": "Draft a template only from this process and stop short of making a full skill package.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0.6, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "document_only": [ + "template only" + ], + "no_build_directive": [ + "stop short of making a full skill" + ] + } + } + } + }, + { + "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", + "family": "long_context_near_neighbor", + "score": 0.328, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.18, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document", + "template only" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.278, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "for later", + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.375, + "negative_penalty": 0.6, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "workflow", + "checklist" + ] + }, + "negative": { + "document_only": [ + "template only" + ], + "no_build_directive": [ + "no skill package", + "discussion only" + ] + } + } + } } ] }, "comparison": { - "baseline_false_positives": 0, + "baseline_false_positives": 1, "baseline_false_negatives": 2, "improved_false_positives": 0, "improved_false_negatives": 0, - "false_positive_delta": 0, + "false_positive_delta": -1, "false_negative_delta": -2, - "baseline_precision": 1.0, + "baseline_precision": 0.958, "improved_precision": 1.0, - "baseline_recall": 0.875, + "baseline_recall": 0.92, "improved_recall": 1.0 } } diff --git a/evals/semantic_config.json b/evals/semantic_config.json index f698c0a3..3bc1f852 100644 --- a/evals/semantic_config.json +++ b/evals/semantic_config.json @@ -19,7 +19,13 @@ "meta skill", "meta-skill", "turn this into a skill", - "turn all of that into one reusable skill package" + "turn all of that into one reusable skill package", + "codify this workflow", + "codify this routine", + "operationalize this workflow as a reusable skill", + "formalize this process into a reusable capability", + "turn this playbook into an agent capability package", + "convert this into a maintained skill" ] }, "transform_workflow": { @@ -33,7 +39,15 @@ "operations checklist", "internal process note", "existing skill draft", - "rough notes" + "rough notes", + "standard operating procedure", + "sop", + "transcript", + "workflow fragments", + "release notes", + "onboarding flow", + "support escalation routine", + "quarterly release routine" ] }, "reuse_package": { @@ -47,7 +61,13 @@ "share it with the team", "team reuse", "internal workflows", - "production-ready" + "production-ready", + "library entry", + "shared capability", + "team library", + "ops library", + "maintained", + "operationalize" ] }, "eval_optimize": { @@ -60,7 +80,12 @@ "benchmark", "validation", "validate", - "packaging checks" + "packaging checks", + "false positives", + "false negatives", + "stress test", + "route boundary", + "hardening" ] }, "iterate_existing_skill": { @@ -73,7 +98,11 @@ "optimize skill triggering", "package this skill", "sharing it with the team", - "before sharing it with the team" + "before sharing it with the team", + "tighten the boundary", + "revise this skill", + "draft skill", + "harden this skill for library reuse" ] }, "multi_asset": { @@ -172,7 +201,12 @@ "not a skill yet", "plain note", "keep it as a plain note", - "no packaging needed" + "no packaging needed", + "not for agent execution", + "no skill package", + "do not convert it into a reusable capability", + "keep it as documentation only", + "discussion only" ] }, "document_only": { @@ -187,7 +221,10 @@ "article", "blog title", "headline", - "landing page" + "landing page", + "reference document", + "knowledge base", + "template only" ] }, "future_outline": { @@ -197,7 +234,10 @@ "future skill", "possible future skill", "before we decide whether to build", - "outline for a possible future skill" + "outline for a possible future skill", + "maybe later", + "for later", + "future-ready outline" ] } } diff --git a/evals/train/trigger_cases.json b/evals/train/trigger_cases.json index a439d64d..0bfd2c10 100644 --- a/evals/train/trigger_cases.json +++ b/evals/train/trigger_cases.json @@ -41,6 +41,22 @@ { "text": "Standardize this recurring workflow as a shareable skill package with references.", "family": "package_for_team" + }, + { + "text": "Codify this recurring support escalation workflow into a maintained skill for the ops library.", + "family": "paraphrase_trigger" + }, + { + "text": "Operationalize this workflow as a reusable skill with validation notes and team library metadata.", + "family": "paraphrase_trigger" + }, + { + "text": "Below are release notes, rough notes, and a transcript from the last handoff. Ignore the noise and turn this runbook into a reusable agent skill with references and scripts so the team can reuse it every quarter.", + "family": "long_context_trigger" + }, + { + "text": "We have a standard operating procedure, checklist, and prompt history scattered across docs. Formalize this process into a reusable capability, add validation, and package it for team reuse.", + "family": "long_context_trigger" } ], "should_not_trigger": [ @@ -75,6 +91,18 @@ { "text": "Explain the difference between a runbook and a workflow.", "family": "explain_only" + }, + { + "text": "Below is a long SOP, but I only want a summary for leadership and a recap of the main points. Do not build anything reusable.", + "family": "long_context_summary_only" + }, + { + "text": "Keep this as documentation only for the knowledge base and rewrite it as a reference document.", + "family": "long_context_document_only" + }, + { + "text": "Translate this long onboarding flow into French and keep it as documentation only with no skill package.", + "family": "translate_only" } ], "near_neighbor": [ @@ -101,6 +129,14 @@ { "text": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", "family": "partial_scaffold_not_full_skill" + }, + { + "text": "We may convert this workflow later, but for now keep it as documentation only and produce a reference document.", + "family": "long_context_near_neighbor" + }, + { + "text": "Draft a template only from this process and stop short of making a full skill package.", + "family": "partial_scaffold_not_full_skill" } ] } diff --git a/evals/trigger_cases.json b/evals/trigger_cases.json index a40816d0..1f54b9aa 100644 --- a/evals/trigger_cases.json +++ b/evals/trigger_cases.json @@ -73,6 +73,42 @@ { "text": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill" + }, + { + "text": "Codify this recurring support escalation workflow into a maintained skill for the ops library.", + "family": "paraphrase_trigger" + }, + { + "text": "Operationalize this workflow as a reusable skill with validation notes and team library metadata.", + "family": "paraphrase_trigger" + }, + { + "text": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", + "family": "paraphrase_trigger" + }, + { + "text": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", + "family": "paraphrase_trigger" + }, + { + "text": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", + "family": "paraphrase_trigger" + }, + { + "text": "Below are release notes, rough notes, and a transcript from the last handoff. Ignore the noise and turn this runbook into a reusable agent skill with references and scripts so the team can reuse it every quarter.", + "family": "long_context_trigger" + }, + { + "text": "We have a standard operating procedure, checklist, and prompt history scattered across docs. Formalize this process into a reusable capability, add validation, and package it for team reuse.", + "family": "long_context_trigger" + }, + { + "text": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", + "family": "long_context_trigger" + }, + { + "text": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", + "family": "long_context_trigger" } ], "should_not_trigger": [ @@ -135,6 +171,34 @@ { "text": "Explain this release checklist in simpler language; do not package it.", "family": "explain_only" + }, + { + "text": "Below is a long SOP, but I only want a summary for leadership and a recap of the main points. Do not build anything reusable.", + "family": "long_context_summary_only" + }, + { + "text": "Keep this as documentation only for the knowledge base and rewrite it as a reference document.", + "family": "long_context_document_only" + }, + { + "text": "Translate this long onboarding flow into French and keep it as documentation only with no skill package.", + "family": "translate_only" + }, + { + "text": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", + "family": "long_context_document_only" + }, + { + "text": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", + "family": "long_context_summary_only" + }, + { + "text": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", + "family": "long_context_document_only" + }, + { + "text": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", + "family": "translate_only" } ], "near_neighbor": [ @@ -193,6 +257,26 @@ { "text": "Turn this into a document package for the wiki, not an agent skill.", "family": "document_export_vs_agent_skill" + }, + { + "text": "We may convert this workflow later, but for now keep it as documentation only and produce a reference document.", + "family": "long_context_near_neighbor" + }, + { + "text": "Draft a template only from this process and stop short of making a full skill package.", + "family": "partial_scaffold_not_full_skill" + }, + { + "text": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", + "family": "long_context_near_neighbor" + }, + { + "text": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", + "family": "future_outline_vs_build" + }, + { + "text": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", + "family": "long_context_near_neighbor" } ] } diff --git a/reports/eval_suite.json b/reports/eval_suite.json new file mode 100644 index 00000000..1ba4b2d0 --- /dev/null +++ b/reports/eval_suite.json @@ -0,0 +1,3947 @@ +{ + "summary": { + "suite_count": 3, + "total_cases": 66, + "family_count": 21, + "false_positives": 0, + "false_negatives": 0, + "average_precision": 1.0, + "average_recall": 1.0 + }, + "family_summary": { + "workflow_to_skill": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "package_for_team": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "meta_skill_creation": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 5, + "passed": 5, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_summary_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "one_off_vs_reusable": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_vs_build": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "partial_scaffold_not_full_skill": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "complex_multi_asset": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_not_package": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 4, + "passed": 4, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "suites": { + "train": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 12, + "passed": 12, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 11, + "passed": 11, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 8, + "passed": 8, + "pass_rate": 1.0 + } + }, + "family_stats": { + "workflow_to_skill": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "package_for_team": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "meta_skill_creation": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_only": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "one_off_vs_reusable": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_vs_build": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "partial_scaffold_not_full_skill": { + "total": 3, + "passed": 3, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Create a skill from this repeated workflow.", + "family": "workflow_to_skill", + "score": 0.549, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.714, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "create a skill" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn this runbook into a reusable agent skill.", + "family": "workflow_to_skill", + "score": 0.729, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.625, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Convert this process note into a reusable skill package for the team.", + "family": "workflow_to_skill", + "score": 0.766, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0, + "lexical_support": 0.667, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "reusable", + "package" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Improve this skill description and add evals.", + "family": "iterate_existing_skill", + "score": 0.353, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.291, + "support_score": 0, + "lexical_support": 0.571, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals" + ], + "iterate_existing_skill": [ + "this skill", + "skill description" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Refactor this prompt into a proper skill package.", + "family": "iterate_existing_skill", + "score": 0.532, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Package this skill for team reuse.", + "family": "package_for_team", + "score": 0.448, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "iterate_existing_skill", + "reuse_package", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.667, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "iterate_existing_skill": [ + "this skill", + "package this skill" + ], + "reuse_package": [ + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Create a meta-skill for packaging internal workflows.", + "family": "meta_skill_creation", + "score": 0.591, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.527, + "support_score": 0, + "lexical_support": 0.571, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "meta skill", + "meta-skill" + ], + "reuse_package": [ + "packaging", + "internal workflows" + ], + "team_operationalize": [ + "internal" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Standardize this recurring workflow as a shareable skill package with references.", + "family": "package_for_team", + "score": 0.749, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0.06, + "lexical_support": 0.273, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "multi_asset": [ + "references" + ], + "reuse_package": [ + "package" + ], + "team_operationalize": [ + "standardize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this recurring support escalation workflow into a maintained skill for the ops library.", + "family": "paraphrase_trigger", + "score": 0.397, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "ops library", + "maintained" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Operationalize this workflow as a reusable skill with validation notes and team library metadata.", + "family": "paraphrase_trigger", + "score": 0.88, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.429, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "operationalize this workflow as a reusable skill" + ], + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "team library", + "operationalize" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below are release notes, rough notes, and a transcript from the last handoff. Ignore the noise and turn this runbook into a reusable agent skill with references and scripts so the team can reuse it every quarter.", + "family": "long_context_trigger", + "score": 0.758, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0.06, + "lexical_support": 0.387, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable", + "reuse" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "runbook", + "rough notes", + "transcript", + "release notes" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "We have a standard operating procedure, checklist, and prompt history scattered across docs. Formalize this process into a reusable capability, add validation, and package it for team reuse.", + "family": "long_context_trigger", + "score": 0.88, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.423, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "formalize this process into a reusable capability" + ], + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ], + "transform_workflow": [ + "process", + "checklist", + "prompt history", + "standard operating procedure" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Explain what a workflow is.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "explain_only": [ + "explain what" + ] + } + } + } + }, + { + "prompt": "Just explain what a skill is.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.333, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "explain what", + "just explain" + ] + } + } + } + }, + { + "prompt": "Summarize this random note.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "summarize_only": [ + "summarize" + ] + } + } + } + }, + { + "prompt": "Translate this README into Japanese.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "translate_only" + ], + "exclusive_negative_concepts": [ + "translate_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.53, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "readme" + ], + "translate_only": [ + "translate", + "into japanese" + ] + } + } + } + }, + { + "prompt": "Write a product headline for this landing page.", + "family": "document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "headline", + "landing page" + ] + } + } + } + }, + { + "prompt": "Draft a blog title for this article.", + "family": "document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "article", + "blog title" + ] + } + } + } + }, + { + "prompt": "Rewrite this paragraph more clearly, but do not package anything.", + "family": "document_only", + "score": 0.175, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.1, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "package" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Explain the difference between a runbook and a workflow.", + "family": "explain_only", + "score": 0.187, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow", + "runbook" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below is a long SOP, but I only want a summary for leadership and a recap of the main points. Do not build anything reusable.", + "family": "long_context_summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.19, + "negative_penalty": 0.72, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "no_build_directive": [ + "do not build" + ], + "summarize_only": [ + "summary", + "only want a summary", + "recap" + ] + } + } + } + }, + { + "prompt": "Keep this as documentation only for the knowledge base and rewrite it as a reference document.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "document", + "reference document", + "knowledge base" + ] + } + } + } + }, + { + "prompt": "Translate this long onboarding flow into French and keep it as documentation only with no skill package.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.235, + "negative_penalty": 0.77, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "onboarding flow" + ] + }, + "negative": { + "no_build_directive": [ + "no skill package", + "keep it as documentation only" + ], + "translate_only": [ + "translate", + "into french" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Create a one-off prompt for this task.", + "family": "one_off_vs_reusable", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "one_off_request" + ], + "exclusive_negative_concepts": [ + "one_off_request" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.429, + "negative_penalty": 0.24, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "one_off_request": [ + "one-off" + ] + } + } + } + }, + { + "prompt": "Write a custom answer for this request without creating a skill package.", + "family": "one_off_vs_reusable", + "score": 0.131, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "one_off_request" + ], + "exclusive_negative_concepts": [ + "one_off_request" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.364, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "one_off_request": [ + "custom answer", + "for this request" + ] + } + } + } + }, + { + "prompt": "Help me brainstorm process ideas without building a skill.", + "family": "brainstorm_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.333, + "negative_penalty": 0.66, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "process" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm", + "ideas" + ], + "no_build_directive": [ + "without building" + ] + } + } + } + }, + { + "prompt": "Improve this README but do not turn it into a skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "document_only": [ + "readme" + ], + "no_build_directive": [ + "do not turn it into a skill" + ] + } + } + } + }, + { + "prompt": "Make a checklist for this task, but not a reusable skill.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0.42, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "no_build_directive": [ + "not a reusable skill" + ] + } + } + } + }, + { + "prompt": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.231, + "negative_penalty": 0.42, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "no_build_directive": [ + "plain note", + "keep it as a plain note" + ] + } + } + } + }, + { + "prompt": "We may convert this workflow later, but for now keep it as documentation only and produce a reference document.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.158, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "reference document" + ], + "no_build_directive": [ + "keep it as documentation only" + ] + } + } + } + }, + { + "prompt": "Draft a template only from this process and stop short of making a full skill package.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0.6, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "document_only": [ + "template only" + ], + "no_build_directive": [ + "stop short of making a full skill" + ] + } + } + } + } + ] + }, + "comparison": { + "baseline_false_positives": 0, + "baseline_false_negatives": 1, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": -1, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 0.917, + "improved_recall": 1.0 + }, + "returncode": 0 + }, + "dev": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 8, + "passed": 8, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + } + }, + "family_stats": { + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "complex_multi_asset": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_not_package": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "partial_scaffold_not_full_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Convert this operations checklist into a reusable skill.", + "family": "workflow_to_skill", + "score": 0.415, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "checklist", + "operations checklist" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Add trigger evals to this skill before sharing it with the team.", + "family": "iterate_existing_skill", + "score": 0.388, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.327, + "support_score": 0, + "lexical_support": 0.333, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "trigger evals" + ], + "iterate_existing_skill": [ + "this skill", + "sharing it with the team", + "before sharing it with the team" + ], + "team_operationalize": [ + "team", + "sharing it with the team" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", + "family": "complex_multi_asset", + "score": 0.843, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package", + "turn all of that into one reusable skill package" + ], + "eval_optimize": [ + "evals", + "packaging checks" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "runbook", + "process", + "prompt history" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", + "family": "complex_multi_asset", + "score": 0.418, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.357, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "production-ready" + ], + "transform_workflow": [ + "workflow", + "workflow fragments" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", + "family": "iterate_existing_skill", + "score": 0.563, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.429, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "distribution" + ], + "team_operationalize": [ + "team", + "distribution" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", + "family": "paraphrase_trigger", + "score": 0.471, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.4, + "support_score": 0, + "lexical_support": 0.538, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "reuse_package": [ + "team library", + "maintained" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", + "family": "paraphrase_trigger", + "score": 0.868, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.267, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this workflow" + ], + "eval_optimize": [ + "validate", + "route boundary" + ], + "reuse_package": [ + "package", + "distribution", + "shared capability" + ], + "team_operationalize": [ + "shared", + "distribution" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", + "family": "long_context_trigger", + "score": 0.891, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0.06, + "lexical_support": 0.37, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill", + "skill package" + ], + "eval_optimize": [ + "validation" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ], + "transform_workflow": [ + "rough notes", + "transcript", + "onboarding flow" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Give me ideas for improving our process.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.24, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "process" + ] + }, + "negative": { + "brainstorm_only": [ + "ideas" + ] + } + } + } + }, + { + "prompt": "Review this note and tell me what it means in plain English.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "tell me what it means", + "plain english" + ] + } + } + } + }, + { + "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0.06, + "lexical_support": 0.2, + "negative_penalty": 0.66, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm", + "improve packaging" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Translate these notes into French and keep the structure exactly as-is.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "translate_only" + ], + "exclusive_negative_concepts": [ + "translate_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.273, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "translate_only": [ + "translate", + "into french" + ] + } + } + } + }, + { + "prompt": "Summarize this workflow and list the main points only.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.111, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "summarize_only": [ + "summarize" + ] + } + } + } + }, + { + "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.182, + "negative_penalty": 0.68, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "explain_only": [ + "explain it", + "in simpler language" + ], + "no_build_directive": [ + "do not convert it into a reusable capability", + "keep it as documentation only" + ] + } + } + } + }, + { + "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", + "family": "long_context_summary_only", + "score": 0.257, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.353, + "negative_penalty": 0.3, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "summarize_only": [ + "summary", + "recap" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Review this process note and explain it, no packaging needed.", + "family": "explain_not_package", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "explain_only": [ + "explain it" + ], + "no_build_directive": [ + "no packaging needed" + ] + } + } + } + }, + { + "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.42, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "no_build_directive": [ + "stop short of making a full skill" + ] + } + } + } + }, + { + "prompt": "Package this explanation as a document, not as an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document" + ], + "no_build_directive": [ + "not as an agent skill" + ] + } + } + } + }, + { + "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "iterate_existing_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.327, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "iterate_existing_skill": [ + "this skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.462, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "document_only": [ + "document", + "reusable-looking document" + ], + "no_build_directive": [ + "without turning them into an agent skill" + ] + } + } + } + }, + { + "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", + "family": "long_context_near_neighbor", + "score": 0.328, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.18, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document", + "template only" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.278, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "for later", + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + } + ] + }, + "comparison": { + "baseline_false_positives": 1, + "baseline_false_negatives": 1, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": -1, + "false_negative_delta": -1, + "baseline_precision": 0.875, + "improved_precision": 1.0, + "baseline_recall": 0.875, + "improved_recall": 1.0 + }, + "returncode": 0 + }, + "holdout": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 5, + "passed": 5, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + } + }, + "family_stats": { + "complex_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "family": "complex_multi_asset", + "score": 0.468, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.4, + "support_score": 0.06, + "lexical_support": 0.316, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable" + ], + "team_operationalize": [ + "internal" + ], + "transform_workflow": [ + "process", + "internal process note" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", + "family": "iterate_existing_skill", + "score": 0.729, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.691, + "support_score": 0, + "lexical_support": 0.412, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "near-neighbor", + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "package", + "distribution" + ], + "team_operationalize": [ + "distribution" + ], + "transform_workflow": [ + "existing skill draft" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", + "family": "workflow_to_skill", + "score": 0.563, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.429, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "library entry" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", + "family": "paraphrase_trigger", + "score": 0.858, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0.06, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "packaging checks" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "quarterly release routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", + "family": "long_context_trigger", + "score": 0.744, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "turn this playbook into an agent capability package" + ], + "reuse_package": [ + "package", + "team library" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "prompt history", + "transcript", + "release notes" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.308, + "negative_penalty": 0.72, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "no_build_directive": [ + "not a reusable skill" + ], + "summarize_only": [ + "summary", + "only want a summary" + ] + } + } + } + }, + { + "prompt": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.091, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "explain_only": [ + "explain this", + "in simpler language" + ], + "no_build_directive": [ + "do not package it" + ] + } + } + } + }, + { + "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.222, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document" + ], + "no_build_directive": [ + "not for agent execution" + ] + } + } + } + }, + { + "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.77, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "no_build_directive": [ + "no skill package", + "keep it as documentation only" + ], + "translate_only": [ + "translate", + "into russian" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "semantic_coverage": 0.309, + "support_score": 0, + "lexical_support": 0.308, + "negative_penalty": 0.46, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ] + }, + "negative": { + "brainstorm_only": [ + "shape an idea" + ], + "future_outline": [ + "before we decide whether to build" + ] + } + } + } + }, + { + "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill", + "outline for a possible future skill" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.538, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document", + "document package", + "wiki" + ], + "no_build_directive": [ + "not an agent skill" + ] + } + } + } + }, + { + "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.375, + "negative_penalty": 0.6, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "workflow", + "checklist" + ] + }, + "negative": { + "document_only": [ + "template only" + ], + "no_build_directive": [ + "no skill package", + "discussion only" + ] + } + } + } + } + ] + }, + "comparison": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "returncode": 0 + } + } +} diff --git a/reports/family_summary.md b/reports/family_summary.md new file mode 100644 index 00000000..a258b507 --- /dev/null +++ b/reports/family_summary.md @@ -0,0 +1,34 @@ +# Eval Family Summary + +- total cases: `66` +- families: `21` +- false positives: `0` +- false negatives: `0` +- average precision: `1.0` +- average recall: `1.0` + +## Family Results + +| Family | Cases | Passed | FP | FN | Pass Rate | +| --- | ---: | ---: | ---: | ---: | ---: | +| `brainstorm_only` | 2 | 2 | 0 | 0 | 1.0 | +| `brainstorm_vs_build` | 1 | 1 | 0 | 0 | 1.0 | +| `complex_multi_asset` | 3 | 3 | 0 | 0 | 1.0 | +| `document_export_vs_agent_skill` | 4 | 4 | 0 | 0 | 1.0 | +| `document_only` | 3 | 3 | 0 | 0 | 1.0 | +| `explain_not_package` | 1 | 1 | 0 | 0 | 1.0 | +| `explain_only` | 5 | 5 | 0 | 0 | 1.0 | +| `future_outline_vs_build` | 4 | 4 | 0 | 0 | 1.0 | +| `iterate_existing_skill` | 5 | 5 | 0 | 0 | 1.0 | +| `long_context_document_only` | 3 | 3 | 0 | 0 | 1.0 | +| `long_context_near_neighbor` | 3 | 3 | 0 | 0 | 1.0 | +| `long_context_summary_only` | 2 | 2 | 0 | 0 | 1.0 | +| `long_context_trigger` | 4 | 4 | 0 | 0 | 1.0 | +| `meta_skill_creation` | 1 | 1 | 0 | 0 | 1.0 | +| `one_off_vs_reusable` | 2 | 2 | 0 | 0 | 1.0 | +| `package_for_team` | 2 | 2 | 0 | 0 | 1.0 | +| `paraphrase_trigger` | 5 | 5 | 0 | 0 | 1.0 | +| `partial_scaffold_not_full_skill` | 4 | 4 | 0 | 0 | 1.0 | +| `summary_only` | 3 | 3 | 0 | 0 | 1.0 | +| `translate_only` | 4 | 4 | 0 | 0 | 1.0 | +| `workflow_to_skill` | 5 | 5 | 0 | 0 | 1.0 | diff --git a/scripts/render_eval_dashboard.py b/scripts/render_eval_dashboard.py new file mode 100644 index 00000000..da7f58d7 --- /dev/null +++ b/scripts/render_eval_dashboard.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +BEGIN_MARKER = "" +END_MARKER = "" + + +def run_eval_suite(args: argparse.Namespace) -> dict: + proc = subprocess.run( + [ + sys.executable, + "scripts/run_eval_suite.py", + "--eval-dir", + args.eval_dir, + "--description-file", + args.description_file, + "--baseline-description-file", + args.baseline_description_file, + ], + capture_output=True, + text=True, + check=True, + ) + return json.loads(proc.stdout) + + +def render_readme_panel(report: dict) -> str: + summary = report["summary"] + suites = report["suites"] + families = report["family_summary"] + + lines = [ + BEGIN_MARKER, + f"- regression corpus: `{summary['total_cases']}` prompts across `{summary['family_count']}` families", + f"- aggregate result: `{summary['false_positives']}` false positives, `{summary['false_negatives']}` false negatives, average precision `{summary['average_precision']}`, average recall `{summary['average_recall']}`", + "- suite status:", + "", + "| Suite | Cases | FP | FN | Precision | Recall |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + for suite_name in ("train", "dev", "holdout"): + suite = suites[suite_name] + cases = sum(bucket["total"] for bucket in suite["bucket_stats"].values()) + lines.append( + f"| {suite_name} | {cases} | {suite['false_positives']} | {suite['false_negatives']} | {suite['precision']} | {suite['recall']} |" + ) + + lines.extend( + [ + "", + "| Family | Cases | Pass Rate |", + "| --- | ---: | ---: |", + ] + ) + for family, stats in sorted(families.items()): + lines.append(f"| `{family}` | {stats['total']} | {stats['pass_rate']} |") + + lines.extend( + [ + "", + "Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md)", + END_MARKER, + ] + ) + return "\n".join(lines) + + +def render_family_summary(report: dict) -> str: + summary = report["summary"] + lines = [ + "# Eval Family Summary", + "", + f"- total cases: `{summary['total_cases']}`", + f"- families: `{summary['family_count']}`", + f"- false positives: `{summary['false_positives']}`", + f"- false negatives: `{summary['false_negatives']}`", + f"- average precision: `{summary['average_precision']}`", + f"- average recall: `{summary['average_recall']}`", + "", + "## Family Results", + "", + "| Family | Cases | Passed | FP | FN | Pass Rate |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + for family, stats in sorted(report["family_summary"].items()): + lines.append( + f"| `{family}` | {stats['total']} | {stats['passed']} | {stats['false_positives']} | {stats['false_negatives']} | {stats['pass_rate']} |" + ) + return "\n".join(lines) + "\n" + + +def update_readme(readme_path: Path, panel: str) -> None: + text = readme_path.read_text(encoding="utf-8") + if BEGIN_MARKER not in text or END_MARKER not in text: + raise SystemExit(f"README markers not found in {readme_path}") + start = text.index(BEGIN_MARKER) + end = text.index(END_MARKER) + len(END_MARKER) + updated = text[:start] + panel + text[end:] + readme_path.write_text(updated, encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render README-visible eval results and full family report.") + parser.add_argument("--eval-dir", default="evals") + parser.add_argument("--description-file", default="evals/improved_description.txt") + parser.add_argument("--baseline-description-file", default="evals/baseline_description.txt") + parser.add_argument("--output-dir", default="reports") + parser.add_argument("--readme", default="README.md") + args = parser.parse_args() + + report = run_eval_suite(args) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + (output_dir / "eval_suite.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output_dir / "family_summary.md").write_text(render_family_summary(report), encoding="utf-8") + update_readme(Path(args.readme), render_readme_panel(report)) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_eval_suite.py b/scripts/run_eval_suite.py index 90766316..c1bdd763 100644 --- a/scripts/run_eval_suite.py +++ b/scripts/run_eval_suite.py @@ -33,6 +33,7 @@ def main() -> None: parser.add_argument("--eval-dir", default="evals", help="Root eval directory") parser.add_argument("--description-file", default="evals/improved_description.txt") parser.add_argument("--baseline-description-file", default="evals/baseline_description.txt") + parser.add_argument("--output-file", help="Optional path to write the combined JSON report") args = parser.parse_args() root = Path(args.eval_dir).resolve() @@ -65,15 +66,21 @@ def main() -> None: for family, stats in family_summary.items(): stats["pass_rate"] = round(stats["passed"] / stats["total"], 3) if stats["total"] else None + total_cases = sum(stats["total"] for stats in family_summary.values()) summary = { "suite_count": len(suites), + "total_cases": total_cases, + "family_count": len(family_summary), "false_positives": aggregate["false_positives"], "false_negatives": aggregate["false_negatives"], "average_precision": round(sum(aggregate["precision"]) / len(aggregate["precision"]), 3) if aggregate["precision"] else None, "average_recall": round(sum(aggregate["recall"]) / len(aggregate["recall"]), 3) if aggregate["recall"] else None, } output = {"summary": summary, "family_summary": family_summary, "suites": suites} - print(json.dumps(output, ensure_ascii=False, indent=2)) + rendered = json.dumps(output, ensure_ascii=False, indent=2) + if args.output_file: + Path(args.output_file).write_text(rendered + "\n", encoding="utf-8") + print(rendered) if summary["false_positives"] > 0 or summary["false_negatives"] > 0: raise SystemExit(2)