{ "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "current_candidate": { "id": "current", "label": "Current", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "strategy": "current", "estimated_tokens": 65, "dev": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 0, -1.0, -1.0, -1.0, -1.0, 65 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, "baseline": { "description": "Create and improve agent skills.", "estimated_tokens": 8, "dev": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 1, "false_negatives": 1, "precision": 0.875, "recall": 0.875, "bucket_stats": { "should_trigger": { "total": 8, "passed": 7, "pass_rate": 0.875 }, "should_not_trigger": { "total": 7, "passed": 7, "pass_rate": 1.0 }, "near_neighbor": { "total": 7, "passed": 6, "pass_rate": 0.857 } }, "family_stats": { "workflow_to_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "iterate_existing_skill": { "total": 2, "passed": 1, "false_positives": 0, "false_negatives": 1, "pass_rate": 0.5 }, "complex_multi_asset": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "paraphrase_trigger": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_not_package": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "partial_scaffold_not_full_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "document_export_vs_agent_skill": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "future_outline_vs_build": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_near_neighbor": { "total": 1, "passed": 0, "false_positives": 1, "false_negatives": 0, "pass_rate": 0.0 } }, "misfires": [ { "bucket": "should_trigger", "family": "iterate_existing_skill", "prompt": "Add trigger evals to this skill before sharing it with the team.", "score": 0.328, "reason": "false_negative", "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill" ], "matched_negative_concepts": [] }, { "bucket": "near_neighbor", "family": "long_context_near_neighbor", "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", "score": 0.333, "reason": "false_positive", "matched_desired_concepts": [ "build_skill", "transform_workflow" ], "matched_negative_concepts": [ "document_only" ] } ], "results": { "should_trigger": [ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", "score": 0.387, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.377, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ], "transform_workflow": [ "checklist", "operations checklist" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", "score": 0.328, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill" ], "extra_positive_concepts": [ "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.302, "support_score": 0.04, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "eval_optimize": [ "evals", "trigger evals" ], "iterate_existing_skill": [ "this skill", "sharing it with the team", "before sharing it with the team" ], "team_operationalize": [ "team", "sharing it with the team" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", "score": 0.844, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.849, "support_score": 0, "lexical_support": 0.042, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package", "turn all of that into one reusable skill package" ], "eval_optimize": [ "evals", "packaging checks" ], "reuse_package": [ "reusable", "package", "packaging" ], "transform_workflow": [ "runbook", "process", "prompt history" ] }, "negative": {} } } }, { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", "score": 0.408, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.377, "support_score": 0.06, "lexical_support": 0.071, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "multi_asset": [ "references", "scripts" ], "reuse_package": [ "production-ready" ], "transform_workflow": [ "workflow", "workflow fragments" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", "score": 0.527, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "reuse_package" ], "extra_positive_concepts": [ "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.491, "support_score": 0.04, "lexical_support": 0.071, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "trigger boundary" ], "iterate_existing_skill": [ "existing skill" ], "reuse_package": [ "distribution" ], "team_operationalize": [ "team", "distribution" ] }, "negative": {} } } }, { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", "score": 0.403, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.377, "support_score": 0.04, "lexical_support": 0.077, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "team library", "maintained" ], "team_operationalize": [ "team" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", "score": 0.856, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.849, "support_score": 0.04, "lexical_support": 0.067, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "codify this workflow" ], "eval_optimize": [ "validate", "route boundary" ], "reuse_package": [ "package", "distribution", "shared capability" ], "team_operationalize": [ "shared", "distribution" ], "transform_workflow": [ "workflow" ] }, "negative": {} } } }, { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", "score": 0.869, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset", "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.849, "support_score": 0.1, "lexical_support": 0.037, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "build a skill", "skill package" ], "eval_optimize": [ "validation" ], "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reuse", "package", "team reuse" ], "team_operationalize": [ "team", "team reuse" ], "transform_workflow": [ "rough notes", "transcript", "onboarding flow" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Give me ideas for improving our process.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.189, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "process" ] }, "negative": { "brainstorm_only": [ "ideas" ] } } } }, { "prompt": "Review this note and tell me what it means in plain English.", "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.083, "negative_penalty": 0.26, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "tell me what it means", "plain english" ] } } } }, { "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "brainstorm_only", "no_build_directive" ], "exclusive_negative_concepts": [ "brainstorm_only", "no_build_directive" ], "semantic_coverage": 0.189, "support_score": 0.06, "lexical_support": 0.067, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "reuse_package": [ "packaging" ] }, "negative": { "brainstorm_only": [ "brainstorm", "improve packaging" ], "no_build_directive": [ "do not generate any skill files" ] } } } }, { "prompt": "Translate these notes into French and keep the structure exactly as-is.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "translate_only" ], "exclusive_negative_concepts": [ "translate_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "translate_only": [ "translate", "into french" ] } } } }, { "prompt": "Summarize this workflow and list the main points only.", "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], "semantic_coverage": 0.189, "support_score": 0, "lexical_support": 0.111, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "workflow" ] }, "negative": { "summarize_only": [ "summarize" ] } } } }, { "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", "family": "long_context_document_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.189, "support_score": 0, "lexical_support": 0.045, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ] }, "negative": { "explain_only": [ "explain it", "in simpler language" ], "no_build_directive": [ "do not convert it into a reusable capability", "keep it as documentation only" ] } } } }, { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", "score": 0.257, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], "semantic_coverage": 0.698, "support_score": 0, "lexical_support": 0.059, "negative_penalty": 0.3, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "process" ] }, "negative": { "summarize_only": [ "summary", "recap" ] } } }, "boundary_case": true } ], "near_neighbor": [ { "prompt": "Review this process note and explain it, no packaging needed.", "family": "explain_not_package", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.377, "support_score": 0, "lexical_support": 0.1, "negative_penalty": 0.68, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "explain_only": [ "explain it" ], "no_build_directive": [ "no packaging needed" ] } } } }, { "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", "family": "partial_scaffold_not_full_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.189, "support_score": 0, "lexical_support": 0.071, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "checklist" ] }, "negative": { "no_build_directive": [ "stop short of making a full skill" ] } } } }, { "prompt": "Package this explanation as a document, not as an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.1, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "package" ] }, "negative": { "document_only": [ "document" ], "no_build_directive": [ "not as an agent skill" ] } } } }, { "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "iterate_existing_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.34, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0.64, "coverage_boost": 0.04, "concept_evidence": { "positive": { "iterate_existing_skill": [ "this skill" ], "reuse_package": [ "package" ] }, "negative": { "future_outline": [ "future-ready outline" ], "no_build_directive": [ "do not build" ] } } } }, { "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.077, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "reusable" ] }, "negative": { "document_only": [ "document", "reusable-looking document" ], "no_build_directive": [ "without turning them into an agent skill" ] } } } }, { "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", "family": "long_context_near_neighbor", "score": 0.333, "predicted_trigger": true, "expected_trigger": false, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only" ], "exclusive_negative_concepts": [], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.05, "negative_penalty": 0.18, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "build a skill" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "document", "wiki", "reference document", "template only" ] } } }, "boundary_case": true }, { "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.056, "negative_penalty": 0.64, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ] }, "negative": { "future_outline": [ "for later", "future-ready outline" ], "no_build_directive": [ "do not build" ] } } } } ] } }, "holdout": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 5, "passed": 5, "pass_rate": 1.0 }, "should_not_trigger": { "total": 4, "passed": 4, "pass_rate": 1.0 }, "near_neighbor": { "total": 4, "passed": 4, "pass_rate": 1.0 } }, "family_stats": { "complex_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "iterate_existing_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "workflow_to_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "paraphrase_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "future_outline_vs_build": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "document_export_vs_agent_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_near_neighbor": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", "score": 0.416, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset", "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.377, "support_score": 0.1, "lexical_support": 0.053, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reusable" ], "team_operationalize": [ "internal" ], "transform_workflow": [ "process", "internal process note" ] }, "negative": {} } } }, { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", "score": 0.7, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.679, "support_score": 0.04, "lexical_support": 0.059, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "evals", "near-neighbor", "trigger boundary" ], "iterate_existing_skill": [ "existing skill" ], "reuse_package": [ "package", "distribution" ], "team_operationalize": [ "distribution" ], "transform_workflow": [ "existing skill draft" ] }, "negative": {} } } }, { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", "score": 0.546, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.528, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "validation" ], "reuse_package": [ "reusable", "library entry" ], "transform_workflow": [ "process" ] }, "negative": {} } } }, { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", "score": 0.861, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.849, "support_score": 0.06, "lexical_support": 0.062, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "eval_optimize": [ "packaging checks" ], "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reusable", "package", "packaging" ], "transform_workflow": [ "quarterly release routine" ] }, "negative": {} } } }, { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", "score": 0.719, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "team_operationalize" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.698, "support_score": 0.04, "lexical_support": 0.08, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "turn this playbook into an agent capability package" ], "reuse_package": [ "package", "team library" ], "team_operationalize": [ "team" ], "transform_workflow": [ "prompt history", "transcript", "release notes" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive", "summarize_only" ], "exclusive_negative_concepts": [ "no_build_directive", "summarize_only" ], "semantic_coverage": 0.377, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ], "transform_workflow": [ "process" ] }, "negative": { "no_build_directive": [ "not a reusable skill" ], "summarize_only": [ "summary", "only want a summary" ] } } } }, { "prompt": "Explain this release checklist in simpler language; do not package it.", "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.377, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0.68, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "package" ], "transform_workflow": [ "checklist" ] }, "negative": { "explain_only": [ "explain this", "in simpler language" ], "no_build_directive": [ "do not package it" ] } } } }, { "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", "family": "long_context_document_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.189, "support_score": 0, "lexical_support": 0.111, "negative_penalty": 0.6, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "document", "wiki", "reference document" ], "no_build_directive": [ "not for agent execution" ] } } } }, { "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive", "translate_only" ], "exclusive_negative_concepts": [ "no_build_directive", "translate_only" ], "semantic_coverage": 0.698, "support_score": 0, "lexical_support": 0.062, "negative_penalty": 0.77, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "runbook" ] }, "negative": { "no_build_directive": [ "no skill package", "keep it as documentation only" ], "translate_only": [ "translate", "into russian" ] } } } } ], "near_neighbor": [ { "prompt": "Help me shape an idea before we decide whether to build a skill.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" ], "exclusive_negative_concepts": [ "brainstorm_only", "future_outline" ], "semantic_coverage": 0.321, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ "build a skill" ] }, "negative": { "brainstorm_only": [ "shape an idea" ], "future_outline": [ "before we decide whether to build" ] } } } }, { "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.071, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "future_outline": [ "future skill", "possible future skill", "outline for a possible future skill" ], "no_build_directive": [ "do not build" ] } } } }, { "prompt": "Turn this into a document package for the wiki, not an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.077, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "package" ] }, "negative": { "document_only": [ "document", "document package", "wiki" ], "no_build_directive": [ "not an agent skill" ] } } } }, { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.698, "support_score": 0, "lexical_support": 0.062, "negative_penalty": 0.6, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "workflow", "checklist" ] }, "negative": { "document_only": [ "template only" ], "no_build_directive": [ "no skill package", "discussion only" ] } } } } ] } } }, "winner": { "id": "current", "label": "Current", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "strategy": "current", "estimated_tokens": 65, "dev": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 0, -1.0, -1.0, -1.0, -1.0, 65 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, "winner_dev_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 8, "passed": 8, "pass_rate": 1.0 }, "should_not_trigger": { "total": 7, "passed": 7, "pass_rate": 1.0 }, "near_neighbor": { "total": 7, "passed": 7, "pass_rate": 1.0 } }, "family_stats": { "workflow_to_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "iterate_existing_skill": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "complex_multi_asset": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "paraphrase_trigger": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_not_package": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "partial_scaffold_not_full_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "document_export_vs_agent_skill": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "future_outline_vs_build": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_near_neighbor": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", "score": 0.415, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.5, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ], "transform_workflow": [ "checklist", "operations checklist" ] }, "negative": {} } } }, { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", "score": 0.388, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.327, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "evals", "trigger evals" ], "iterate_existing_skill": [ "this skill", "sharing it with the team", "before sharing it with the team" ], "team_operationalize": [ "team", "sharing it with the team" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", "score": 0.843, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0, "lexical_support": 0.375, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package", "turn all of that into one reusable skill package" ], "eval_optimize": [ "evals", "packaging checks" ], "reuse_package": [ "reusable", "package", "packaging" ], "transform_workflow": [ "runbook", "process", "prompt history" ] }, "negative": {} } } }, { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", "score": 0.418, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0.06, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "multi_asset": [ "references", "scripts" ], "reuse_package": [ "production-ready" ], "transform_workflow": [ "workflow", "workflow fragments" ] }, "negative": {} } } }, { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", "score": 0.557, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "reuse_package", "team_operationalize" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "trigger boundary" ], "iterate_existing_skill": [ "existing skill" ], "reuse_package": [ "distribution" ], "team_operationalize": [ "team", "distribution" ] }, "negative": {} } } }, { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", "score": 0.471, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.4, "support_score": 0, "lexical_support": 0.538, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "reuse_package": [ "team library", "maintained" ], "team_operationalize": [ "team" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } } }, { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", "score": 0.868, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.855, "support_score": 0, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "codify this workflow" ], "eval_optimize": [ "validate", "route boundary" ], "reuse_package": [ "package", "distribution", "shared capability" ], "team_operationalize": [ "shared", "distribution" ], "transform_workflow": [ "workflow" ] }, "negative": {} } } }, { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", "score": 0.888, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.855, "support_score": 0.06, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "build a skill", "skill package" ], "eval_optimize": [ "validation" ], "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reuse", "package", "team reuse" ], "team_operationalize": [ "team", "team reuse" ], "transform_workflow": [ "rough notes", "transcript", "onboarding flow" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Give me ideas for improving our process.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "process" ] }, "negative": { "brainstorm_only": [ "ideas" ] } } } }, { "prompt": "Review this note and tell me what it means in plain English.", "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.083, "negative_penalty": 0.26, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "tell me what it means", "plain english" ] } } } }, { "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "brainstorm_only", "no_build_directive" ], "exclusive_negative_concepts": [ "brainstorm_only", "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0.06, "lexical_support": 0.2, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "reuse_package": [ "packaging" ] }, "negative": { "brainstorm_only": [ "brainstorm", "improve packaging" ], "no_build_directive": [ "do not generate any skill files" ] } } } }, { "prompt": "Translate these notes into French and keep the structure exactly as-is.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "translate_only" ], "exclusive_negative_concepts": [ "translate_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "translate_only": [ "translate", "into french" ] } } } }, { "prompt": "Summarize this workflow and list the main points only.", "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.111, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "workflow" ] }, "negative": { "summarize_only": [ "summarize" ] } } } }, { "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", "family": "long_context_document_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.182, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ] }, "negative": { "explain_only": [ "explain it", "in simpler language" ], "no_build_directive": [ "do not convert it into a reusable capability", "keep it as documentation only" ] } } } }, { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", "score": 0.257, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], "semantic_coverage": 0.673, "support_score": 0, "lexical_support": 0.353, "negative_penalty": 0.3, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "process" ] }, "negative": { "summarize_only": [ "summary", "recap" ] } } }, "boundary_case": true } ], "near_neighbor": [ { "prompt": "Review this process note and explain it, no packaging needed.", "family": "explain_not_package", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "explain_only": [ "explain it" ], "no_build_directive": [ "no packaging needed" ] } } } }, { "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", "family": "partial_scaffold_not_full_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "checklist" ] }, "negative": { "no_build_directive": [ "stop short of making a full skill" ] } } } }, { "prompt": "Package this explanation as a document, not as an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "package" ] }, "negative": { "document_only": [ "document" ], "no_build_directive": [ "not as an agent skill" ] } } } }, { "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "iterate_existing_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.327, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.64, "coverage_boost": 0.04, "concept_evidence": { "positive": { "iterate_existing_skill": [ "this skill" ], "reuse_package": [ "package" ] }, "negative": { "future_outline": [ "future-ready outline" ], "no_build_directive": [ "do not build" ] } } } }, { "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.462, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "reusable" ] }, "negative": { "document_only": [ "document", "reusable-looking document" ], "no_build_directive": [ "without turning them into an agent skill" ] } } } }, { "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", "family": "long_context_near_neighbor", "score": 0.328, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only" ], "exclusive_negative_concepts": [], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.18, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "build a skill" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "document", "wiki", "reference document", "template only" ] } } }, "boundary_case": true }, { "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.278, "negative_penalty": 0.64, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ] }, "negative": { "future_outline": [ "for later", "future-ready outline" ], "no_build_directive": [ "do not build" ] } } } } ] } }, "winner_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 5, "passed": 5, "pass_rate": 1.0 }, "should_not_trigger": { "total": 4, "passed": 4, "pass_rate": 1.0 }, "near_neighbor": { "total": 4, "passed": 4, "pass_rate": 1.0 } }, "family_stats": { "complex_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "iterate_existing_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "workflow_to_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "paraphrase_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "future_outline_vs_build": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "document_export_vs_agent_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_near_neighbor": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", "score": 0.468, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.4, "support_score": 0.06, "lexical_support": 0.316, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reusable" ], "team_operationalize": [ "internal" ], "transform_workflow": [ "process", "internal process note" ] }, "negative": {} } } }, { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", "score": 0.729, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.691, "support_score": 0, "lexical_support": 0.412, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "evals", "near-neighbor", "trigger boundary" ], "iterate_existing_skill": [ "existing skill" ], "reuse_package": [ "package", "distribution" ], "team_operationalize": [ "distribution" ], "transform_workflow": [ "existing skill draft" ] }, "negative": {} } } }, { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", "score": 0.563, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.429, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "validation" ], "reuse_package": [ "reusable", "library entry" ], "transform_workflow": [ "process" ] }, "negative": {} } } }, { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", "score": 0.858, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0.06, "lexical_support": 0.375, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "eval_optimize": [ "packaging checks" ], "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reusable", "package", "packaging" ], "transform_workflow": [ "quarterly release routine" ] }, "negative": {} } } }, { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", "score": 0.744, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.709, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "turn this playbook into an agent capability package" ], "reuse_package": [ "package", "team library" ], "team_operationalize": [ "team" ], "transform_workflow": [ "prompt history", "transcript", "release notes" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive", "summarize_only" ], "exclusive_negative_concepts": [ "no_build_directive", "summarize_only" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.308, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ], "transform_workflow": [ "process" ] }, "negative": { "no_build_directive": [ "not a reusable skill" ], "summarize_only": [ "summary", "only want a summary" ] } } } }, { "prompt": "Explain this release checklist in simpler language; do not package it.", "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "package" ], "transform_workflow": [ "checklist" ] }, "negative": { "explain_only": [ "explain this", "in simpler language" ], "no_build_directive": [ "do not package it" ] } } } }, { "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", "family": "long_context_document_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "document", "wiki", "reference document" ], "no_build_directive": [ "not for agent execution" ] } } } }, { "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive", "translate_only" ], "exclusive_negative_concepts": [ "no_build_directive", "translate_only" ], "semantic_coverage": 0.673, "support_score": 0, "lexical_support": 0.25, "negative_penalty": 0.77, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "runbook" ] }, "negative": { "no_build_directive": [ "no skill package", "keep it as documentation only" ], "translate_only": [ "translate", "into russian" ] } } } } ], "near_neighbor": [ { "prompt": "Help me shape an idea before we decide whether to build a skill.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" ], "exclusive_negative_concepts": [ "brainstorm_only", "future_outline" ], "semantic_coverage": 0.309, "support_score": 0, "lexical_support": 0.308, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ "build a skill" ] }, "negative": { "brainstorm_only": [ "shape an idea" ], "future_outline": [ "before we decide whether to build" ] } } } }, { "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.357, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "future_outline": [ "future skill", "possible future skill", "outline for a possible future skill" ], "no_build_directive": [ "do not build" ] } } } }, { "prompt": "Turn this into a document package for the wiki, not an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.538, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "package" ] }, "negative": { "document_only": [ "document", "document package", "wiki" ], "no_build_directive": [ "not an agent skill" ] } } } }, { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.673, "support_score": 0, "lexical_support": 0.375, "negative_penalty": 0.6, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "workflow", "checklist" ] }, "negative": { "document_only": [ "template only" ], "no_build_directive": [ "no skill package", "discussion only" ] } } } } ] } }, "current_dev_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 8, "passed": 8, "pass_rate": 1.0 }, "should_not_trigger": { "total": 7, "passed": 7, "pass_rate": 1.0 }, "near_neighbor": { "total": 7, "passed": 7, "pass_rate": 1.0 } }, "family_stats": { "workflow_to_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "iterate_existing_skill": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "complex_multi_asset": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "paraphrase_trigger": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_not_package": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "partial_scaffold_not_full_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "document_export_vs_agent_skill": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "future_outline_vs_build": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_near_neighbor": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", "score": 0.415, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.5, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ], "transform_workflow": [ "checklist", "operations checklist" ] }, "negative": {} } } }, { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", "score": 0.388, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.327, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "evals", "trigger evals" ], "iterate_existing_skill": [ "this skill", "sharing it with the team", "before sharing it with the team" ], "team_operationalize": [ "team", "sharing it with the team" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", "score": 0.843, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0, "lexical_support": 0.375, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package", "turn all of that into one reusable skill package" ], "eval_optimize": [ "evals", "packaging checks" ], "reuse_package": [ "reusable", "package", "packaging" ], "transform_workflow": [ "runbook", "process", "prompt history" ] }, "negative": {} } } }, { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", "score": 0.418, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0.06, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "multi_asset": [ "references", "scripts" ], "reuse_package": [ "production-ready" ], "transform_workflow": [ "workflow", "workflow fragments" ] }, "negative": {} } } }, { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", "score": 0.557, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "reuse_package", "team_operationalize" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "trigger boundary" ], "iterate_existing_skill": [ "existing skill" ], "reuse_package": [ "distribution" ], "team_operationalize": [ "team", "distribution" ] }, "negative": {} } } }, { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", "score": 0.471, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.4, "support_score": 0, "lexical_support": 0.538, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "reuse_package": [ "team library", "maintained" ], "team_operationalize": [ "team" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } } }, { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", "score": 0.868, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.855, "support_score": 0, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "codify this workflow" ], "eval_optimize": [ "validate", "route boundary" ], "reuse_package": [ "package", "distribution", "shared capability" ], "team_operationalize": [ "shared", "distribution" ], "transform_workflow": [ "workflow" ] }, "negative": {} } } }, { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", "score": 0.888, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.855, "support_score": 0.06, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "build a skill", "skill package" ], "eval_optimize": [ "validation" ], "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reuse", "package", "team reuse" ], "team_operationalize": [ "team", "team reuse" ], "transform_workflow": [ "rough notes", "transcript", "onboarding flow" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Give me ideas for improving our process.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "process" ] }, "negative": { "brainstorm_only": [ "ideas" ] } } } }, { "prompt": "Review this note and tell me what it means in plain English.", "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.083, "negative_penalty": 0.26, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "tell me what it means", "plain english" ] } } } }, { "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "brainstorm_only", "no_build_directive" ], "exclusive_negative_concepts": [ "brainstorm_only", "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0.06, "lexical_support": 0.2, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "reuse_package": [ "packaging" ] }, "negative": { "brainstorm_only": [ "brainstorm", "improve packaging" ], "no_build_directive": [ "do not generate any skill files" ] } } } }, { "prompt": "Translate these notes into French and keep the structure exactly as-is.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "translate_only" ], "exclusive_negative_concepts": [ "translate_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "translate_only": [ "translate", "into french" ] } } } }, { "prompt": "Summarize this workflow and list the main points only.", "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.111, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "workflow" ] }, "negative": { "summarize_only": [ "summarize" ] } } } }, { "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", "family": "long_context_document_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.182, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ] }, "negative": { "explain_only": [ "explain it", "in simpler language" ], "no_build_directive": [ "do not convert it into a reusable capability", "keep it as documentation only" ] } } } }, { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", "score": 0.257, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], "semantic_coverage": 0.673, "support_score": 0, "lexical_support": 0.353, "negative_penalty": 0.3, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "process" ] }, "negative": { "summarize_only": [ "summary", "recap" ] } } }, "boundary_case": true } ], "near_neighbor": [ { "prompt": "Review this process note and explain it, no packaging needed.", "family": "explain_not_package", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "explain_only": [ "explain it" ], "no_build_directive": [ "no packaging needed" ] } } } }, { "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", "family": "partial_scaffold_not_full_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "checklist" ] }, "negative": { "no_build_directive": [ "stop short of making a full skill" ] } } } }, { "prompt": "Package this explanation as a document, not as an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "package" ] }, "negative": { "document_only": [ "document" ], "no_build_directive": [ "not as an agent skill" ] } } } }, { "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "iterate_existing_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.327, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.64, "coverage_boost": 0.04, "concept_evidence": { "positive": { "iterate_existing_skill": [ "this skill" ], "reuse_package": [ "package" ] }, "negative": { "future_outline": [ "future-ready outline" ], "no_build_directive": [ "do not build" ] } } } }, { "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.462, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "reusable" ] }, "negative": { "document_only": [ "document", "reusable-looking document" ], "no_build_directive": [ "without turning them into an agent skill" ] } } } }, { "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", "family": "long_context_near_neighbor", "score": 0.328, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only" ], "exclusive_negative_concepts": [], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.18, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "build a skill" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "document", "wiki", "reference document", "template only" ] } } }, "boundary_case": true }, { "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.278, "negative_penalty": 0.64, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ] }, "negative": { "future_outline": [ "for later", "future-ready outline" ], "no_build_directive": [ "do not build" ] } } } } ] } }, "current_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 5, "passed": 5, "pass_rate": 1.0 }, "should_not_trigger": { "total": 4, "passed": 4, "pass_rate": 1.0 }, "near_neighbor": { "total": 4, "passed": 4, "pass_rate": 1.0 } }, "family_stats": { "complex_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "iterate_existing_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "workflow_to_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "paraphrase_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_trigger": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "summary_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "future_outline_vs_build": { "total": 2, "passed": 2, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "document_export_vs_agent_skill": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "long_context_near_neighbor": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", "score": 0.468, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.4, "support_score": 0.06, "lexical_support": 0.316, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reusable" ], "team_operationalize": [ "internal" ], "transform_workflow": [ "process", "internal process note" ] }, "negative": {} } } }, { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", "score": 0.729, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.691, "support_score": 0, "lexical_support": 0.412, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "evals", "near-neighbor", "trigger boundary" ], "iterate_existing_skill": [ "existing skill" ], "reuse_package": [ "package", "distribution" ], "team_operationalize": [ "distribution" ], "transform_workflow": [ "existing skill draft" ] }, "negative": {} } } }, { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", "score": 0.563, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.509, "support_score": 0, "lexical_support": 0.429, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "eval_optimize": [ "validation" ], "reuse_package": [ "reusable", "library entry" ], "transform_workflow": [ "process" ] }, "negative": {} } } }, { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", "score": 0.858, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0.06, "lexical_support": 0.375, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "eval_optimize": [ "packaging checks" ], "multi_asset": [ "references", "scripts" ], "reuse_package": [ "reusable", "package", "packaging" ], "transform_workflow": [ "quarterly release routine" ] }, "negative": {} } } }, { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", "score": 0.744, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.709, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "turn this playbook into an agent capability package" ], "reuse_package": [ "package", "team library" ], "team_operationalize": [ "team" ], "transform_workflow": [ "prompt history", "transcript", "release notes" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", "family": "summary_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive", "summarize_only" ], "exclusive_negative_concepts": [ "no_build_directive", "summarize_only" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.308, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable" ], "transform_workflow": [ "process" ] }, "negative": { "no_build_directive": [ "not a reusable skill" ], "summarize_only": [ "summary", "only want a summary" ] } } } }, { "prompt": "Explain this release checklist in simpler language; do not package it.", "family": "explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only", "no_build_directive" ], "exclusive_negative_concepts": [ "explain_only", "no_build_directive" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "package" ], "transform_workflow": [ "checklist" ] }, "negative": { "explain_only": [ "explain this", "in simpler language" ], "no_build_directive": [ "do not package it" ] } } } }, { "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", "family": "long_context_document_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "document", "wiki", "reference document" ], "no_build_directive": [ "not for agent execution" ] } } } }, { "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "no_build_directive", "translate_only" ], "exclusive_negative_concepts": [ "no_build_directive", "translate_only" ], "semantic_coverage": 0.673, "support_score": 0, "lexical_support": 0.25, "negative_penalty": 0.77, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "runbook" ] }, "negative": { "no_build_directive": [ "no skill package", "keep it as documentation only" ], "translate_only": [ "translate", "into russian" ] } } } } ], "near_neighbor": [ { "prompt": "Help me shape an idea before we decide whether to build a skill.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" ], "exclusive_negative_concepts": [ "brainstorm_only", "future_outline" ], "semantic_coverage": 0.309, "support_score": 0, "lexical_support": 0.308, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ "build a skill" ] }, "negative": { "brainstorm_only": [ "shape an idea" ], "future_outline": [ "before we decide whether to build" ] } } } }, { "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", "family": "future_outline_vs_build", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.357, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "future_outline": [ "future skill", "possible future skill", "outline for a possible future skill" ], "no_build_directive": [ "do not build" ] } } } }, { "prompt": "Turn this into a document package for the wiki, not an agent skill.", "family": "document_export_vs_agent_skill", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.491, "support_score": 0, "lexical_support": 0.538, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ "agent skill" ], "reuse_package": [ "package" ] }, "negative": { "document_only": [ "document", "document package", "wiki" ], "no_build_directive": [ "not an agent skill" ] } } } }, { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.673, "support_score": 0, "lexical_support": 0.375, "negative_penalty": 0.6, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "reuse_package": [ "package" ], "transform_workflow": [ "workflow", "checklist" ] }, "negative": { "document_only": [ "template only" ], "no_build_directive": [ "no skill package", "discussion only" ] } } } } ] } }, "winner_blind_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_future_outline": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", "score": 0.392, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "maintained", "operationalize" ], "transform_workflow": [ "support escalation routine" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable", "package" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } }, "boundary_case": true } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", "score": 0.024, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "document_only" ], "exclusive_negative_concepts": [], "semantic_coverage": 0.182, "support_score": 0.06, "lexical_support": 0.273, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "wiki" ] } } } }, { "prompt": "Explain the differences between prompts and skills in plain English.", "family": "blind_explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0.26, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "plain english" ] } } } } ], "near_neighbor": [ { "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", "family": "blind_future_outline", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "future_outline" ], "exclusive_negative_concepts": [ "future_outline" ], "semantic_coverage": 0.0, "support_score": 0.06, "lexical_support": 0.308, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ] }, "negative": { "future_outline": [ "future skill" ] } } } }, { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", "score": 0.026, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline" ], "exclusive_negative_concepts": [ "future_outline" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "future_outline": [ "for later" ] } } } } ] } }, "current_blind_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_future_outline": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", "score": 0.392, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "maintained", "operationalize" ], "transform_workflow": [ "support escalation routine" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable", "package" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } }, "boundary_case": true } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", "score": 0.024, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "document_only" ], "exclusive_negative_concepts": [], "semantic_coverage": 0.182, "support_score": 0.06, "lexical_support": 0.273, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "wiki" ] } } } }, { "prompt": "Explain the differences between prompts and skills in plain English.", "family": "blind_explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0.26, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "plain english" ] } } } } ], "near_neighbor": [ { "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", "family": "blind_future_outline", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "future_outline" ], "exclusive_negative_concepts": [ "future_outline" ], "semantic_coverage": 0.0, "support_score": 0.06, "lexical_support": 0.308, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ] }, "negative": { "future_outline": [ "future skill" ] } } } }, { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", "score": 0.026, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline" ], "exclusive_negative_concepts": [ "future_outline" ], "semantic_coverage": 0.364, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "future_outline": [ "for later" ] } } } } ] } }, "baseline_blind_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_future_outline": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", "score": 0.393, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.377, "support_score": 0, "lexical_support": 0.071, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "maintained", "operationalize" ], "transform_workflow": [ "support escalation routine" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", "score": 0.393, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.377, "support_score": 0, "lexical_support": 0.067, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "reusable", "package" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } }, "boundary_case": true } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", "score": 0.016, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "document_only" ], "exclusive_negative_concepts": [], "semantic_coverage": 0.189, "support_score": 0.06, "lexical_support": 0.091, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "wiki" ] } } } }, { "prompt": "Explain the differences between prompts and skills in plain English.", "family": "blind_explain_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.26, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "plain english" ] } } } } ], "near_neighbor": [ { "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", "family": "blind_future_outline", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "future_outline" ], "exclusive_negative_concepts": [ "future_outline" ], "semantic_coverage": 0.0, "support_score": 0.06, "lexical_support": 0.0, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { "positive": { "multi_asset": [ "files" ] }, "negative": { "future_outline": [ "future skill" ] } } } }, { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", "score": 0.024, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline" ], "exclusive_negative_concepts": [ "future_outline" ], "semantic_coverage": 0.377, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.22, "coverage_boost": 0.04, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "future_outline": [ "for later" ] } } } } ] } }, "winner_judge_blind_holdout_report": { "judge": "rubric-blind-v1", "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_future_outline": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 3, "negative_vote": 0, "margin": 3, "confidence": 0.74, "concept_evidence": { "positive": { "reuse_package": [ "maintained", "operationalize" ], "transform_workflow": [ "support escalation routine" ] }, "negative": {} } } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [ "notes" ], "artifact_hits": [], "capability_overlap": 2, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 4, "negative_vote": 0, "margin": 4, "confidence": 0.82, "concept_evidence": { "positive": { "reuse_package": [ "reusable", "package" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow" ], "strong_positive_concepts": [ "transform_workflow" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 2, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [ "document_only" ], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 2, "negative_vote": 1, "margin": 1, "confidence": 0.58, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "wiki" ] } } }, "boundary_case": true }, { "prompt": "Explain the differences between prompts and skills in plain English.", "family": "blind_explain_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [ "prompts" ], "artifact_hits": [ "prompts" ], "capability_overlap": 2, "exclusive_negative_concepts": [ "explain_only" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "explain", "plain english" ], "positive_vote": 2, "negative_vote": 3, "margin": -1, "confidence": 0.58, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "plain english" ] } } }, "boundary_case": true } ], "near_neighbor": [ { "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", "family": "blind_future_outline", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [ "notes" ], "artifact_hits": [], "capability_overlap": 0, "exclusive_negative_concepts": [ "future_outline" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 1, "negative_vote": 2, "margin": -1, "confidence": 0.58, "concept_evidence": { "positive": { "multi_asset": [ "files" ] }, "negative": { "future_outline": [ "future skill" ] } } }, "boundary_case": true }, { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [ "future_outline" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 3, "negative_vote": 2, "margin": 1, "confidence": 0.66, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "future_outline": [ "for later" ] } } }, "boundary_case": true } ] }, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.66, "rubric_version": "blind-v1" } }, "current_judge_blind_holdout_report": { "judge": "rubric-blind-v1", "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_future_outline": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 3, "negative_vote": 0, "margin": 3, "confidence": 0.74, "concept_evidence": { "positive": { "reuse_package": [ "maintained", "operationalize" ], "transform_workflow": [ "support escalation routine" ] }, "negative": {} } } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [ "notes" ], "artifact_hits": [], "capability_overlap": 2, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 4, "negative_vote": 0, "margin": 4, "confidence": 0.82, "concept_evidence": { "positive": { "reuse_package": [ "reusable", "package" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow" ], "strong_positive_concepts": [ "transform_workflow" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 2, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [ "document_only" ], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 2, "negative_vote": 1, "margin": 1, "confidence": 0.58, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "wiki" ] } } }, "boundary_case": true }, { "prompt": "Explain the differences between prompts and skills in plain English.", "family": "blind_explain_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [ "prompts" ], "artifact_hits": [ "prompts" ], "capability_overlap": 2, "exclusive_negative_concepts": [ "explain_only" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "explain", "plain english" ], "positive_vote": 2, "negative_vote": 3, "margin": -1, "confidence": 0.58, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "plain english" ] } } }, "boundary_case": true } ], "near_neighbor": [ { "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", "family": "blind_future_outline", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [ "notes" ], "artifact_hits": [], "capability_overlap": 0, "exclusive_negative_concepts": [ "future_outline" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 1, "negative_vote": 2, "margin": -1, "confidence": 0.58, "concept_evidence": { "positive": { "multi_asset": [ "files" ] }, "negative": { "future_outline": [ "future skill" ] } } }, "boundary_case": true }, { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [ "future_outline" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 3, "negative_vote": 2, "margin": 1, "confidence": 0.66, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "future_outline": [ "for later" ] } } }, "boundary_case": true } ] }, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.66, "rubric_version": "blind-v1" } }, "baseline_judge_blind_holdout_report": { "judge": "rubric-blind-v1", "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_multi_asset": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_document_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_future_outline": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 3, "negative_vote": 0, "margin": 3, "confidence": 0.74, "concept_evidence": { "positive": { "reuse_package": [ "maintained", "operationalize" ], "transform_workflow": [ "support escalation routine" ] }, "negative": {} } } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [ "notes" ], "artifact_hits": [], "capability_overlap": 2, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 4, "negative_vote": 0, "margin": 4, "confidence": 0.82, "concept_evidence": { "positive": { "reuse_package": [ "reusable", "package" ], "transform_workflow": [ "sop", "transcript" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow" ], "strong_positive_concepts": [ "transform_workflow" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 2, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [ "document_only" ], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 2, "negative_vote": 1, "margin": 1, "confidence": 0.58, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "transform_workflow": [ "workflow" ] }, "negative": { "document_only": [ "wiki" ] } } }, "boundary_case": true }, { "prompt": "Explain the differences between prompts and skills in plain English.", "family": "blind_explain_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [ "prompts" ], "artifact_hits": [ "prompts" ], "capability_overlap": 2, "exclusive_negative_concepts": [ "explain_only" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "explain", "plain english" ], "positive_vote": 2, "negative_vote": 3, "margin": -1, "confidence": 0.58, "concept_evidence": { "positive": {}, "negative": { "explain_only": [ "plain english" ] } } }, "boundary_case": true } ], "near_neighbor": [ { "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", "family": "blind_future_outline", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [ "notes" ], "artifact_hits": [], "capability_overlap": 0, "exclusive_negative_concepts": [ "future_outline" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 1, "negative_vote": 2, "margin": -1, "confidence": 0.58, "concept_evidence": { "positive": { "multi_asset": [ "files" ] }, "negative": { "future_outline": [ "future skill" ] } } }, "boundary_case": true }, { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "transform_workflow", "reuse_package" ], "strong_positive_concepts": [ "transform_workflow", "reuse_package" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [ "future_outline" ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 3, "negative_vote": 2, "margin": 1, "confidence": 0.66, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ], "transform_workflow": [ "process" ] }, "negative": { "future_outline": [ "for later" ] } } }, "boundary_case": true } ] }, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.66, "rubric_version": "blind-v1" } }, "winner_adversarial_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "adversarial_noisy_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_multi_asset_boundary": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_document_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_summary_translation_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_future_outline_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_brainstorm_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", "score": 0.834, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0, "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "eval_optimize": [ "hardening" ], "reuse_package": [ "package", "maintained", "operationalize" ], "transform_workflow": [ "workflow" ] }, "negative": {} } } }, { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", "score": 0.842, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0, "lexical_support": 0.368, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "codify this routine" ], "eval_optimize": [ "stress test", "route boundary" ], "reuse_package": [ "reusable" ], "transform_workflow": [ "prompt history" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Create a wiki package from this runbook and do not generate any skill files.", "family": "adversarial_document_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.364, "support_score": 0.06, "lexical_support": 0.429, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "reuse_package": [ "package" ], "transform_workflow": [ "runbook" ] }, "negative": { "document_only": [ "wiki" ], "no_build_directive": [ "do not generate any skill files" ] } } } }, { "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", "family": "adversarial_summary_translation_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive", "summarize_only", "translate_only" ], "exclusive_negative_concepts": [ "no_build_directive", "summarize_only", "translate_only" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "sop" ] }, "negative": { "document_only": [ "knowledge base" ], "no_build_directive": [ "keep it as documentation only" ], "summarize_only": [ "summarize" ], "translate_only": [ "translate", "into japanese" ] } } } } ], "near_neighbor": [ { "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", "family": "adversarial_future_outline_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.25, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "future_outline": [ "future skill", "possible future skill" ], "no_build_directive": [ "stop short of making a full skill", "plain note", "keep it as a plain note" ] } } } }, { "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", "family": "adversarial_brainstorm_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "brainstorm_only", "future_outline", "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.182, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ] }, "negative": { "brainstorm_only": [ "brainstorm" ], "future_outline": [ "for later" ], "no_build_directive": [ "discussion only" ] } } } } ] } }, "current_adversarial_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "adversarial_noisy_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_multi_asset_boundary": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_document_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_summary_translation_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_future_outline_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_brainstorm_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", "score": 0.834, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0, "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "eval_optimize": [ "hardening" ], "reuse_package": [ "package", "maintained", "operationalize" ], "transform_workflow": [ "workflow" ] }, "negative": {} } } }, { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", "score": 0.842, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.818, "support_score": 0, "lexical_support": 0.368, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "codify this routine" ], "eval_optimize": [ "stress test", "route boundary" ], "reuse_package": [ "reusable" ], "transform_workflow": [ "prompt history" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Create a wiki package from this runbook and do not generate any skill files.", "family": "adversarial_document_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.364, "support_score": 0.06, "lexical_support": 0.429, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "reuse_package": [ "package" ], "transform_workflow": [ "runbook" ] }, "negative": { "document_only": [ "wiki" ], "no_build_directive": [ "do not generate any skill files" ] } } } }, { "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", "family": "adversarial_summary_translation_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive", "summarize_only", "translate_only" ], "exclusive_negative_concepts": [ "no_build_directive", "summarize_only", "translate_only" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "sop" ] }, "negative": { "document_only": [ "knowledge base" ], "no_build_directive": [ "keep it as documentation only" ], "summarize_only": [ "summarize" ], "translate_only": [ "translate", "into japanese" ] } } } } ], "near_neighbor": [ { "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", "family": "adversarial_future_outline_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.25, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "future_outline": [ "future skill", "possible future skill" ], "no_build_directive": [ "stop short of making a full skill", "plain note", "keep it as a plain note" ] } } } }, { "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", "family": "adversarial_brainstorm_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "brainstorm_only", "future_outline", "no_build_directive" ], "semantic_coverage": 0.182, "support_score": 0, "lexical_support": 0.182, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ] }, "negative": { "brainstorm_only": [ "brainstorm" ], "future_outline": [ "for later" ], "no_build_directive": [ "discussion only" ] } } } } ] } }, "baseline_adversarial_holdout_report": { "threshold": 0.33, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "adversarial_noisy_operationalize": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_multi_asset_boundary": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_document_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_summary_translation_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_future_outline_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_brainstorm_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", "score": 0.845, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.849, "support_score": 0, "lexical_support": 0.053, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "skill package" ], "eval_optimize": [ "hardening" ], "reuse_package": [ "package", "maintained", "operationalize" ], "transform_workflow": [ "workflow" ] }, "negative": {} } } }, { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", "score": 0.845, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.849, "support_score": 0, "lexical_support": 0.053, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "build_skill": [ "codify this routine" ], "eval_optimize": [ "stress test", "route boundary" ], "reuse_package": [ "reusable" ], "transform_workflow": [ "prompt history" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Create a wiki package from this runbook and do not generate any skill files.", "family": "adversarial_document_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ "multi_asset" ], "matched_negative_concepts": [ "document_only", "no_build_directive" ], "exclusive_negative_concepts": [ "no_build_directive" ], "semantic_coverage": 0.377, "support_score": 0.06, "lexical_support": 0.143, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { "positive": { "multi_asset": [ "files" ], "reuse_package": [ "package" ], "transform_workflow": [ "runbook" ] }, "negative": { "document_only": [ "wiki" ], "no_build_directive": [ "do not generate any skill files" ] } } } }, { "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", "family": "adversarial_summary_translation_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "transform_workflow" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "document_only", "no_build_directive", "summarize_only", "translate_only" ], "exclusive_negative_concepts": [ "no_build_directive", "summarize_only", "translate_only" ], "semantic_coverage": 0.189, "support_score": 0, "lexical_support": 0.067, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { "positive": { "transform_workflow": [ "sop" ] }, "negative": { "document_only": [ "knowledge base" ], "no_build_directive": [ "keep it as documentation only" ], "summarize_only": [ "summarize" ], "translate_only": [ "translate", "into japanese" ] } } } } ], "near_neighbor": [ { "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", "family": "adversarial_future_outline_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "future_outline", "no_build_directive" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.05, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "future_outline": [ "future skill", "possible future skill" ], "no_build_directive": [ "stop short of making a full skill", "plain note", "keep it as a plain note" ] } } } }, { "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", "family": "adversarial_brainstorm_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "build_skill", "transform_workflow", "reuse_package", "eval_optimize", "iterate_existing_skill" ], "matched_desired_concepts": [ "reuse_package" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline", "no_build_directive" ], "exclusive_negative_concepts": [ "brainstorm_only", "future_outline", "no_build_directive" ], "semantic_coverage": 0.189, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { "positive": { "reuse_package": [ "packaging" ] }, "negative": { "brainstorm_only": [ "brainstorm" ], "future_outline": [ "for later" ], "no_build_directive": [ "discussion only" ] } } } } ] } }, "candidates": [ { "id": "current", "label": "Current", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "strategy": "current", "estimated_tokens": 65, "dev": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 0, -1.0, -1.0, -1.0, -1.0, 65 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "minimal", "label": "Minimal", "description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", "strategy": "minimal_template", "estimated_tokens": 41, "dev": { "false_positives": 2, "false_negatives": 1, "precision": 0.778, "recall": 0.875, "near_neighbor_pass_rate": 0.857, "should_not_trigger_pass_rate": 0.857 }, "selection_key": [ 2, 1, -0.857, -0.857, -0.778, -0.875, 41 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "balanced", "label": "Balanced", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", "strategy": "balanced_template", "estimated_tokens": 54, "dev": { "false_positives": 2, "false_negatives": 1, "precision": 0.778, "recall": 0.875, "near_neighbor_pass_rate": 0.857, "should_not_trigger_pass_rate": 0.857 }, "selection_key": [ 2, 1, -0.857, -0.857, -0.778, -0.875, 54 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "artifact_aware", "label": "Artifact Aware", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", "strategy": "artifact_template", "estimated_tokens": 72, "dev": { "false_positives": 2, "false_negatives": 1, "precision": 0.778, "recall": 0.875, "near_neighbor_pass_rate": 0.857, "should_not_trigger_pass_rate": 0.857 }, "selection_key": [ 2, 1, -0.857, -0.857, -0.778, -0.875, 72 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "boundary", "label": "Boundary", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", "strategy": "boundary_template", "estimated_tokens": 78, "dev": { "false_positives": 2, "false_negatives": 1, "precision": 0.778, "recall": 0.875, "near_neighbor_pass_rate": 0.857, "should_not_trigger_pass_rate": 0.857 }, "selection_key": [ 2, 1, -0.857, -0.857, -0.778, -0.875, 78 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "guardrail", "label": "Guardrail", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", "strategy": "guardrail_template", "estimated_tokens": 50, "dev": { "false_positives": 5, "false_negatives": 1, "precision": 0.583, "recall": 0.875, "near_neighbor_pass_rate": 0.429, "should_not_trigger_pass_rate": 0.857 }, "selection_key": [ 5, 1, -0.429, -0.857, -0.583, -0.875, 50 ], "holdout": { "false_positives": 2, "false_negatives": 0, "precision": 0.714, "recall": 1.0, "near_neighbor_pass_rate": 0.5, "should_not_trigger_pass_rate": 1.0 } } ], "selection_logic": { "priority": [ "fewest false positives", "fewest false negatives", "highest near-neighbor pass rate", "highest negative pass rate", "highest precision", "highest recall", "shortest description" ] }, "comparison": { "winner_vs_current_dev": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_dev": { "baseline_false_positives": 1, "baseline_false_negatives": 1, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": -1, "false_negative_delta": -1, "baseline_precision": 0.875, "improved_precision": 1.0, "baseline_recall": 0.875, "improved_recall": 1.0 }, "winner_vs_baseline_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_judge_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_judge_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_adversarial_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_adversarial_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 } }, "acceptance_gates": { "selection_basis": "dev only", "holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.33, "mean_positive_score": 0.672, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, "min_positive_score": 0.468, "max_non_trigger_score": 0.0, "score_gap": 0.468, "threshold_margin": 0.138, "positive_threshold_buffer": 0.138, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "current_calibration": { "threshold": 0.33, "mean_positive_score": 0.672, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, "min_positive_score": 0.468, "max_non_trigger_score": 0.0, "score_gap": 0.468, "threshold_margin": 0.138, "positive_threshold_buffer": 0.138, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "baseline_calibration": { "threshold": 0.33, "mean_positive_score": 0.648, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, "min_positive_score": 0.416, "max_non_trigger_score": 0.0, "score_gap": 0.416, "threshold_margin": 0.086, "positive_threshold_buffer": 0.086, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "winner_family_health": { "family_count": 12, "clean_family_count": 12, "failing_families": [], "weakest_family": { "family": "workflow_to_skill", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 12, "clean_family_count": 12, "failing_families": [], "weakest_family": { "family": "workflow_to_skill", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 12, "clean_family_count": 12, "failing_families": [], "weakest_family": { "family": "workflow_to_skill", "pass_rate": 1.0, "errors": 0 } } }, "blind_holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.33, "mean_positive_score": 0.399, "mean_non_trigger_score": 0.013, "mean_near_neighbor_score": 0.013, "min_positive_score": 0.392, "max_non_trigger_score": 0.026, "score_gap": 0.366, "threshold_margin": 0.062, "positive_threshold_buffer": 0.062, "negative_threshold_buffer": 0.304, "boundary_case_count": 2, "boundary_case_rate": 0.333, "risk_band": "watch" }, "current_calibration": { "threshold": 0.33, "mean_positive_score": 0.399, "mean_non_trigger_score": 0.013, "mean_near_neighbor_score": 0.013, "min_positive_score": 0.392, "max_non_trigger_score": 0.026, "score_gap": 0.366, "threshold_margin": 0.062, "positive_threshold_buffer": 0.062, "negative_threshold_buffer": 0.304, "boundary_case_count": 2, "boundary_case_rate": 0.333, "risk_band": "watch" }, "baseline_calibration": { "threshold": 0.33, "mean_positive_score": 0.393, "mean_non_trigger_score": 0.01, "mean_near_neighbor_score": 0.012, "min_positive_score": 0.393, "max_non_trigger_score": 0.024, "score_gap": 0.369, "threshold_margin": 0.063, "positive_threshold_buffer": 0.063, "negative_threshold_buffer": 0.306, "boundary_case_count": 2, "boundary_case_rate": 0.333, "risk_band": "watch" }, "winner_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_operationalize", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_operationalize", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_operationalize", "pass_rate": 1.0, "errors": 0 } } }, "judge_blind_holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.66, "rubric_version": "blind-v1" } }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.66, "rubric_version": "blind-v1" } }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.66, "rubric_version": "blind-v1" } }, "winner_calibration": null, "current_calibration": null, "baseline_calibration": null, "winner_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_operationalize", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_operationalize", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_operationalize", "pass_rate": 1.0, "errors": 0 } } }, "adversarial_holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.33, "mean_positive_score": 0.838, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, "min_positive_score": 0.834, "max_non_trigger_score": 0.0, "score_gap": 0.834, "threshold_margin": 0.33, "positive_threshold_buffer": 0.504, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "current_calibration": { "threshold": 0.33, "mean_positive_score": 0.838, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, "min_positive_score": 0.834, "max_non_trigger_score": 0.0, "score_gap": 0.834, "threshold_margin": 0.33, "positive_threshold_buffer": 0.504, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "baseline_calibration": { "threshold": 0.33, "mean_positive_score": 0.845, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, "min_positive_score": 0.845, "max_non_trigger_score": 0.0, "score_gap": 0.845, "threshold_margin": 0.33, "positive_threshold_buffer": 0.515, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "winner_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "adversarial_summary_translation_collision", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "adversarial_summary_translation_collision", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "adversarial_summary_translation_collision", "pass_rate": 1.0, "errors": 0 } } } }, "summary": { "winner_label": "Current", "winner_tokens": 65, "current_tokens": 65, "winner_dev_total_errors": 0, "current_dev_total_errors": 0, "winner_holdout_total_errors": 0, "current_holdout_total_errors": 0, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_total_errors": 0, "current_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_total_errors": 0, "current_adversarial_holdout_total_errors": 0, "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", "winner_adversarial_score_gap": 0.834, "candidate_count": 6, "baseline_tokens": 8, "baseline_dev_total_errors": 2, "baseline_holdout_total_errors": 0, "baseline_blind_holdout_total_errors": 0, "baseline_judge_blind_holdout_total_errors": 0, "baseline_adversarial_holdout_total_errors": 0 } }