yao-meta-skill/evals/trigger_cases.json

{
  "recommended_threshold": 0.33,
  "negative_patterns": [
    "one-off",
    "do not turn",
    "without building",
    "brainstorm",
    "just explain",
    "headline",
    "summarize"
  ],
  "should_trigger": [
    {
      "text": "Create a skill from this repeated workflow.",
      "family": "workflow_to_skill"
    },
    {
      "text": "Turn this runbook into a reusable agent skill.",
      "family": "workflow_to_skill"
    },
    {
      "text": "Convert this process note into a reusable skill package for the team.",
      "family": "workflow_to_skill"
    },
    {
      "text": "Improve this skill description and add evals.",
      "family": "iterate_existing_skill"
    },
    {
      "text": "Refactor this prompt into a proper skill package.",
      "family": "iterate_existing_skill"
    },
    {
      "text": "Package this skill for team reuse.",
      "family": "package_for_team"
    },
    {
      "text": "Create a meta-skill for packaging internal workflows.",
      "family": "meta_skill_creation"
    },
    {
      "text": "Standardize this recurring workflow as a shareable skill package with references.",
      "family": "package_for_team"
    },
    {
      "text": "Convert this operations checklist into a reusable skill.",
      "family": "workflow_to_skill"
    },
    {
      "text": "Add trigger evals to this skill before sharing it with the team.",
      "family": "iterate_existing_skill"
    },
    {
      "text": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
      "family": "complex_multi_asset"
    },
    {
      "text": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
      "family": "complex_multi_asset"
    },
    {
      "text": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
      "family": "iterate_existing_skill"
    },
    {
      "text": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
      "family": "complex_multi_asset"
    },
    {
      "text": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
      "family": "iterate_existing_skill"
    },
    {
      "text": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
      "family": "workflow_to_skill"
    },
    {
      "text": "Codify this recurring support escalation workflow into a maintained skill for the ops library.",
      "family": "paraphrase_trigger"
    },
    {
      "text": "Operationalize this workflow as a reusable skill with validation notes and team library metadata.",
      "family": "paraphrase_trigger"
    },
    {
      "text": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
      "family": "paraphrase_trigger"
    },
    {
      "text": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
      "family": "paraphrase_trigger"
    },
    {
      "text": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
      "family": "paraphrase_trigger"
    },
    {
      "text": "Below are release notes, rough notes, and a transcript from the last handoff. Ignore the noise and turn this runbook into a reusable agent skill with references and scripts so the team can reuse it every quarter.",
      "family": "long_context_trigger"
    },
    {
      "text": "We have a standard operating procedure, checklist, and prompt history scattered across docs. Formalize this process into a reusable capability, add validation, and package it for team reuse.",
      "family": "long_context_trigger"
    },
    {
      "text": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
      "family": "long_context_trigger"
    },
    {
      "text": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
      "family": "long_context_trigger"
    }
  ],
  "should_not_trigger": [
    {
      "text": "Explain what a workflow is.",
      "family": "explain_only"
    },
    {
      "text": "Just explain what a skill is.",
      "family": "explain_only"
    },
    {
      "text": "Summarize this random note.",
      "family": "summary_only"
    },
    {
      "text": "Translate this README into Japanese.",
      "family": "translate_only"
    },
    {
      "text": "Write a product headline for this landing page.",
      "family": "document_only"
    },
    {
      "text": "Draft a blog title for this article.",
      "family": "document_only"
    },
    {
      "text": "Rewrite this paragraph more clearly, but do not package anything.",
      "family": "document_only"
    },
    {
      "text": "Explain the difference between a runbook and a workflow.",
      "family": "explain_only"
    },
    {
      "text": "Give me ideas for improving our process.",
      "family": "brainstorm_only"
    },
    {
      "text": "Review this note and tell me what it means in plain English.",
      "family": "explain_only"
    },
    {
      "text": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
      "family": "brainstorm_only"
    },
    {
      "text": "Translate these notes into French and keep the structure exactly as-is.",
      "family": "translate_only"
    },
    {
      "text": "Summarize this workflow and list the main points only.",
      "family": "summary_only"
    },
    {
      "text": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
      "family": "summary_only"
    },
    {
      "text": "Explain this release checklist in simpler language; do not package it.",
      "family": "explain_only"
    },
    {
      "text": "Below is a long SOP, but I only want a summary for leadership and a recap of the main points. Do not build anything reusable.",
      "family": "long_context_summary_only"
    },
    {
      "text": "Keep this as documentation only for the knowledge base and rewrite it as a reference document.",
      "family": "long_context_document_only"
    },
    {
      "text": "Translate this long onboarding flow into French and keep it as documentation only with no skill package.",
      "family": "translate_only"
    },
    {
      "text": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
      "family": "long_context_document_only"
    },
    {
      "text": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
      "family": "long_context_summary_only"
    },
    {
      "text": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
      "family": "long_context_document_only"
    },
    {
      "text": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
      "family": "translate_only"
    }
  ],
  "near_neighbor": [
    {
      "text": "Create a one-off prompt for this task.",
      "family": "one_off_vs_reusable"
    },
    {
      "text": "Write a custom answer for this request without creating a skill package.",
      "family": "one_off_vs_reusable"
    },
    {
      "text": "Help me brainstorm process ideas without building a skill.",
      "family": "brainstorm_vs_build"
    },
    {
      "text": "Improve this README but do not turn it into a skill.",
      "family": "document_export_vs_agent_skill"
    },
    {
      "text": "Make a checklist for this task, but not a reusable skill.",
      "family": "partial_scaffold_not_full_skill"
    },
    {
      "text": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.",
      "family": "partial_scaffold_not_full_skill"
    },
    {
      "text": "Review this process note and explain it, no packaging needed.",
      "family": "explain_not_package"
    },
    {
      "text": "Turn this into a checklist and template, but stop short of making a full skill.",
      "family": "partial_scaffold_not_full_skill"
    },
    {
      "text": "Package this explanation as a document, not as an agent skill.",
      "family": "document_export_vs_agent_skill"
    },
    {
      "text": "Make a future-ready outline for this skill idea, but do not build the package.",
      "family": "future_outline_vs_build"
    },
    {
      "text": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
      "family": "document_export_vs_agent_skill"
    },
    {
      "text": "Help me shape an idea before we decide whether to build a skill.",
      "family": "future_outline_vs_build"
    },
    {
      "text": "Create an outline for a possible future skill, but do not build the skill yet.",
      "family": "future_outline_vs_build"
    },
    {
      "text": "Turn this into a document package for the wiki, not an agent skill.",
      "family": "document_export_vs_agent_skill"
    },
    {
      "text": "We may convert this workflow later, but for now keep it as documentation only and produce a reference document.",
      "family": "long_context_near_neighbor"
    },
    {
      "text": "Draft a template only from this process and stop short of making a full skill package.",
      "family": "partial_scaffold_not_full_skill"
    },
    {
      "text": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
      "family": "long_context_near_neighbor"
    },
    {
      "text": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
      "family": "future_outline_vs_build"
    },
    {
      "text": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
      "family": "long_context_near_neighbor"
    }
  ]
}