From 91abffaad604f04717809bc2ad12aaede84d41ec Mon Sep 17 00:00:00 2001 From: yaojingang Date: Thu, 23 Apr 2026 15:37:11 +0800 Subject: [PATCH] feat: expand chinese trigger coverage --- evals/improved_description.txt | 2 +- evals/semantic_config.json | 167 +++++++++++++++++++++++++++++---- evals/trigger_cases.json | 58 +++++++++++- scripts/trigger_eval.py | 6 +- 4 files changed, 210 insertions(+), 23 deletions(-) diff --git a/evals/improved_description.txt b/evals/improved_description.txt index 93713dcf..a8fdc966 100644 --- a/evals/improved_description.txt +++ b/evals/improved_description.txt @@ -1 +1 @@ -Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. Also trigger on common Chinese asks such as 做一个 skill, 把流程沉淀成可复用能力, 封装成团队可复用的 skill, 优化已有 skill, 补 trigger 评测, or 收紧触发边界. diff --git a/evals/semantic_config.json b/evals/semantic_config.json index e363bc4e..80803467 100644 --- a/evals/semantic_config.json +++ b/evals/semantic_config.json @@ -6,27 +6,46 @@ "prompts", "transcripts", "docs", - "notes" + "notes", + "流程", + "提示词", + "对话记录", + "文档", + "笔记" ], "trigger_actions": [ "create a skill", "turn a repeated process into a reusable skill", "improve an existing skill", "add evals", - "package a skill for team reuse" + "package a skill for team reuse", + "做一个 skill", + "把流程沉淀成可复用 skill", + "优化现有 skill", + "补 trigger 评测", + "封装成团队可复用 skill" ], "exclusions": [ "summary-only requests", "translation-only requests", "plain documentation tasks", - "brainstorming without building a skill" + "brainstorming without building a skill", + "只要总结", + "只做翻译", + "只做文档", + "先 brainstorm 不建 skill" ], "artifacts": [ "workflows", "prompts", "transcripts", "docs", - "rough notes" + "rough notes", + "流程", + "提示词", + "转录", + "文档", + "零散笔记" ] }, "fallback_positive_concepts": [ @@ -55,7 +74,20 @@ "operationalize this workflow as a reusable skill", "formalize this process into a reusable capability", "turn this playbook into an agent capability package", - "convert this into a maintained skill" + "convert this into a maintained skill", + "做一个 skill", + "创建一个 skill", + "新建一个 skill", + "帮我做个 skill", + "做成一个 skill", + "封装成一个 skill", + "沉淀成一个 skill", + "整理成一个 skill", + "抽成一个 skill", + "可复用 skill", + "agent 能力包", + "能力包", + "skill 包" ] }, "transform_workflow": { @@ -77,7 +109,19 @@ "release notes", "onboarding flow", "support escalation routine", - "quarterly release routine" + "quarterly release routine", + "流程", + "工作流", + "操作流程", + "做事流程", + "方法", + "经验", + "sop 文档", + "会议纪要", + "对话记录", + "零散材料", + "提示词", + "话术" ] }, "reuse_package": { @@ -97,7 +141,19 @@ "team library", "ops library", "maintained", - "operationalize" + "operationalize", + "可复用", + "复用", + "封装", + "沉淀", + "标准化", + "标准化下来", + "团队复用", + "给团队用", + "方法包", + "团队能力包", + "沉淀下来", + "skill 包" ] }, "eval_optimize": { @@ -115,7 +171,18 @@ "false negatives", "stress test", "route boundary", - "hardening" + "hardening", + "评测", + "补评测", + "补 eval", + "触发测试", + "收紧触发边界", + "路由边界", + "误触发", + "漏触发", + "质量门", + "打包检查", + "边界测试" ] }, "iterate_existing_skill": { @@ -132,7 +199,18 @@ "tighten the boundary", "revise this skill", "draft skill", - "harden this skill for library reuse" + "harden this skill for library reuse", + "优化这个 skill", + "改进这个 skill", + "重构这个 skill", + "补一下评测", + "补 trigger 测试", + "加强路由判断", + "让它适合团队复用", + "收紧这个 skill 的边界", + "已有 skill", + "现有 skill", + "优化已有 skill" ] }, "multi_asset": { @@ -143,7 +221,13 @@ "manifest", "folder", "directory", - "files" + "files", + "参考资料", + "脚本", + "目录", + "文件夹", + "文档", + "话术" ] }, "team_operationalize": { @@ -155,7 +239,13 @@ "shared", "distribution", "sharing it with the team", - "team reuse" + "team reuse", + "团队", + "内部", + "统一标准", + "共用", + "给团队用", + "团队共享" ] } }, @@ -170,7 +260,12 @@ "explain this", "in simpler language", "tell me what it means", - "plain english" + "plain english", + "解释一下", + "给我讲讲", + "什么意思", + "说清楚一点", + "帮我理解一下" ] }, "summarize_only": { @@ -180,7 +275,11 @@ "summary", "summarize", "only want a summary", - "recap" + "recap", + "总结一下", + "概括一下", + "提炼重点", + "简单总结" ] }, "translate_only": { @@ -190,7 +289,11 @@ "translate", "into japanese", "into french", - "into russian" + "into russian", + "翻译", + "译成中文", + "译成英文", + "译成日文" ] }, "brainstorm_only": { @@ -200,7 +303,11 @@ "brainstorm", "ideas", "improve packaging", - "shape an idea" + "shape an idea", + "头脑风暴", + "给点想法", + "一起想想", + "先想方向" ] }, "one_off_request": { @@ -210,7 +317,11 @@ "one-off", "custom answer", "for this request", - "just this task" + "just this task", + "一次性", + "就这一次", + "只处理这次", + "不用复用" ] }, "no_build_directive": { @@ -229,6 +340,9 @@ "do not package it", "stop short of making a full skill", "not a skill yet", + "not now", + "later but not now", + "maybe build later", "plain note", "keep it as a plain note", "no packaging needed", @@ -236,7 +350,15 @@ "no skill package", "do not convert it into a reusable capability", "keep it as documentation only", - "discussion only" + "discussion only", + "先不要做成 skill", + "不要做成 skill", + "不用封装", + "不要打包", + "不要沉淀成能力", + "先别做成可复用能力", + "只做文档", + "只做说明" ] }, "document_only": { @@ -254,7 +376,12 @@ "landing page", "reference document", "knowledge base", - "template only" + "template only", + "文档", + "说明文档", + "知识库", + "文章", + "汇报" ] }, "future_outline": { @@ -267,7 +394,9 @@ "outline for a possible future skill", "maybe later", "for later", - "future-ready outline" + "future-ready outline", + "not now", + "later but not now" ] } } diff --git a/evals/trigger_cases.json b/evals/trigger_cases.json index 1f54b9aa..9be8184a 100644 --- a/evals/trigger_cases.json +++ b/evals/trigger_cases.json @@ -7,7 +7,11 @@ "brainstorm", "just explain", "headline", - "summarize" + "summarize", + "不要做成 skill", + "只要总结", + "翻译一下", + "头脑风暴" ], "should_trigger": [ { @@ -109,6 +113,26 @@ { "text": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger" + }, + { + "text": "用 yao-meta-skill 帮我把这套重复流程沉淀成一个可复用 skill。", + "family": "cn_workflow_to_skill" + }, + { + "text": "把这套 SOP、提示词和对话记录封装成团队可复用的 agent 能力。", + "family": "cn_complex_multi_asset" + }, + { + "text": "优化这个已有 skill，补一下 trigger 评测和打包检查。", + "family": "cn_iterate_existing_skill" + }, + { + "text": "帮我把零散材料和脚本整理成一个给团队复用的 skill 包。", + "family": "cn_package_for_team" + }, + { + "text": "把这个客服升级流程标准化成一个可执行的 skill，并加上边界测试。", + "family": "cn_eval_and_package" } ], "should_not_trigger": [ @@ -199,6 +223,22 @@ { "text": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", "family": "translate_only" + }, + { + "text": "解释一下这个 workflow 是什么意思。", + "family": "cn_explain_only" + }, + { + "text": "把这段文档总结一下就行，不要做成 skill。", + "family": "cn_summary_only" + }, + { + "text": "把这些笔记翻译成中文，保持原样。", + "family": "cn_translate_only" + }, + { + "text": "先一起头脑风暴几个方向，不要封装成 skill。", + "family": "cn_brainstorm_only" } ], "near_neighbor": [ @@ -277,6 +317,22 @@ { "text": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor" + }, + { + "text": "先做一个可复用风格的清单模板，但不要做成 skill。", + "family": "cn_partial_scaffold_not_full_skill" + }, + { + "text": "帮我整理成 wiki 文档，不要做成 agent 能力。", + "family": "cn_document_export_vs_agent_skill" + }, + { + "text": "给我一个未来可能做成 skill 的大纲，先不要真的生成包。", + "family": "cn_future_outline_vs_build" + }, + { + "text": "这次只给我一个一次性 prompt，不用复用成 skill。", + "family": "cn_one_off_vs_reusable" } ] } diff --git a/scripts/trigger_eval.py b/scripts/trigger_eval.py index 5bcbcbef..ff610aff 100644 --- a/scripts/trigger_eval.py +++ b/scripts/trigger_eval.py @@ -5,13 +5,13 @@ import re from pathlib import Path -WORD_RE = re.compile(r"[a-z0-9][a-z0-9_-]*") +WORD_RE = re.compile(r"[\w\u4e00-\u9fff]+", re.UNICODE) DEFAULT_CONFIG_PATH = Path("evals/semantic_config.json") def normalize(text: str) -> str: text = text.lower() - text = re.sub(r"[^a-z0-9]+", " ", text) + text = re.sub(r"[^\w\u4e00-\u9fff]+", " ", text, flags=re.UNICODE) return re.sub(r"\s+", " ", text).strip() @@ -52,6 +52,8 @@ def phrase_present(text: str, phrase: str) -> bool: phrase = normalize(phrase) if not phrase: return False + if re.search(r"[\u4e00-\u9fff]", phrase): + return phrase in text return f" {phrase} " in f" {text} "