feat: expand chinese trigger coverage

This commit is contained in:
yaojingang
2026-04-23 15:37:11 +08:00
parent 7126bcb2e5
commit 91abffaad6
4 changed files with 210 additions and 23 deletions
+4 -2
View File
@@ -5,13 +5,13 @@ import re
from pathlib import Path
WORD_RE = re.compile(r"[a-z0-9][a-z0-9_-]*")
WORD_RE = re.compile(r"[\w\u4e00-\u9fff]+", re.UNICODE)
DEFAULT_CONFIG_PATH = Path("evals/semantic_config.json")
def normalize(text: str) -> str:
text = text.lower()
text = re.sub(r"[^a-z0-9]+", " ", text)
text = re.sub(r"[^\w\u4e00-\u9fff]+", " ", text, flags=re.UNICODE)
return re.sub(r"\s+", " ", text).strip()
@@ -52,6 +52,8 @@ def phrase_present(text: str, phrase: str) -> bool:
phrase = normalize(phrase)
if not phrase:
return False
if re.search(r"[\u4e00-\u9fff]", phrase):
return phrase in text
return f" {phrase} " in f" {text} "