chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1,50 @@
+# Copyright 2017 Google Inc.
+
+
+# Adapted from https://github.com/google/TextNormalizationCoveringGrammars
+# Russian minimally supervised number grammar.
+
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NON_BREAKING_SPACE,
+    DAMO_SPACE,
+)
+from fun_text_processing.text_normalization.ru.utils import get_abs_path
+
+RU_LOWER_ALPHA = "абвгдеёжзийклмнопрстуфхцчшщъыьэюя"
+RU_UPPER_ALPHA = RU_LOWER_ALPHA.upper()
+RU_LOWER_ALPHA = pynini.union(*RU_LOWER_ALPHA).optimize()
+RU_UPPER_ALPHA = pynini.union(*RU_UPPER_ALPHA).optimize()
+RU_ALPHA = (RU_LOWER_ALPHA | RU_UPPER_ALPHA).optimize()
+
+RU_STRESSED_MAP = [
+    ("А́", "А'"),
+    ("Е́", "Е'"),
+    ("Ё́", "Е'"),
+    ("И́", "И'"),
+    ("О́", "О'"),
+    ("У́", "У'"),
+    ("Ы́", "Ы'"),
+    ("Э́", "Э'"),
+    ("Ю́", "Ю'"),
+    ("Я́", "Я'"),
+    ("а́", "а'"),
+    ("е́", "е'"),
+    ("ё́", "е'"),
+    ("и́", "и'"),
+    ("о́", "о'"),
+    ("у́", "у'"),
+    ("ы́", "ы'"),
+    ("э́", "э'"),
+    ("ю́", "ю'"),
+    ("я́", "я'"),
+    ("ё", "е"),
+    ("Ё", "Е"),
+]
+
+REWRITE_STRESSED = pynini.closure(
+    pynini.string_map(RU_STRESSED_MAP).optimize() | RU_ALPHA
+).optimize()
+TO_CYRILLIC = pynini.string_file(get_abs_path("data/latin_to_cyrillic.tsv")).optimize()
+TO_LATIN = pynini.invert(TO_CYRILLIC).optimize()
+RU_ALPHA_OR_SPACE = pynini.union(RU_ALPHA, DAMO_SPACE, DAMO_NON_BREAKING_SPACE).optimize()