chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1 @@
+
@@ -0,0 +1,16 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class Cardinal(GraphFst):
+    """
+    tokens { cardinal { integer: "一二三" } } -> 一二三
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
+
+        graph = pynutil.delete('integer: "') + pynini.closure(FUN_NOT_QUOTE) + pynutil.delete('"')
+
+        self.fst = self.delete_tokens(graph).optimize()
@@ -0,0 +1,14 @@
+from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class Char(GraphFst):
+    """
+    tokens { char: "你" } -> 你
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="char", kind="verbalize", deterministic=deterministic)
+
+        graph = pynutil.delete('name: "') + FUN_NOT_QUOTE + pynutil.delete('"')
+        self.fst = graph.optimize()
@@ -0,0 +1,61 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import (
+    FUN_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from fun_text_processing.text_normalization.zh.utils import UNIT_1e01, get_abs_path
+from pynini.lib import pynutil
+
+
+class Date(GraphFst):
+    """
+    tokens { date { year: "2002" month: "01" day: "28"} }  ->  二零零二年一月二十八日
+    tokens { date { year: "2002" } } ->  二零零八年
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="date", kind="verbalize", deterministic=deterministic)
+        date_type0 = pynutil.delete('year: "') + pynini.closure(FUN_NOT_QUOTE) + pynutil.delete('"')
+        graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv"))
+        graph_teen = pynini.string_file(get_abs_path("data/number/digit_teen.tsv"))
+        graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv"))
+        graph_no_zero = pynini.cross("0", "")
+        graph_year = pynini.closure(graph_digit | graph_zero, 2, 4)
+        graph_digit_no_zero = graph_digit | graph_no_zero
+        graph_2_digit_date = (graph_teen + pynutil.insert(UNIT_1e01) + graph_digit_no_zero) | (
+            graph_no_zero + graph_digit
+        )
+
+        date_type1 = (
+            pynutil.delete('year: "')
+            + graph_year
+            + pynutil.insert("年")
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete('month: "')
+            + graph_2_digit_date
+            + pynutil.insert("月")
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete('day: "')
+            + graph_2_digit_date
+            + pynutil.insert("日")
+            + pynutil.delete('"')
+        )
+
+        date_type2 = (
+            pynutil.delete('year: "')
+            + graph_year
+            + pynutil.insert("年")
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete('month: "')
+            + graph_2_digit_date
+            + pynutil.insert("月")
+            + pynutil.delete('"')
+        )
+
+        graph = date_type0 | date_type1 | date_type2
+
+        self.fst = self.delete_tokens(graph).optimize()
@@ -0,0 +1,20 @@
+from fun_text_processing.text_normalization.zh.graph_utils import GraphFst
+from fun_text_processing.text_normalization.zh.taggers.cardinal import Cardinal
+from pynini.lib import pynutil
+
+
+class Fraction(GraphFst):
+    """
+    tokens { fraction { denominator: "5" numerator: "1" } } -> 五分之一
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="fraction", kind="verbalize", deterministic=deterministic)
+
+        denominator = (
+            pynutil.delete('denominator: "') + Cardinal().graph_cardinal + pynutil.delete('"')
+        )
+        numerator = pynutil.delete('numerator: "') + Cardinal().graph_cardinal + pynutil.delete('"')
+        graph = denominator + pynutil.delete(" ") + pynutil.insert("分之") + numerator
+
+        self.fst = self.delete_tokens(graph).optimize()
@@ -0,0 +1,16 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class MathSymbol(GraphFst):
+    """
+    tokens { sign: "加" }  -> 加
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="sign", kind="verbalize", deterministic=deterministic)
+
+        graph = pynutil.delete('score: "') + pynini.closure(FUN_NOT_QUOTE) + pynutil.delete('"')
+
+        self.fst = graph.optimize()
@@ -0,0 +1,41 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import (
+    FUN_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class Measure(GraphFst):
+    """
+    tokens { measure { cardinal: "一" } units: "千克" } } ->  一千克
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
+
+        graph = (
+            pynutil.delete("cardinal {")
+            + delete_space
+            + pynutil.delete('integer: "')
+            + pynini.closure(FUN_NOT_QUOTE)
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete("}")
+            + delete_space
+            + pynutil.delete('units: "')
+            + pynini.closure(FUN_NOT_QUOTE)
+            + pynutil.delete('"')
+        )
+        percent_graph = (
+            pynutil.delete("decimal { ")
+            + pynutil.delete('integer_part: "')
+            + pynutil.insert("百分之")
+            + pynini.closure(FUN_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph |= percent_graph
+        self.fst = self.delete_tokens(graph).optimize()
@@ -0,0 +1,31 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import (
+    FUN_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class Money(GraphFst):
+    """
+    tokens { money { integer_part: "一点五" fractional_part: "元" } } ->  一点五元
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="money", kind="verbalize", deterministic=deterministic)
+
+        cur = (
+            pynutil.delete('fractional_part: "')
+            + pynini.closure(FUN_NOT_QUOTE)
+            + pynutil.delete('"')
+        )
+        num = (
+            pynutil.delete('integer_part: "')
+            + pynini.closure(FUN_NOT_QUOTE)
+            + pynutil.delete('"')
+            + delete_space
+        )
+        graph = num + cur
+
+        self.fst = self.delete_tokens(graph).optimize()
@@ -0,0 +1,73 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import (
+    FUN_ALPHA,
+    FUN_DIGIT,
+    FUN_PUNCT,
+    FUN_SIGMA,
+    FUN_WHITE_SPACE,
+    GraphFst,
+)
+from fun_text_processing.text_normalization.zh.utils import get_abs_path
+from pynini.lib import pynutil, utf8
+
+
+class PostProcessor(GraphFst):
+    """
+    Postprocessing of TN, now contains:
+        1. punctuation removal
+        2. letter case conversion
+        3. oov tagger
+    """
+
+    def __init__(
+        self,
+        remove_puncts: bool = False,
+        to_upper: bool = False,
+        to_lower: bool = False,
+        tag_oov: bool = False,
+    ):
+        super().__init__(name="PostProcessor", kind="processor")
+
+        graph = pynini.cdrewrite("", "", "", FUN_SIGMA)
+        if remove_puncts:
+            remove_puncts_graph = pynutil.delete(
+                pynini.union(
+                    FUN_PUNCT, pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv"))
+                )
+            )
+            graph @= pynini.cdrewrite(remove_puncts_graph, "", "", FUN_SIGMA).optimize()
+
+        if to_upper or to_lower:
+            if to_upper:
+                conv_cases_graph = pynini.inverse(
+                    pynini.string_file(get_abs_path("data/char/upper_to_lower.tsv"))
+                )
+            else:
+                conv_cases_graph = pynini.string_file(get_abs_path("data/char/upper_to_lower.tsv"))
+
+            graph @= pynini.cdrewrite(conv_cases_graph, "", "", FUN_SIGMA).optimize()
+
+        if tag_oov:
+            zh_charset_std = pynini.string_file(
+                get_abs_path("data/char/charset_national_standard_2013_8105.tsv")
+            )
+            zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv"))
+
+            zh_charset = (
+                zh_charset_std
+                | zh_charset_ext
+                | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv"))
+            )
+            en_charset = FUN_DIGIT | FUN_ALPHA | FUN_PUNCT | FUN_WHITE_SPACE
+            charset = zh_charset | en_charset
+
+            with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f:
+                tags = f.readline().strip().split("\t")
+                assert len(tags) == 2
+                ltag, rtag = tags
+
+            oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset)
+            tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag)
+            graph @= pynini.cdrewrite(tag_oov_graph, "", "", FUN_SIGMA).optimize()
+
+        self.fst = graph.optimize()
@@ -0,0 +1,88 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import (
+    FUN_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from fun_text_processing.text_normalization.zh.utils import UNIT_1e01, get_abs_path
+from pynini.lib import pynutil
+
+
+class Time(GraphFst):
+    """
+    tokens { time { h: "1" m: "02" s: "36" } } -> 一点零二分三十六秒
+    tokens { time { suffix "am"  hours: "1" minutes: "02" seconds: "36" } } -> 上午一点零二分三十六秒
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="time", kind="verbalize", deterministic=deterministic)
+        graph_digit = pynini.string_file(get_abs_path("data/number/digit.tsv"))
+        graph_teen = pynini.string_file(get_abs_path("data/number/digit_teen.tsv"))
+        graph_zero = pynini.string_file(get_abs_path("data/number/zero.tsv"))
+        graph_no_zero = pynini.cross("0", "")
+
+        graph_digit_no_zero = graph_digit | graph_no_zero
+
+        graph_2_digit_zero_none = pynini.cross("0", "") + pynini.cross("0", "")
+        graph_2_digit_zero = pynini.cross("00", "零")
+
+        graph_2_digit_time = (graph_teen + pynutil.insert(UNIT_1e01) + graph_digit_no_zero) | (
+            graph_zero + graph_digit
+        )
+        h = graph_2_digit_time | graph_2_digit_zero | graph_digit
+        m = graph_2_digit_time | graph_2_digit_zero
+        s = graph_2_digit_time | graph_2_digit_zero
+
+        # 6:25
+        h_m = (
+            pynutil.delete('hours: "')
+            + h
+            + pynutil.insert("点")
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete('minutes: "')
+            + (graph_2_digit_time)
+            + pynutil.insert("分")
+            + pynutil.delete('"')
+        )
+
+        # 23:00
+        h_00 = (
+            pynutil.delete('hours: "')
+            + h
+            + pynutil.insert("点")
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete('minutes: "')
+            + (graph_2_digit_zero_none)
+            + pynutil.delete('"')
+        )
+
+        # 9:12:52
+        h_m_s = (
+            pynutil.delete('hours: "')
+            + h
+            + pynutil.insert("点")
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete('minutes: "')
+            + m
+            + pynutil.insert("分")
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete('seconds: "')
+            + s
+            + pynutil.insert("秒")
+            + pynutil.delete('"')
+        )
+
+        graph = h_m | h_m_s | h_00
+        graph_suffix = (
+            pynutil.delete('suffix: "')
+            + pynini.closure(FUN_NOT_QUOTE)
+            + pynutil.delete('"')
+            + delete_space
+            + graph
+        )
+        graph |= graph_suffix
+        self.fst = self.delete_tokens(graph).optimize()
@@ -0,0 +1,49 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import GraphFst
+from fun_text_processing.text_normalization.zh.verbalizers.cardinal import Cardinal
+from fun_text_processing.text_normalization.zh.verbalizers.char import Char
+from fun_text_processing.text_normalization.zh.verbalizers.date import Date
+from fun_text_processing.text_normalization.zh.verbalizers.fraction import Fraction
+from fun_text_processing.text_normalization.zh.verbalizers.math_symbol import MathSymbol
+from fun_text_processing.text_normalization.zh.verbalizers.measure import Measure
+from fun_text_processing.text_normalization.zh.verbalizers.money import Money
+from fun_text_processing.text_normalization.zh.verbalizers.time import Time
+from fun_text_processing.text_normalization.zh.verbalizers.whitelist import Whitelist
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple options (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
+
+        date = Date(deterministic=deterministic)
+        cardinal = Cardinal(deterministic=deterministic)
+        char = Char(deterministic=deterministic)
+        fraction = Fraction(deterministic=deterministic)
+        math_symbol = MathSymbol(deterministic=deterministic)
+        money = Money(deterministic=deterministic)
+        measure = Measure(deterministic=deterministic)
+        time = Time(deterministic=deterministic)
+        whitelist = Whitelist(deterministic=deterministic)
+
+        graph = pynini.union(
+            date.fst,
+            cardinal.fst,
+            fraction.fst,
+            char.fst,
+            math_symbol.fst,
+            money.fst,
+            measure.fst,
+            time.fst,
+            whitelist.fst,
+        )
+
+        self.fst = graph.optimize()
@@ -0,0 +1,51 @@
+import os
+
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import (
+    GraphFst,
+    delete_space,
+    generator_main,
+)
+from fun_text_processing.text_normalization.zh.verbalizers.postprocessor import PostProcessor
+from fun_text_processing.text_normalization.zh.verbalizers.verbalize import VerbalizeFst
+from pynini.lib import pynutil
+
+# import logging
+
+
+class VerbalizeFinalFst(GraphFst):
+    """ """
+
+    def __init__(
+        self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False
+    ):
+        super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            far_file = os.path.join(
+                cache_dir, f"zh_tn_{deterministic}_deterministic_verbalizer.far"
+            )
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["verbalize"]
+        else:
+            token_graph = VerbalizeFst(deterministic=deterministic)
+            token_verbalizer = (
+                pynutil.delete("tokens {")
+                + delete_space
+                + token_graph.fst
+                + delete_space
+                + pynutil.delete(" }")
+            )
+            verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space)
+
+            postprocessor = PostProcessor(
+                remove_puncts=False,
+                to_upper=False,
+                to_lower=False,
+                tag_oov=False,
+            )
+
+            self.fst = (verbalizer @ postprocessor.fst).optimize()
+            if far_file:
+                generator_main(far_file, {"verbalize": self.fst})
@@ -0,0 +1,16 @@
+import pynini
+from fun_text_processing.text_normalization.zh.graph_utils import FUN_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class Whitelist(GraphFst):
+    """
+    tokens { whitelist: "ATM" } -> A T M
+    """
+
+    def __init__(self, deterministic: bool = True, lm: bool = False):
+        super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic)
+        remove_erhua = pynutil.delete('erhua: "') + pynutil.delete("儿") + pynutil.delete('"')
+        whitelist = pynutil.delete('name: "') + pynini.closure(FUN_NOT_QUOTE) + pynutil.delete('"')
+        graph = remove_erhua | whitelist
+        self.fst = graph.optimize()