chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1 @@
+
@@ -0,0 +1,32 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing cardinals
+        e.g. cardinal { integer: "тысяча один" } -> "тысяча один"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
+        optional_sign = pynini.closure(pynini.cross('negative: "true" ', "минус "), 0, 1)
+        optional_quantity_part = pynini.closure(
+            pynini.accep(" ")
+            + pynutil.delete('quantity: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"'),
+            0,
+            1,
+        )
+        integer = (
+            pynutil.delete('integer: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+        self.graph = optional_sign + integer + optional_quantity_part
+        delete_tokens = self.delete_tokens(self.graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,22 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+from fun_text_processing.text_normalization.ru.alphabet import RU_ALPHA
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for verbalizing date, e.g.
+        tokens { date { day: "первое мая" } } -> "первое мая"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="date", kind="verbalize", deterministic=deterministic)
+
+        graph = pynutil.delete('day: "') + pynini.closure(RU_ALPHA | " ", 1) + pynutil.delete('"')
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,46 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing decimal, e.g.
+        tokens { decimal { integer_part: "одно целая" fractional_part: "восемь сотых} } ->
+            "одно целая восемь сотых"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="decimal", kind="verbalize", deterministic=deterministic)
+
+        optional_sign = pynini.closure(pynini.cross('negative: "true" ', "минус "), 0, 1)
+        integer = pynutil.delete(' "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        integer_part = pynutil.delete("integer_part:") + integer
+        fractional_part = pynutil.delete("fractional_part:") + integer
+        optional_quantity_part = pynini.closure(
+            pynini.accep(" ")
+            + pynutil.delete('quantity: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"'),
+            0,
+            1,
+        )
+
+        self.graph = (
+            optional_sign
+            + integer_part
+            + pynini.accep(" ")
+            + fractional_part
+            + optional_quantity_part
+            + delete_space
+        )
+        delete_tokens = self.delete_tokens(self.graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,22 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+from fun_text_processing.text_normalization.ru.alphabet import RU_ALPHA
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. electronic { username: "эй би собака эн ди точка ру" } -> "эй би собака эн ди точка ру"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+        for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
+
+        graph = pynutil.delete('username: "') + pynini.closure(RU_ALPHA | " ") + pynutil.delete('"')
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,34 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NON_BREAKING_SPACE,
+    DAMO_SPACE,
+    GraphFst,
+    delete_space,
+)
+from fun_text_processing.text_normalization.ru.alphabet import RU_ALPHA
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure, e.g.
+        measure { cardinal { integer: "два килограма" } } -> "два килограма"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
+
+        graph = (
+            pynutil.delete(' cardinal { integer: "')
+            + pynini.closure(RU_ALPHA | DAMO_SPACE | DAMO_NON_BREAKING_SPACE)
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete("}")
+        )
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,24 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+from fun_text_processing.text_normalization.ru.alphabet import RU_ALPHA
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing money, e.g.
+        money {  "пять рублей" } -> пять рублей
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="money", kind="verbalize", deterministic=deterministic)
+
+        graph = pynini.closure(RU_ALPHA | " ")
+        delete_tokens = self.delete_tokens(
+            pynutil.delete('integer_part: "') + graph + pynutil.delete('"')
+        )
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,22 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing roman numerals
+        e.g. ordinal { integer: "второе" } } -> "второе"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic)
+
+        value = pynini.closure(DAMO_NOT_QUOTE)
+        graph = pynutil.delete('integer: "') + value + pynutil.delete('"')
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,26 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+from fun_text_processing.text_normalization.ru.alphabet import RU_ALPHA
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for verbalizing telephone, e.g.
+        telephone { number_part: "восемь девятьсот тринадцать девятьсот восемьдесят три пятьдесят шесть ноль один" } -> "восемь девятьсот тринадцать девятьсот восемьдесят три пятьдесят шесть ноль один"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="telephone", kind="verbalize", deterministic=deterministic)
+
+        graph = (
+            pynutil.delete('number_part: "')
+            + pynini.closure(RU_ALPHA | " ", 1)
+            + pynutil.delete('"')
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,51 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. time { hours: "два часа пятнадцать минут" } -> "два часа пятнадцать минут"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+        for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="time", kind="verbalize", deterministic=deterministic)
+
+        hour = (
+            pynutil.delete("hours:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        minutes = (
+            pynutil.delete("minutes:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        self.graph = (
+            hour
+            + delete_space
+            + insert_space
+            + minutes
+            + delete_space
+            + pynutil.delete("preserve_order: true")
+        )
+        self.graph |= hour + delete_space
+        self.graph |= minutes + delete_space + insert_space + hour + delete_space
+
+        delete_tokens = self.delete_tokens(self.graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,55 @@
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
+from fun_text_processing.text_normalization.ru.verbalizers.cardinal import CardinalFst
+from fun_text_processing.text_normalization.ru.verbalizers.date import DateFst
+from fun_text_processing.text_normalization.ru.verbalizers.decimal import DecimalFst
+from fun_text_processing.text_normalization.ru.verbalizers.electronic import ElectronicFst
+from fun_text_processing.text_normalization.ru.verbalizers.measure import MeasureFst
+from fun_text_processing.text_normalization.ru.verbalizers.money import MoneyFst
+from fun_text_processing.text_normalization.ru.verbalizers.ordinal import OrdinalFst
+from fun_text_processing.text_normalization.ru.verbalizers.telephone import TelephoneFst
+from fun_text_processing.text_normalization.ru.verbalizers.time import TimeFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple options (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
+        cardinal = CardinalFst()
+        cardinal_graph = cardinal.fst
+        ordinal_graph = OrdinalFst().fst
+        decimal = DecimalFst()
+        decimal_graph = decimal.fst
+        date = DateFst()
+        date_graph = date.fst
+        measure = MeasureFst()
+        measure_graph = measure.fst
+        electronic = ElectronicFst()
+        electronic_graph = electronic.fst
+        whitelist_graph = WhiteListFst().fst
+        money_graph = MoneyFst().fst
+        telephone_graph = TelephoneFst().fst
+        time_graph = TimeFst().fst
+
+        graph = (
+            measure_graph
+            | cardinal_graph
+            | decimal_graph
+            | ordinal_graph
+            | date_graph
+            | electronic_graph
+            | money_graph
+            | whitelist_graph
+            | telephone_graph
+            | time_graph
+        )
+        self.fst = graph
@@ -0,0 +1,62 @@
+import os
+
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+    generator_main,
+)
+from fun_text_processing.text_normalization.en.verbalizers.word import WordFst
+from fun_text_processing.text_normalization.ru.verbalizers.verbalize import VerbalizeFst
+from pynini.lib import pynutil
+
+import logging
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence, e.g.
+    tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple options (used for audio-based normalization)
+        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
+        overwrite_cache: set to True to overwrite .far files
+    """
+
+    def __init__(
+        self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False
+    ):
+        super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
+
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            far_file = os.path.join(
+                cache_dir, f"ru_tn_{deterministic}_deterministic_verbalizer.far"
+            )
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["verbalize"]
+            logging.info(f"VerbalizeFinalFst graph was restored from {far_file}.")
+        else:
+
+            verbalize = VerbalizeFst().fst
+            word = WordFst().fst
+            types = verbalize | word
+            graph = (
+                pynutil.delete("tokens")
+                + delete_space
+                + pynutil.delete("{")
+                + delete_space
+                + types
+                + delete_space
+                + pynutil.delete("}")
+            )
+            graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+            self.fst = graph.optimize()
+
+            if far_file:
+                generator_main(far_file, {"verbalize": self.fst})
+                logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.")