chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1 @@
+
@@ -0,0 +1,41 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing roman numerals
+        e.g. cardinal { integer: "1 001" } -> 1 001
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
+
+        optional_sign = pynini.closure(
+            pynutil.delete("negative:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space,
+            0,
+            1,
+        )
+
+        graph = (
+            optional_sign
+            + pynutil.delete('integer: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,16 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for verbalizing date, e.g.
+        date { day: "02.03.89" }  -> "02.03.89"
+    """
+
+    def __init__(self):
+        super().__init__(name="date", kind="verbalize")
+        graph = pynutil.delete('day: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        delete_tokens = self.delete_tokens(graph.optimize())
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,35 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SPACE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing decimal, e.g.
+        decimal { negative: "true" integer_part: "3," fractional_part: "2" } -> -3,2
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="decimal", kind="verbalize", deterministic=deterministic)
+
+        optional_sign = pynini.closure(pynini.cross('negative: "true" ', "-"), 0, 1)
+
+        integer = pynutil.delete(' "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        integer_part = pynutil.delete("integer_part:") + integer
+        fractional_part = pynutil.delete("fractional_part:") + integer
+        optional_quantity = pynini.closure(
+            pynini.accep(DAMO_SPACE) + pynutil.delete("quantity:") + integer, 0, 1
+        )
+
+        graph = optional_sign + integer_part + delete_space + fractional_part + optional_quantity
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,19 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. electronic { username: "ab@nd.ru" } -> "ab@nd.ru"
+    """
+
+    def __init__(self):
+        super().__init__(name="electronic", kind="verbalize")
+
+        graph = (
+            pynutil.delete('username: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,27 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure
+        e.g. measure { cardinal { integer: "2 кг" } } -> "2 кг"
+    """
+
+    def __init__(self):
+        super().__init__(name="measure", kind="verbalize")
+
+        graph = (
+            pynutil.delete(' cardinal { integer: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + delete_space
+            + pynutil.delete("}")
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,21 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. money { integer_part: "2 руб." } -> "2 руб."
+    """
+
+    def __init__(self):
+        super().__init__(name="money", kind="verbalize")
+
+        graph = (
+            pynutil.delete('integer_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,22 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing ordinal numbers
+        e.g. ordinal { integer: "2" } -> "2"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic)
+
+        value = pynini.closure(DAMO_NOT_QUOTE)
+        graph = pynutil.delete('integer: "') + value + pynutil.delete('"')
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,21 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for verbalizing telephone
+        e.g. telephone { number_part: "8-913-983-56-01" } -> "8-913-983-56-01"
+    """
+
+    def __init__(self):
+        super().__init__(name="telephone", kind="verbalize")
+
+        graph = (
+            pynutil.delete('number_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,40 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing time
+        e.g. time { hours: "02:15" } -> "02:15"
+    """
+
+    def __init__(self):
+        super().__init__(name="time", kind="verbalize")
+        hour = (
+            pynutil.delete("hours: ")
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        minutes = (
+            pynutil.delete("minutes: ")
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph_preserve_order = (
+            pynutil.delete('hours: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+
+        # for cases that require permutations for the correct verbalization
+        graph_reverse_order = hour + delete_space + pynutil.insert(":") + minutes + delete_space
+
+        graph = graph_preserve_order | graph_reverse_order
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,50 @@
+from fun_text_processing.inverse_text_normalization.en.verbalizers.whitelist import WhiteListFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.date import DateFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.ordinal import OrdinalFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.time import TimeFst
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize", kind="verbalize")
+        cardinal = CardinalFst()
+        cardinal_graph = cardinal.fst
+        ordinal = OrdinalFst()
+        ordinal_graph = ordinal.fst
+        decimal = DecimalFst()
+        decimal_graph = decimal.fst
+        whitelist_graph = WhiteListFst().fst
+        electronic_graph = ElectronicFst().fst
+        money_graph = MoneyFst().fst
+        date_graph = DateFst().fst
+        measure_graph = MeasureFst().fst
+        telephone_graph = TelephoneFst().fst
+        time_graph = TimeFst().fst
+
+        graph = (
+            whitelist_graph
+            | cardinal_graph
+            | ordinal_graph
+            | decimal_graph
+            | electronic_graph
+            | date_graph
+            | money_graph
+            | measure_graph
+            | telephone_graph
+            | time_graph
+        )
+
+        self.fst = graph
@@ -0,0 +1,33 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst
+from fun_text_processing.inverse_text_normalization.ru.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.text_normalization.en.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence, e.g.
+    tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize_final", kind="verbalize")
+        verbalize = VerbalizeFst().fst
+        word = WordFst().fst
+        types = verbalize | word
+        graph = (
+            pynutil.delete("tokens")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + types
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+        self.fst = graph