chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1 @@
+
@@ -0,0 +1,23 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing cardinal
+        e.g. cardinal { integer: "23" negative: "-" } -> -23
+
+    Args:
+        tn_cardinal_verbalizer: TN cardinal verbalizer
+    """
+
+    def __init__(self, tn_cardinal_verbalizer: GraphFst, deterministic: bool = True):
+        super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
+        self.numbers = tn_cardinal_verbalizer.numbers
+        optional_sign = pynini.closure(
+            pynutil.delete('negative: "') + DAMO_NOT_QUOTE + pynutil.delete('" '), 0, 1
+        )
+        graph = optional_sign + self.numbers
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,37 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_preserve_order,
+)
+from pynini.lib import pynutil
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing decimal, e.g.
+        decimal { negative: "true" integer_part: "12"  fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
+
+    Args:
+        tn_decimal_verbalizer: TN decimal verbalizer
+    """
+
+    def __init__(self, tn_decimal_verbalizer: GraphFst, deterministic: bool = True):
+        super().__init__(name="decimal", kind="verbalize", deterministic=deterministic)
+        delete_space = pynutil.delete(" ")
+        optional_sign = pynini.closure(
+            pynutil.delete('negative: "') + DAMO_NOT_QUOTE + pynutil.delete('"') + delete_space,
+            0,
+            1,
+        )
+        optional_integer = pynini.closure(tn_decimal_verbalizer.integer, 0, 1)
+        optional_fractional = pynini.closure(
+            delete_space + pynutil.insert(",") + tn_decimal_verbalizer.fractional_default, 0, 1
+        )
+        graph = (
+            optional_integer + optional_fractional + tn_decimal_verbalizer.optional_quantity
+        ).optimize()
+        self.numbers = optional_sign + graph
+        graph = self.numbers + delete_preserve_order
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,50 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure, e.g.
+        measure { cardinal { negative: "true" integer: "12" } units: "kg" } -> -12 kg
+        measure { decimal { integer_part: "1/2" } units: "kg" } -> 1/2 kg
+        measure { decimal { integer_part: "1" fractional_part: "2" quantity: "million" } units: "kg" } -> 1,2 million kg
+
+    Args:
+        decimal: ITN Decimal verbalizer
+        cardinal: ITN Cardinal verbalizer
+    """
+
+    def __init__(self, decimal: GraphFst, cardinal: GraphFst, deterministic: bool = True):
+        super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
+        unit = (
+            pynutil.delete("units:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+            + delete_space
+        )
+        graph_decimal = (
+            pynutil.delete("decimal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + decimal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph_cardinal = (
+            pynutil.delete("cardinal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + cardinal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+
+        graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert(" ") + unit
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,26 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing money, e.g.
+        money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05
+
+    Args:
+        decimal: ITN Decimal verbalizer
+    """
+
+    def __init__(self, decimal: GraphFst, deterministic: bool = True):
+        super().__init__(name="money", kind="verbalize", deterministic=deterministic)
+        unit = (
+            pynutil.delete("currency:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = unit + delete_space + decimal.numbers
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,52 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing time, e.g.
+        time { hours: "8" minutes: "30" zone: "e s t" } -> 08:30 Uhr est
+        time { hours: "8" } -> 8 Uhr
+        time { hours: "8" minutes: "30" seconds: "10" } -> 08:30:10 Uhr
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="time", kind="verbalize", deterministic=deterministic)
+
+        add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
+            pynutil.insert("0") + DAMO_DIGIT
+        )
+        hour = pynutil.delete('hours: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
+        minute = pynutil.delete('minutes: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
+
+        second = pynutil.delete('seconds: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
+        zone = (
+            pynutil.delete('zone: "')
+            + pynini.closure(DAMO_ALPHA + delete_space)
+            + DAMO_ALPHA
+            + pynutil.delete('"')
+        )
+        optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1)
+        graph = (
+            delete_space
+            + pynutil.insert(":")
+            + (minute @ add_leading_zero_to_double_digit)
+            + pynini.closure(
+                delete_space + pynutil.insert(":") + (second @ add_leading_zero_to_double_digit),
+                0,
+                1,
+            )
+            + pynutil.insert(" Uhr")
+            + optional_zone
+        )
+        graph_h = hour + pynutil.insert(" Uhr") + optional_zone
+        graph_hm = hour @ add_leading_zero_to_double_digit + graph
+        graph_hms = hour @ add_leading_zero_to_double_digit + graph
+        final_graph = graph_hm | graph_hms | graph_h
+        self.fst = self.delete_tokens(final_graph).optimize()
@@ -0,0 +1,35 @@
+from fun_text_processing.inverse_text_normalization.de.verbalizers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.de.verbalizers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.de.verbalizers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.de.verbalizers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.de.verbalizers.time import TimeFst
+from fun_text_processing.text_normalization.de.verbalizers.cardinal import (
+    CardinalFst as TNCardinalVerbalizer,
+)
+from fun_text_processing.text_normalization.de.verbalizers.decimal import (
+    DecimalFst as TNDecimalVerbalizer,
+)
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
+        tn_cardinal_verbalizer = TNCardinalVerbalizer(deterministic=False)
+        tn_decimal_verbalizer = TNDecimalVerbalizer(deterministic=False)
+
+        cardinal = CardinalFst(tn_cardinal_verbalizer=tn_cardinal_verbalizer)
+        cardinal_graph = cardinal.fst
+        decimal = DecimalFst(tn_decimal_verbalizer=tn_decimal_verbalizer)
+        decimal_graph = decimal.fst
+        measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst
+        money_graph = MoneyFst(decimal=decimal).fst
+        time_graph = TimeFst().fst
+        graph = time_graph | money_graph | measure_graph | decimal_graph | cardinal_graph
+        self.fst = graph
@@ -0,0 +1,33 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.de.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst
+from fun_text_processing.text_normalization.en.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence, e.g.
+    tokens { name: "jetzt" } tokens { name: "ist" } tokens { time { hours: "12" minutes: "30" } } -> jetzt ist 12:30 Uhr
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
+        verbalize = VerbalizeFst().fst
+        word = WordFst().fst
+        types = verbalize | word
+        graph = (
+            pynutil.delete("tokens")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + types
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+        self.fst = graph