chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1 @@
+
@@ -0,0 +1,38 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing cardinal
+        e.g. cardinal { negative: "-" integer: "23" } -> -23
+    """
+
+    def __init__(self):
+        super().__init__(name="cardinal", kind="verbalize")
+        optional_sign = pynini.closure(
+            pynutil.delete("negative:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space,
+            0,
+            1,
+        )
+        graph = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        self.numbers = graph
+        graph = optional_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,62 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for verbalizing date, e.g.
+        date { day: "1" month: "janvier" preserve_order: true } -> 1 de enero
+    """
+
+    def __init__(self):
+        super().__init__(name="date", kind="verbalize")
+
+        convert_primer = pynini.cross("1", "1ᵉʳ")
+        day = (
+            pynutil.delete("day:")
+            + delete_space
+            + pynutil.delete('"')
+            + (
+                pynini.closure(DAMO_NOT_QUOTE, 1) | pynutil.add_weight(convert_primer, -1)
+            )  # first of the month is ordinal
+            + pynutil.delete('"')
+        )
+        month = (
+            pynutil.delete("month:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        year = (
+            pynutil.delete("year:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        # day month
+        graph_dm = day + delete_extra_space + month
+        graph_dmy = graph_dm + delete_extra_space + year
+
+        optional_preserve_order = pynini.closure(
+            pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
+            | pynutil.delete("field_order:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space
+        )
+
+        final_graph = (graph_dm | graph_dmy) + delete_space + optional_preserve_order
+
+        delete_tokens = self.delete_tokens(final_graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,81 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_DIGIT,
+    DAMO_NON_BREAKING_SPACE,
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class NumberParser(GraphFst):
+    """
+    Finite state transducer for parsing strings of digis. Breaks up digit strings into groups of three for
+        strings of digits of four or more (inclusive). Groupings are separated by non-breaking space.
+    e.g. '1000' -> '1 000'
+    e.g. '1000,33333' -> '1 000,333 33
+    """
+
+    def __init__(self):
+        super().__init__(name="parser", kind="verbalize")
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing decimal, e.g.
+        decimal { negative: "true" integer_part: "12"  fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
+    """
+
+    def __init__(self):
+        super().__init__(name="decimal", kind="verbalize")
+
+        # Need parser to group digits by threes
+        exactly_three_digits = DAMO_DIGIT**3
+        at_most_three_digits = pynini.closure(DAMO_DIGIT, 1, 3)
+
+        space_every_three_integer = (
+            at_most_three_digits
+            + (pynutil.insert(DAMO_NON_BREAKING_SPACE) + exactly_three_digits).closure()
+        )
+        space_every_three_decimal = (
+            pynini.accep(",")
+            + (exactly_three_digits + pynutil.insert(DAMO_NON_BREAKING_SPACE)).closure()
+            + at_most_three_digits
+        )
+        group_by_threes = space_every_three_integer | space_every_three_decimal
+        self.group_by_threes = group_by_threes
+
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        integer = (
+            pynutil.delete("integer_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        integer = integer @ group_by_threes
+        optional_integer = pynini.closure(integer + delete_space, 0, 1)
+        fractional = (
+            pynutil.insert(",")
+            + pynutil.delete("fractional_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        fractional = fractional @ group_by_threes
+        optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
+        quantity = (
+            pynutil.delete("quantity:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1)
+        graph = (optional_integer + optional_fractional + optional_quantity).optimize()
+        self.numbers = graph
+        graph = optional_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,35 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu
+    """
+
+    def __init__(self):
+        super().__init__(name="electronic", kind="verbalize")
+        user_name = (
+            pynutil.delete("username:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        domain = (
+            pynutil.delete("domain:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = user_name + delete_space + pynutil.insert("@") + domain
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,43 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for verbalizing fraction
+        e.g. fraction { integer_part: "1" numerator: "2" denominator: "3" } } -> 1 2/3
+
+    """
+
+    def __init__(self):
+        super().__init__(name="fraction", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        integer = (
+            pynutil.delete('integer_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + insert_space
+        )
+        numerator = (
+            pynutil.delete('numerator: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+
+        denominator = (
+            pynutil.insert("/")
+            + pynutil.delete('denominator: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = (
+            pynini.closure(integer + delete_space, 0, 1) + numerator + delete_space + denominator
+        ).optimize()
+        self.numbers = graph
+        delete_tokens = self.delete_tokens(optional_sign + graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,68 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure, e.g.
+        measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg
+
+    Args:
+        decimal: DecimalFst
+        cardinal: CardinalFst
+        fraction: FractionFst
+    """
+
+    def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst):
+        super().__init__(name="measure", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
+        unit = (
+            pynutil.delete("units:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+            + delete_space
+        )
+        graph_decimal = (
+            pynutil.delete("decimal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + decimal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph_cardinal = (
+            pynutil.delete("cardinal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + cardinal.numbers
+            @ decimal.group_by_threes  # measurements most obey three by three spacing
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph_fraction = (
+            pynutil.delete("fraction {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + fraction.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+
+        graph = (
+            (graph_cardinal | graph_decimal | graph_fraction)
+            + delete_space
+            + pynutil.insert(" ")
+            + unit
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,31 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing money, e.g.
+        money { integer_part: "12" fractional_part: "05" currency: "$" } -> 12.05 $
+
+    Args:
+        decimal: DecimalFst
+    """
+
+    def __init__(self, decimal: GraphFst):
+        super().__init__(name="money", kind="verbalize")
+        unit = (
+            pynutil.delete("currency:")
+            + delete_extra_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        graph = decimal.numbers + delete_space + unit
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,71 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_DIGIT,
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing ordinal, e.g.
+        ordinal { integer: "13" morphosyntactic_features: "e" } -> 13ᵉ
+
+    Given 'special' terms for ordinals (e.g. siècle), renders
+        amount in conventional format. e.g.
+
+        ordinal { integer: "13" morphosyntactic_features: "e/siècle" } -> XIIIᵉ
+    """
+
+    def __init__(self):
+        super().__init__(name="ordinal", kind="verbalize")
+        graph_integer = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        replace_suffix = pynini.union(
+            pynini.cross("e", "ᵉ"),  # only delete first quote since there may be more features
+            pynini.cross("d", "ᵈ"),
+            pynini.cross("r", "ʳ"),
+            pynini.cross("s", "ˢ"),
+        )
+        replace_suffix = pynutil.delete(' morphosyntactic_features: "') + replace_suffix.plus
+
+        graph_arabic = graph_integer + replace_suffix.plus
+
+        # For roman.
+        graph_roman_digits = pynini.string_file(
+            get_abs_path("data/roman/digits_large.tsv")
+        ).invert()
+        graph_roman_ties = pynini.string_file(get_abs_path("data/roman/ties_large.tsv")).invert()
+        graph_roman_hundreds = pynini.string_file(
+            get_abs_path("data/roman/hundreds_large.tsv")
+        ).invert()
+        graph_roman_zero_digit = pynutil.delete("0")
+
+        graph_roman_hundreds = DAMO_DIGIT**3 @ (
+            graph_roman_hundreds
+            + pynini.union(graph_roman_ties, graph_roman_zero_digit)
+            + pynini.union(graph_roman_digits, graph_roman_zero_digit)
+        )
+        graph_roman_ties = DAMO_DIGIT**2 @ (
+            graph_roman_ties + pynini.union(graph_roman_digits, graph_roman_zero_digit)
+        )
+        graph_roman_digits = DAMO_DIGIT @ graph_roman_digits
+
+        graph_roman_integers = graph_roman_hundreds | graph_roman_ties | graph_roman_digits
+
+        graph_roman = (graph_integer @ graph_roman_integers) + replace_suffix
+        graph_roman += pynini.cross("/", " ") + "siècle"
+
+        graph = (graph_roman | graph_arabic) + pynutil.delete('"')
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,22 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for verbalizing telephone, e.g.
+        telephone { number_part: "02 33 43 53 22" }
+        -> 02 33 43 53 22
+    """
+
+    def __init__(self):
+        super().__init__(name="telephone", kind="verbalize")
+
+        number_part = (
+            pynutil.delete('number_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        delete_tokens = self.delete_tokens(number_part)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,61 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_DIGIT,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing time, e.g.
+        time { hours: "8" minutes: "30" suffix: "du matin"} -> 8 h 30
+        time { hours: "8" minutes: "30" } -> 8 h 30
+        time { hours: "8" minutes: "30" suffix: "du soir"} -> 20 h 30
+    """
+
+    def __init__(self):
+        super().__init__(name="time", kind="verbalize")
+
+        hour_to_night = pynini.string_file(get_abs_path("data/time/hour_to_night.tsv"))
+
+        day_suffixes = pynutil.delete('suffix: "am"')
+        night_suffixes = pynutil.delete('suffix: "pm"')
+
+        hour = (
+            pynutil.delete("hours:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1, 2)
+            + pynutil.delete('"')
+        )
+        minute = (
+            pynutil.delete("minutes:")
+            + delete_extra_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1, 2)
+            + pynutil.delete('"')
+        )
+
+        graph = (
+            hour
+            + delete_extra_space
+            + pynutil.insert("h")
+            + minute.ques
+            + delete_space
+            + day_suffixes.ques
+        )
+
+        graph |= (
+            hour @ hour_to_night
+            + delete_extra_space
+            + pynutil.insert("h")
+            + minute.ques
+            + delete_space
+            + night_suffixes
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()
@@ -0,0 +1,51 @@
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.date import DateFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.fraction import FractionFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.ordinal import OrdinalFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.time import TimeFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.whitelist import WhiteListFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize", kind="verbalize")
+        cardinal = CardinalFst()
+        cardinal_graph = cardinal.fst
+        ordinal_graph = OrdinalFst().fst
+        decimal = DecimalFst()
+        decimal_graph = decimal.fst
+        fraction = FractionFst()
+        fraction_graph = fraction.fst
+        measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction).fst
+        money_graph = MoneyFst(decimal=decimal).fst
+        time_graph = TimeFst().fst
+        date_graph = DateFst().fst
+        whitelist_graph = WhiteListFst().fst
+        telephone_graph = TelephoneFst().fst
+        electronic_graph = ElectronicFst().fst
+        graph = (
+            time_graph
+            | date_graph
+            | money_graph
+            | measure_graph
+            | fraction_graph
+            | ordinal_graph
+            | decimal_graph
+            | cardinal_graph
+            | whitelist_graph
+            | telephone_graph
+            | electronic_graph
+        )
+        self.fst = graph
@@ -0,0 +1,33 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.fr.verbalizers.word import WordFst
+from pynini.lib import pynutil
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence, e.g.
+    tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize_final", kind="verbalize")
+        verbalize = VerbalizeFst().fst
+        word = WordFst().fst
+        types = verbalize | word
+        graph = (
+            pynutil.delete("tokens")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + types
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+        self.fst = graph
@@ -0,0 +1,27 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for verbalizing whitelist
+        e.g. tokens { name: "mrs." } -> mrs.
+    """
+
+    def __init__(self):
+        super().__init__(name="whitelist", kind="verbalize")
+        graph = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+        self.fst = graph.optimize()
@@ -0,0 +1,29 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for verbalizing plain tokens
+        e.g. tokens { name: "sleep" } -> sleep
+    """
+
+    def __init__(self):
+        super().__init__(name="word", kind="verbalize")
+        chars = pynini.closure(DAMO_CHAR - " ", 1)
+        char = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + chars
+            + pynutil.delete('"')
+        )
+        graph = char @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+
+        self.fst = graph.optimize()