chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1,95 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_NOT_SPACE,
+    DAMO_SIGMA,
+    TO_UPPER,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+    insert_space,
+)
+from fun_text_processing.text_normalization.en.utils import get_abs_path
+from pynini.examples import plurals
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> c d f one at a b c dot e d u
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+        for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
+        graph_digit_no_zero = pynini.invert(
+            pynini.string_file(get_abs_path("data/number/digit.tsv"))
+        ).optimize()
+        graph_zero = pynini.cross("0", "zero")
+
+        if not deterministic:
+            graph_zero |= pynini.cross("0", "o") | pynini.cross("0", "oh")
+
+        graph_digit = graph_digit_no_zero | graph_zero
+        graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize()
+
+        default_chars_symbols = pynini.cdrewrite(
+            pynutil.insert(" ") + (graph_symbols | graph_digit) + pynutil.insert(" "),
+            "",
+            "",
+            DAMO_SIGMA,
+        )
+        default_chars_symbols = pynini.compose(
+            pynini.closure(DAMO_NOT_SPACE), default_chars_symbols.optimize()
+        ).optimize()
+
+        user_name = (
+            pynutil.delete("username:")
+            + delete_space
+            + pynutil.delete('"')
+            + default_chars_symbols
+            + pynutil.delete('"')
+        )
+
+        domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
+
+        domain = (
+            default_chars_symbols
+            + insert_space
+            + plurals._priority_union(
+                domain_common,
+                pynutil.add_weight(pynini.cross(".", "dot"), weight=0.0001),
+                DAMO_SIGMA,
+            )
+            + pynini.closure(
+                insert_space
+                + (pynini.cdrewrite(TO_UPPER, "", "", DAMO_SIGMA) @ default_chars_symbols),
+                0,
+                1,
+            )
+        )
+        domain = (
+            pynutil.delete("domain:")
+            + delete_space
+            + pynutil.delete('"')
+            + domain
+            + delete_space
+            + pynutil.delete('"')
+        ).optimize()
+
+        protocol = (
+            pynutil.delete('protocol: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+        graph = (
+            pynini.closure(protocol + delete_space, 0, 1)
+            + pynini.closure(user_name + delete_space + pynutil.insert(" at ") + delete_space, 0, 1)
+            + domain
+            + delete_space
+        ).optimize() @ pynini.cdrewrite(delete_extra_space, "", "", DAMO_SIGMA)
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()