chore: import upstream snapshot with attribution

fun_text_processing/text_normalization/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/data_loader_utils.py

@@ -0,0 +1,350 @@
+import json
+import re
+import string
+from collections import defaultdict, namedtuple
+from typing import Dict, List, Optional, Set, Tuple
+from unicodedata import category
+
+import logging
+
+
+EOS_TYPE = "EOS"
+PUNCT_TYPE = "PUNCT"
+PLAIN_TYPE = "PLAIN"
+Instance = namedtuple("Instance", "token_type un_normalized normalized")
+known_types = [
+    "PLAIN",
+    "DATE",
+    "CARDINAL",
+    "LETTERS",
+    "VERBATIM",
+    "MEASURE",
+    "DECIMAL",
+    "ORDINAL",
+    "DIGIT",
+    "MONEY",
+    "TELEPHONE",
+    "ELECTRONIC",
+    "FRACTION",
+    "TIME",
+    "ADDRESS",
+]
+
+
+def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]:
+    """
+    https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish
+    Loads text file in the Kaggle Google text normalization file format: <semiotic class>\t<unnormalized text>\t<`self` if trivial class or normalized text>
+    E.g.
+    PLAIN   Brillantaisia   <self>
+    PLAIN   is      <self>
+    PLAIN   a       <self>
+    PLAIN   genus   <self>
+    PLAIN   of      <self>
+    PLAIN   plant   <self>
+    PLAIN   in      <self>
+    PLAIN   family  <self>
+    PLAIN   Acanthaceae     <self>
+    PUNCT   .       sil
+    <eos>   <eos>
+
+    Args:
+        file_path: file path to text file
+
+    Returns: flat list of instances
+    """
+    res = []
+    with open(file_path, "r") as fp:
+        for line in fp:
+            parts = line.strip().split("\t")
+            if parts[0] == "<eos>":
+                res.append(Instance(token_type=EOS_TYPE, un_normalized="", normalized=""))
+            else:
+                l_type, l_token, l_normalized = parts
+                l_token = l_token.lower()
+                l_normalized = l_normalized.lower()
+
+                if l_type == PLAIN_TYPE:
+                    res.append(
+                        Instance(token_type=l_type, un_normalized=l_token, normalized=l_token)
+                    )
+                elif l_type != PUNCT_TYPE:
+                    res.append(
+                        Instance(token_type=l_type, un_normalized=l_token, normalized=l_normalized)
+                    )
+    return res
+
+
+def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file) -> List[Instance]:
+    """
+    Load given list of text files using the `load_func` function.
+
+    Args:
+        file_paths: list of file paths
+        load_func: loading function
+
+    Returns: flat list of instances
+    """
+    res = []
+    for file_path in file_paths:
+        res.extend(load_func(file_path=file_path))
+    return res
+
+
+def clean_generic(text: str) -> str:
+    """
+    Cleans text without affecting semiotic classes.
+
+    Args:
+        text: string
+
+    Returns: cleaned string
+    """
+    text = text.strip()
+    text = text.lower()
+    return text
+
+
+def evaluate(
+    preds: List[str], labels: List[str], input: Optional[List[str]] = None, verbose: bool = True
+) -> float:
+    """
+    Evaluates accuracy given predictions and labels.
+
+    Args:
+        preds: predictions
+        labels: labels
+        input: optional, only needed for verbosity
+        verbose: if true prints [input], golden labels and predictions
+
+    Returns accuracy
+    """
+    acc = 0
+    nums = len(preds)
+    for i in range(nums):
+        pred_norm = clean_generic(preds[i])
+        label_norm = clean_generic(labels[i])
+        if pred_norm == label_norm:
+            acc = acc + 1
+        else:
+            if input:
+                print(f"inpu: {json.dumps(input[i])}")
+            print(f"gold: {json.dumps(label_norm)}")
+            print(f"pred: {json.dumps(pred_norm)}")
+    return acc / nums
+
+
+def training_data_to_tokens(
+    data: List[Instance], category: Optional[str] = None
+) -> Dict[str, Tuple[List[str], List[str]]]:
+    """
+    Filters the instance list by category if provided and converts it into a map from token type to list of un_normalized and normalized strings
+
+    Args:
+        data: list of instances
+        category: optional semiotic class category name
+
+    Returns Dict: token type -> (list of un_normalized strings, list of normalized strings)
+    """
+    result = defaultdict(lambda: ([], []))
+    for instance in data:
+        if instance.token_type != EOS_TYPE:
+            if category is None or instance.token_type == category:
+                result[instance.token_type][0].append(instance.un_normalized)
+                result[instance.token_type][1].append(instance.normalized)
+    return result
+
+
+def training_data_to_sentences(data: List[Instance]) -> Tuple[List[str], List[str], List[Set[str]]]:
+    """
+    Takes instance list, creates list of sentences split by EOS_Token
+    Args:
+        data: list of instances
+    Returns (list of unnormalized sentences, list of normalized sentences, list of sets of categories in a sentence)
+    """
+    # split data at EOS boundaries
+    sentences = []
+    sentence = []
+    categories = []
+    sentence_categories = set()
+
+    for instance in data:
+        if instance.token_type == EOS_TYPE:
+            sentences.append(sentence)
+            sentence = []
+            categories.append(sentence_categories)
+            sentence_categories = set()
+        else:
+            sentence.append(instance)
+            sentence_categories.update([instance.token_type])
+    un_normalized = [
+        " ".join([instance.un_normalized for instance in sentence]) for sentence in sentences
+    ]
+    normalized = [
+        " ".join([instance.normalized for instance in sentence]) for sentence in sentences
+    ]
+    return un_normalized, normalized, categories
+
+
+def post_process_punctuation(text: str) -> str:
+    """
+    Normalized quotes and spaces
+
+    Args:
+        text: text
+
+    Returns: text with normalized spaces and quotes
+    """
+    text = (
+        text.replace("( ", "(")
+        .replace(" )", ")")
+        .replace("{ ", "{")
+        .replace(" }", "}")
+        .replace("[ ", "[")
+        .replace(" ]", "]")
+        .replace("  ", " ")
+        .replace("”", '"')
+        .replace("’", "'")
+        .replace("»", '"')
+        .replace("«", '"')
+        .replace("\\", "")
+        .replace("„", '"')
+        .replace("´", "'")
+        .replace("’", "'")
+        .replace("“", '"')
+        .replace("‘", "'")
+        .replace("`", "'")
+        .replace("- -", "--")
+    )
+
+    for punct in "!,.:;?":
+        text = text.replace(f" {punct}", punct)
+    return text.strip()
+
+
+def pre_process(text: str) -> str:
+    """
+    Optional text preprocessing before normalization (part of TTS TN pipeline)
+
+    Args:
+        text: string that may include semiotic classes
+
+    Returns: text with spaces around punctuation marks
+    """
+    space_both = "[]"
+    for punct in space_both:
+        text = text.replace(punct, " " + punct + " ")
+
+    # remove extra space
+    text = re.sub(r" +", " ", text)
+    return text
+
+
+def load_file(file_path: str) -> List[str]:
+    """
+    Loads given text file with separate lines into list of string.
+
+    Args:
+        file_path: file path
+
+    Returns: flat list of string
+    """
+    res = []
+    with open(file_path, "r") as fp:
+        for line in fp:
+            res.append(line)
+    return res
+
+
+def write_file(file_path: str, data: List[str]):
+    """
+    Writes out list of string to file.
+
+    Args:
+        file_path: file path
+        data: list of string
+
+    """
+    with open(file_path, "w") as fp:
+        for line in data:
+            fp.write(line + "\n")
+
+
+def post_process_punct(input: str, normalized_text: str, add_unicode_punct: bool = False):
+    """
+    Post-processing of the normalized output to match input in terms of spaces around punctuation marks.
+    After NN normalization, Moses detokenization puts a space after
+    punctuation marks, and attaches an opening quote "'" to the word to the right.
+    E.g., input to the TN NN model is "12 test' example",
+    after normalization and detokenization -> "twelve test 'example" (the quote is considered to be an opening quote,
+    but it doesn't match the input and can cause issues during TTS voice generation.)
+    The current function will match the punctuation and spaces of the normalized text with the input sequence.
+    "12 test' example" -> "twelve test 'example" -> "twelve test' example" (the quote was shifted to match the input).
+
+    Args:
+        input: input text (original input to the NN, before normalization or tokenization)
+        normalized_text: output text (output of the TN NN model)
+        add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation (increases post processing time)
+    """
+    # in the post-processing WFST graph "``" are repalced with '"" quotes (otherwise single quotes "`" won't be handled correctly)
+    # this function fixes spaces around them based on input sequence, so here we're making the same double quote replacement
+    # to make sure these new double quotes work with this function
+    if "``" in input and "``" not in normalized_text:
+        input = input.replace("``", '"')
+    input = [x for x in input]
+    normalized_text = [x for x in normalized_text]
+    punct_marks = [x for x in string.punctuation if x in input]
+
+    if add_unicode_punct:
+        punct_unicode = [
+            chr(i)
+            for i in range(sys.maxunicode)
+            if category(chr(i)).startswith("P") and chr(i) not in punct_default and chr(i) in input
+        ]
+        punct_marks = punct_marks.extend(punct_unicode)
+
+    for punct in punct_marks:
+        try:
+            equal = True
+            if input.count(punct) != normalized_text.count(punct):
+                equal = False
+            idx_in, idx_out = 0, 0
+            while punct in input[idx_in:]:
+                idx_out = normalized_text.index(punct, idx_out)
+                idx_in = input.index(punct, idx_in)
+
+                def _is_valid(idx_out, idx_in, normalized_text, input):
+                    """Check if previous or next word match (for cases when punctuation marks are part of
+                    semiotic token, i.e. some punctuation can be missing in the normalized text)"""
+                    return (
+                        idx_out > 0
+                        and idx_in > 0
+                        and normalized_text[idx_out - 1] == input[idx_in - 1]
+                    ) or (
+                        idx_out < len(normalized_text) - 1
+                        and idx_in < len(input) - 1
+                        and normalized_text[idx_out + 1] == input[idx_in + 1]
+                    )
+
+                if not equal and not _is_valid(idx_out, idx_in, normalized_text, input):
+                    idx_in += 1
+                    continue
+                if idx_in > 0 and idx_out > 0:
+                    if normalized_text[idx_out - 1] == " " and input[idx_in - 1] != " ":
+                        normalized_text[idx_out - 1] = ""
+
+                    elif normalized_text[idx_out - 1] != " " and input[idx_in - 1] == " ":
+                        normalized_text[idx_out - 1] += " "
+
+                if idx_in < len(input) - 1 and idx_out < len(normalized_text) - 1:
+                    if normalized_text[idx_out + 1] == " " and input[idx_in + 1] != " ":
+                        normalized_text[idx_out + 1] = ""
+                    elif normalized_text[idx_out + 1] != " " and input[idx_in + 1] == " ":
+                        normalized_text[idx_out] = normalized_text[idx_out] + " "
+                idx_out += 1
+                idx_in += 1
+        except:
+            logging.debug(f"Skipping post-processing of {''.join(normalized_text)} for '{punct}'")
+
+    normalized_text = "".join(normalized_text)
+    return re.sub(r" +", " ", normalized_text)

fun_text_processing/text_normalization/de/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/electronic/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/electronic/domain.tsv

@@ -0,0 +1,9 @@
+.com	punkt com
+.uk	punkt uk
+.fr	punkt fr
+.net	dot net
+.br	punkt br
+.in	punkt in
+.ru	punkt ru
+.de	punkt de
+.it	punkt it

fun_text_processing/text_normalization/de/data/electronic/server_name.tsv

@@ -0,0 +1,12 @@
+gmail	g mail
+nvidia
+outlook
+hotmail
+yahoo
+live
+yandex
+orange
+wanadoo
+web
+comcast
+aol

fun_text_processing/text_normalization/de/data/electronic/symbols.tsv

@@ -0,0 +1,20 @@
+.	punkt
+:	doppelpunkt
+-	bindestrich
+_	unterstrich
+!	ausrufezeichen
+#	raute
+$	dollar zeichen
+%	prozent zeichen
+&	und
+'	apostroph
+*	asterisk
++	plus
+/	slash
+=	gleichheitszeichen
+?	fragezeichen
+^	zirkumflex
+{	linke klammer auf
+}	rechte klammer zu
+~	tilde
+,	komma

fun_text_processing/text_normalization/de/data/fractions.tsv

@@ -0,0 +1,30 @@
+halb	zwei
+drittel	drei
+viertel	vier
+fünftel	fünf
+sechstel	sechs
+siebtel	sieben
+achtel	acht
+neuntel	neun
+zehntel	zehn
+elftel	elf
+zwölftel	zwölf
+dreizehntel	dreizehn
+vierzehntel	vierzehn
+fünfzehntel	fünfzehn
+sechzehntel	sechzehn
+siebzehntel	siebzehn
+achtzehntel	achtzehn
+neunzehntel	neunzehn
+zwanzigstel	zwanzig
+dreißigstel	dreißig
+vierzigstel	vierzig
+fünfzigstel	fünfzig
+sechzigstel	sechzig
+siebzigstel	siebzig
+achtzigstel	achtzig
+neunzigstel	neunzig
+hundertstel	hundert
+tausendstel	tausend
+millionstel	million
+milliardstel	milliarde

fun_text_processing/text_normalization/de/data/measure/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/measure/measurements.tsv

@@ -0,0 +1,82 @@
+%	prozent
+f	fahrenheit
+c	celsius
+°C	grad celsius
+°F	grad fahrenheit
+K	kelvin
+km	kilometer
+m	meter
+cm	zentimeter
+mm	millimeter
+μm	mikrometer
+nm	nanometer
+dm	dezimeter
+pm	pikometer
+hm	hektometer
+ha	hektar
+mi	meile
+m²	quadrat meter	-0.0001
+km²	quadrat kilometer	-0.0001
+mm²	quadrat millimeter	-0.0001
+cm²	quadrat zentimeter	-0.0001
+m³	kubik meter	-0.0001
+km³	kubik kilometer	-0.0001
+mm³	kubik millimeter	-0.0001
+cm³	kubik zentimeter	-0.0001
+m2	quadrat meter
+km2	quadrat kilometer
+mm2	quadrat millimeter
+cm2	quadrat zentimeter
+m3	kubik meter
+km3	kubik kilometer
+mm3	kubik millimeter
+cm3	kubik zentimeter
+ft	fuß
+g	gramm
+µg	mikrogramm
+mg	milligramm
+kg	kilogramm
+lb	pfund
+oz	unze
+cwt	zentner
+gr	korn
+dr	drachne
+μg	mikrogramm
+pg	petagramm
+h	stunde
+s	sekunde
+min	minute
+ds	decisekunde
+ms	millisekunde
+μs	mikrosekunde
+hz	hertz
+kw	kilowatt
+kwh	kilowattstunde
+ghz	gigahertz
+khz	kilohertz
+mhz	megahertz
+v	volt
+mc	megacoulomb
+mA	milliampere
+A	ampere
+tw	terawatt
+mv	millivolt
+mw	megawatt
+gw	gigawatt
+ω	ohm
+db	dezibel
+gb	gigabyte
+kb	kilobit
+pb	petabit
+mb	megabyte
+kb	kilobyte
+tb	terabyte
+kv	kilovolt
+mv	megavolt
+kn	kilonewton
+ml	milliliter
+l	liter
+ma	megaampere
+bar	bar
+kcal	kilokalorie
+cal	kalorie

fun_text_processing/text_normalization/de/data/measure/suppletive.tsv

@@ -0,0 +1,10 @@
+meile	meilen
+unze	unzen
+drachne	drachnen
+stunde	stunden
+sekunde	sekunden
+minute	minuten
+minute	minuten
+bit	bits
+byte	bytes
+kalorie	kalorien

fun_text_processing/text_normalization/de/data/money/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/money/currency.tsv

@@ -0,0 +1,25 @@
+€	euro
+$	dollar
+$	us dollar
+£	pfund
+₩	won
+nzd	neuseeland dollar
+rs	rupie
+chf	schweizer franken
+dkk	dänische krone
+fim	finnische mark
+aed	dirham
+¥	yen
+czk	tschechische krone
+mro	ouguiya
+pkr	pakistanische rupie
+crc	colon
+hkd	hong kong dollar
+npr	nepalesische rupee
+awg	aruba florin
+nok	norwegische krone
+tzs	tansania schilling
+sek	schwedisch krone
+cyp	zypern pfund
+dm	d-mark
+dm	deutsche mark

fun_text_processing/text_normalization/de/data/money/currency_minor_plural.tsv

@@ -0,0 +1,3 @@
+$	cent
+€	cent
+£	pence

fun_text_processing/text_normalization/de/data/money/currency_minor_singular.tsv

@@ -0,0 +1,3 @@
+$	cent
+€	cent
+£	penny

fun_text_processing/text_normalization/de/data/months/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/months/abbr_to_name.tsv

@@ -0,0 +1,13 @@
+jan	januar
+feb	februar
+mär	märz
+apr	april
+mai	mai
+jun	juni
+jul	juli
+aug	august
+sep	september
+sept	september
+okt	oktober
+nov	november
+dez	dezember

fun_text_processing/text_normalization/de/data/months/numbers.tsv

@@ -0,0 +1,24 @@
+1	januar
+2	februar
+3	märz
+4	april
+5	mai
+6	juni
+7	juli
+8	august
+9	september
+10	oktober
+11	november
+12	dezember
+01	januar
+02	februar
+03	märz
+04	april
+05	mai
+06	juni
+07	juli
+08	august
+09	september
+10	oktober
+11	november
+12	dezember

fun_text_processing/text_normalization/de/data/numbers/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/numbers/digit.tsv

@@ -0,0 +1,8 @@
+zwei	2
+drei	3
+vier	4
+fünf	5
+sechs	6
+sieben	7
+acht	8
+neun	9

fun_text_processing/text_normalization/de/data/numbers/ones.tsv

@@ -0,0 +1,3 @@
+eine	1
+ein	1
+eins	1	-0.0001

fun_text_processing/text_normalization/de/data/numbers/quantities.tsv

@@ -0,0 +1,12 @@
+million
+millionen
+milliarde
+milliarden
+billion
+billionen
+billiarde
+billiarden
+trillion
+trillionen
+trilliarde
+trilliarde

fun_text_processing/text_normalization/de/data/numbers/teen.tsv

@@ -0,0 +1,10 @@
+zehn	10
+elf	11
+zwölf	12
+dreizehn	13
+vierzehn	14
+fünfzehn	15
+sechzehn	16
+siebzehn	17
+achtzehn	18
+neunzehn	19

fun_text_processing/text_normalization/de/data/numbers/ties.tsv

@@ -0,0 +1,8 @@
+zwanzig	2
+dreißig	3
+vierzig	4
+fünfzig	5
+sechzig	6
+siebzig	7
+achtzig	8
+neunzig	9

fun_text_processing/text_normalization/de/data/numbers/zero.tsv

@@ -0,0 +1 @@
+null	0

fun_text_processing/text_normalization/de/data/ordinals/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/ordinals/digit.tsv

@@ -0,0 +1,9 @@
+ers	eins
+zwei	zwei
+drit	drei
+vier	vier
+fünf	fünf
+sechs	sechs
+sieb	sieben
+ach	acht
+neun	neun

fun_text_processing/text_normalization/de/data/ordinals/thousands.tsv

@@ -0,0 +1,4 @@
+hunderts	hundert
+tausends	tausend
+millions	million
+milliards	milliarde

fun_text_processing/text_normalization/de/data/ordinals/ties.tsv

@@ -0,0 +1,8 @@
+zwanzigs	zwanzig
+dreißigs	dreißig
+vierzigs	vierzig
+fünfzigs	fünfzig
+sechzigs	sechzig
+siebzigs	siebzig
+achtzigs	achtzig
+neunzigs	neunzig

fun_text_processing/text_normalization/de/data/time/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/data/time/hour_to.tsv

@@ -0,0 +1,12 @@
+1	12
+2	1
+3	2
+4	3
+5	4
+6	5
+7	6
+8	7
+9	8
+10	9
+11	10
+12	11

fun_text_processing/text_normalization/de/data/time/hour_to_night.tsv

@@ -0,0 +1,13 @@
+12	0
+1	13
+2	14
+3	15
+4	16
+5	17
+6	18
+7	19
+8	20
+9	21
+10	22
+11	23
+12	24

fun_text_processing/text_normalization/de/data/time/minute_to.tsv

@@ -0,0 +1,59 @@
+1	59
+2	58
+3	57
+4	56
+5	55
+6	54
+7	53
+8	52
+9	51
+10	50
+11	49
+12	48
+13	47
+14	46
+15	45
+16	44
+17	43
+18	42
+19	41
+20	40
+21	39
+22	38
+23	37
+24	36
+25	35
+26	34
+27	33
+28	32
+29	31
+30	30
+31	29
+32	28
+33	27
+34	26
+35	25
+36	24
+37	23
+38	22
+39	21
+40	20
+41	19
+42	18
+43	17
+44	16
+45	15
+46	14
+47	13
+48	12
+49	11
+50	10
+51	9
+52	8
+53	7
+54	6
+55	5
+56	4
+57	3
+58	2
+59	1

fun_text_processing/text_normalization/de/data/time/time_zone.tsv

@@ -0,0 +1,7 @@
+cst	c s t
+cet	c e t
+pst	p s t
+est	e s t
+pt	p t
+et	e t
+gmt	g m t

fun_text_processing/text_normalization/de/data/whitelist.tsv

@@ -0,0 +1,7 @@
+z.B.	zum beispiel
+d.h.	dass heißt
+Dr.	doktor
+Mr.	mister
+Mrs.	misses
+Ms.	miss
+Nr.	nummer

fun_text_processing/text_normalization/de/taggers/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/taggers/cardinal.py

@@ -0,0 +1,215 @@
+from collections import defaultdict
+
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_DIGIT,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+AND = "und"
+
+
+def get_ties_digit(digit_path: str, tie_path: str) -> "pynini.FstLike":
+    """
+    getting all inverse normalizations for numbers between 21 - 100
+
+    Args:
+        digit_path: file to digit tsv
+        tie_path: file to tie tsv, e.g. 20, 30, etc.
+    Returns:
+        res: fst that converts numbers to their verbalization
+    """
+
+    digits = defaultdict(list)
+    ties = defaultdict(list)
+    for k, v in load_labels(digit_path):
+        digits[v].append(k)
+    digits["1"] = ["ein"]
+
+    for k, v in load_labels(tie_path):
+        ties[v].append(k)
+
+    d = []
+    for i in range(21, 100):
+        s = str(i)
+        if s[1] == "0":
+            continue
+
+        for di in digits[s[1]]:
+            for ti in ties[s[0]]:
+                word = di + f" {AND} " + ti
+                d.append((word, s))
+
+    res = pynini.string_map(d)
+    return res
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for classifying cardinals, e.g.
+        "101" ->  cardinal { integer: "ein hundert und zehn" }
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = False):
+        super().__init__(name="cardinal", kind="classify", deterministic=deterministic)
+
+        graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert()
+        graph_digit_no_one = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert()
+        graph_one = pynini.string_file(get_abs_path("data/numbers/ones.tsv")).invert()
+        graph_digit = graph_digit_no_one | graph_one
+        self.digit = (graph_digit | graph_zero).optimize()
+        graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).invert()
+
+        graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).invert()
+        # separator = "."
+
+        def tens_no_zero():
+            return (
+                pynutil.delete("0") + graph_digit
+                | get_ties_digit(
+                    get_abs_path("data/numbers/digit.tsv"), get_abs_path("data/numbers/ties.tsv")
+                ).invert()
+                | graph_teen
+                | (graph_ties + pynutil.delete("0"))
+            )
+
+        def hundred_non_zero():
+            return (graph_digit_no_one + insert_space | pynini.cross("1", "ein ")) + pynutil.insert(
+                "hundert"
+            ) + (
+                pynini.closure(insert_space + pynutil.insert(AND, weight=0.0001), 0, 1)
+                + insert_space
+                + tens_no_zero()
+                | pynutil.delete("00")
+            ) | pynutil.delete(
+                "0"
+            ) + tens_no_zero()
+
+        def thousand():
+            return (
+                hundred_non_zero() + insert_space + pynutil.insert("tausend")
+                | pynutil.delete("000")
+            ) + (insert_space + hundred_non_zero() | pynutil.delete("000"))
+
+        optional_plural_quantity_en = pynini.closure(pynutil.insert("en", weight=-0.0001), 0, 1)
+        optional_plural_quantity_n = pynini.closure(pynutil.insert("n", weight=-0.0001), 0, 1)
+        graph_million = pynini.union(
+            hundred_non_zero()
+            + insert_space
+            + pynutil.insert("million")
+            + optional_plural_quantity_en,
+            pynutil.delete("000"),
+        )
+
+        graph_billion = pynini.union(
+            hundred_non_zero()
+            + insert_space
+            + pynutil.insert("milliarde")
+            + optional_plural_quantity_n,
+            pynutil.delete("000"),
+        )
+
+        graph_trillion = pynini.union(
+            hundred_non_zero()
+            + insert_space
+            + pynutil.insert("billion")
+            + optional_plural_quantity_en,
+            pynutil.delete("000"),
+        )
+
+        graph_quadrillion = pynini.union(
+            hundred_non_zero()
+            + insert_space
+            + pynutil.insert("billiarde")
+            + optional_plural_quantity_n,
+            pynutil.delete("000"),
+        )
+
+        graph_quintillion = pynini.union(
+            hundred_non_zero()
+            + insert_space
+            + pynutil.insert("trillion")
+            + optional_plural_quantity_en,
+            pynutil.delete("000"),
+        )
+
+        graph_sextillion = pynini.union(
+            hundred_non_zero()
+            + insert_space
+            + pynutil.insert("trilliarde")
+            + optional_plural_quantity_n,
+            pynutil.delete("000"),
+        )
+        graph = pynini.union(
+            graph_sextillion
+            + insert_space
+            + graph_quintillion
+            + insert_space
+            + graph_quadrillion
+            + insert_space
+            + graph_trillion
+            + insert_space
+            + graph_billion
+            + insert_space
+            + graph_million
+            + insert_space
+            + thousand()
+        )
+
+        fix_syntax = [
+            ("eins tausend", "ein tausend"),
+            ("eins millionen", "eine million"),
+            ("eins milliarden", "eine milliarde"),
+            ("eins billionen", "eine billion"),
+            ("eins billiarden", "eine billiarde"),
+        ]
+        fix_syntax = pynini.union(*[pynini.cross(*x) for x in fix_syntax])
+        self.graph = (
+            ((DAMO_DIGIT - "0" + pynini.closure(DAMO_DIGIT, 0)) - "0" - "1")
+            @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", DAMO_SIGMA)
+            @ DAMO_DIGIT**24
+            @ graph
+            @ pynini.cdrewrite(delete_space, "[BOS]", "", DAMO_SIGMA)
+            @ pynini.cdrewrite(delete_space, "", "[EOS]", DAMO_SIGMA)
+            @ pynini.cdrewrite(pynini.cross("  ", " "), "", "", DAMO_SIGMA)
+            @ pynini.cdrewrite(fix_syntax, "[BOS]", "", DAMO_SIGMA)
+        )
+        self.graph |= graph_zero | pynini.cross("1", "eins")
+
+        # self.graph = pynini.cdrewrite(pynutil.delete(separator), "", "", DAMO_SIGMA) @ self.graph
+        self.graph = self.graph.optimize()
+
+        self.graph_hundred_component_at_least_one_none_zero_digit = (
+            ((DAMO_DIGIT - "0" + pynini.closure(DAMO_DIGIT, 0)) - "0" - "1")
+            @ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", DAMO_SIGMA)
+            @ DAMO_DIGIT**3
+            @ hundred_non_zero()
+        ) | pynini.cross("1", "eins")
+
+        self.graph_hundred_component_at_least_one_none_zero_digit = (
+            self.graph_hundred_component_at_least_one_none_zero_digit.optimize()
+        )
+
+        self.two_digit_non_zero = (
+            pynini.closure(DAMO_DIGIT, 1, 2)
+            @ self.graph_hundred_component_at_least_one_none_zero_digit
+        )
+
+        optional_minus_graph = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
+        )
+
+        final_graph = (
+            optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
+        )
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/date.py

@@ -0,0 +1,130 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_CHAR,
+    DAMO_DIGIT,
+    TO_LOWER,
+    GraphFst,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))).optimize()
+graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize()
+ties_graph = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv"))).optimize()
+delete_leading_zero = (pynutil.delete("0") | (DAMO_DIGIT - "0")) + DAMO_DIGIT
+
+
+def get_year_graph(cardinal: GraphFst) -> "pynini.FstLike":
+    """
+    Returns year verbalizations as fst
+
+     < 2000 neunzehn (hundert) (vier und zwanzig), >= 2000 regular cardinal
+    **00 ** hundert
+
+    Args:
+        delete_leading_zero: removed leading zero
+        cardinal: cardinal GraphFst
+    """
+
+    year_gt_2000 = (pynini.union("21", "20") + DAMO_DIGIT**2) @ cardinal.graph
+
+    graph_two_digit = delete_leading_zero @ cardinal.two_digit_non_zero
+    hundred = pynutil.insert("hundert")
+    graph_double_double = (
+        (pynini.accep("1") + DAMO_DIGIT) @ graph_two_digit
+        + insert_space
+        + pynini.closure(hundred + insert_space, 0, 1)
+        + graph_two_digit
+    )
+    # for 20**
+    graph_double_double |= pynini.accep("20") @ graph_two_digit + insert_space + graph_two_digit
+    graph = (
+        graph_double_double
+        | (pynini.accep("1") + DAMO_DIGIT) @ graph_two_digit
+        + insert_space
+        + pynutil.delete("00")
+        + hundred
+        | year_gt_2000
+    )
+    return graph
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for classifying date, e.g.
+        "01.04.2010" -> date { day: "erster" month: "april" year: "zwei tausend zehn" preserve_order: true }
+        "1994" -> date { year: "neunzehn vier und neuzig" }
+        "1900" -> date { year: "neunzehn hundert" }
+
+    Args:
+        cardinal: cardinal GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, cardinal: GraphFst, deterministic: bool):
+        super().__init__(name="date", kind="classify", deterministic=deterministic)
+
+        month_abbr_graph = load_labels(get_abs_path("data/months/abbr_to_name.tsv"))
+        number_to_month = pynini.string_file(get_abs_path("data/months/numbers.tsv")).optimize()
+        month_graph = pynini.union(*[x[1] for x in month_abbr_graph]).optimize()
+        month_abbr_graph = pynini.string_map(month_abbr_graph)
+        month_abbr_graph = (
+            pynutil.add_weight(month_abbr_graph, weight=0.0001)
+            | ((TO_LOWER + pynini.closure(DAMO_CHAR)) @ month_abbr_graph)
+        ) + pynini.closure(pynutil.delete(".", weight=-0.0001), 0, 1)
+
+        self.month_abbr = month_abbr_graph
+        month_graph |= (TO_LOWER + pynini.closure(DAMO_CHAR)) @ month_graph
+        # jan.-> januar, Jan-> januar, januar-> januar
+        month_graph |= month_abbr_graph
+
+        numbers = cardinal.graph_hundred_component_at_least_one_none_zero_digit
+        optional_leading_zero = delete_leading_zero | DAMO_DIGIT
+        # 01, 31, 1
+        digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ numbers
+        day = (pynutil.insert('day: "') + digit_day + pynutil.insert('"')).optimize()
+
+        digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)])
+        number_to_month = digit_month @ number_to_month
+        digit_month @= numbers
+
+        month_name = (pynutil.insert('month: "') + month_graph + pynutil.insert('"')).optimize()
+        month_number = (
+            pynutil.insert('month: "')
+            + (pynutil.add_weight(digit_month, weight=0.0001) | number_to_month)
+            + pynutil.insert('"')
+        ).optimize()
+
+        # prefer cardinal over year
+        year = pynutil.add_weight(get_year_graph(cardinal=cardinal), weight=0.001)
+        self.year = year
+
+        year_only = pynutil.insert('year: "') + year + pynutil.insert('"')
+
+        graph_dmy = (
+            day
+            + pynutil.delete(".")
+            + pynini.closure(pynutil.delete(" "), 0, 1)
+            + insert_space
+            + month_name
+            + pynini.closure(pynini.accep(" ") + year_only, 0, 1)
+        )
+
+        separators = ["."]
+        for sep in separators:
+            year_optional = pynini.closure(pynini.cross(sep, " ") + year_only, 0, 1)
+            new_graph = day + pynini.cross(sep, " ") + month_number + year_optional
+            graph_dmy |= new_graph
+
+        dash = "-"
+        day_optional = pynini.closure(pynini.cross(dash, " ") + day, 0, 1)
+        graph_ymd = year_only + pynini.cross(dash, " ") + month_number + day_optional
+
+        final_graph = graph_dmy + pynutil.insert(" preserve_order: true")
+        final_graph |= year_only
+        final_graph |= graph_ymd
+
+        self.final_graph = final_graph.optimize()
+        self.fst = self.add_tokens(self.final_graph).optimize()

fun_text_processing/text_normalization/de/taggers/decimal.py

@@ -0,0 +1,82 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space
+from pynini.lib import pynutil
+
+quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv"))
+
+
+def get_quantity(
+    decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike"
+) -> "pynini.FstLike":
+    """
+    Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
+    e.g. 1 million -> integer_part: "eine" quantity: "million"
+    e.g. 1.4 million -> integer_part: "eins" fractional_part: "vier" quantity: "million"
+
+    Args:
+        decimal: decimal FST
+        cardinal_up_to_hundred: cardinal FST
+    """
+    numbers = cardinal_up_to_hundred
+
+    res = (
+        pynutil.insert('integer_part: "')
+        + numbers
+        + pynutil.insert('"')
+        + pynini.accep(" ")
+        + pynutil.insert('quantity: "')
+        + quantities
+        + pynutil.insert('"')
+    )
+    res |= (
+        decimal
+        + pynini.accep(" ")
+        + pynutil.insert('quantity: "')
+        + quantities
+        + pynutil.insert('"')
+    )
+    return res
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for classifying decimal, e.g.
+        -11,4006 billion -> decimal { negative: "true" integer_part: "elf"  fractional_part: "vier null null sechs" quantity: "billion" preserve_order: true }
+        1 billion -> decimal { integer_part: "eins" quantity: "billion" preserve_order: true }
+    Args:
+        cardinal: CardinalFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, cardinal: GraphFst, deterministic: bool = True):
+        super().__init__(name="decimal", kind="classify", deterministic=deterministic)
+
+        graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert()
+        graph_digit |= pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert()
+        graph_digit |= pynini.cross("1", "eins")
+        self.graph = graph_digit + pynini.closure(insert_space + graph_digit).optimize()
+
+        point = pynutil.delete(",")
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
+        )
+
+        self.graph_fractional = (
+            pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"')
+        )
+        self.graph_integer = (
+            pynutil.insert('integer_part: "') + cardinal.graph + pynutil.insert('"')
+        )
+        final_graph_wo_sign = self.graph_integer + point + insert_space + self.graph_fractional
+
+        self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(
+            final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit
+        )
+        final_graph = optional_graph_negative + self.final_graph_wo_negative
+        final_graph += pynutil.insert(" preserve_order: true")
+
+        final_graph = self.add_tokens(final_graph)
+
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/electronic.py

@@ -0,0 +1,64 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    GraphFst,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for classifying electronic: email addresses
+        e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true }
+        e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true }
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="electronic", kind="classify", deterministic=deterministic)
+
+        dot = pynini.accep(".")
+        accepted_common_domains = [
+            x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))
+        ]
+        accepted_common_domains = pynini.union(*accepted_common_domains)
+        accepted_symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))]
+        accepted_symbols = pynini.union(*accepted_symbols) - dot
+        accepted_characters = pynini.closure(DAMO_ALPHA | DAMO_DIGIT | accepted_symbols)
+
+        # email
+        username = (
+            pynutil.insert('username: "')
+            + accepted_characters
+            + pynutil.insert('"')
+            + pynini.cross("@", " ")
+        )
+        domain_graph = accepted_characters + dot + accepted_characters
+        domain_graph = pynutil.insert('domain: "') + domain_graph + pynutil.insert('"')
+        domain_common_graph = (
+            pynutil.insert('domain: "')
+            + accepted_characters
+            + accepted_common_domains
+            + pynini.closure(
+                (accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1
+            )
+            + pynutil.insert('"')
+        )
+        graph = (username + domain_graph) | domain_common_graph
+
+        # url
+        protocol_start = pynini.accep("https://") | pynini.accep("http://")
+        protocol_end = pynini.accep("www.")
+        protocol = protocol_start | protocol_end | (protocol_start + protocol_end)
+        protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"')
+        graph |= protocol + insert_space + (domain_graph | domain_common_graph)
+        self.graph = graph
+
+        final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true"))
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/fraction.py

@@ -0,0 +1,42 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for classifying fraction
+    "23 4/6" ->
+    fraction { integer: "drei und zwanzig" numerator: "vier" denominator: "sechs" preserve_order: true }
+
+    Args:
+        cardinal: cardinal GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, cardinal, deterministic: bool = True):
+        super().__init__(name="fraction", kind="classify", deterministic=deterministic)
+        cardinal_graph = cardinal.graph
+
+        self.optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
+        )
+        self.integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
+        self.numerator = (
+            pynutil.insert('numerator: "')
+            + cardinal_graph
+            + pynini.cross(pynini.union("/", " / "), '" ')
+        )
+        self.denominator = pynutil.insert('denominator: "') + cardinal_graph + pynutil.insert('"')
+
+        self.graph = (
+            self.optional_graph_negative
+            + pynini.closure(self.integer + pynini.accep(" "), 0, 1)
+            + self.numerator
+            + self.denominator
+        )
+
+        graph = self.graph + pynutil.insert(" preserve_order: true")
+        final_graph = self.add_tokens(graph)
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/measure.py

@@ -0,0 +1,185 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    DAMO_NON_BREAKING_SPACE,
+    DAMO_SIGMA,
+    GraphFst,
+    convert_space,
+    insert_space,
+)
+from pynini.examples import plurals
+from pynini.lib import pynutil
+
+unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv"))
+suppletive = pynini.string_file(get_abs_path("data/measure/suppletive.tsv"))
+
+
+def singular_to_plural():
+    # plural endung n/en maskuline Nomen mit den Endungen e, ent, and, ant, ist, or
+    _n = DAMO_SIGMA + pynini.union("e") + pynutil.insert("n")
+    _en = (
+        DAMO_SIGMA
+        + pynini.union(
+            "ent", "and", "ant", "ist", "or", "ion", "ik", "heit", "keit", "schaft", "tät", "ung"
+        )
+        + pynutil.insert("en")
+    )
+    _nen = DAMO_SIGMA + pynini.union("in") + (pynutil.insert("e") | pynutil.insert("nen"))
+    _fremd = DAMO_SIGMA + pynini.union("ma", "um", "us") + pynutil.insert("en")
+    # maskuline Nomen mit den Endungen eur, ich, ier, ig, ling, ör
+    _e = DAMO_SIGMA + pynini.union("eur", "ich", "ier", "ig", "ling", "ör") + pynutil.insert("e")
+    _s = DAMO_SIGMA + pynini.union("a", "i", "o", "u", "y") + pynutil.insert("s")
+
+    graph_plural = plurals._priority_union(
+        suppletive, pynini.union(_n, _en, _nen, _fremd, _e, _s), DAMO_SIGMA
+    ).optimize()
+
+    return graph_plural
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for classifying measure,  e.g.
+        "2,4 oz" -> measure { cardinal { integer_part: "zwei" fractional_part: "vier" units: "unzen" preserve_order: true } }
+        "1 oz" -> measure { cardinal { integer: "zwei" units: "unze" preserve_order: true } }
+        "1 million oz" -> measure { cardinal { integer: "eins" quantity: "million" units: "unze" preserve_order: true } }
+        This class also converts words containing numbers and letters
+        e.g. "a-8" —> "a acht"
+        e.g. "1,2-a" —> "ein komma zwei a"
+
+    Args:
+        cardinal: CardinalFst
+        decimal: DecimalFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(
+        self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True
+    ):
+        super().__init__(name="measure", kind="classify", deterministic=deterministic)
+        cardinal_graph = cardinal.graph
+
+        graph_unit_singular = convert_space(unit_singular)
+        graph_unit_plural = graph_unit_singular @ pynini.cdrewrite(
+            convert_space(suppletive), "", "[EOS]", DAMO_SIGMA
+        )
+        optional_graph_negative = pynini.closure("-", 0, 1)
+
+        graph_unit_denominator = (
+            pynini.cross("/", "pro") + pynutil.insert(DAMO_NON_BREAKING_SPACE) + graph_unit_singular
+        )
+
+        optional_unit_denominator = pynini.closure(
+            pynutil.insert(DAMO_NON_BREAKING_SPACE) + graph_unit_denominator,
+            0,
+            1,
+        )
+
+        unit_plural = (
+            pynutil.insert('units: "')
+            + (graph_unit_plural + (optional_unit_denominator) | graph_unit_denominator)
+            + pynutil.insert('"')
+        )
+
+        unit_singular_graph = (
+            pynutil.insert('units: "')
+            + ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator)
+            + pynutil.insert('"')
+        )
+
+        subgraph_decimal = (
+            decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural
+        )
+
+        subgraph_cardinal = (
+            (optional_graph_negative + (pynini.closure(DAMO_DIGIT) - "1")) @ cardinal.fst
+            + insert_space
+            + pynini.closure(pynutil.delete(" "), 0, 1)
+            + unit_plural
+        )
+
+        subgraph_cardinal |= (
+            (optional_graph_negative + pynini.accep("1"))
+            @ cardinal.fst
+            @ pynini.cdrewrite(pynini.cross("eins", "ein"), "", "", DAMO_SIGMA)
+            + insert_space
+            + pynini.closure(pynutil.delete(" "), 0, 1)
+            + unit_singular_graph
+        )
+
+        subgraph_fraction = (
+            fraction.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural
+        )
+
+        cardinal_dash_alpha = (
+            pynutil.insert('cardinal { integer: "')
+            + cardinal_graph
+            + pynutil.delete("-")
+            + pynutil.insert('" } units: "')
+            + pynini.closure(DAMO_ALPHA, 1)
+            + pynutil.insert('"')
+        )
+
+        alpha_dash_cardinal = (
+            pynutil.insert('units: "')
+            + pynini.closure(DAMO_ALPHA, 1)
+            + pynutil.delete("-")
+            + pynutil.insert('"')
+            + pynutil.insert(' cardinal { integer: "')
+            + cardinal_graph
+            + pynutil.insert('" }')
+        )
+
+        decimal_dash_alpha = (
+            pynutil.insert("decimal { ")
+            + decimal.final_graph_wo_negative
+            + pynutil.delete("-")
+            + pynutil.insert(' } units: "')
+            + pynini.closure(DAMO_ALPHA, 1)
+            + pynutil.insert('"')
+        )
+
+        decimal_times = (
+            pynutil.insert("decimal { ")
+            + decimal.final_graph_wo_negative
+            + pynutil.insert(' } units: "')
+            + pynini.union("x", "X")
+            + pynutil.insert('"')
+        )
+
+        cardinal_times = (
+            pynutil.insert('cardinal { integer: "')
+            + cardinal_graph
+            + pynutil.insert('" } units: "')
+            + pynini.union("x", "X")
+            + pynutil.insert('"')
+        )
+
+        alpha_dash_decimal = (
+            pynutil.insert('units: "')
+            + pynini.closure(DAMO_ALPHA, 1)
+            + pynutil.delete("-")
+            + pynutil.insert('"')
+            + pynutil.insert(" decimal { ")
+            + decimal.final_graph_wo_negative
+            + pynutil.insert(" }")
+        )
+
+        final_graph = (
+            subgraph_decimal
+            | subgraph_cardinal
+            | cardinal_dash_alpha
+            | alpha_dash_cardinal
+            | decimal_dash_alpha
+            | decimal_times
+            | alpha_dash_decimal
+            | subgraph_fraction
+            | cardinal_times
+        )
+        final_graph += pynutil.insert(" preserve_order: true")
+        final_graph = self.add_tokens(final_graph)
+
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/money.py

@@ -0,0 +1,160 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    DAMO_SIGMA,
+    GraphFst,
+    convert_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv"))
+min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv"))
+maj_singular = pynini.string_file((get_abs_path("data/money/currency.tsv")))
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for classifying money, e.g.
+        "€1" -> money { currency_maj: "euro" integer_part: "ein"}
+        "€1,000" -> money { currency_maj: "euro" integer_part: "ein" }
+        "€1,001" -> money { currency_maj: "euro" integer_part: "eins" fractional_part: "null null eins"}
+        "£1,4" -> money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" preserve_order: true}
+               -> money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" currency_min: "pence" preserve_order: true}
+        "£0,01" -> money { fractional_part: "ein" currency_min: "penny" preserve_order: true}
+        "£0,01 million" -> money { currency_maj: "pfund" integer_part: "null" fractional_part: "null eins" quantity: "million"}
+
+    Args:
+        cardinal: CardinalFst
+        decimal: DecimalFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True):
+        super().__init__(name="money", kind="classify", deterministic=deterministic)
+        cardinal_graph = cardinal.graph
+        graph_decimal_final = decimal.fst
+
+        maj_singular_labels = load_labels(get_abs_path("data/money/currency.tsv"))
+        maj_singular_graph = convert_space(maj_singular)
+        maj_plural_graph = maj_singular_graph
+
+        graph_maj_singular = (
+            pynutil.insert('currency_maj: "') + maj_singular_graph + pynutil.insert('"')
+        )
+        graph_maj_plural = (
+            pynutil.insert('currency_maj: "') + maj_plural_graph + pynutil.insert('"')
+        )
+
+        optional_delete_fractional_zeros = pynini.closure(
+            pynutil.delete(",") + pynini.closure(pynutil.delete("0"), 1), 0, 1
+        )
+        graph_integer_one = (
+            pynutil.insert('integer_part: "') + pynini.cross("1", "ein") + pynutil.insert('"')
+        )
+
+        # only for decimals where third decimal after comma is non-zero or with quantity
+        decimal_delete_last_zeros = (
+            pynini.closure(DAMO_DIGIT, 1)
+            + pynini.accep(",")
+            + pynini.closure(DAMO_DIGIT, 2)
+            + (DAMO_DIGIT - "0")
+            + pynini.closure(pynutil.delete("0"))
+        )
+        decimal_with_quantity = DAMO_SIGMA + DAMO_ALPHA
+        graph_decimal = (
+            graph_maj_plural
+            + insert_space
+            + (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final
+        )
+
+        graph_integer = (
+            pynutil.insert('integer_part: "')
+            + ((DAMO_SIGMA - "1") @ cardinal_graph)
+            + pynutil.insert('"')
+        )
+
+        graph_integer_only = graph_maj_singular + insert_space + graph_integer_one
+        graph_integer_only |= graph_maj_plural + insert_space + graph_integer
+
+        graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_decimal
+
+        # remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits
+        # e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10
+        # not accepted: 002, 00, 0,
+        two_digits_fractional_part = (
+            pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(pynutil.delete("0"))
+        ) @ (
+            (pynutil.delete("0") + (DAMO_DIGIT - "0"))
+            | ((DAMO_DIGIT - "0") + pynutil.insert("0"))
+            | ((DAMO_DIGIT - "0") + DAMO_DIGIT)
+        )
+
+        graph_min_singular = pynutil.insert(' currency_min: "') + min_singular + pynutil.insert('"')
+        graph_min_plural = pynutil.insert(' currency_min: "') + min_plural + pynutil.insert('"')
+
+        # format ** euro ** cent
+        decimal_graph_with_minor = None
+        for curr_symbol, _ in maj_singular_labels:
+            preserve_order = pynutil.insert(" preserve_order: true")
+            integer_plus_maj = (
+                graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural
+            )
+            integer_plus_maj |= (
+                graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular
+            )
+            # non zero integer part
+            integer_plus_maj = (pynini.closure(DAMO_DIGIT) - "0") @ integer_plus_maj
+
+            graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "ein")
+            graph_fractional_one = (
+                pynutil.insert('fractional_part: "') + graph_fractional_one + pynutil.insert('"')
+            )
+            graph_fractional = (
+                two_digits_fractional_part
+                @ (pynini.closure(DAMO_DIGIT, 1, 2) - "1")
+                @ cardinal.two_digit_non_zero
+            )
+            graph_fractional = (
+                pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"')
+            )
+
+            fractional_plus_min = (
+                graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural
+            )
+            fractional_plus_min |= (
+                graph_fractional_one
+                + insert_space
+                + pynutil.insert(curr_symbol) @ graph_min_singular
+            )
+
+            decimal_graph_with_minor_curr = (
+                integer_plus_maj + pynini.cross(",", " ") + fractional_plus_min
+            )
+            decimal_graph_with_minor_curr |= pynutil.add_weight(
+                integer_plus_maj
+                + pynini.cross(",", " ")
+                + pynutil.insert('fractional_part: "')
+                + two_digits_fractional_part @ cardinal.two_digit_non_zero
+                + pynutil.insert('"'),
+                weight=0.0001,
+            )
+
+            decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min
+            decimal_graph_with_minor_curr = (
+                pynutil.delete(curr_symbol) + decimal_graph_with_minor_curr + preserve_order
+            )
+
+            decimal_graph_with_minor = (
+                decimal_graph_with_minor_curr
+                if decimal_graph_with_minor is None
+                else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr)
+            )
+
+        final_graph = graph | decimal_graph_with_minor
+
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/ordinal.py

@@ -0,0 +1,37 @@
+# Adapted from https://github.com/google/TextNormalizationCoveringGrammars
+# Russian minimally supervised number grammar.
+
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for classifying cardinals, e.g.
+        "2." -> ordinal { integer: "zwei" } }
+        "2tes" -> ordinal { integer: "zwei" } }
+
+    Args:
+        cardinal: cardinal GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, cardinal: GraphFst, deterministic=False):
+        super().__init__(name="ordinal", kind="classify", deterministic=deterministic)
+
+        cardinal_graph = cardinal.graph
+        endings = ["ter", "tes", "tem", "te", "ten"]
+        self.graph = (
+            (
+                pynini.closure(DAMO_DIGIT | pynini.accep("."))
+                + pynutil.delete(
+                    pynutil.add_weight(pynini.union(*endings), weight=0.0001) | pynini.accep(".")
+                )
+            )
+            @ cardinal_graph
+        ).optimize()
+        final_graph = pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/telephone.py

@@ -0,0 +1,66 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst, insert_space
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for classifying telephone, which includes country code, number part and extension
+
+    E.g
+    "+49 1234-1233" -> telephone { country_code: "plus neun und vierzig" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true }
+    "(012) 1234-1233" -> telephone { country_code: "null eins zwei" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true }
+    (0**)
+
+    Args:
+        cardinal: cardinal GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, cardinal: GraphFst, deterministic: bool = True):
+        super().__init__(name="telephone", kind="classify", deterministic=deterministic)
+
+        graph_zero = pynini.invert(
+            pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        ).optimize()
+        graph_digit_no_zero = pynini.invert(
+            pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+        ).optimize() | pynini.cross("1", "eins")
+        graph_digit = graph_digit_no_zero | graph_zero
+
+        numbers_with_single_digits = pynini.closure(graph_digit + insert_space) + graph_digit
+
+        two_digit_and_zero = (DAMO_DIGIT**2 @ cardinal.two_digit_non_zero) | graph_zero
+        # def add_space_after_two_digit():
+        #     return pynini.closure(two_digit_and_zero + insert_space) + (
+        #         two_digit_and_zero
+        #     )
+
+        country_code = pynini.closure(pynini.cross("+", "plus "), 0, 1) + two_digit_and_zero
+        country_code |= (
+            pynutil.delete("(")
+            + graph_zero
+            + insert_space
+            + numbers_with_single_digits
+            + pynutil.delete(")")
+        )
+        country_code |= graph_zero + insert_space + numbers_with_single_digits
+
+        country_code = pynutil.insert('country_code: "') + country_code + pynutil.insert('"')
+
+        del_separator = pynini.cross(pynini.union("-", " "), " ")
+        # numbers_with_two_digits = pynini.closure(graph_digit + insert_space) + add_space_after_two_digit() + pynini.closure(insert_space + graph_digit)
+        # numbers = numbers_with_two_digits + pynini.closure(del_separator + numbers_with_two_digits, 0, 1)
+        numbers = numbers_with_single_digits + pynini.closure(
+            del_separator + numbers_with_single_digits, 0, 1
+        )
+        number_length = pynini.closure((DAMO_DIGIT | pynini.union("-", " ", ")", "(")), 7)
+        number_part = pynini.compose(number_length, numbers)
+        number = pynutil.insert('number_part: "') + number_part + pynutil.insert('"')
+
+        graph = country_code + pynini.accep(" ") + number
+        self.graph = graph
+        final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true"))
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/time.py

@@ -0,0 +1,94 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_DIGIT,
+    GraphFst,
+    convert_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for classifying time, e.g.
+        "02:15 Uhr est" -> time { hours: "2" minutes: "15" zone: "e s t"}
+        "2 Uhr" -> time { hours: "2" }
+        "09:00 Uhr" -> time { hours: "2" }
+        "02:15:10 Uhr" -> time { hours: "2" minutes: "15" seconds: "10"}
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="time", kind="classify", deterministic=deterministic)
+
+        final_suffix = pynutil.delete(" ") + pynutil.delete("Uhr") | pynutil.delete("uhr")
+        time_zone_graph = pynini.string_file(get_abs_path("data/time/time_zone.tsv"))
+
+        labels_hour = [str(x) for x in range(0, 25)]
+        labels_minute_single = [str(x) for x in range(1, 10)]
+        labels_minute_double = [str(x) for x in range(10, 60)]
+
+        delete_leading_zero_to_double_digit = (
+            pynutil.delete("0") | (DAMO_DIGIT - "0")
+        ) + DAMO_DIGIT
+
+        graph_hour = pynini.union(*labels_hour)
+
+        graph_minute_single = pynini.union(*labels_minute_single)
+        graph_minute_double = pynini.union(*labels_minute_double)
+
+        final_graph_hour_only = pynutil.insert('hours: "') + graph_hour + pynutil.insert('"')
+        final_graph_hour = (
+            pynutil.insert('hours: "')
+            + delete_leading_zero_to_double_digit @ graph_hour
+            + pynutil.insert('"')
+        )
+        final_graph_minute = (
+            pynutil.insert('minutes: "')
+            + (pynutil.delete("0") + graph_minute_single | graph_minute_double)
+            + pynutil.insert('"')
+        )
+        final_graph_second = (
+            pynutil.insert('seconds: "')
+            + (pynutil.delete("0") + graph_minute_single | graph_minute_double)
+            + pynutil.insert('"')
+        )
+        final_time_zone_optional = pynini.closure(
+            pynini.accep(" ")
+            + pynutil.insert('zone: "')
+            + convert_space(time_zone_graph)
+            + pynutil.insert('"'),
+            0,
+            1,
+        )
+
+        # 02:30 Uhr
+        graph_hm = (
+            final_graph_hour
+            + pynutil.delete(":")
+            + (pynutil.delete("00") | (insert_space + final_graph_minute))
+            + final_suffix
+            + final_time_zone_optional
+        )
+
+        # 10:30:05 Uhr,
+        graph_hms = (
+            final_graph_hour
+            + pynutil.delete(":")
+            + (pynini.cross("00", ' minutes: "0"') | (insert_space + final_graph_minute))
+            + pynutil.delete(":")
+            + (pynini.cross("00", ' seconds: "0"') | (insert_space + final_graph_second))
+            + final_suffix
+            + final_time_zone_optional
+            + pynutil.insert(" preserve_order: true")
+        )
+
+        # 2 Uhr est
+        graph_h = final_graph_hour_only + final_suffix + final_time_zone_optional
+        final_graph = (graph_hm | graph_h | graph_hms).optimize()
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/text_normalization/de/taggers/tokenize_and_classify.py

@@ -0,0 +1,145 @@
+import os
+
+import pynini
+from fun_text_processing.text_normalization.de.taggers.cardinal import CardinalFst
+from fun_text_processing.text_normalization.de.taggers.date import DateFst
+from fun_text_processing.text_normalization.de.taggers.decimal import DecimalFst
+from fun_text_processing.text_normalization.de.taggers.electronic import ElectronicFst
+from fun_text_processing.text_normalization.de.taggers.fraction import FractionFst
+from fun_text_processing.text_normalization.de.taggers.measure import MeasureFst
+from fun_text_processing.text_normalization.de.taggers.money import MoneyFst
+from fun_text_processing.text_normalization.de.taggers.ordinal import OrdinalFst
+from fun_text_processing.text_normalization.de.taggers.telephone import TelephoneFst
+from fun_text_processing.text_normalization.de.taggers.time import TimeFst
+from fun_text_processing.text_normalization.de.taggers.whitelist import WhiteListFst
+from fun_text_processing.text_normalization.de.taggers.word import WordFst
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_CHAR,
+    DAMO_DIGIT,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+    generator_main,
+)
+from fun_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
+from pynini.lib import pynutil
+
+import logging
+
+
+class ClassifyFst(GraphFst):
+    """
+    Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+
+    Args:
+        input_case: accepting either "lower_cased" or "cased" input.
+        deterministic: if True will provide a single transduction option,
+            for False multiple options (used for audio-based normalization)
+        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
+        overwrite_cache: set to True to overwrite .far files
+        whitelist: path to a file with whitelist replacements
+    """
+
+    def __init__(
+        self,
+        input_case: str,
+        deterministic: bool = False,
+        cache_dir: str = None,
+        overwrite_cache: bool = False,
+        whitelist: str = None,
+    ):
+        super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic)
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            whitelist_file = os.path.basename(whitelist) if whitelist else ""
+            far_file = os.path.join(
+                cache_dir, f"_{input_case}_de_tn_{deterministic}_deterministic{whitelist_file}.far"
+            )
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
+            no_digits = pynini.closure(pynini.difference(DAMO_CHAR, DAMO_DIGIT))
+            self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize()
+            logging.info(f"ClassifyFst.fst was restored from {far_file}.")
+        else:
+            logging.info(f"Creating ClassifyFst grammars. This might take some time...")
+
+            self.cardinal = CardinalFst(deterministic=deterministic)
+            cardinal_graph = self.cardinal.fst
+
+            self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic)
+            ordinal_graph = self.ordinal.fst
+
+            self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic)
+            decimal_graph = self.decimal.fst
+
+            self.fraction = FractionFst(cardinal=self.cardinal, deterministic=deterministic)
+            fraction_graph = self.fraction.fst
+            self.measure = MeasureFst(
+                cardinal=self.cardinal,
+                decimal=self.decimal,
+                fraction=self.fraction,
+                deterministic=deterministic,
+            )
+            measure_graph = self.measure.fst
+            self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic)
+            date_graph = self.date.fst
+            word_graph = WordFst(deterministic=deterministic).fst
+            self.time = TimeFst(deterministic=deterministic)
+            time_graph = self.time.fst
+            self.telephone = TelephoneFst(cardinal=self.cardinal, deterministic=deterministic)
+            telephone_graph = self.telephone.fst
+            self.electronic = ElectronicFst(deterministic=deterministic)
+            electronic_graph = self.electronic.fst
+            self.money = MoneyFst(
+                cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic
+            )
+            money_graph = self.money.fst
+            self.whitelist = WhiteListFst(
+                input_case=input_case, deterministic=deterministic, input_file=whitelist
+            )
+            whitelist_graph = self.whitelist.fst
+            punct_graph = PunctuationFst(deterministic=deterministic).fst
+
+            classify = (
+                pynutil.add_weight(whitelist_graph, 1.01)
+                | pynutil.add_weight(time_graph, 1.1)
+                | pynutil.add_weight(measure_graph, 1.1)
+                | pynutil.add_weight(cardinal_graph, 1.1)
+                | pynutil.add_weight(fraction_graph, 1.1)
+                | pynutil.add_weight(date_graph, 1.1)
+                | pynutil.add_weight(ordinal_graph, 1.1)
+                | pynutil.add_weight(decimal_graph, 1.1)
+                | pynutil.add_weight(money_graph, 1.1)
+                | pynutil.add_weight(telephone_graph, 1.1)
+                | pynutil.add_weight(electronic_graph, 1.1)
+            )
+
+            classify |= pynutil.add_weight(word_graph, 100)
+
+            punct = (
+                pynutil.insert("tokens { ")
+                + pynutil.add_weight(punct_graph, weight=1.1)
+                + pynutil.insert(" }")
+            )
+            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
+            token_plus_punct = (
+                pynini.closure(punct + pynutil.insert(" "))
+                + token
+                + pynini.closure(pynutil.insert(" ") + punct)
+            )
+
+            graph = token_plus_punct + pynini.closure(
+                pynutil.add_weight(delete_extra_space, 1.1) + token_plus_punct
+            )
+            graph = delete_space + graph + delete_space
+
+            self.fst = graph.optimize()
+            no_digits = pynini.closure(pynini.difference(DAMO_CHAR, DAMO_DIGIT))
+            self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize()
+
+            if far_file:
+                generator_main(far_file, {"tokenize_and_classify": self.fst})
+                logging.info(f"ClassifyFst grammars are saved to {far_file}.")

fun_text_processing/text_normalization/de/taggers/whitelist.py

@@ -0,0 +1,52 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for classifying whitelist, e.g.
+        "Mr." -> tokens { name: "mister" }
+    This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
+
+    Args:
+        input_case: accepting either "lower_cased" or "cased" input.
+        deterministic: if True will provide a single transduction option,
+            for False multiple options (used for audio-based normalization)
+        input_file: path to a file with whitelist replacements
+    """
+
+    def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None):
+        super().__init__(name="whitelist", kind="classify", deterministic=deterministic)
+
+        def _get_whitelist_graph(input_case, file):
+            whitelist = load_labels(file)
+            if input_case == "lower_cased":
+                whitelist = [[x[0].lower()] + x[1:] for x in whitelist]
+            graph = pynini.string_map(whitelist)
+            return graph
+
+        graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv"))
+        if not deterministic and input_case != "lower_cased":
+            graph |= pynutil.add_weight(
+                _get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")),
+                weight=0.0001,
+            )
+
+        if input_file:
+            whitelist_provided = _get_whitelist_graph(input_case, input_file)
+            if not deterministic:
+                graph |= whitelist_provided
+            else:
+                graph = whitelist_provided
+
+        if not deterministic:
+            units_graph = _get_whitelist_graph(
+                input_case, file=get_abs_path("data/measure/measurements.tsv")
+            )
+            graph |= units_graph
+
+        self.graph = graph
+        self.final_graph = convert_space(self.graph).optimize()
+        self.fst = (pynutil.insert('name: "') + self.final_graph + pynutil.insert('"')).optimize()

fun_text_processing/text_normalization/de/taggers/word.py

@@ -0,0 +1,19 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for classifying word.
+        e.g. sleep -> tokens { name: "sleep" }
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="word", kind="classify")
+        word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
+        self.fst = word.optimize()

fun_text_processing/text_normalization/de/utils.py

@@ -0,0 +1,34 @@
+import csv
+import os
+
+import logging
+
+
+def get_abs_path(rel_path):
+    """
+    Get absolute path
+
+    Args:
+        rel_path: relative path to this file
+
+    Returns absolute path
+    """
+    abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path
+
+    if not os.path.exists(abs_path):
+        logging.warning(f"{abs_path} does not exist")
+    return abs_path
+
+
+def load_labels(abs_path):
+    """
+    loads relative path file as dictionary
+
+    Args:
+        abs_path: absolute path
+
+    Returns dictionary of mappings
+    """
+    label_tsv = open(abs_path, encoding="utf-8")
+    labels = list(csv.reader(label_tsv, delimiter="\t"))
+    return labels

fun_text_processing/text_normalization/de/verbalizers/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/de/verbalizers/cardinal.py

@@ -0,0 +1,28 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing cardinals
+        e.g. cardinal { integer: "zwei" } -> "zwei"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
+        optional_sign = pynini.closure(pynini.cross('negative: "true" ', "minus "), 0, 1)
+        self.optional_sign = optional_sign
+        integer = pynini.closure(DAMO_NOT_QUOTE, 1)
+
+        self.integer = pynutil.delete(' "') + integer + pynutil.delete('"')
+
+        integer = pynutil.delete("integer:") + self.integer
+        self.numbers = integer
+        graph = optional_sign + self.numbers
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/date.py

@@ -0,0 +1,55 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_preserve_order,
+)
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for verbalizing date, e.g.
+        date { day: "vier" month: "april" year: "zwei tausend zwei" } -> "vierter april zwei tausend zwei"
+        date { day: "vier" month: "mai" year: "zwei tausend zwei" } -> "vierter mai zwei tausend zwei"
+
+    Args:
+        ordinal: ordinal verbalizer GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, ordinal: GraphFst, deterministic: bool = True):
+        super().__init__(name="date", kind="verbalize", deterministic=deterministic)
+
+        day_cardinal = (
+            pynutil.delete('day: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+        day = day_cardinal @ pynini.cdrewrite(
+            ordinal.ordinal_stem, "", "[EOS]", DAMO_SIGMA
+        ) + pynutil.insert("ter")
+
+        months_names = pynini.union(
+            *[x[1] for x in load_labels(get_abs_path("data/months/abbr_to_name.tsv"))]
+        )
+        month = pynutil.delete('month: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        final_month = month @ months_names
+        final_month |= month @ pynini.difference(DAMO_SIGMA, months_names) @ pynini.cdrewrite(
+            ordinal.ordinal_stem, "", "[EOS]", DAMO_SIGMA
+        ) + pynutil.insert("ter")
+
+        year = pynutil.delete('year: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+
+        # day month year
+        graph_dmy = (
+            day + pynini.accep(" ") + final_month + pynini.closure(pynini.accep(" ") + year, 0, 1)
+        )
+        graph_dmy |= final_month + pynini.accep(" ") + year
+
+        self.graph = graph_dmy | year
+        final_graph = self.graph + delete_preserve_order
+
+        delete_tokens = self.delete_tokens(final_graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/decimal.py

@@ -0,0 +1,57 @@
+import pynini
+from fun_text_processing.text_normalization.de.taggers.decimal import quantities
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_preserve_order,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for classifying decimal, e.g.
+        decimal { negative: "true" integer_part: "elf"  fractional_part: "vier null sechs" quantity: "billionen" } -> minus elf komma vier null sechs billionen
+        decimal { integer_part: "eins" quantity: "billion" } -> eins billion
+
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="decimal", kind="classify", deterministic=deterministic)
+
+        delete_space = pynutil.delete(" ")
+        self.optional_sign = pynini.closure(
+            pynini.cross('negative: "true"', "minus ") + delete_space, 0, 1
+        )
+        self.integer = (
+            pynutil.delete('integer_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        self.fractional_default = (
+            pynutil.delete('fractional_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        self.fractional = pynutil.insert(" komma ") + self.fractional_default
+
+        self.quantity = (
+            delete_space
+            + insert_space
+            + pynutil.delete('quantity: "')
+            + quantities
+            + pynutil.delete('"')
+        )
+        self.optional_quantity = pynini.closure(self.quantity, 0, 1)
+
+        graph = self.optional_sign + (
+            self.integer + self.quantity
+            | self.integer + delete_space + self.fractional + self.optional_quantity
+        )
+
+        self.numbers = graph
+        graph += delete_preserve_order
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/electronic.py

@@ -0,0 +1,64 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_preserve_order,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c at hotmail punkt com"
+                                                           -> "a b c at h o t m a i l punkt c o m"
+                                                           -> "a b c at hotmail punkt c o m"
+                                                           -> "a b c at h o t m a i l punkt com"
+    Args:
+        deterministic: if True will provide a single transduction option,
+        for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
+        graph_digit_no_zero = pynini.invert(
+            pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+        ).optimize() | pynini.cross("1", "eins")
+        graph_zero = pynini.invert(
+            pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        ).optimize()
+        graph_digit = graph_digit_no_zero | graph_zero
+        graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).optimize()
+        server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv"))
+        domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
+
+        def add_space_after_char():
+            return pynini.closure(DAMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + (
+                DAMO_NOT_QUOTE - pynini.accep(" ")
+            )
+
+        verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", DAMO_SIGMA)
+
+        user_name = pynutil.delete('username: "') + add_space_after_char() + pynutil.delete('"')
+        user_name @= verbalize_characters
+
+        convert_defaults = (
+            pynutil.add_weight(DAMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common
+        )
+        domain = convert_defaults + pynini.closure(insert_space + convert_defaults)
+        domain @= verbalize_characters
+
+        domain = pynutil.delete('domain: "') + domain + pynutil.delete('"')
+        protocol = (
+            pynutil.delete('protocol: "')
+            + add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", DAMO_SIGMA)
+            + pynutil.delete('"')
+        )
+        self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | (
+            user_name + pynini.accep(" ") + pynutil.insert("at ") + domain
+        )
+        delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/fraction.py

@@ -0,0 +1,68 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_preserve_order,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for verbalizing fraction
+        e.g. fraction { integer: "drei" numerator: "eins" denominator: "zwei" }-> drei ein halb
+        e.g. fraction { numerator: "vier" denominator: "zwei" } -> vier halbe
+        e.g. fraction { numerator: "drei" denominator: "vier" } -> drei viertel
+
+    Args:
+        ordinal: ordinal GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, ordinal: GraphFst, deterministic: bool = True):
+        super().__init__(name="fraction", kind="verbalize", deterministic=deterministic)
+
+        optional_sign = pynini.closure(
+            pynini.cross('negative: "true"', "minus ") + pynutil.delete(" "), 0, 1
+        )
+        change_one = pynini.cdrewrite(
+            pynutil.add_weight(pynini.cross("eins", "ein"), weight=-0.0001),
+            "[BOS]",
+            "[EOS]",
+            DAMO_SIGMA,
+        )
+        change_numerator_two = pynini.cdrewrite(
+            pynini.cross("zweitel", "halbe"), "[BOS]", "[EOS]", DAMO_SIGMA
+        )
+        integer = pynutil.delete('integer_part: "') + change_one + pynutil.delete('" ')
+        numerator = pynutil.delete('numerator: "') + change_one + pynutil.delete('" ')
+        denominator = (
+            pynutil.delete('denominator: "')
+            + pynini.closure(DAMO_NOT_QUOTE)
+            @ (
+                pynini.cdrewrite(
+                    pynini.closure(ordinal.ordinal_stem, 0, 1), "", "[EOS]", DAMO_SIGMA
+                )
+                + pynutil.insert("tel")
+            )
+            @ change_numerator_two
+            + pynutil.delete('"')
+        )
+
+        integer += insert_space + pynini.closure(pynutil.insert("und ", weight=0.001), 0, 1)
+
+        denominator_one_half = pynini.cdrewrite(
+            pynini.cross("ein halbe", "ein halb"), "[BOS]", "[EOS]", DAMO_SIGMA
+        )
+
+        fraction_default = (numerator + insert_space + denominator) @ denominator_one_half
+
+        self.graph = optional_sign + pynini.closure(integer, 0, 1) + fraction_default
+
+        graph = self.graph + delete_preserve_order
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/measure.py

@@ -0,0 +1,41 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_preserve_order,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure, e.g.
+        measure { cardinal { integer: "zwei" units: "unzen" } } -> "zwei unzen"
+        measure { cardinal { integer_part: "zwei" quantity: "millionen" units: "unzen" } } -> "zwei millionen unzen"
+
+    Args:
+        decimal: decimal GraphFst
+        cardinal: cardinal GraphFst
+        fraction: fraction GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(
+        self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool
+    ):
+        super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
+        unit = pynutil.delete('units: "') + pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('"')
+
+        graph_decimal = decimal.fst
+        graph_cardinal = cardinal.fst
+        graph_fraction = fraction.fst
+
+        graph = (graph_cardinal | graph_decimal | graph_fraction) + pynini.accep(" ") + unit
+
+        graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal)
+        graph += delete_preserve_order
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/money.py

@@ -0,0 +1,77 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_preserve_order,
+)
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing money, e.g.
+        money { currency_maj: "euro" integer_part: "ein"} -> "ein euro"
+        money { currency_maj: "euro" integer_part: "eins" fractional_part: "null null eins"} -> "eins komma null null eins euro"
+        money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" preserve_order: true} -> "ein pfund vierzig"
+        money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" currency_min: "pence" preserve_order: true} -> "ein pfund vierzig pence"
+        money { fractional_part: "ein" currency_min: "penny" preserve_order: true} -> "ein penny"
+        money { currency_maj: "pfund" integer_part: "null" fractional_part: "null eins" quantity: "million"} -> "null komma null eins million pfund"
+
+    Args:
+        decimal: GraphFst
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, decimal: GraphFst, deterministic: bool = True):
+        super().__init__(name="money", kind="verbalize", deterministic=deterministic)
+
+        keep_space = pynini.accep(" ")
+
+        maj = (
+            pynutil.delete('currency_maj: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        min = (
+            pynutil.delete('currency_min: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        fractional_part = (
+            pynutil.delete('fractional_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        integer_part = (
+            pynutil.delete('integer_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_add_and = pynini.closure(pynutil.insert("und "), 0, 1)
+
+        #  *** currency_maj
+        graph_integer = integer_part + keep_space + maj
+
+        #  *** currency_maj + (***) | ((und) *** current_min)
+        graph_integer_with_minor = (
+            integer_part
+            + keep_space
+            + maj
+            + keep_space
+            + (fractional_part | (optional_add_and + fractional_part + keep_space + min))
+            + delete_preserve_order
+        )
+
+        # *** komma *** currency_maj
+        graph_decimal = decimal.fst + keep_space + maj
+
+        # *** current_min
+        graph_minor = fractional_part + keep_space + min + delete_preserve_order
+
+        graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/ordinal.py

@@ -0,0 +1,45 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SIGMA,
+    GraphFst,
+)
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing roman numerals
+        e.g. ordinal { integer: "vier" } } -> "vierter"
+                                           -> "viertes" ...
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic)
+        graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")).invert()
+        graph_ties = pynini.string_file(get_abs_path("data/ordinals/ties.tsv")).invert()
+        graph_thousands = pynini.string_file(get_abs_path("data/ordinals/thousands.tsv")).invert()
+
+        graph = (
+            pynutil.delete('integer: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+
+        suffixes = pynini.union("ten", "tem", "ter", "tes", "te")
+        convert_rest = pynutil.insert(suffixes, weight=0.01)
+        self.ordinal_stem = graph_digit | graph_ties | graph_thousands
+
+        suffix = pynini.cdrewrite(
+            pynini.closure(self.ordinal_stem, 0, 1) + convert_rest,
+            "",
+            "[EOS]",
+            DAMO_SIGMA,
+        ).optimize()
+        self.graph = pynini.compose(graph, suffix)
+        self.suffix = suffix
+        delete_tokens = self.delete_tokens(self.graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/telephone.py

@@ -0,0 +1,40 @@
+import pynini
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_preserve_order,
+)
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for verbalizing telephone, e.g.
+        telephone { country_code: "plus neun und vierzig" number_part: "null eins eins eins null null null" }
+            -> "plus neun und vierzig null eins eins eins null null null"
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="telephone", kind="verbalize", deterministic=deterministic)
+
+        country_code = (
+            pynutil.delete('country_code: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        number_part = (
+            pynutil.delete('number_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        self.graph = country_code + pynini.accep(" ") + number_part
+
+        graph = self.graph + delete_preserve_order
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/time.py

@@ -0,0 +1,126 @@
+import pynini
+from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
+from fun_text_processing.text_normalization.en.graph_utils import (
+    DAMO_DIGIT,
+    DAMO_SIGMA,
+    GraphFst,
+    convert_space,
+    delete_preserve_order,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic, e.g.
+        time { hours: "2" minutes: "15"} -> "zwei uhr fünfzehn"
+        time { minutes: "15" hours: "2" } -> "viertel nach zwei"
+        time { minutes: "15" hours: "2" } -> "fünfzehn nach zwei"
+        time { hours: "14" minutes: "15"} -> "vierzehn uhr fünfzehn"
+        time { minutes: "15" hours: "14" } -> "viertel nach zwei"
+        time { minutes: "15" hours: "14" } -> "fünfzehn nach drei"
+        time { minutes: "45" hours: "14" } -> "viertel vor drei"
+
+    Args:
+        cardinal_tagger: cardinal_tagger tagger GraphFst
+        deterministic: if True will provide a single transduction option,
+        for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True):
+        super().__init__(name="time", kind="verbalize", deterministic=deterministic)
+
+        # add weight so when using inverse text normalization this conversion is depriotized
+        night_to_early = pynutil.add_weight(
+            pynini.invert(
+                pynini.string_file(get_abs_path("data/time/hour_to_night.tsv"))
+            ).optimize(),
+            weight=0.0001,
+        )
+        hour_to = pynini.invert(
+            pynini.string_file(get_abs_path("data/time/hour_to.tsv"))
+        ).optimize()
+        minute_to = pynini.invert(
+            pynini.string_file(get_abs_path("data/time/minute_to.tsv"))
+        ).optimize()
+        time_zone_graph = pynini.invert(
+            convert_space(
+                pynini.union(*[x[1] for x in load_labels(get_abs_path("data/time/time_zone.tsv"))])
+            )
+        )
+
+        graph_zero = pynini.invert(
+            pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        ).optimize()
+        number_verbalization = graph_zero | cardinal_tagger.two_digit_non_zero
+        hour = pynutil.delete('hours: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
+        hour_verbalized = hour @ number_verbalization @ pynini.cdrewrite(
+            pynini.cross("eins", "ein"), "[BOS]", "[EOS]", DAMO_SIGMA
+        ) + pynutil.insert(" uhr")
+        minute = pynutil.delete('minutes: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
+        zone = pynutil.delete('zone: "') + time_zone_graph + pynutil.delete('"')
+        optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1)
+        second = pynutil.delete('seconds: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
+        graph_hms = (
+            hour_verbalized
+            + pynini.accep(" ")
+            + minute @ number_verbalization
+            + pynutil.insert(" minuten")
+            + pynini.accep(" ")
+            + second @ number_verbalization
+            + pynutil.insert(" sekunden")
+            + optional_zone
+        )
+        graph_hms @= pynini.cdrewrite(
+            pynini.cross("eins minuten", "eine minute")
+            | pynini.cross("eins sekunden", "eine sekunde"),
+            pynini.union(" ", "[BOS]"),
+            "",
+            DAMO_SIGMA,
+        )
+
+        min_30 = [str(x) for x in range(1, 31)]
+        min_30 = pynini.union(*min_30)
+        min_29 = [str(x) for x in range(1, 30)]
+        min_29 = pynini.union(*min_29)
+
+        graph_h = hour_verbalized
+        graph_hm = hour_verbalized + pynini.accep(" ") + minute @ number_verbalization
+
+        graph_m_past_h = (
+            minute @ min_30 @ (number_verbalization | pynini.cross("15", "viertel"))
+            + pynini.accep(" ")
+            + pynutil.insert("nach ")
+            # + hour @ number_verbalization
+            + hour
+            @ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
+            @ number_verbalization
+        )
+        graph_m30_h = (
+            minute @ pynini.cross("30", "halb")
+            + pynini.accep(" ")
+            + hour
+            @ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
+            @ hour_to
+            @ number_verbalization
+        )
+        graph_m_to_h = (
+            minute @ minute_to @ min_29 @ (number_verbalization | pynini.cross("15", "viertel"))
+            + pynini.accep(" ")
+            + pynutil.insert("vor ")
+            + hour
+            @ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
+            @ hour_to
+            @ number_verbalization
+        )
+
+        self.graph = (
+            graph_hms
+            | graph_h
+            | graph_hm
+            | pynutil.add_weight(graph_m_past_h, weight=0.0001)
+            | pynutil.add_weight(graph_m30_h, weight=0.0001)
+            | pynutil.add_weight(graph_m_to_h, weight=0.0001)
+        ) + optional_zone
+        delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
+        self.fst = delete_tokens.optimize()

fun_text_processing/text_normalization/de/verbalizers/verbalize.py

@@ -0,0 +1,64 @@
+from fun_text_processing.text_normalization.de.taggers.cardinal import CardinalFst as CardinalTagger
+from fun_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst
+from fun_text_processing.text_normalization.de.verbalizers.date import DateFst
+from fun_text_processing.text_normalization.de.verbalizers.decimal import DecimalFst
+from fun_text_processing.text_normalization.de.verbalizers.electronic import ElectronicFst
+from fun_text_processing.text_normalization.de.verbalizers.fraction import FractionFst
+from fun_text_processing.text_normalization.de.verbalizers.measure import MeasureFst
+from fun_text_processing.text_normalization.de.verbalizers.money import MoneyFst
+from fun_text_processing.text_normalization.de.verbalizers.ordinal import OrdinalFst
+from fun_text_processing.text_normalization.de.verbalizers.telephone import TelephoneFst
+from fun_text_processing.text_normalization.de.verbalizers.time import TimeFst
+from fun_text_processing.text_normalization.en.graph_utils import GraphFst
+from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple options (used for audio-based normalization)
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
+        cardinal_tagger = CardinalTagger(deterministic=deterministic)
+        cardinal = CardinalFst(deterministic=deterministic)
+        cardinal_graph = cardinal.fst
+        ordinal = OrdinalFst(deterministic=deterministic)
+        ordinal_graph = ordinal.fst
+        decimal = DecimalFst(deterministic=deterministic)
+        decimal_graph = decimal.fst
+        fraction = FractionFst(ordinal=ordinal, deterministic=deterministic)
+        fraction_graph = fraction.fst
+        date = DateFst(ordinal=ordinal)
+        date_graph = date.fst
+        measure = MeasureFst(
+            cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic
+        )
+        measure_graph = measure.fst
+        electronic = ElectronicFst(deterministic=deterministic)
+        electronic_graph = electronic.fst
+        whitelist_graph = WhiteListFst(deterministic=deterministic).fst
+        money_graph = MoneyFst(decimal=decimal).fst
+        telephone_graph = TelephoneFst(deterministic=deterministic).fst
+        time_graph = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic).fst
+
+        graph = (
+            cardinal_graph
+            | measure_graph
+            | decimal_graph
+            | ordinal_graph
+            | date_graph
+            | electronic_graph
+            | money_graph
+            | fraction_graph
+            | whitelist_graph
+            | telephone_graph
+            | time_graph
+        )
+        self.fst = graph

fun_text_processing/text_normalization/de/verbalizers/verbalize_final.py

@@ -0,0 +1,61 @@
+import os
+
+import pynini
+from fun_text_processing.text_normalization.de.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.text_normalization.en.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+    generator_main,
+)
+from fun_text_processing.text_normalization.en.verbalizers.word import WordFst
+from pynini.lib import pynutil
+
+import logging
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple options (used for audio-based normalization)
+        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
+        overwrite_cache: set to True to overwrite .far files
+    """
+
+    def __init__(
+        self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False
+    ):
+        super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
+
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            far_file = os.path.join(
+                cache_dir, f"de_tn_{deterministic}_deterministic_verbalizer.far"
+            )
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["verbalize"]
+            logging.info(f"VerbalizeFinalFst graph was restored from {far_file}.")
+        else:
+            verbalize = VerbalizeFst(deterministic=deterministic).fst
+            word = WordFst(deterministic=deterministic).fst
+
+            types = verbalize | word
+            graph = (
+                pynutil.delete("tokens")
+                + delete_space
+                + pynutil.delete("{")
+                + delete_space
+                + types
+                + delete_space
+                + pynutil.delete("}")
+            )
+            graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+
+            self.fst = graph.optimize()
+            if far_file:
+                generator_main(far_file, {"verbalize": self.fst})
+                logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.")

fun_text_processing/text_normalization/en/__init__.py

@@ -0,0 +1,3 @@
+from fun_text_processing.text_normalization.en.taggers.tokenize_and_classify import ClassifyFst
+from fun_text_processing.text_normalization.en.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.text_normalization.en.verbalizers.verbalize_final import VerbalizeFinalFst

fun_text_processing/text_normalization/en/clean_eval_data.py

@@ -0,0 +1,384 @@
+from argparse import ArgumentParser
+from typing import List
+
+import regex as re
+from fun_text_processing.text_normalization.data_loader_utils import (
+    EOS_TYPE,
+    Instance,
+    load_files,
+    training_data_to_sentences,
+)
+
+
+"""
+This file is for evaluation purposes.
+filter_loaded_data() cleans data (list of instances) for text normalization. Filters and cleaners can be specified for each semiotic class individually.
+For example, normalized text should only include characters and whitespace characters but no punctuation. 
+            Cardinal unnormalized instances should contain at least one integer and all other characters are removed.
+"""
+
+
+class Filter:
+    """
+    Filter class
+
+    Args:
+        class_type: semiotic class used in dataset
+        process_func: function to transform text
+        filter_func:  function to filter text
+
+    """
+
+    def __init__(self, class_type: str, process_func: object, filter_func: object):
+        self.class_type = class_type
+        self.process_func = process_func
+        self.filter_func = filter_func
+
+    def filter(self, instance: Instance) -> bool:
+        """
+        filter function
+
+        Args:
+            filters given instance with filter function
+
+        Returns: True if given instance fulfills criteria or does not belong to class type
+        """
+        if instance.token_type != self.class_type:
+            return True
+        return self.filter_func(instance)
+
+    def process(self, instance: Instance) -> Instance:
+        """
+        process function
+
+        Args:
+            processes given instance with process function
+
+        Returns: processed instance if instance belongs to expected class type or original instance
+        """
+        if instance.token_type != self.class_type:
+            return instance
+        return self.process_func(instance)
+
+
+def filter_cardinal_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_cardinal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r"[^0-9]", "", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_ordinal_1(instance: Instance) -> bool:
+    ok = re.search(r"(st|nd|rd|th)\s*$", instance.un_normalized)
+    return ok
+
+
+def process_ordinal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r"[,\s]", "", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_decimal_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_decimal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_measure_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_measure_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    un_normalized = re.sub(r"m2", "m²", un_normalized)
+    un_normalized = re.sub(r"(\d)([^\d.\s])", r"\1 \2", un_normalized)
+    normalized = re.sub(r"[^a-z\s]", "", normalized)
+    normalized = re.sub(r"per ([a-z\s]*)s$", r"per \1", normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_money_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_money_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    un_normalized = re.sub(r"a\$", r"$", un_normalized)
+    un_normalized = re.sub(r"us\$", r"$", un_normalized)
+    un_normalized = re.sub(r"(\d)m\s*$", r"\1 million", un_normalized)
+    un_normalized = re.sub(r"(\d)bn?\s*$", r"\1 billion", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_time_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_time_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r": ", ":", un_normalized)
+    un_normalized = re.sub(r"(\d)\s?a\s?m\s?", r"\1 a.m.", un_normalized)
+    un_normalized = re.sub(r"(\d)\s?p\s?m\s?", r"\1 p.m.", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_plain_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_plain_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_punct_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_punct_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_date_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_date_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_letters_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_letters_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_verbatim_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_verbatim_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_digit_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_digit_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_telephone_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_telephone_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_electronic_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_electronic_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_fraction_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_fraction_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_address_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_address_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+filters = []
+filters.append(
+    Filter(class_type="CARDINAL", process_func=process_cardinal_1, filter_func=filter_cardinal_1)
+)
+filters.append(
+    Filter(class_type="ORDINAL", process_func=process_ordinal_1, filter_func=filter_ordinal_1)
+)
+filters.append(
+    Filter(class_type="DECIMAL", process_func=process_decimal_1, filter_func=filter_decimal_1)
+)
+filters.append(
+    Filter(class_type="MEASURE", process_func=process_measure_1, filter_func=filter_measure_1)
+)
+filters.append(Filter(class_type="MONEY", process_func=process_money_1, filter_func=filter_money_1))
+filters.append(Filter(class_type="TIME", process_func=process_time_1, filter_func=filter_time_1))
+
+filters.append(Filter(class_type="DATE", process_func=process_date_1, filter_func=filter_date_1))
+filters.append(Filter(class_type="PLAIN", process_func=process_plain_1, filter_func=filter_plain_1))
+filters.append(Filter(class_type="PUNCT", process_func=process_punct_1, filter_func=filter_punct_1))
+filters.append(
+    Filter(class_type="LETTERS", process_func=process_letters_1, filter_func=filter_letters_1)
+)
+filters.append(
+    Filter(class_type="VERBATIM", process_func=process_verbatim_1, filter_func=filter_verbatim_1)
+)
+filters.append(Filter(class_type="DIGIT", process_func=process_digit_1, filter_func=filter_digit_1))
+filters.append(
+    Filter(class_type="TELEPHONE", process_func=process_telephone_1, filter_func=filter_telephone_1)
+)
+filters.append(
+    Filter(
+        class_type="ELECTRONIC", process_func=process_electronic_1, filter_func=filter_electronic_1
+    )
+)
+filters.append(
+    Filter(class_type="FRACTION", process_func=process_fraction_1, filter_func=filter_fraction_1)
+)
+filters.append(
+    Filter(class_type="ADDRESS", process_func=process_address_1, filter_func=filter_address_1)
+)
+filters.append(Filter(class_type=EOS_TYPE, process_func=lambda x: x, filter_func=lambda x: True))
+
+
+def filter_loaded_data(data: List[Instance], verbose: bool = False) -> List[Instance]:
+    """
+    Filters list of instances
+
+    Args:
+        data: list of instances
+
+    Returns: filtered and transformed list of instances
+    """
+    updates_instances = []
+    for instance in data:
+        updated_instance = False
+        for fil in filters:
+            if tl.class_type == instance.token_type and tl.filter(instance):
+                instance = fil.process(instance)
+                updated_instance = True
+        if updated_instance:
+            if verbose:
+                print(instance)
+            updates_instances.append(instance)
+    return updates_instances
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--input", help="input file path", type=str, default="./en_with_types/output-00001-of-00100"
+    )
+    parser.add_argument("--verbose", help="print filtered instances", action="store_true")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    file_path = args.input
+
+    print("Loading training data: " + file_path)
+    instance_list = load_files([file_path])  # List of instances
+    filtered_instance_list = filter_loaded_data(instance_list, args.verbose)
+    training_data_to_sentences(filtered_instance_list)

fun_text_processing/text_normalization/en/data/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/address/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/address/address_word.tsv

@@ -0,0 +1,14 @@
+st	Street
+street	Street
+expy	Expressway
+fwy	Freeway
+hwy	Highway
+dr	Drive
+ct	Court
+ave	Avenue
+av	Avenue
+cir	Circle
+blvd	Boulevard
+alley	Alley
+way	Way
+jct	Junction

fun_text_processing/text_normalization/en/data/address/state.tsv

@@ -0,0 +1,52 @@
+Alabama	AL
+Alaska	AK
+Arizona	AZ
+Arkansas	AR
+California	CA
+Colorado	CO
+Connecticut	CT
+Delaware	DE
+Florida	FL
+Georgia	GA
+Hawaii	HI
+Idaho	ID
+Illinois	IL
+Indiana	IN
+Indiana	IND
+Iowa	IA
+Kansas	KS
+Kentucky	KY
+Louisiana	LA
+Maine	ME
+Maryland	MD
+Massachusetts	MA
+Michigan	MI
+Minnesota	MN
+Mississippi	MS
+Missouri	MO
+Montana	MT
+Nebraska	NE
+Nevada	NV
+New Hampshire	NH
+New Jersey	NJ
+New Mexico	NM
+New York	NY
+North Carolina	NC
+North Dakota	ND
+Ohio	OH
+Oklahoma	OK
+Oregon	OR
+Pennsylvania	PA
+Rhode Island	RI
+South Carolina	SC
+South Dakota	SD
+Tennessee	TN
+Tennessee	TENN
+Texas	TX
+Utah	UT
+Vermont	VT
+Virginia	VA
+Washington	WA
+West Virginia	WV
+Wisconsin	WI
+Wyoming	WY

fun_text_processing/text_normalization/en/data/date/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/date/day.tsv

@@ -0,0 +1,31 @@
+one
+two
+three
+four
+five
+six
+seven
+eight
+nine
+ten
+eleven
+twelve
+thirteen
+fourteen
+fifteen
+sixteen
+seventeen
+eighteen
+nineteen
+twenty
+twenty one
+twenty two
+twenty three
+twenty four
+twenty five
+twenty six
+twenty seven
+twenty eight
+twenty nine
+thirty
+thirty one

fun_text_processing/text_normalization/en/data/date/month_abbr.tsv

@@ -0,0 +1,12 @@
+jan	january
+feb	february
+mar	march
+apr	april
+jun	june
+jul	july
+aug	august
+sep	september
+sept	september
+oct	october
+nov	november
+dec	december

fun_text_processing/text_normalization/en/data/date/month_name.tsv

@@ -0,0 +1,12 @@
+january
+february
+march
+april
+may
+june
+july
+august
+september
+october
+november
+december

fun_text_processing/text_normalization/en/data/date/month_number.tsv

@@ -0,0 +1,24 @@
+1	january
+2	february
+3	march
+4	april
+5	may
+6	june
+7	july
+8	august
+9	september
+10	october
+11	november
+12	december
+01	january
+02	february
+03	march
+04	april
+05	may
+06	june
+07	july
+08	august
+09	september
+10	october
+11	november
+12	december

fun_text_processing/text_normalization/en/data/date/year_suffix.tsv

@@ -0,0 +1,16 @@
+A. D	AD
+A.D	AD
+a. d	AD
+a.d	AD
+a. d.	AD
+a.d.	AD
+B. C	BC
+B.C	BC
+b. c	BC
+b.c	BC
+A. D.	AD
+A.D.	AD
+B. C.	BC
+B.C.	BC
+b. c.	BC
+b.c.	BC

fun_text_processing/text_normalization/en/data/electronic/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/electronic/domain.tsv

@@ -0,0 +1,12 @@
+.com	dot com
+.org	dot org
+.gov	dot gov
+.uk	dot UK
+.fr	dot FR
+.net	dot net
+.br	dot BR
+.in	dot IN
+.ru	dot RU
+.de	dot DE
+.it	dot IT
+.jpg	dot jpeg

fun_text_processing/text_normalization/en/data/electronic/symbol.tsv

@@ -0,0 +1,21 @@
+.	dot
+-	dash
+_	underscore
+!	exclamation mark
+#	number sign
+$	dollar sign
+%	percent sign
+&	ampersand
+'	quote
+*	asterisk
++	plus
+/	slash
+=	equal sign
+?	question mark
+^	circumflex
+`	right single quote
+{	left brace
+|	vertical bar
+}	right brace
+~	tilde
+,	comma

fun_text_processing/text_normalization/en/data/measure/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/measure/math_operation.tsv

@@ -0,0 +1,8 @@
++	plus
+-	minus
+/	divided
+÷	divided
+:	divided
+×	times
+*	times
+·	times

fun_text_processing/text_normalization/en/data/measure/unit.tsv

@@ -0,0 +1,127 @@
+amu	atomic mass unit
+bar	bar
+°	degree
+º	degree
+°c	degree Celsius
+°C	degree Celsius
+ºc	degree Celsius
+ºC	degree Celsius
+℃	degree Celsius
+cm2	square centimeter
+cm²	square centimeter
+cm3	cubic centimeter
+cm³	cubic centimeter
+cm	centimeter
+cwt	hundredweight
+db	decibel
+dm3	cubic decimeter
+dm³	cubic decimeter
+dm	decimeter
+ds	decisecond
+°f	degree Fahrenheit
+°F	degree Fahrenheit
+℉	degree Fahrenheit
+ft	foot
+ghz	gigahertz
+gw	gigawatt
+gwh	gigawatt hour
+hz	hertz
+"	inch
+kbps	kilobit per second
+kcal	kilo calory
+kgf	kilogram force
+kg	kilogram
+khz	kilohertz
+km2	square kilometer
+km²	square kilometer
+km3	cubic kilometer
+km³	cubic kilometer
+km	kilometer
+kpa	kilopascal
+kwh	kilowatt hour
+kw	kilowatt
+kW	kilowatt
+lb	pound
+lbs	pound
+m2	square meter
+m²	square meter
+m3	cubic meter
+m³	cubic meter
+mbps	megabit per second
+mg	milligram
+mhz	megahertz
+mi2	square mile
+mi²	square mile
+mi3	cubic mile
+mi³	cubic mile
+cu mi	cubic mile
+mi	mile
+min	minute
+ml	milliliter
+mm2	square millimeter
+mm²	square millimeter
+mol	mole
+mpa	megapascal
+mph	mile per hour
+ng	nanogram
+nm	nanometer
+ns	nanosecond
+oz	ounce
+pa	pascal
+%	percent
+rad	radian
+rpm	revolution per minute
+sq ft	square foot
+sq mi	square mile
+sv	sievert
+tb	terabyte
+tj	terajoule
+tl	teraliter
+v	volt
+yd	yard
+μg	microgram
+μm	micrometer
+μs	microsecond
+ω	ohm
+atm	ATM
+au	AU
+bq	BQ
+cc	CC
+cd	CD
+da	DA
+eb	EB
+ev	EV
+f	F
+gb	GB
+g	G
+gl	GL
+gpa	GPA
+gy	GY
+ha	HA
+h	H
+hl	HL
+hp	GP
+hs	HS
+kb	KB
+kl	KL
+kn	KN
+kt	KT
+kv	KV
+lm	LM
+ma	MA
+mA	MA
+mb	MB
+mc	MC
+mf	MF
+m	M
+mm	MM
+ms	MS
+mv	MV
+mw	MW
+pb	PB
+pg	PG
+ps	PS
+s	S
+tb	TB
+tb	YB
+zb	ZB

fun_text_processing/text_normalization/en/data/measure/unit_alternatives.tsv

@@ -0,0 +1,43 @@
+atm	atmosphere
+bq	becquerel
+cd	candela
+da	dalton
+eb	exabyte
+f	degree Fahrenheit
+gb	gigabyte
+g	gram
+gl	gigaliter
+ha	hectare
+h	hour
+hl	hectoliter
+hp	horsepower
+hp	horsepower
+kb	kilobit
+kb	kilobyte
+ma	megaampere
+mA	megaampere
+ma	milliampere
+mA	milliampere
+mb	megabyte
+mc	megacoulomb
+mf	megafarad
+m	meter
+m	minute
+mm	millimeter
+mm	millimeter
+mm	millimeter
+ms	megasecond
+ms	mega siemens
+ms	millisecond
+mv	millivolt
+mV	millivolt
+mw	megawatt
+mW	megawatt
+pb	petabyte
+pg	petagram
+ps	petasecond
+s	second
+tb	terabyte
+tb	terabyte
+yb	yottabyte
+zb	zettabyte

fun_text_processing/text_normalization/en/data/money/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/money/currency_major.tsv

@@ -0,0 +1,39 @@
+$	dollar
+$	us dollar
+US$	us dollar
+฿	Thai Baht
+£	pound
+€	euro
+₩	won
+nzd	new zealand dollar
+rs	rupee
+chf	swiss franc
+dkk	danish kroner
+fim	finnish markka
+aed	arab emirates dirham
+¥	yen
+czk	czech koruna
+mro	mauritanian ouguiya
+pkr	pakistani rupee
+crc	costa rican colon
+hk$	hong kong dollar
+npr	nepalese rupee
+awg	aruban florin
+nok	norwegian kroner
+tzs	tanzanian shilling
+sek	swedish kronor
+cyp	cypriot pound
+r	real
+sar	saudi riyal
+cve	cape verde escudo
+rsd	serbian dinar
+dm	german mark
+shp	saint helena pounds
+php	philippine peso
+cad	canadian dollar
+ssp	south sudanese pound
+scr	seychelles rupee
+mvr	maldivian rufiyaa
+DH	dirham
+Dh	dirham
+Dhs.	dirham

fun_text_processing/text_normalization/en/data/money/currency_minor_plural.tsv

@@ -0,0 +1,4 @@
+$	cents
+US$	cents
+€	cents
+£	pence

fun_text_processing/text_normalization/en/data/money/currency_minor_singular.tsv

@@ -0,0 +1,3 @@
+$	cent
+€	cent
+£	penny

fun_text_processing/text_normalization/en/data/money/per_unit.tsv

@@ -0,0 +1,2 @@
+/ea	each
+/dozen

fun_text_processing/text_normalization/en/data/number/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/number/digit.tsv

@@ -0,0 +1,9 @@
+one	1
+two	2
+three	3
+four	4
+five	5
+six	6
+seven	7
+eight	8
+nine	9

fun_text_processing/text_normalization/en/data/number/fraction.tsv

@@ -0,0 +1,18 @@
+¼	1/4
+½	1/2
+¾	3/4
+⅐	1/7
+⅑	1/9
+⅒	1/10
+⅓	1/3
+⅔	2/3
+⅕	1/5
+⅖	2/5
+⅗	3/5
+⅘	4/5
+⅙	1/6
+⅚	5/6
+⅛	1/8
+⅜	3/8
+⅝	5/8
+⅞	7/8

fun_text_processing/text_normalization/en/data/number/hundred.tsv

@@ -0,0 +1 @@
+hundred

fun_text_processing/text_normalization/en/data/number/quantity_abbr.tsv

@@ -0,0 +1,10 @@
+M	million
+MLN	million
+m	million
+mln	million
+B	billion
+b	billion
+BN	billion
+bn	billion
+K	thousand
+k	thousand

fun_text_processing/text_normalization/en/data/number/teen.tsv

@@ -0,0 +1,10 @@
+ten	10
+eleven	11
+twelve	12
+thirteen	13
+fourteen	14
+fifteen	15
+sixteen	16
+seventeen	17
+eighteen	18
+nineteen	19

fun_text_processing/text_normalization/en/data/number/thousand.tsv

@@ -0,0 +1,22 @@
+thousand
+million
+billion
+trillion
+quadrillion
+quintillion
+sextillion
+septillion
+octillion
+nonillion
+decillion
+undecillion
+duodecillion
+tredecillion
+quattuordecillion
+quindecillion
+sexdecillion
+septendecillion
+octodecillion
+novemdecillion
+vigintillion
+centillion

fun_text_processing/text_normalization/en/data/number/ty.tsv

@@ -0,0 +1,8 @@
+twenty	2
+thirty	3
+forty	4
+fifty	5
+sixty	6
+seventy	7
+eighty	8
+ninety	9

fun_text_processing/text_normalization/en/data/number/zero.tsv

@@ -0,0 +1 @@
+zero	0

fun_text_processing/text_normalization/en/data/ordinal/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/text_normalization/en/data/ordinal/digit.tsv

@@ -0,0 +1,9 @@
+first	one
+second	two
+third	three
+fourth	four
+fifth	five
+sixth	sixth
+seventh	seven
+eighth	eight
+ninth	nine