chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,7 @@
|
||||
from fun_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import (
|
||||
ClassifyFst,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import (
|
||||
VerbalizeFinalFst,
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying cardinals. Numbers below ten are not converted.
|
||||
Allows both compound numeral strings or separated by whitespace.
|
||||
"und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number.
|
||||
|
||||
e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } }
|
||||
e.g. minus dreiundzwanzig -> cardinal { integer: "23" } }
|
||||
e.g. dreizehn -> cardinal { integer: "13" } }
|
||||
e.g. ein hundert -> cardinal { integer: "100" } }
|
||||
e.g. einhundert -> cardinal { integer: "100" } }
|
||||
e.g. ein tausend -> cardinal { integer: "1000" } }
|
||||
e.g. eintausend -> cardinal { integer: "1000" } }
|
||||
e.g. ein tausend zwanzig -> cardinal { integer: "1020" } }
|
||||
|
||||
Args:
|
||||
tn_cardinal_tagger: TN cardinal tagger
|
||||
"""
|
||||
|
||||
def __init__(self, tn_cardinal_tagger: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="cardinal", kind="classify", deterministic=deterministic)
|
||||
|
||||
# add_space_between_chars = pynini.cdrewrite(pynini.closure(insert_space, 0, 1), DAMO_CHAR, DAMO_CHAR, DAMO_SIGMA)
|
||||
optional_delete_space = pynini.closure(DAMO_SIGMA | pynutil.delete(" "))
|
||||
|
||||
graph = (tn_cardinal_tagger.graph @ optional_delete_space).invert().optimize()
|
||||
self.graph_hundred_component_at_least_one_none_zero_digit = (
|
||||
(
|
||||
tn_cardinal_tagger.graph_hundred_component_at_least_one_none_zero_digit
|
||||
@ optional_delete_space
|
||||
)
|
||||
.invert()
|
||||
.optimize()
|
||||
)
|
||||
|
||||
self.graph_ties = (
|
||||
(tn_cardinal_tagger.two_digit_non_zero @ optional_delete_space).invert().optimize()
|
||||
)
|
||||
# this is to make sure if there is an ambiguity with decimal, decimal is chosen, e.g. 1000000 vs. 1 million
|
||||
graph = pynutil.add_weight(graph, weight=0.001)
|
||||
self.graph_no_exception = graph
|
||||
self.digit = (
|
||||
pynini.arcmap(tn_cardinal_tagger.digit, map_type="rmweight").invert().optimize()
|
||||
)
|
||||
graph_exception = pynini.project(self.digit, "input")
|
||||
self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph
|
||||
|
||||
self.optional_minus_graph = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("minus ", '"-" '), 0, 1
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
self.optional_minus_graph
|
||||
+ pynutil.insert('integer: "')
|
||||
+ self.graph
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,84 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying date, in the form of (day) month (year) or year
|
||||
e.g. vierundzwanzigster juli zwei tausend dreizehn -> tokens { name: "24. Jul. 2013" }
|
||||
e.g. neunzehnachtzig -> tokens { name: "1980" }
|
||||
e.g. vierzehnter januar -> tokens { name: "14. Jan." }
|
||||
e.g. zweiter dritter -> tokens { name: "02.03." }
|
||||
e.g. januar neunzehnachtzig -> tokens { name: "Jan. 1980" }
|
||||
e.g. zwanzigzwanzig -> tokens { name: "2020" }
|
||||
|
||||
Args:
|
||||
itn_cardinal_tagger: ITN cardinal tagger
|
||||
tn_date_tagger: TN date tagger
|
||||
tn_date_verbalizer: TN date verbalizer
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
itn_cardinal_tagger: GraphFst,
|
||||
tn_date_tagger: GraphFst,
|
||||
tn_date_verbalizer: GraphFst,
|
||||
deterministic: bool = True,
|
||||
):
|
||||
super().__init__(name="date", kind="classify", deterministic=deterministic)
|
||||
|
||||
add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
|
||||
pynutil.insert("0") + DAMO_DIGIT
|
||||
)
|
||||
optional_delete_space = pynini.closure(DAMO_SIGMA | pynutil.delete(" ", weight=0.0001))
|
||||
tagger = tn_date_verbalizer.graph.invert().optimize()
|
||||
|
||||
delete_day_marker = (
|
||||
pynutil.delete('day: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
) @ itn_cardinal_tagger.graph_no_exception
|
||||
|
||||
month_as_number = (
|
||||
pynutil.delete('month: "')
|
||||
+ itn_cardinal_tagger.graph_no_exception
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
month_as_string = (
|
||||
pynutil.delete('month: "') + tn_date_tagger.month_abbr.invert() + pynutil.delete('"')
|
||||
)
|
||||
|
||||
convert_year = (tn_date_tagger.year @ optional_delete_space).invert().optimize()
|
||||
delete_year_marker = (
|
||||
pynutil.delete('year: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
) @ convert_year
|
||||
|
||||
# day. month as string (year)
|
||||
verbalizer = (
|
||||
pynini.closure(delete_day_marker + pynutil.insert(".") + pynini.accep(" "), 0, 1)
|
||||
+ month_as_string
|
||||
+ pynini.closure(pynini.accep(" ") + delete_year_marker, 0, 1)
|
||||
)
|
||||
|
||||
# day. month as number (year)
|
||||
verbalizer |= (
|
||||
delete_day_marker @ add_leading_zero_to_double_digit
|
||||
+ pynutil.insert(".")
|
||||
+ pynutil.delete(" ")
|
||||
+ month_as_number @ add_leading_zero_to_double_digit
|
||||
+ pynutil.insert(".")
|
||||
+ pynini.closure(pynutil.delete(" ") + delete_year_marker, 0, 1)
|
||||
)
|
||||
|
||||
# year
|
||||
verbalizer |= delete_year_marker
|
||||
|
||||
final_graph = tagger @ verbalizer
|
||||
|
||||
graph = pynutil.insert('name: "') + convert_space(final_graph) + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,52 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.taggers.decimal import get_quantity, quantities
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal
|
||||
e.g. minus elf komma zwei null null sechs billionen -> decimal { negative: "true" integer_part: "11" fractional_part: "2006" quantity: "billionen" }
|
||||
e.g. eine billion -> decimal { integer_part: "1" quantity: "billion" }
|
||||
Args:
|
||||
itn_cardinal_tagger: ITN Cardinal tagger
|
||||
tn_decimal_tagger: TN decimal tagger
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, itn_cardinal_tagger: GraphFst, tn_decimal_tagger: GraphFst, deterministic: bool = True
|
||||
):
|
||||
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
||||
|
||||
self.graph = tn_decimal_tagger.graph.invert().optimize()
|
||||
|
||||
delete_point = pynutil.delete(" komma")
|
||||
|
||||
allow_spelling = pynini.cdrewrite(
|
||||
pynini.cross("eine ", "eins ") + quantities, "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
|
||||
graph_fractional = pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"')
|
||||
graph_integer = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ itn_cardinal_tagger.graph_no_exception
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
final_graph_wo_sign = graph_integer + delete_point + pynini.accep(" ") + graph_fractional
|
||||
|
||||
self.final_graph_wo_negative = (
|
||||
allow_spelling
|
||||
@ (
|
||||
final_graph_wo_sign
|
||||
| get_quantity(
|
||||
final_graph_wo_sign,
|
||||
itn_cardinal_tagger.graph_hundred_component_at_least_one_none_zero_digit,
|
||||
)
|
||||
).optimize()
|
||||
)
|
||||
|
||||
final_graph = itn_cardinal_tagger.optional_minus_graph + self.final_graph_wo_negative
|
||||
final_graph += pynutil.insert(" preserve_order: true")
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,29 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying electronic: email addresses, etc.
|
||||
e.g. c d f eins at a b c punkt e d u -> tokens { name: "cdf1.abc.edu" }
|
||||
|
||||
Args:
|
||||
tn_electronic_tagger: TN eletronic tagger
|
||||
tn_electronic_verbalizer: TN eletronic verbalizer
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tn_electronic_tagger: GraphFst,
|
||||
tn_electronic_verbalizer: GraphFst,
|
||||
deterministic: bool = True,
|
||||
):
|
||||
super().__init__(name="electronic", kind="classify", deterministic=deterministic)
|
||||
|
||||
tagger = pynini.invert(tn_electronic_verbalizer.graph).optimize()
|
||||
verbalizer = pynini.invert(tn_electronic_tagger.graph).optimize()
|
||||
final_graph = tagger @ verbalizer
|
||||
|
||||
graph = pynutil.insert('name: "') + final_graph + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,66 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying fraction
|
||||
e.g. ein halb -> tokens { name: "1/2" }
|
||||
e.g. ein ein halb -> tokens { name: "1 1/2" }
|
||||
e.g. drei zwei ein hundertstel -> tokens { name: "3 2/100" }
|
||||
|
||||
Args:
|
||||
itn_cardinal_tagger: ITN cardinal tagger
|
||||
tn_fraction_verbalizer: TN fraction verbalizer
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
itn_cardinal_tagger: GraphFst,
|
||||
tn_fraction_verbalizer: GraphFst,
|
||||
deterministic: bool = True,
|
||||
):
|
||||
super().__init__(name="fraction", kind="classify", deterministic=deterministic)
|
||||
tagger = tn_fraction_verbalizer.graph.invert().optimize()
|
||||
|
||||
delete_optional_sign = pynini.closure(
|
||||
pynutil.delete("negative: ") + pynini.cross('"true" ', "-"), 0, 1
|
||||
)
|
||||
delete_integer_marker = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
) @ itn_cardinal_tagger.graph_no_exception
|
||||
|
||||
delete_numerator_marker = (
|
||||
pynutil.delete('numerator: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
) @ itn_cardinal_tagger.graph_no_exception
|
||||
|
||||
delete_denominator_marker = (
|
||||
pynutil.insert("/")
|
||||
+ (
|
||||
pynutil.delete('denominator: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
@ itn_cardinal_tagger.graph_no_exception
|
||||
)
|
||||
|
||||
graph = (
|
||||
pynini.closure(delete_integer_marker + pynini.accep(" "), 0, 1)
|
||||
+ delete_numerator_marker
|
||||
+ delete_space
|
||||
+ delete_denominator_marker
|
||||
).optimize()
|
||||
verbalizer = delete_optional_sign + graph
|
||||
|
||||
self.graph = tagger @ verbalizer
|
||||
|
||||
graph = pynutil.insert('name: "') + convert_space(self.graph) + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,98 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.taggers.measure import (
|
||||
singular_to_plural,
|
||||
unit_singular,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying measure. Allows for plural form for unit.
|
||||
e.g. minus elf kilogramm -> measure { cardinal { negative: "true" integer: "11" } units: "kg" }
|
||||
e.g. drei stunden -> measure { cardinal { integer: "3" } units: "h" }
|
||||
e.g. ein halb kilogramm -> measure { decimal { integer_part: "1/2" } units: "kg" }
|
||||
e.g. eins komma zwei kilogramm -> measure { decimal { integer_part: "1" fractional_part: "2" } units: "kg" }
|
||||
|
||||
Args:
|
||||
itn_cardinal_tagger: ITN Cardinal tagger
|
||||
itn_decimal_tagger: ITN Decimal tagger
|
||||
itn_fraction_tagger: ITN Fraction tagger
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
itn_cardinal_tagger: GraphFst,
|
||||
itn_decimal_tagger: GraphFst,
|
||||
itn_fraction_tagger: GraphFst,
|
||||
deterministic: bool = True,
|
||||
):
|
||||
super().__init__(name="measure", kind="classify", deterministic=deterministic)
|
||||
|
||||
cardinal_graph = (
|
||||
pynini.cdrewrite(
|
||||
pynini.cross(pynini.union("ein", "eine"), "eins"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
@ itn_cardinal_tagger.graph_no_exception
|
||||
)
|
||||
|
||||
graph_unit_singular = pynini.invert(unit_singular) # singular -> abbr
|
||||
unit = (
|
||||
pynini.invert(singular_to_plural()) @ graph_unit_singular
|
||||
) | graph_unit_singular # plural -> abbr
|
||||
unit = convert_space(unit)
|
||||
graph_unit_singular = convert_space(graph_unit_singular)
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("minus", '"true"') + delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
unit_misc = pynutil.insert("/") + pynutil.delete("pro") + delete_space + graph_unit_singular
|
||||
|
||||
unit = (
|
||||
pynutil.insert('units: "')
|
||||
+ (unit | unit_misc | pynutil.add_weight(unit + delete_space + unit_misc, 0.01))
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
subgraph_decimal = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ optional_graph_negative
|
||||
+ itn_decimal_tagger.final_graph_wo_negative
|
||||
+ pynutil.insert(" }")
|
||||
+ delete_extra_space
|
||||
+ unit
|
||||
)
|
||||
|
||||
subgraph_fraction = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ optional_graph_negative
|
||||
+ pynutil.insert('integer_part: "')
|
||||
+ itn_fraction_tagger.graph
|
||||
+ pynutil.insert('" }')
|
||||
+ delete_extra_space
|
||||
+ unit
|
||||
)
|
||||
|
||||
subgraph_cardinal = (
|
||||
pynutil.insert("cardinal { ")
|
||||
+ optional_graph_negative
|
||||
+ pynutil.insert('integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(" }")
|
||||
+ delete_extra_space
|
||||
+ unit
|
||||
)
|
||||
final_graph = subgraph_cardinal | subgraph_decimal | subgraph_fraction
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,90 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.taggers.money import (
|
||||
maj_singular,
|
||||
min_plural,
|
||||
min_singular,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying money
|
||||
e.g. elf euro und vier cent -> money { integer_part: "11" fractional_part: 04 currency: "€" }
|
||||
|
||||
Args:
|
||||
itn_cardinal_tagger: ITN Cardinal Tagger
|
||||
itn_decimal_tagger: ITN Decimal Tagger
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
itn_cardinal_tagger: GraphFst,
|
||||
itn_decimal_tagger: GraphFst,
|
||||
deterministic: bool = True,
|
||||
):
|
||||
super().__init__(name="money", kind="classify", deterministic=deterministic)
|
||||
cardinal_graph = (
|
||||
pynini.cdrewrite(
|
||||
pynini.cross(pynini.union("ein", "eine"), "eins"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
@ itn_cardinal_tagger.graph_no_exception
|
||||
)
|
||||
graph_decimal_final = itn_decimal_tagger.final_graph_wo_negative
|
||||
|
||||
graph_unit = pynini.invert(maj_singular)
|
||||
graph_unit = pynutil.insert('currency: "') + convert_space(graph_unit) + pynutil.insert('"')
|
||||
|
||||
add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
|
||||
pynutil.insert("0") + DAMO_DIGIT
|
||||
)
|
||||
min_unit = pynini.project(min_singular | min_plural, "output")
|
||||
# elf euro (und) vier cent, vier cent
|
||||
cents_standalone = (
|
||||
pynutil.insert('fractional_part: "')
|
||||
+ cardinal_graph @ add_leading_zero_to_double_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete(min_unit)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
optional_cents_standalone = pynini.closure(
|
||||
delete_space
|
||||
+ pynini.closure(pynutil.delete("und") + delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ cents_standalone,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
# elf euro vierzig, only after integer
|
||||
optional_cents_suffix = pynini.closure(
|
||||
delete_extra_space
|
||||
+ pynutil.insert('fractional_part: "')
|
||||
+ pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7)
|
||||
+ pynutil.insert('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
graph_integer = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ delete_extra_space
|
||||
+ graph_unit
|
||||
+ (optional_cents_standalone | optional_cents_suffix)
|
||||
)
|
||||
graph_decimal = graph_decimal_final + delete_extra_space + graph_unit
|
||||
graph_decimal |= pynutil.insert('currency: "€" integer_part: "0" ') + cents_standalone
|
||||
final_graph = graph_integer | graph_decimal
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,33 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying ordinal
|
||||
e.g. dreizehnter -> tokens { name: "13." }
|
||||
|
||||
Args:
|
||||
itn_cardinal_tagger: ITN Cardinal Tagger
|
||||
tn_ordinal_verbalizer: TN Ordinal Verbalizer
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
itn_cardinal_tagger: GraphFst,
|
||||
tn_ordinal_verbalizer: GraphFst,
|
||||
deterministic: bool = True,
|
||||
):
|
||||
super().__init__(name="ordinal", kind="classify", deterministic=deterministic)
|
||||
|
||||
tagger = tn_ordinal_verbalizer.graph.invert().optimize()
|
||||
|
||||
graph = (
|
||||
pynutil.delete('integer: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
) @ itn_cardinal_tagger.graph
|
||||
|
||||
final_graph = tagger @ graph + pynutil.insert(".")
|
||||
|
||||
graph = pynutil.insert('name: "') + final_graph + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,43 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
GraphFst,
|
||||
convert_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying telephone numbers, e.g.
|
||||
null vier eins eins eins zwei drei vier eins zwei drei vier -> tokens { name: "(0411) 1234-1234" }
|
||||
|
||||
Args:
|
||||
tn_cardinal_tagger: TN Cardinal Tagger
|
||||
"""
|
||||
|
||||
def __init__(self, tn_cardinal_tagger: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="telephone", kind="classify", deterministic=deterministic)
|
||||
separator = pynini.accep(" ") # between components
|
||||
digit = pynini.union(*list(map(str, range(1, 10)))) @ tn_cardinal_tagger.two_digit_non_zero
|
||||
zero = pynini.cross("0", "null")
|
||||
|
||||
number_part = (
|
||||
pynutil.delete("(")
|
||||
+ zero
|
||||
+ insert_space
|
||||
+ pynini.closure(digit + insert_space, 2, 2)
|
||||
+ digit
|
||||
+ pynutil.delete(")")
|
||||
+ separator
|
||||
+ pynini.closure(digit + insert_space, 3, 3)
|
||||
+ digit
|
||||
+ pynutil.delete("-")
|
||||
+ insert_space
|
||||
+ pynini.closure(digit + insert_space, 3, 3)
|
||||
+ digit
|
||||
)
|
||||
graph = convert_space(pynini.invert(number_part))
|
||||
final_graph = pynutil.insert('name: "') + graph + pynutil.insert('"')
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,28 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying time
|
||||
e.g. acht uhr e s t-> time { hours: "8" zone: "e s t" }
|
||||
e.g. dreizehn uhr -> time { hours: "13" }
|
||||
e.g. dreizehn uhr zehn -> time { hours: "13" minutes: "10" }
|
||||
e.g. viertel vor zwölf -> time { minutes: "45" hours: "11" }
|
||||
e.g. viertel nach zwölf -> time { minutes: "15" hours: "12" }
|
||||
e.g. halb zwölf -> time { minutes: "30" hours: "11" }
|
||||
e.g. drei vor zwölf -> time { minutes: "57" hours: "11" }
|
||||
e.g. drei nach zwölf -> time { minutes: "3" hours: "12" }
|
||||
e.g. drei uhr zehn minuten zehn sekunden -> time { hours: "3" hours: "10" sekunden: "10"}
|
||||
|
||||
Args:
|
||||
tn_time_verbalizer: TN time verbalizer
|
||||
"""
|
||||
|
||||
def __init__(self, tn_time_verbalizer: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="classify", deterministic=deterministic)
|
||||
# lazy way to make sure compounds work
|
||||
optional_delete_space = pynini.closure(DAMO_SIGMA | pynutil.delete(" ", weight=0.0001))
|
||||
graph = (tn_time_verbalizer.graph @ optional_delete_space).invert().optimize()
|
||||
self.fst = self.add_tokens(graph).optimize()
|
||||
@@ -0,0 +1,162 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.date import DateFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.electronic import ElectronicFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.fraction import FractionFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.ordinal import OrdinalFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.telephone import TelephoneFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.time import TimeFst
|
||||
from fun_text_processing.inverse_text_normalization.de.taggers.whitelist import WhiteListFst
|
||||
from fun_text_processing.inverse_text_normalization.en.taggers.punctuation import PunctuationFst
|
||||
from fun_text_processing.inverse_text_normalization.en.taggers.word import WordFst
|
||||
from fun_text_processing.text_normalization.de.taggers.cardinal import (
|
||||
CardinalFst as TNCardinalTagger,
|
||||
)
|
||||
from fun_text_processing.text_normalization.de.taggers.date import DateFst as TNDateTagger
|
||||
from fun_text_processing.text_normalization.de.taggers.decimal import DecimalFst as TNDecimalTagger
|
||||
from fun_text_processing.text_normalization.de.taggers.electronic import (
|
||||
ElectronicFst as TNElectronicTagger,
|
||||
)
|
||||
from fun_text_processing.text_normalization.de.taggers.whitelist import (
|
||||
WhiteListFst as TNWhitelistTagger,
|
||||
)
|
||||
from fun_text_processing.text_normalization.de.verbalizers.date import DateFst as TNDateVerbalizer
|
||||
from fun_text_processing.text_normalization.de.verbalizers.electronic import (
|
||||
ElectronicFst as TNElectronicVerbalizer,
|
||||
)
|
||||
from fun_text_processing.text_normalization.de.verbalizers.fraction import (
|
||||
FractionFst as TNFractionVerbalizer,
|
||||
)
|
||||
from fun_text_processing.text_normalization.de.verbalizers.ordinal import (
|
||||
OrdinalFst as TNOrdinalVerbalizer,
|
||||
)
|
||||
from fun_text_processing.text_normalization.de.verbalizers.time import TimeFst as TNTimeVerbalizer
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class ClassifyFst(GraphFst):
|
||||
"""
|
||||
Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, cache_dir: str = None, overwrite_cache: bool = False, deterministic: bool = True
|
||||
):
|
||||
super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic)
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(cache_dir, "_de_itn.far")
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
|
||||
logging.info(f"ClassifyFst.fst was restored from {far_file}.")
|
||||
else:
|
||||
logging.info(f"Creating ClassifyFst grammars.")
|
||||
tn_cardinal_tagger = TNCardinalTagger(deterministic=False)
|
||||
tn_date_tagger = TNDateTagger(cardinal=tn_cardinal_tagger, deterministic=False)
|
||||
tn_decimal_tagger = TNDecimalTagger(cardinal=tn_cardinal_tagger, deterministic=False)
|
||||
tn_ordinal_verbalizer = TNOrdinalVerbalizer(deterministic=False)
|
||||
tn_fraction_verbalizer = TNFractionVerbalizer(
|
||||
ordinal=tn_ordinal_verbalizer, deterministic=False
|
||||
)
|
||||
tn_time_verbalizer = TNTimeVerbalizer(
|
||||
cardinal_tagger=tn_cardinal_tagger, deterministic=False
|
||||
)
|
||||
tn_date_verbalizer = TNDateVerbalizer(
|
||||
ordinal=tn_ordinal_verbalizer, deterministic=False
|
||||
)
|
||||
tn_electronic_tagger = TNElectronicTagger(deterministic=False)
|
||||
tn_electronic_verbalizer = TNElectronicVerbalizer(deterministic=False)
|
||||
tn_whitelist_tagger = TNWhitelistTagger(input_case="cased", deterministic=False)
|
||||
|
||||
cardinal = CardinalFst(tn_cardinal_tagger=tn_cardinal_tagger)
|
||||
cardinal_graph = cardinal.fst
|
||||
|
||||
ordinal = OrdinalFst(
|
||||
itn_cardinal_tagger=cardinal, tn_ordinal_verbalizer=tn_ordinal_verbalizer
|
||||
)
|
||||
ordinal_graph = ordinal.fst
|
||||
decimal = DecimalFst(itn_cardinal_tagger=cardinal, tn_decimal_tagger=tn_decimal_tagger)
|
||||
decimal_graph = decimal.fst
|
||||
|
||||
fraction = FractionFst(
|
||||
itn_cardinal_tagger=cardinal, tn_fraction_verbalizer=tn_fraction_verbalizer
|
||||
)
|
||||
fraction_graph = fraction.fst
|
||||
|
||||
measure_graph = MeasureFst(
|
||||
itn_cardinal_tagger=cardinal,
|
||||
itn_decimal_tagger=decimal,
|
||||
itn_fraction_tagger=fraction,
|
||||
).fst
|
||||
date_graph = DateFst(
|
||||
itn_cardinal_tagger=cardinal,
|
||||
tn_date_verbalizer=tn_date_verbalizer,
|
||||
tn_date_tagger=tn_date_tagger,
|
||||
).fst
|
||||
word_graph = WordFst().fst
|
||||
time_graph = TimeFst(tn_time_verbalizer=tn_time_verbalizer).fst
|
||||
money_graph = MoneyFst(itn_cardinal_tagger=cardinal, itn_decimal_tagger=decimal).fst
|
||||
whitelist_graph = WhiteListFst(tn_whitelist_tagger=tn_whitelist_tagger).fst
|
||||
punct_graph = PunctuationFst().fst
|
||||
electronic_graph = ElectronicFst(
|
||||
tn_electronic_tagger=tn_electronic_tagger,
|
||||
tn_electronic_verbalizer=tn_electronic_verbalizer,
|
||||
).fst
|
||||
telephone_graph = TelephoneFst(tn_cardinal_tagger=tn_cardinal_tagger).fst
|
||||
|
||||
classify = (
|
||||
pynutil.add_weight(cardinal_graph, 1.1)
|
||||
| pynutil.add_weight(whitelist_graph, 1.0)
|
||||
| pynutil.add_weight(time_graph, 1.1)
|
||||
| pynutil.add_weight(date_graph, 1.1)
|
||||
| pynutil.add_weight(decimal_graph, 1.1)
|
||||
| pynutil.add_weight(measure_graph, 1.1)
|
||||
| pynutil.add_weight(ordinal_graph, 1.1)
|
||||
| pynutil.add_weight(fraction_graph, 1.1)
|
||||
| pynutil.add_weight(money_graph, 1.1)
|
||||
| pynutil.add_weight(telephone_graph, 1.1)
|
||||
| pynutil.add_weight(electronic_graph, 1.1)
|
||||
| pynutil.add_weight(word_graph, 100)
|
||||
)
|
||||
|
||||
punct = (
|
||||
pynutil.insert("tokens { ")
|
||||
+ pynutil.add_weight(punct_graph, weight=1.1)
|
||||
+ pynutil.insert(" }")
|
||||
)
|
||||
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
|
||||
token_plus_punct = (
|
||||
pynini.closure(punct + pynutil.insert(" "))
|
||||
+ token
|
||||
+ pynini.closure(pynutil.insert(" ") + punct)
|
||||
)
|
||||
|
||||
graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
|
||||
graph = delete_space + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
|
||||
if far_file:
|
||||
generator_main(far_file, {"tokenize_and_classify": self.fst})
|
||||
logging.info(f"ClassifyFst grammars are saved to {far_file}.")
|
||||
@@ -0,0 +1,19 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying whitelisted tokens
|
||||
e.g. misses -> tokens { name: "Mrs." }
|
||||
Args:
|
||||
tn_whitelist_tagger: TN whitelist tagger
|
||||
"""
|
||||
|
||||
def __init__(self, tn_whitelist_tagger: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="whitelist", kind="classify", deterministic=deterministic)
|
||||
|
||||
whitelist = pynini.invert(tn_whitelist_tagger.graph)
|
||||
graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinal
|
||||
e.g. cardinal { integer: "23" negative: "-" } -> -23
|
||||
|
||||
Args:
|
||||
tn_cardinal_verbalizer: TN cardinal verbalizer
|
||||
"""
|
||||
|
||||
def __init__(self, tn_cardinal_verbalizer: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
|
||||
self.numbers = tn_cardinal_verbalizer.numbers
|
||||
optional_sign = pynini.closure(
|
||||
pynutil.delete('negative: "') + DAMO_NOT_QUOTE + pynutil.delete('" '), 0, 1
|
||||
)
|
||||
graph = optional_sign + self.numbers
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,37 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "12" fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
|
||||
|
||||
Args:
|
||||
tn_decimal_verbalizer: TN decimal verbalizer
|
||||
"""
|
||||
|
||||
def __init__(self, tn_decimal_verbalizer: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="decimal", kind="verbalize", deterministic=deterministic)
|
||||
delete_space = pynutil.delete(" ")
|
||||
optional_sign = pynini.closure(
|
||||
pynutil.delete('negative: "') + DAMO_NOT_QUOTE + pynutil.delete('"') + delete_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
optional_integer = pynini.closure(tn_decimal_verbalizer.integer, 0, 1)
|
||||
optional_fractional = pynini.closure(
|
||||
delete_space + pynutil.insert(",") + tn_decimal_verbalizer.fractional_default, 0, 1
|
||||
)
|
||||
graph = (
|
||||
optional_integer + optional_fractional + tn_decimal_verbalizer.optional_quantity
|
||||
).optimize()
|
||||
self.numbers = optional_sign + graph
|
||||
graph = self.numbers + delete_preserve_order
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,50 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { cardinal { negative: "true" integer: "12" } units: "kg" } -> -12 kg
|
||||
measure { decimal { integer_part: "1/2" } units: "kg" } -> 1/2 kg
|
||||
measure { decimal { integer_part: "1" fractional_part: "2" quantity: "million" } units: "kg" } -> 1,2 million kg
|
||||
|
||||
Args:
|
||||
decimal: ITN Decimal verbalizer
|
||||
cardinal: ITN Cardinal verbalizer
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, cardinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
|
||||
unit = (
|
||||
pynutil.delete("units:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space
|
||||
)
|
||||
graph_decimal = (
|
||||
pynutil.delete("decimal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ decimal.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph_cardinal = (
|
||||
pynutil.delete("cardinal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ cardinal.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
|
||||
graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert(" ") + unit
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,26 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_CHAR, GraphFst, delete_space
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05
|
||||
|
||||
Args:
|
||||
decimal: ITN Decimal verbalizer
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="money", kind="verbalize", deterministic=deterministic)
|
||||
unit = (
|
||||
pynutil.delete("currency:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = unit + delete_space + decimal.numbers
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,52 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing time, e.g.
|
||||
time { hours: "8" minutes: "30" zone: "e s t" } -> 08:30 Uhr est
|
||||
time { hours: "8" } -> 8 Uhr
|
||||
time { hours: "8" minutes: "30" seconds: "10" } -> 08:30:10 Uhr
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
|
||||
pynutil.insert("0") + DAMO_DIGIT
|
||||
)
|
||||
hour = pynutil.delete('hours: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
minute = pynutil.delete('minutes: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
|
||||
second = pynutil.delete('seconds: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
zone = (
|
||||
pynutil.delete('zone: "')
|
||||
+ pynini.closure(DAMO_ALPHA + delete_space)
|
||||
+ DAMO_ALPHA
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1)
|
||||
graph = (
|
||||
delete_space
|
||||
+ pynutil.insert(":")
|
||||
+ (minute @ add_leading_zero_to_double_digit)
|
||||
+ pynini.closure(
|
||||
delete_space + pynutil.insert(":") + (second @ add_leading_zero_to_double_digit),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
+ pynutil.insert(" Uhr")
|
||||
+ optional_zone
|
||||
)
|
||||
graph_h = hour + pynutil.insert(" Uhr") + optional_zone
|
||||
graph_hm = hour @ add_leading_zero_to_double_digit + graph
|
||||
graph_hms = hour @ add_leading_zero_to_double_digit + graph
|
||||
final_graph = graph_hm | graph_hms | graph_h
|
||||
self.fst = self.delete_tokens(final_graph).optimize()
|
||||
@@ -0,0 +1,35 @@
|
||||
from fun_text_processing.inverse_text_normalization.de.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.de.verbalizers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.de.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.de.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.de.verbalizers.time import TimeFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.cardinal import (
|
||||
CardinalFst as TNCardinalVerbalizer,
|
||||
)
|
||||
from fun_text_processing.text_normalization.de.verbalizers.decimal import (
|
||||
DecimalFst as TNDecimalVerbalizer,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
|
||||
tn_cardinal_verbalizer = TNCardinalVerbalizer(deterministic=False)
|
||||
tn_decimal_verbalizer = TNDecimalVerbalizer(deterministic=False)
|
||||
|
||||
cardinal = CardinalFst(tn_cardinal_verbalizer=tn_cardinal_verbalizer)
|
||||
cardinal_graph = cardinal.fst
|
||||
decimal = DecimalFst(tn_decimal_verbalizer=tn_decimal_verbalizer)
|
||||
decimal_graph = decimal.fst
|
||||
measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst
|
||||
money_graph = MoneyFst(decimal=decimal).fst
|
||||
time_graph = TimeFst().fst
|
||||
graph = time_graph | money_graph | measure_graph | decimal_graph | cardinal_graph
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,33 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.de.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.en.verbalizers.word import WordFst
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence, e.g.
|
||||
tokens { name: "jetzt" } tokens { name: "ist" } tokens { time { hours: "12" minutes: "30" } } -> jetzt ist 12:30 Uhr
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
|
||||
verbalize = VerbalizeFst().fst
|
||||
word = WordFst().fst
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
self.fst = graph
|
||||
Reference in New Issue
Block a user