chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,28 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinals
|
||||
e.g. cardinal { integer: "zwei" } -> "zwei"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true" ', "minus "), 0, 1)
|
||||
self.optional_sign = optional_sign
|
||||
integer = pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
|
||||
self.integer = pynutil.delete(' "') + integer + pynutil.delete('"')
|
||||
|
||||
integer = pynutil.delete("integer:") + self.integer
|
||||
self.numbers = integer
|
||||
graph = optional_sign + self.numbers
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,55 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing date, e.g.
|
||||
date { day: "vier" month: "april" year: "zwei tausend zwei" } -> "vierter april zwei tausend zwei"
|
||||
date { day: "vier" month: "mai" year: "zwei tausend zwei" } -> "vierter mai zwei tausend zwei"
|
||||
|
||||
Args:
|
||||
ordinal: ordinal verbalizer GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, ordinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="date", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
day_cardinal = (
|
||||
pynutil.delete('day: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
day = day_cardinal @ pynini.cdrewrite(
|
||||
ordinal.ordinal_stem, "", "[EOS]", DAMO_SIGMA
|
||||
) + pynutil.insert("ter")
|
||||
|
||||
months_names = pynini.union(
|
||||
*[x[1] for x in load_labels(get_abs_path("data/months/abbr_to_name.tsv"))]
|
||||
)
|
||||
month = pynutil.delete('month: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
final_month = month @ months_names
|
||||
final_month |= month @ pynini.difference(DAMO_SIGMA, months_names) @ pynini.cdrewrite(
|
||||
ordinal.ordinal_stem, "", "[EOS]", DAMO_SIGMA
|
||||
) + pynutil.insert("ter")
|
||||
|
||||
year = pynutil.delete('year: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
|
||||
# day month year
|
||||
graph_dmy = (
|
||||
day + pynini.accep(" ") + final_month + pynini.closure(pynini.accep(" ") + year, 0, 1)
|
||||
)
|
||||
graph_dmy |= final_month + pynini.accep(" ") + year
|
||||
|
||||
self.graph = graph_dmy | year
|
||||
final_graph = self.graph + delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(final_graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,57 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.taggers.decimal import quantities
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "elf" fractional_part: "vier null sechs" quantity: "billionen" } -> minus elf komma vier null sechs billionen
|
||||
decimal { integer_part: "eins" quantity: "billion" } -> eins billion
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
||||
|
||||
delete_space = pynutil.delete(" ")
|
||||
self.optional_sign = pynini.closure(
|
||||
pynini.cross('negative: "true"', "minus ") + delete_space, 0, 1
|
||||
)
|
||||
self.integer = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.fractional_default = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
self.fractional = pynutil.insert(" komma ") + self.fractional_default
|
||||
|
||||
self.quantity = (
|
||||
delete_space
|
||||
+ insert_space
|
||||
+ pynutil.delete('quantity: "')
|
||||
+ quantities
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.optional_quantity = pynini.closure(self.quantity, 0, 1)
|
||||
|
||||
graph = self.optional_sign + (
|
||||
self.integer + self.quantity
|
||||
| self.integer + delete_space + self.fractional + self.optional_quantity
|
||||
)
|
||||
|
||||
self.numbers = graph
|
||||
graph += delete_preserve_order
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,64 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic
|
||||
e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c at hotmail punkt com"
|
||||
-> "a b c at h o t m a i l punkt c o m"
|
||||
-> "a b c at hotmail punkt c o m"
|
||||
-> "a b c at h o t m a i l punkt com"
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
|
||||
graph_digit_no_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
).optimize() | pynini.cross("1", "eins")
|
||||
graph_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
).optimize()
|
||||
graph_digit = graph_digit_no_zero | graph_zero
|
||||
graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).optimize()
|
||||
server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv"))
|
||||
domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
||||
|
||||
def add_space_after_char():
|
||||
return pynini.closure(DAMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + (
|
||||
DAMO_NOT_QUOTE - pynini.accep(" ")
|
||||
)
|
||||
|
||||
verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", DAMO_SIGMA)
|
||||
|
||||
user_name = pynutil.delete('username: "') + add_space_after_char() + pynutil.delete('"')
|
||||
user_name @= verbalize_characters
|
||||
|
||||
convert_defaults = (
|
||||
pynutil.add_weight(DAMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common
|
||||
)
|
||||
domain = convert_defaults + pynini.closure(insert_space + convert_defaults)
|
||||
domain @= verbalize_characters
|
||||
|
||||
domain = pynutil.delete('domain: "') + domain + pynutil.delete('"')
|
||||
protocol = (
|
||||
pynutil.delete('protocol: "')
|
||||
+ add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", DAMO_SIGMA)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | (
|
||||
user_name + pynini.accep(" ") + pynutil.insert("at ") + domain
|
||||
)
|
||||
delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,68 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing fraction
|
||||
e.g. fraction { integer: "drei" numerator: "eins" denominator: "zwei" }-> drei ein halb
|
||||
e.g. fraction { numerator: "vier" denominator: "zwei" } -> vier halbe
|
||||
e.g. fraction { numerator: "drei" denominator: "vier" } -> drei viertel
|
||||
|
||||
Args:
|
||||
ordinal: ordinal GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, ordinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="fraction", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
optional_sign = pynini.closure(
|
||||
pynini.cross('negative: "true"', "minus ") + pynutil.delete(" "), 0, 1
|
||||
)
|
||||
change_one = pynini.cdrewrite(
|
||||
pynutil.add_weight(pynini.cross("eins", "ein"), weight=-0.0001),
|
||||
"[BOS]",
|
||||
"[EOS]",
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
change_numerator_two = pynini.cdrewrite(
|
||||
pynini.cross("zweitel", "halbe"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
integer = pynutil.delete('integer_part: "') + change_one + pynutil.delete('" ')
|
||||
numerator = pynutil.delete('numerator: "') + change_one + pynutil.delete('" ')
|
||||
denominator = (
|
||||
pynutil.delete('denominator: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE)
|
||||
@ (
|
||||
pynini.cdrewrite(
|
||||
pynini.closure(ordinal.ordinal_stem, 0, 1), "", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
+ pynutil.insert("tel")
|
||||
)
|
||||
@ change_numerator_two
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
integer += insert_space + pynini.closure(pynutil.insert("und ", weight=0.001), 0, 1)
|
||||
|
||||
denominator_one_half = pynini.cdrewrite(
|
||||
pynini.cross("ein halbe", "ein halb"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
|
||||
fraction_default = (numerator + insert_space + denominator) @ denominator_one_half
|
||||
|
||||
self.graph = optional_sign + pynini.closure(integer, 0, 1) + fraction_default
|
||||
|
||||
graph = self.graph + delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,41 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { cardinal { integer: "zwei" units: "unzen" } } -> "zwei unzen"
|
||||
measure { cardinal { integer_part: "zwei" quantity: "millionen" units: "unzen" } } -> "zwei millionen unzen"
|
||||
|
||||
Args:
|
||||
decimal: decimal GraphFst
|
||||
cardinal: cardinal GraphFst
|
||||
fraction: fraction GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool
|
||||
):
|
||||
super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
|
||||
unit = pynutil.delete('units: "') + pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('"')
|
||||
|
||||
graph_decimal = decimal.fst
|
||||
graph_cardinal = cardinal.fst
|
||||
graph_fraction = fraction.fst
|
||||
|
||||
graph = (graph_cardinal | graph_decimal | graph_fraction) + pynini.accep(" ") + unit
|
||||
|
||||
graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal)
|
||||
graph += delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,77 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { currency_maj: "euro" integer_part: "ein"} -> "ein euro"
|
||||
money { currency_maj: "euro" integer_part: "eins" fractional_part: "null null eins"} -> "eins komma null null eins euro"
|
||||
money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" preserve_order: true} -> "ein pfund vierzig"
|
||||
money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" currency_min: "pence" preserve_order: true} -> "ein pfund vierzig pence"
|
||||
money { fractional_part: "ein" currency_min: "penny" preserve_order: true} -> "ein penny"
|
||||
money { currency_maj: "pfund" integer_part: "null" fractional_part: "null eins" quantity: "million"} -> "null komma null eins million pfund"
|
||||
|
||||
Args:
|
||||
decimal: GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="money", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
keep_space = pynini.accep(" ")
|
||||
|
||||
maj = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
min = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
fractional_part = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
integer_part = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_add_and = pynini.closure(pynutil.insert("und "), 0, 1)
|
||||
|
||||
# *** currency_maj
|
||||
graph_integer = integer_part + keep_space + maj
|
||||
|
||||
# *** currency_maj + (***) | ((und) *** current_min)
|
||||
graph_integer_with_minor = (
|
||||
integer_part
|
||||
+ keep_space
|
||||
+ maj
|
||||
+ keep_space
|
||||
+ (fractional_part | (optional_add_and + fractional_part + keep_space + min))
|
||||
+ delete_preserve_order
|
||||
)
|
||||
|
||||
# *** komma *** currency_maj
|
||||
graph_decimal = decimal.fst + keep_space + maj
|
||||
|
||||
# *** current_min
|
||||
graph_minor = fractional_part + keep_space + min + delete_preserve_order
|
||||
|
||||
graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,45 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing roman numerals
|
||||
e.g. ordinal { integer: "vier" } } -> "vierter"
|
||||
-> "viertes" ...
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic)
|
||||
graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")).invert()
|
||||
graph_ties = pynini.string_file(get_abs_path("data/ordinals/ties.tsv")).invert()
|
||||
graph_thousands = pynini.string_file(get_abs_path("data/ordinals/thousands.tsv")).invert()
|
||||
|
||||
graph = (
|
||||
pynutil.delete('integer: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
suffixes = pynini.union("ten", "tem", "ter", "tes", "te")
|
||||
convert_rest = pynutil.insert(suffixes, weight=0.01)
|
||||
self.ordinal_stem = graph_digit | graph_ties | graph_thousands
|
||||
|
||||
suffix = pynini.cdrewrite(
|
||||
pynini.closure(self.ordinal_stem, 0, 1) + convert_rest,
|
||||
"",
|
||||
"[EOS]",
|
||||
DAMO_SIGMA,
|
||||
).optimize()
|
||||
self.graph = pynini.compose(graph, suffix)
|
||||
self.suffix = suffix
|
||||
delete_tokens = self.delete_tokens(self.graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,40 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing telephone, e.g.
|
||||
telephone { country_code: "plus neun und vierzig" number_part: "null eins eins eins null null null" }
|
||||
-> "plus neun und vierzig null eins eins eins null null null"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="telephone", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
country_code = (
|
||||
pynutil.delete('country_code: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
number_part = (
|
||||
pynutil.delete('number_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
self.graph = country_code + pynini.accep(" ") + number_part
|
||||
|
||||
graph = self.graph + delete_preserve_order
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,126 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic, e.g.
|
||||
time { hours: "2" minutes: "15"} -> "zwei uhr fünfzehn"
|
||||
time { minutes: "15" hours: "2" } -> "viertel nach zwei"
|
||||
time { minutes: "15" hours: "2" } -> "fünfzehn nach zwei"
|
||||
time { hours: "14" minutes: "15"} -> "vierzehn uhr fünfzehn"
|
||||
time { minutes: "15" hours: "14" } -> "viertel nach zwei"
|
||||
time { minutes: "15" hours: "14" } -> "fünfzehn nach drei"
|
||||
time { minutes: "45" hours: "14" } -> "viertel vor drei"
|
||||
|
||||
Args:
|
||||
cardinal_tagger: cardinal_tagger tagger GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
# add weight so when using inverse text normalization this conversion is depriotized
|
||||
night_to_early = pynutil.add_weight(
|
||||
pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/hour_to_night.tsv"))
|
||||
).optimize(),
|
||||
weight=0.0001,
|
||||
)
|
||||
hour_to = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/hour_to.tsv"))
|
||||
).optimize()
|
||||
minute_to = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/minute_to.tsv"))
|
||||
).optimize()
|
||||
time_zone_graph = pynini.invert(
|
||||
convert_space(
|
||||
pynini.union(*[x[1] for x in load_labels(get_abs_path("data/time/time_zone.tsv"))])
|
||||
)
|
||||
)
|
||||
|
||||
graph_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
).optimize()
|
||||
number_verbalization = graph_zero | cardinal_tagger.two_digit_non_zero
|
||||
hour = pynutil.delete('hours: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
hour_verbalized = hour @ number_verbalization @ pynini.cdrewrite(
|
||||
pynini.cross("eins", "ein"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
) + pynutil.insert(" uhr")
|
||||
minute = pynutil.delete('minutes: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
zone = pynutil.delete('zone: "') + time_zone_graph + pynutil.delete('"')
|
||||
optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1)
|
||||
second = pynutil.delete('seconds: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
graph_hms = (
|
||||
hour_verbalized
|
||||
+ pynini.accep(" ")
|
||||
+ minute @ number_verbalization
|
||||
+ pynutil.insert(" minuten")
|
||||
+ pynini.accep(" ")
|
||||
+ second @ number_verbalization
|
||||
+ pynutil.insert(" sekunden")
|
||||
+ optional_zone
|
||||
)
|
||||
graph_hms @= pynini.cdrewrite(
|
||||
pynini.cross("eins minuten", "eine minute")
|
||||
| pynini.cross("eins sekunden", "eine sekunde"),
|
||||
pynini.union(" ", "[BOS]"),
|
||||
"",
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
|
||||
min_30 = [str(x) for x in range(1, 31)]
|
||||
min_30 = pynini.union(*min_30)
|
||||
min_29 = [str(x) for x in range(1, 30)]
|
||||
min_29 = pynini.union(*min_29)
|
||||
|
||||
graph_h = hour_verbalized
|
||||
graph_hm = hour_verbalized + pynini.accep(" ") + minute @ number_verbalization
|
||||
|
||||
graph_m_past_h = (
|
||||
minute @ min_30 @ (number_verbalization | pynini.cross("15", "viertel"))
|
||||
+ pynini.accep(" ")
|
||||
+ pynutil.insert("nach ")
|
||||
# + hour @ number_verbalization
|
||||
+ hour
|
||||
@ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
@ number_verbalization
|
||||
)
|
||||
graph_m30_h = (
|
||||
minute @ pynini.cross("30", "halb")
|
||||
+ pynini.accep(" ")
|
||||
+ hour
|
||||
@ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
@ hour_to
|
||||
@ number_verbalization
|
||||
)
|
||||
graph_m_to_h = (
|
||||
minute @ minute_to @ min_29 @ (number_verbalization | pynini.cross("15", "viertel"))
|
||||
+ pynini.accep(" ")
|
||||
+ pynutil.insert("vor ")
|
||||
+ hour
|
||||
@ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
@ hour_to
|
||||
@ number_verbalization
|
||||
)
|
||||
|
||||
self.graph = (
|
||||
graph_hms
|
||||
| graph_h
|
||||
| graph_hm
|
||||
| pynutil.add_weight(graph_m_past_h, weight=0.0001)
|
||||
| pynutil.add_weight(graph_m30_h, weight=0.0001)
|
||||
| pynutil.add_weight(graph_m_to_h, weight=0.0001)
|
||||
) + optional_zone
|
||||
delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,64 @@
|
||||
from fun_text_processing.text_normalization.de.taggers.cardinal import CardinalFst as CardinalTagger
|
||||
from fun_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.date import DateFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.decimal import DecimalFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.electronic import ElectronicFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.fraction import FractionFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.ordinal import OrdinalFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.telephone import TelephoneFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.time import TimeFst
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
|
||||
from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
|
||||
cardinal_tagger = CardinalTagger(deterministic=deterministic)
|
||||
cardinal = CardinalFst(deterministic=deterministic)
|
||||
cardinal_graph = cardinal.fst
|
||||
ordinal = OrdinalFst(deterministic=deterministic)
|
||||
ordinal_graph = ordinal.fst
|
||||
decimal = DecimalFst(deterministic=deterministic)
|
||||
decimal_graph = decimal.fst
|
||||
fraction = FractionFst(ordinal=ordinal, deterministic=deterministic)
|
||||
fraction_graph = fraction.fst
|
||||
date = DateFst(ordinal=ordinal)
|
||||
date_graph = date.fst
|
||||
measure = MeasureFst(
|
||||
cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic
|
||||
)
|
||||
measure_graph = measure.fst
|
||||
electronic = ElectronicFst(deterministic=deterministic)
|
||||
electronic_graph = electronic.fst
|
||||
whitelist_graph = WhiteListFst(deterministic=deterministic).fst
|
||||
money_graph = MoneyFst(decimal=decimal).fst
|
||||
telephone_graph = TelephoneFst(deterministic=deterministic).fst
|
||||
time_graph = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic).fst
|
||||
|
||||
graph = (
|
||||
cardinal_graph
|
||||
| measure_graph
|
||||
| decimal_graph
|
||||
| ordinal_graph
|
||||
| date_graph
|
||||
| electronic_graph
|
||||
| money_graph
|
||||
| fraction_graph
|
||||
| whitelist_graph
|
||||
| telephone_graph
|
||||
| time_graph
|
||||
)
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.verbalizers.word import WordFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False
|
||||
):
|
||||
super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(
|
||||
cache_dir, f"de_tn_{deterministic}_deterministic_verbalizer.far"
|
||||
)
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["verbalize"]
|
||||
logging.info(f"VerbalizeFinalFst graph was restored from {far_file}.")
|
||||
else:
|
||||
verbalize = VerbalizeFst(deterministic=deterministic).fst
|
||||
word = WordFst(deterministic=deterministic).fst
|
||||
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
if far_file:
|
||||
generator_main(far_file, {"verbalize": self.fst})
|
||||
logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.")
|
||||
Reference in New Issue
Block a user