chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,47 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
add_cardinal_apocope_fem,
|
||||
shift_cardinal_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinals
|
||||
e.g. cardinal { integer: "dos" } -> "dos"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true" ', "menos "), 0, 1)
|
||||
self.optional_sign = optional_sign
|
||||
|
||||
integer = pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
self.integer = pynutil.delete(' "') + integer + pynutil.delete('"')
|
||||
|
||||
integer = pynutil.delete("integer:") + self.integer
|
||||
|
||||
graph_masc = optional_sign + integer
|
||||
graph_fem = shift_cardinal_gender(graph_masc)
|
||||
|
||||
self.graph_masc = pynini.optimize(graph_masc)
|
||||
self.graph_fem = pynini.optimize(graph_fem)
|
||||
|
||||
# Adding adjustment for fem gender (choice of gender will be random)
|
||||
graph = graph_masc | graph_fem
|
||||
|
||||
if not deterministic:
|
||||
# For alternate renderings when apocope is omitted (i.e. cardinal stands alone)
|
||||
graph |= strip_cardinal_apocope(graph_masc)
|
||||
# "una" will drop to "un" in unique contexts
|
||||
graph |= add_cardinal_apocope_fem(graph_fem)
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,75 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope
|
||||
from fun_text_processing.text_normalization.es.taggers.date import articles
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing date, e.g.
|
||||
date { day: "treinta y uno" month: "marzo" year: "dos mil" } -> "treinta y uno de marzo de dos mil"
|
||||
date { day: "uno" month: "mayo" year: "del mil novecientos noventa" } -> "primero de mayo del mil novecientos noventa"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="date", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
day_cardinal = (
|
||||
pynutil.delete('day: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
day = strip_cardinal_apocope(day_cardinal)
|
||||
|
||||
primero = pynini.cdrewrite(pynini.cross("uno", "primero"), "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
day = (
|
||||
(day @ primero) if deterministic else pynini.union(day, day @ primero)
|
||||
) # Primero for first day is traditional, but will vary depending on region
|
||||
|
||||
month = pynutil.delete('month: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
|
||||
year = (
|
||||
pynutil.delete('year: "')
|
||||
+ articles
|
||||
+ DAMO_SPACE
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
# Insert preposition if wasn't originally with the year. This would mean a space was present
|
||||
year = pynutil.add_weight(year, -0.001)
|
||||
year |= (
|
||||
pynutil.delete('year: "')
|
||||
+ pynutil.insert("de ")
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
# day month year
|
||||
graph_dmy = (
|
||||
day
|
||||
+ pynini.cross(DAMO_SPACE, " de ")
|
||||
+ month
|
||||
+ pynini.closure(pynini.accep(" ") + year, 0, 1)
|
||||
)
|
||||
|
||||
graph_mdy = month + DAMO_SPACE + day + pynini.closure(DAMO_SPACE + year, 0, 1)
|
||||
if deterministic:
|
||||
graph_mdy += pynutil.delete(
|
||||
" preserve_order: true"
|
||||
) # Only accepts this if was explicitly passed
|
||||
|
||||
self.graph = graph_dmy | graph_mdy
|
||||
final_graph = self.graph + delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(final_graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,89 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es import LOCALIZATION
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
add_cardinal_apocope_fem,
|
||||
shift_cardinal_gender,
|
||||
shift_number_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones
|
||||
decimal { integer_part: "un" quantity: "billón" } -> un billón
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
||||
|
||||
optional_sign = pynini.closure(
|
||||
pynini.cross('negative: "true"', "menos ") + delete_space, 0, 1
|
||||
)
|
||||
integer = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
fractional_default = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
conjunction = (
|
||||
pynutil.insert(" punto ") if LOCALIZATION == "am" else pynutil.insert(" coma ")
|
||||
)
|
||||
if not deterministic:
|
||||
conjunction |= pynutil.insert(pynini.union(" con ", " y "))
|
||||
fractional_default |= strip_cardinal_apocope(fractional_default)
|
||||
fractional = conjunction + fractional_default
|
||||
|
||||
quantity = (
|
||||
delete_space
|
||||
+ insert_space
|
||||
+ pynutil.delete('quantity: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_quantity = pynini.closure(quantity, 0, 1)
|
||||
|
||||
graph_masc = optional_sign + pynini.union(
|
||||
(integer + quantity), (integer + delete_space + fractional + optional_quantity)
|
||||
)
|
||||
|
||||
# Allowing permutation for fem gender, don't include quantity since "million","billion", etc.. are masculine
|
||||
graph_fem = optional_sign + (
|
||||
shift_cardinal_gender(integer) + delete_space + shift_number_gender(fractional)
|
||||
)
|
||||
if not deterministic: # "una" will drop to "un" in certain cases
|
||||
graph_fem |= add_cardinal_apocope_fem(graph_fem)
|
||||
|
||||
self.numbers_only_quantity = (
|
||||
optional_sign
|
||||
+ pynini.union(
|
||||
(integer + quantity), (integer + delete_space + fractional + quantity)
|
||||
).optimize()
|
||||
)
|
||||
|
||||
self.graph_masc = (graph_masc + delete_preserve_order).optimize()
|
||||
self.graph_fem = (graph_fem + delete_preserve_order).optimize()
|
||||
|
||||
graph = graph_masc | graph_fem
|
||||
|
||||
graph += delete_preserve_order
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,67 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv")))
|
||||
zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv")))
|
||||
|
||||
graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv"))
|
||||
server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv"))
|
||||
domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic
|
||||
e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c arroba hotmail punto com"
|
||||
-> "a b c arroba h o t m a i l punto c o m"
|
||||
-> "a b c arroba hotmail punto c o m"
|
||||
-> "a b c at h o t m a i l punto com"
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
graph_digit_no_zero = (
|
||||
digit_no_zero
|
||||
@ pynini.cdrewrite(pynini.cross("un", "uno"), "", "", DAMO_SIGMA).optimize()
|
||||
)
|
||||
graph_digit = graph_digit_no_zero | zero
|
||||
|
||||
def add_space_after_char():
|
||||
return pynini.closure(DAMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + (
|
||||
DAMO_NOT_QUOTE - pynini.accep(" ")
|
||||
)
|
||||
|
||||
verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", DAMO_SIGMA)
|
||||
|
||||
user_name = pynutil.delete('username: "') + add_space_after_char() + pynutil.delete('"')
|
||||
user_name @= verbalize_characters
|
||||
|
||||
convert_defaults = (
|
||||
pynutil.add_weight(DAMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common
|
||||
)
|
||||
domain = convert_defaults + pynini.closure(insert_space + convert_defaults)
|
||||
domain @= verbalize_characters
|
||||
|
||||
domain = pynutil.delete('domain: "') + domain + pynutil.delete('"')
|
||||
protocol = (
|
||||
pynutil.delete('protocol: "')
|
||||
+ add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", DAMO_SIGMA)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | (
|
||||
user_name + pynini.accep(" ") + pynutil.insert("arroba ") + domain
|
||||
)
|
||||
delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,195 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_NOT_SPACE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
accents,
|
||||
shift_cardinal_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing fraction
|
||||
e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } ->
|
||||
treinta y tres y cuatro quintos
|
||||
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="fraction", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
# Derivational strings append 'avo' as a suffix. Adding space for processing aid
|
||||
fraction_stem = pynutil.insert(" avo")
|
||||
plural = pynutil.insert("s")
|
||||
conjunction = pynutil.insert(" y ")
|
||||
|
||||
integer = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ strip_cardinal_apocope(pynini.closure(DAMO_NOT_QUOTE))
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
numerator_one = pynutil.delete('numerator: "') + pynini.accep("un") + pynutil.delete('" ')
|
||||
numerator = (
|
||||
pynutil.delete('numerator: "')
|
||||
+ pynini.difference(pynini.closure(DAMO_NOT_QUOTE), "un")
|
||||
+ pynutil.delete('" ')
|
||||
)
|
||||
|
||||
denominator_add_stem = pynutil.delete('denominator: "') + (
|
||||
pynini.closure(DAMO_NOT_QUOTE)
|
||||
+ fraction_stem
|
||||
+ pynutil.delete('" morphosyntactic_features: "add_root"')
|
||||
)
|
||||
denominator_ordinal = pynutil.delete('denominator: "') + (
|
||||
pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('" morphosyntactic_features: "ordinal"')
|
||||
)
|
||||
denominator_cardinal = pynutil.delete('denominator: "') + (
|
||||
pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
denominator_singular = pynini.union(denominator_add_stem, denominator_ordinal)
|
||||
if not deterministic:
|
||||
# Occasional exceptions
|
||||
denominator_singular |= denominator_add_stem @ pynini.string_map(
|
||||
[("once avo", "undécimo"), ("doce avo", "duodécimo")]
|
||||
)
|
||||
denominator_plural = denominator_singular + plural
|
||||
|
||||
# Merging operations
|
||||
merge = pynini.cdrewrite(
|
||||
pynini.cross(" y ", "i"), "", "", DAMO_SIGMA
|
||||
) # The denominator must be a single word, with the conjunction "y" replaced by i
|
||||
merge @= pynini.cdrewrite(
|
||||
delete_space, "", pynini.difference(DAMO_CHAR, "parte"), DAMO_SIGMA
|
||||
)
|
||||
|
||||
# The merger can produce duplicate vowels. This is not allowed in orthography
|
||||
delete_duplicates = pynini.string_map([("aa", "a"), ("oo", "o")]) # Removes vowels
|
||||
delete_duplicates = pynini.cdrewrite(delete_duplicates, "", "", DAMO_SIGMA)
|
||||
|
||||
remove_accents = pynini.cdrewrite(
|
||||
accents,
|
||||
pynini.union(DAMO_SPACE, pynini.accep("[BOS]")) + pynini.closure(DAMO_NOT_SPACE),
|
||||
pynini.closure(DAMO_NOT_SPACE) + pynini.union("avo", "ava", "ésimo", "ésima"),
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
merge_into_single_word = merge @ remove_accents @ delete_duplicates
|
||||
|
||||
fraction_default = (
|
||||
numerator + delete_space + insert_space + (denominator_plural @ merge_into_single_word)
|
||||
)
|
||||
|
||||
fraction_with_one = (
|
||||
numerator_one
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (denominator_singular @ merge_into_single_word)
|
||||
)
|
||||
|
||||
fraction_with_cardinal = strip_cardinal_apocope(numerator | numerator_one)
|
||||
fraction_with_cardinal += (
|
||||
delete_space + pynutil.insert(" sobre ") + strip_cardinal_apocope(denominator_cardinal)
|
||||
)
|
||||
|
||||
if not deterministic:
|
||||
# There is an alternative rendering where ordinals act as adjectives for 'parte'. This requires use of the feminine
|
||||
# Other rules will manage use of "un" at end, so just worry about endings
|
||||
exceptions = pynini.string_map([("tercia", "tercera")])
|
||||
apply_exceptions = pynini.cdrewrite(exceptions, "", "", DAMO_SIGMA)
|
||||
vowel_change = pynini.cdrewrite(
|
||||
pynini.cross("o", "a"), "", pynini.accep("[EOS]"), DAMO_SIGMA
|
||||
)
|
||||
|
||||
denominator_singular_fem = (
|
||||
shift_cardinal_gender(denominator_singular) @ vowel_change @ apply_exceptions
|
||||
)
|
||||
denominator_plural_fem = denominator_singular_fem + plural
|
||||
|
||||
numerator_one_fem = shift_cardinal_gender(numerator_one)
|
||||
numerator_fem = shift_cardinal_gender(numerator)
|
||||
|
||||
fraction_with_cardinal |= (
|
||||
(numerator_one_fem | numerator_fem)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" sobre ")
|
||||
+ shift_cardinal_gender(denominator_cardinal)
|
||||
)
|
||||
|
||||
# Still need to manage stems
|
||||
merge_stem = pynini.cdrewrite(
|
||||
delete_space, "", pynini.union("avo", "ava", "avos", "avas"), DAMO_SIGMA
|
||||
) # For managing alternative spacing
|
||||
merge_stem @= remove_accents @ delete_duplicates
|
||||
|
||||
fraction_with_one_fem = numerator_one_fem + delete_space + insert_space
|
||||
fraction_with_one_fem += pynini.union(
|
||||
denominator_singular_fem @ merge_stem,
|
||||
denominator_singular_fem @ merge_into_single_word,
|
||||
) # Both forms exists
|
||||
fraction_with_one_fem += pynutil.insert(" parte")
|
||||
fraction_with_one_fem @= pynini.cdrewrite(
|
||||
pynini.cross("una media", "media"), "", "", DAMO_SIGMA
|
||||
) # "media" not "una media"
|
||||
|
||||
fraction_default_fem = numerator_fem + delete_space + insert_space
|
||||
fraction_default_fem += pynini.union(
|
||||
denominator_plural_fem @ merge_stem, denominator_plural_fem @ merge_into_single_word
|
||||
)
|
||||
fraction_default_fem += pynutil.insert(" partes")
|
||||
|
||||
fraction_default |= (
|
||||
numerator + delete_space + insert_space + denominator_plural @ merge_stem
|
||||
) # Case of no merger
|
||||
fraction_default |= fraction_default_fem
|
||||
|
||||
fraction_with_one |= (
|
||||
numerator_one + delete_space + insert_space + denominator_singular @ merge_stem
|
||||
)
|
||||
fraction_with_one |= fraction_with_one_fem
|
||||
|
||||
fraction_with_one @= pynini.cdrewrite(
|
||||
pynini.cross("un medio", "medio"), "", "", DAMO_SIGMA
|
||||
) # "medio" not "un medio"
|
||||
|
||||
fraction = fraction_with_one | fraction_default | fraction_with_cardinal
|
||||
graph_masc = pynini.closure(integer + delete_space + conjunction, 0, 1) + fraction
|
||||
|
||||
# Manage cases of fem gender (only shows on integer except for "medio")
|
||||
integer_fem = shift_cardinal_gender(integer)
|
||||
fraction_default |= (
|
||||
shift_cardinal_gender(numerator)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (denominator_plural @ pynini.cross("medios", "medias"))
|
||||
)
|
||||
fraction_with_one |= (
|
||||
pynutil.delete(numerator_one)
|
||||
+ delete_space
|
||||
+ (denominator_singular @ pynini.cross("medio", "media"))
|
||||
)
|
||||
|
||||
fraction_fem = fraction_with_one | fraction_default | fraction_with_cardinal
|
||||
graph_fem = pynini.closure(integer_fem + delete_space + conjunction, 0, 1) + fraction_fem
|
||||
|
||||
self.graph_masc = pynini.optimize(graph_masc)
|
||||
self.graph_fem = pynini.optimize(graph_fem)
|
||||
|
||||
self.graph = graph_masc | graph_fem
|
||||
|
||||
delete_tokens = self.delete_tokens(self.graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,118 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
DAMO_WHITE_SPACE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import ones
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv"))
|
||||
unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv"))
|
||||
|
||||
unit_singular_fem = pynini.project(unit_plural_fem, "input")
|
||||
unit_singular_masc = pynini.project(unit_plural_masc, "input")
|
||||
|
||||
unit_plural_fem = pynini.project(unit_plural_fem, "output")
|
||||
unit_plural_masc = pynini.project(unit_plural_masc, "output")
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { cardinal { integer: "dos" units: "gramos" } } -> "dos gramos"
|
||||
measure { decimal { integer_part: "dos" quantity: "millones" units: "gramos" } } -> "dos millones de gramos"
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
cardinal: CardinalFst
|
||||
fraction: FractionFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool
|
||||
):
|
||||
super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
graph_decimal_masc = decimal.delete_tokens(decimal.graph_masc)
|
||||
graph_decimal_fem = decimal.delete_tokens(decimal.graph_fem)
|
||||
graph_cardinal_masc = cardinal.delete_tokens(cardinal.graph_masc)
|
||||
graph_cardinal_fem = cardinal.delete_tokens(cardinal.graph_fem)
|
||||
graph_fraction_fem = fraction.delete_tokens(fraction.graph_fem)
|
||||
graph_fraction_masc = fraction.delete_tokens(fraction.graph_masc)
|
||||
|
||||
unit_masc = (unit_plural_masc | unit_singular_masc) + pynini.closure(
|
||||
DAMO_WHITE_SPACE + "por" + pynini.closure(DAMO_NOT_QUOTE, 1), 0, 1
|
||||
)
|
||||
unit_masc |= "por" + pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
unit_masc = (
|
||||
pynutil.delete('units: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE) @ unit_masc)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
unit_fem = (unit_plural_fem | unit_singular_fem) + pynini.closure(
|
||||
DAMO_WHITE_SPACE + "por" + pynini.closure(DAMO_NOT_QUOTE, 1), 0, 1
|
||||
)
|
||||
unit_fem = (
|
||||
pynutil.delete('units: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE) @ unit_fem)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph_masc = (graph_cardinal_masc | graph_decimal_masc) + DAMO_WHITE_SPACE + unit_masc
|
||||
graph_masc |= graph_fraction_masc + DAMO_WHITE_SPACE + pynutil.insert("de ") + unit_masc
|
||||
graph_masc |= pynutil.add_weight(
|
||||
graph_fraction_masc @ (DAMO_SIGMA + pynini.union("medio", "medios"))
|
||||
+ DAMO_WHITE_SPACE
|
||||
+ unit_masc,
|
||||
-0.001,
|
||||
) # "medio litro" not "medio de litro"
|
||||
|
||||
graph_fem = (graph_cardinal_fem | graph_decimal_fem) + DAMO_WHITE_SPACE + unit_fem
|
||||
graph_fem |= graph_fraction_fem + DAMO_WHITE_SPACE + pynutil.insert("de ") + unit_fem
|
||||
graph_fem |= pynutil.add_weight(
|
||||
graph_fraction_fem @ (DAMO_SIGMA + pynini.union("media", "medias"))
|
||||
+ DAMO_WHITE_SPACE
|
||||
+ unit_fem,
|
||||
-0.001,
|
||||
)
|
||||
|
||||
graph = graph_masc | graph_fem
|
||||
|
||||
graph = (
|
||||
pynini.cdrewrite(
|
||||
pynutil.insert(" de"),
|
||||
'quantity: "' + pynini.closure(DAMO_NOT_QUOTE, 1),
|
||||
'"',
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
@ graph
|
||||
) # billones de xyz
|
||||
|
||||
graph @= pynini.cdrewrite(
|
||||
pynini.cross(ones, "uno"), "", DAMO_WHITE_SPACE + "por", DAMO_SIGMA
|
||||
)
|
||||
|
||||
# To manage alphanumeric combonations ("a-8, 5x"), we let them use a weighted default path.
|
||||
alpha_num_unit = (
|
||||
pynutil.delete('units: "') + pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('"')
|
||||
)
|
||||
graph_alpha_num = pynini.union(
|
||||
(graph_cardinal_masc | graph_decimal_masc) + DAMO_SPACE + alpha_num_unit,
|
||||
alpha_num_unit + delete_extra_space + (graph_cardinal_masc | graph_decimal_masc),
|
||||
)
|
||||
|
||||
graph |= pynutil.add_weight(graph_alpha_num, 0.01)
|
||||
|
||||
graph += delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,169 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
shift_cardinal_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv")))
|
||||
masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv")))
|
||||
|
||||
fem_singular = pynini.project(fem, "input")
|
||||
masc_singular = pynini.project(masc, "input")
|
||||
|
||||
fem_plural = pynini.project(fem, "output")
|
||||
masc_plural = pynini.project(masc, "output")
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { currency_maj: "euro" integer_part: "un"} -> "un euro"
|
||||
money { currency_maj: "euro" integer_part: "un" fractional_part: "cero cero un"} -> "uno coma cero cero uno euros"
|
||||
money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" preserve_order: true} -> "una libra cuarenta"
|
||||
money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" currency_min: "peniques" preserve_order: true} -> "una libra con cuarenta peniques"
|
||||
money { fractional_part: "un" currency_min: "penique" preserve_order: true} -> "un penique"
|
||||
|
||||
Args:
|
||||
decimal: GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="money", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
maj_singular_masc = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
maj_singular_fem = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
maj_plural_masc = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
maj_plural_fem = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
maj_masc = maj_plural_masc | maj_singular_masc # Tagger kept quantity resolution stable
|
||||
maj_fem = maj_plural_fem | maj_singular_fem
|
||||
|
||||
min_singular_masc = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
min_singular_fem = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
min_plural_masc = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
min_plural_fem = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
min_masc = min_plural_masc | min_singular_masc
|
||||
min_fem = min_plural_fem | min_singular_fem
|
||||
|
||||
fractional_part = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
integer_part = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_add_and = pynini.closure(pynutil.insert(pynini.union("con ", "y ")), 0, 1)
|
||||
|
||||
# *** currency_maj
|
||||
graph_integer_masc = integer_part + DAMO_SPACE + maj_masc
|
||||
graph_integer_fem = shift_cardinal_gender(integer_part) + DAMO_SPACE + maj_fem
|
||||
|
||||
graph_integer = graph_integer_fem | graph_integer_masc
|
||||
|
||||
# *** currency_maj + (***) | ((con) *** current_min)
|
||||
graph_integer_with_minor_masc = (
|
||||
graph_integer_masc
|
||||
+ DAMO_SPACE
|
||||
+ pynini.union(
|
||||
optional_add_and + strip_cardinal_apocope(fractional_part),
|
||||
(optional_add_and + fractional_part + DAMO_SPACE + min_masc),
|
||||
(optional_add_and + shift_cardinal_gender(fractional_part) + DAMO_SPACE + min_fem),
|
||||
) # Could be minor currency that is different gender
|
||||
+ delete_preserve_order
|
||||
)
|
||||
|
||||
graph_integer_with_minor_fem = (
|
||||
graph_integer_fem
|
||||
+ DAMO_SPACE
|
||||
+ pynini.union(
|
||||
optional_add_and + shift_cardinal_gender(fractional_part),
|
||||
(optional_add_and + fractional_part + DAMO_SPACE + min_masc),
|
||||
(optional_add_and + shift_cardinal_gender(fractional_part) + DAMO_SPACE + min_fem),
|
||||
) # Could be minor currency that is different gender
|
||||
+ delete_preserve_order
|
||||
)
|
||||
|
||||
graph_integer_with_minor = graph_integer_with_minor_fem | graph_integer_with_minor_masc
|
||||
|
||||
## *** coma *** currency_maj
|
||||
graph_decimal_masc = decimal.graph_masc + DAMO_SPACE + maj_masc
|
||||
|
||||
graph_decimal_fem = decimal.graph_fem
|
||||
graph_decimal_fem |= (
|
||||
decimal.numbers_only_quantity
|
||||
) # can still have "x billions" with fem currency
|
||||
graph_decimal_fem += DAMO_SPACE + maj_fem
|
||||
|
||||
graph_decimal = graph_decimal_fem | graph_decimal_masc
|
||||
graph_decimal = (
|
||||
pynini.cdrewrite(
|
||||
pynutil.insert(" de"),
|
||||
'quantity: "' + pynini.closure(DAMO_NOT_QUOTE, 1),
|
||||
'"',
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
@ graph_decimal
|
||||
) # formally it's millones/billones de ***
|
||||
|
||||
# *** current_min
|
||||
graph_minor_masc = fractional_part + DAMO_SPACE + min_masc + delete_preserve_order
|
||||
graph_minor_fem = (
|
||||
shift_cardinal_gender(fractional_part) + DAMO_SPACE + min_fem + delete_preserve_order
|
||||
)
|
||||
|
||||
graph_minor = graph_minor_fem | graph_minor_masc
|
||||
|
||||
graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,68 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import shift_number_gender
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing ordinals
|
||||
e.g. ordinal { integer: "tercer" } } -> "tercero"
|
||||
-> "tercera"
|
||||
-> "tercer"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
graph = (
|
||||
pynutil.delete('integer: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
# masculne gender we leave as is
|
||||
graph_masc = graph + pynutil.delete(' morphosyntactic_features: "gender_masc')
|
||||
|
||||
# shift gender
|
||||
graph_fem_ending = graph @ pynini.cdrewrite(
|
||||
pynini.cross("o", "a"), "", DAMO_SPACE | pynini.accep("[EOS]"), DAMO_SIGMA
|
||||
)
|
||||
graph_fem = shift_number_gender(graph_fem_ending) + pynutil.delete(
|
||||
' morphosyntactic_features: "gender_fem'
|
||||
)
|
||||
|
||||
# Apocope just changes tercero and primero. May occur if someone wrote 11.er (uncommon)
|
||||
graph_apocope = (
|
||||
pynini.cross("tercero", "tercer")
|
||||
| pynini.cross("primero", "primer")
|
||||
| pynini.cross("undécimo", "decimoprimer")
|
||||
) # In case someone wrote 11.er with deterministic
|
||||
graph_apocope = (
|
||||
graph @ pynini.cdrewrite(graph_apocope, "", "", DAMO_SIGMA)
|
||||
) + pynutil.delete(' morphosyntactic_features: "apocope')
|
||||
|
||||
graph = graph_apocope | graph_masc | graph_fem
|
||||
|
||||
if not deterministic:
|
||||
# Plural graph
|
||||
graph_plural = pynini.cdrewrite(
|
||||
pynutil.insert("s"),
|
||||
pynini.union("o", "a"),
|
||||
DAMO_SPACE | pynini.accep("[EOS]"),
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
|
||||
graph |= (graph @ graph_plural) + pynutil.delete("/plural")
|
||||
|
||||
self.graph = graph + pynutil.delete('"')
|
||||
|
||||
delete_tokens = self.delete_tokens(self.graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,26 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing telephone, e.g.
|
||||
telephone { number_part: "uno dos tres uno dos tres cinco seis siete ocho" }
|
||||
-> uno dos tres uno dos tres cinco seis siete ocho
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="telephone", kind="verbalize")
|
||||
|
||||
number_part = (
|
||||
pynutil.delete('number_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
delete_tokens = self.delete_tokens(number_part)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,249 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
alt_minutes = pynini.string_file(get_abs_path("data/time/alt_minutes.tsv"))
|
||||
|
||||
morning_times = pynini.string_file(get_abs_path("data/time/morning_times.tsv"))
|
||||
afternoon_times = pynini.string_file(get_abs_path("data/time/afternoon_times.tsv"))
|
||||
evening_times = pynini.string_file(get_abs_path("data/time/evening_times.tsv"))
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing time, e.g.
|
||||
time { hours: "doce" minutes: "media" suffix: "a m" } -> doce y media de la noche
|
||||
time { hours: "doce" } -> twelve o'clock
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
change_minutes = pynini.cdrewrite(
|
||||
alt_minutes, pynini.accep("[BOS]"), pynini.accep("[EOS]"), DAMO_SIGMA
|
||||
)
|
||||
|
||||
morning_phrases = pynini.cross("am", "de la mañana")
|
||||
afternoon_phrases = pynini.cross("pm", "de la tarde")
|
||||
evening_phrases = pynini.cross("pm", "de la noche")
|
||||
|
||||
# For the 12's
|
||||
mid_times = pynini.accep("doce")
|
||||
mid_phrases = (
|
||||
pynini.string_map([("pm", "del mediodía"), ("am", "de la noche")])
|
||||
if deterministic
|
||||
else pynini.string_map(
|
||||
[
|
||||
("pm", "de la mañana"),
|
||||
("pm", "del día"),
|
||||
("pm", "del mediodía"),
|
||||
("am", "de la noche"),
|
||||
("am", "de la medianoche"),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
hour = (
|
||||
pynutil.delete("hours:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
minute = (
|
||||
pynutil.delete("minutes:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
minute = (
|
||||
(minute @ change_minutes)
|
||||
if deterministic
|
||||
else pynini.union(minute, minute @ change_minutes)
|
||||
)
|
||||
|
||||
suffix = (
|
||||
pynutil.delete("suffix:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
zone = (
|
||||
pynutil.delete("zone:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1)
|
||||
second = (
|
||||
pynutil.delete("seconds:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph_hms = (
|
||||
hour
|
||||
+ pynutil.insert(" horas ")
|
||||
+ delete_space
|
||||
+ minute
|
||||
+ pynutil.insert(" minutos y ")
|
||||
+ delete_space
|
||||
+ second
|
||||
+ pynutil.insert(" segundos")
|
||||
)
|
||||
|
||||
graph_hm = hour + delete_space + pynutil.insert(" y ") + minute
|
||||
graph_hm |= pynini.union(
|
||||
(hour @ morning_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ morning_phrases),
|
||||
(hour @ afternoon_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ afternoon_phrases),
|
||||
(hour @ evening_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ evening_phrases),
|
||||
(hour @ mid_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ mid_phrases),
|
||||
)
|
||||
|
||||
graph_h = pynini.union(
|
||||
hour,
|
||||
(hour @ morning_times) + delete_space + insert_space + (suffix @ morning_phrases),
|
||||
(hour @ afternoon_times) + delete_space + insert_space + (suffix @ afternoon_phrases),
|
||||
(hour @ evening_times) + delete_space + insert_space + (suffix @ evening_phrases),
|
||||
(hour @ mid_times) + delete_space + insert_space + (suffix @ mid_phrases),
|
||||
)
|
||||
|
||||
graph = (graph_hms | graph_hm | graph_h) + optional_zone
|
||||
|
||||
if not deterministic:
|
||||
graph_style_1 = pynutil.delete(' style: "1"')
|
||||
graph_style_2 = pynutil.delete(' style: "2"')
|
||||
|
||||
graph_menos = hour + delete_space + pynutil.insert(" menos ") + minute + graph_style_1
|
||||
graph_menos |= (
|
||||
(hour @ morning_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ morning_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos |= (
|
||||
(hour @ afternoon_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ afternoon_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos |= (
|
||||
(hour @ evening_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ evening_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos |= (
|
||||
(hour @ mid_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ mid_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos += optional_zone
|
||||
|
||||
graph_para = minute + pynutil.insert(" para las ") + delete_space + hour + graph_style_2
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ morning_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ morning_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ afternoon_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ afternoon_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ evening_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ evening_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ mid_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ mid_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para += optional_zone
|
||||
graph_para @= pynini.cdrewrite(
|
||||
pynini.cross(" las ", " la "), "para", "una", DAMO_SIGMA
|
||||
) # Need agreement with one
|
||||
|
||||
graph |= graph_menos | graph_para
|
||||
delete_tokens = self.delete_tokens(graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,62 @@
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
|
||||
from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.date import DateFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.decimals import DecimalFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.electronic import ElectronicFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.fraction import FractionFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.ordinal import OrdinalFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.telephone import TelephoneFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.time import TimeFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
|
||||
cardinal = CardinalFst(deterministic=deterministic)
|
||||
cardinal_graph = cardinal.fst
|
||||
ordinal = OrdinalFst(deterministic=deterministic)
|
||||
ordinal_graph = ordinal.fst
|
||||
decimal = DecimalFst(deterministic=deterministic)
|
||||
decimal_graph = decimal.fst
|
||||
fraction = FractionFst(deterministic=deterministic)
|
||||
fraction_graph = fraction.fst
|
||||
date = DateFst(deterministic=deterministic)
|
||||
date_graph = date.fst
|
||||
measure = MeasureFst(
|
||||
cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic
|
||||
)
|
||||
measure_graph = measure.fst
|
||||
electronic = ElectronicFst(deterministic=deterministic)
|
||||
electronic_graph = electronic.fst
|
||||
whitelist_graph = WhiteListFst(deterministic=deterministic).fst
|
||||
money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst
|
||||
telephone_graph = TelephoneFst(deterministic=deterministic).fst
|
||||
time_graph = TimeFst(deterministic=deterministic).fst
|
||||
|
||||
graph = (
|
||||
cardinal_graph
|
||||
| measure_graph
|
||||
| decimal_graph
|
||||
| ordinal_graph
|
||||
| date_graph
|
||||
| electronic_graph
|
||||
| money_graph
|
||||
| fraction_graph
|
||||
| whitelist_graph
|
||||
| telephone_graph
|
||||
| time_graph
|
||||
)
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.verbalizers.word import WordFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.verbalize import VerbalizeFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False
|
||||
):
|
||||
super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(
|
||||
cache_dir, f"es_tn_{deterministic}_deterministic_verbalizer.far"
|
||||
)
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["verbalize"]
|
||||
logging.info(f"VerbalizeFinalFst graph was restored from {far_file}.")
|
||||
else:
|
||||
|
||||
verbalize = VerbalizeFst(deterministic=deterministic).fst
|
||||
word = WordFst(deterministic=deterministic).fst
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
if far_file:
|
||||
generator_main(far_file, {"verbalize": self.fst})
|
||||
logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.")
|
||||
Reference in New Issue
Block a user