chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,22 @@
|
||||
# Localization Note
|
||||
|
||||
Depending on locale, Spanish number strings will vary in formatting. In the EU and South American countries, it is common to use a period (".") or space to delineate groupings of three
|
||||
digits. e.g.
|
||||
`1.000.000` -> "un millón"
|
||||
`1 000 000` -> "un millón"
|
||||
|
||||
and commas (",") to seperate cardinal and decimal strings. e.g.
|
||||
|
||||
`1,00` -> "uno coma cero cero"
|
||||
|
||||
While Central and Northern America will use commas (",") to delineate groupings of three digits, e.g.
|
||||
`1,000,000` -> "un millón"
|
||||
|
||||
and periods (".") to seperate cardinal and decimal strings. e.g.
|
||||
|
||||
`1.00` -> "uno coma cero cero"
|
||||
|
||||
As inclusion of both forms will create inherrent ambiguity for verbalization, this module defaults to the former formatting (periods for cardinal delineation and commas for decimals).
|
||||
|
||||
To toggle the alternate formatting, you may edit the `LOCALIZATION` variable in `fun_text_processing.text_normalization.es.__init__` with the value of `'am'`. This will perform necessary
|
||||
adjustments to all affected classes.
|
||||
@@ -0,0 +1 @@
|
||||
LOCALIZATION = "eu" # Set to am for alternate formatting
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
1 enero
|
||||
2 febrero
|
||||
3 marzo
|
||||
4 abril
|
||||
5 mayo
|
||||
6 junio
|
||||
7 julio
|
||||
8 agosto
|
||||
9 septiembre
|
||||
10 octubre
|
||||
11 noviembre
|
||||
12 diciembre
|
||||
|
@@ -0,0 +1,14 @@
|
||||
.com punto com
|
||||
.org punto org
|
||||
.gov punto gov
|
||||
.uk punto uk
|
||||
.fr punto fr
|
||||
.net punto net
|
||||
.br punto br
|
||||
.in punto in
|
||||
.ru punto ru
|
||||
.de punto de
|
||||
.it punto it
|
||||
.es punto es
|
||||
.mx punto mx
|
||||
.us punto us
|
||||
|
@@ -0,0 +1,11 @@
|
||||
gmail
|
||||
nvidia
|
||||
outlook
|
||||
hotmail
|
||||
yahoo
|
||||
live
|
||||
yandex
|
||||
orange
|
||||
wanadoo
|
||||
web
|
||||
comcast
|
||||
|
@@ -0,0 +1,22 @@
|
||||
. punto
|
||||
- guión
|
||||
_ barra baja
|
||||
! signo de exclamación
|
||||
# almohadilla
|
||||
$ signo de dólar
|
||||
% signo de porcentaje
|
||||
& et
|
||||
' comilla
|
||||
* asterisco
|
||||
+ signo más
|
||||
/ barra
|
||||
= signo igual
|
||||
? signo de interrogación
|
||||
^ acento circunflej
|
||||
` acento grave
|
||||
{ llave izquierda
|
||||
| pleca
|
||||
} llave derecha
|
||||
~ tilde
|
||||
, coma
|
||||
: dos puntos
|
||||
|
@@ -0,0 +1,2 @@
|
||||
segundo medio
|
||||
tercero tercio
|
||||
|
@@ -0,0 +1,9 @@
|
||||
millón millonésimo
|
||||
millones millonésimo
|
||||
billón billonésimo
|
||||
billones billonésimo
|
||||
trillón trillonésimo
|
||||
trillones trillonésimo
|
||||
mil millones mil millonésimo
|
||||
mil billones mil billonésimo
|
||||
mil trillones mil trillonésimo
|
||||
|
@@ -0,0 +1,6 @@
|
||||
décimo décima
|
||||
centésimo centésima
|
||||
milésimo milésima
|
||||
millonésimo millonésima
|
||||
billonésimo billonésima
|
||||
trillonésimo trillonésima
|
||||
|
@@ -0,0 +1,17 @@
|
||||
cm centímetro
|
||||
g gramo
|
||||
h hora
|
||||
kg kilogramo
|
||||
km kilómetro
|
||||
km² kilómetro cuadrado
|
||||
l litro
|
||||
m metro
|
||||
m² metro cuadrado
|
||||
m³ metro cubico
|
||||
mph milla por hora
|
||||
ml mililitro
|
||||
mm milímetro
|
||||
ms milisegundo
|
||||
min minuto
|
||||
% por ciento
|
||||
s segundo
|
||||
|
@@ -0,0 +1,3 @@
|
||||
hora horas
|
||||
milla por hora millas por hora
|
||||
milla millas
|
||||
|
@@ -0,0 +1,15 @@
|
||||
centímetro centímetros
|
||||
gramo gramos
|
||||
kilogramo kilogramos
|
||||
kilómetro kilómetros
|
||||
kilómetro cuadrado kilómetros cuadrados
|
||||
litro litros
|
||||
metro metros
|
||||
metro cuadrado metros cuadrados
|
||||
metro cubico metros cubicos
|
||||
mililitro mililitros
|
||||
milímetro milímetros
|
||||
milisegundo milisegundos
|
||||
minuto minutos
|
||||
segundo segundos
|
||||
por ciento por ciento
|
||||
|
@@ -0,0 +1,29 @@
|
||||
€ euro
|
||||
$ dólar
|
||||
£ libra
|
||||
₩ won
|
||||
nzd dólar de Nueva Zelanda
|
||||
rs rupia
|
||||
chf franco suizo
|
||||
fr franco
|
||||
dkk corona danesa
|
||||
kr corona
|
||||
fim marco finlandes
|
||||
mk marco
|
||||
aed dírham
|
||||
dh dírham
|
||||
dhs dírham
|
||||
¥ yen
|
||||
czk corona checa
|
||||
kč corona checa
|
||||
crc colón costarricense
|
||||
₡ colón
|
||||
hkd dólar de Hong Kong
|
||||
awg florín
|
||||
nok corona noruega
|
||||
sek corona sueca
|
||||
R$ real
|
||||
BRL real
|
||||
ESP peseta
|
||||
₧ peseta
|
||||
Ptas peseta
|
||||
|
@@ -0,0 +1,4 @@
|
||||
$ centavo
|
||||
€ céntimo
|
||||
£ penique
|
||||
¥ centavo
|
||||
|
@@ -0,0 +1,8 @@
|
||||
rupia rupias
|
||||
libra libras
|
||||
corona danesa coronas danesas
|
||||
corona coronas
|
||||
corona checa coronas checas
|
||||
corona noruega coronas noruegas
|
||||
corona sueca coronas suecas
|
||||
peseta pesetas
|
||||
|
@@ -0,0 +1,19 @@
|
||||
euro euros
|
||||
dólar dólares
|
||||
dólar estadounidense dólares estadounidenses
|
||||
won wones
|
||||
dólar de Nueva Zelanda dólares de Nueva Zelanda
|
||||
franco suizo francos suizos
|
||||
franco francos
|
||||
marco finlandes marcos finlandeses
|
||||
marco marcos
|
||||
dírham dírhams
|
||||
dírham dírhams
|
||||
dírham dírhams
|
||||
yen yenes
|
||||
colón costarricense colónes costarricenses
|
||||
colón colónes
|
||||
dólar de Hong Kong dólares de Hong Kong
|
||||
centavo centavos
|
||||
céntimo céntimos
|
||||
penique peniques
|
||||
|
@@ -0,0 +1,9 @@
|
||||
un 1
|
||||
dos 2
|
||||
tres 3
|
||||
cuatro 4
|
||||
cinco 5
|
||||
seis 6
|
||||
siete 7
|
||||
ocho 8
|
||||
nueve 9
|
||||
|
@@ -0,0 +1,8 @@
|
||||
doscientos 2
|
||||
trescientos 3
|
||||
cuatrocientos 4
|
||||
quinientos 5
|
||||
seiscientos 6
|
||||
setecientos 7
|
||||
ochocientos 8
|
||||
novecientos 9
|
||||
|
@@ -0,0 +1,6 @@
|
||||
millón
|
||||
millones
|
||||
billón
|
||||
billones
|
||||
trillón
|
||||
trillones
|
||||
|
@@ -0,0 +1,10 @@
|
||||
diez 10
|
||||
once 11
|
||||
doce 12
|
||||
trece 13
|
||||
catorce 14
|
||||
quince 15
|
||||
dieciséis 16
|
||||
diecisiete 17
|
||||
dieciocho 18
|
||||
diecinueve 19
|
||||
|
@@ -0,0 +1,7 @@
|
||||
treinta 3
|
||||
cuarenta 4
|
||||
cincuenta 5
|
||||
sesenta 6
|
||||
setenta 7
|
||||
ochenta 8
|
||||
noventa 9
|
||||
|
@@ -0,0 +1,10 @@
|
||||
veinte 20
|
||||
veintiún 21
|
||||
veintidós 22
|
||||
veintitrés 23
|
||||
veinticuatro 24
|
||||
veinticinco 25
|
||||
veintiséis 26
|
||||
veintisiete 27
|
||||
veintiocho 28
|
||||
veintinueve 29
|
||||
|
@@ -0,0 +1 @@
|
||||
cero 0
|
||||
|
@@ -0,0 +1,11 @@
|
||||
primero uno
|
||||
primera una
|
||||
primero un
|
||||
segundo dos
|
||||
tercero tres
|
||||
cuarto cuatro
|
||||
quinto cinco
|
||||
sexto seis
|
||||
séptimo siete
|
||||
octavo ocho
|
||||
noveno nueve
|
||||
|
@@ -0,0 +1,18 @@
|
||||
centésimo ciento
|
||||
centésimo cien
|
||||
ducentésimo doscientos
|
||||
ducentésima doscientas
|
||||
tricentésimo trescientos
|
||||
tricentésima trescientas
|
||||
cuadringentésimo cuatrocientos
|
||||
cuadringentésima cuatrocientas
|
||||
quingentésimo quinientos
|
||||
quingentésima quinientas
|
||||
sexcentésimo seiscientos
|
||||
sexcentésima seiscientas
|
||||
septingentésimo setecientos
|
||||
septingentésima setecientas
|
||||
octingentésimo ochocientos
|
||||
octingentésima ochocientas
|
||||
noningentésimo novecientos
|
||||
noningentésima novecientas
|
||||
|
@@ -0,0 +1,10 @@
|
||||
décimo diez
|
||||
undécimo once
|
||||
duodécimo doce
|
||||
decimotercero trece
|
||||
decimocuarto catorce
|
||||
decimoquinto quince
|
||||
decimosexto dieciséis
|
||||
decimoséptimo diecisiete
|
||||
decimooctavo dieciocho
|
||||
decimonoveno diecinueve
|
||||
|
@@ -0,0 +1,8 @@
|
||||
vigésimo veinte
|
||||
trigésimo treinta
|
||||
cuadragésimo cuarenta
|
||||
quincuagésimo cincuenta
|
||||
sexagésimo sesenta
|
||||
septuagésimo setenta
|
||||
octogésimo ochenta
|
||||
nonagésimo noventa
|
||||
|
@@ -0,0 +1,11 @@
|
||||
vigesimoprimero veintiuno
|
||||
vigesimoprimera veintiuna
|
||||
vigesimoprimero veintiún
|
||||
vigesimosegundo veintidós
|
||||
vigesimotercero veintitrés
|
||||
vigesimocuarto veinticuatro
|
||||
vigesimoquinto veinticinco
|
||||
vigesimosexto veintiséis
|
||||
vigesimoséptimo veintisiete
|
||||
vigesimooctavo veintiocho
|
||||
vigesimonoveno veintinueve
|
||||
|
@@ -0,0 +1,9 @@
|
||||
i 1
|
||||
ii 2
|
||||
iii 3
|
||||
iv 4
|
||||
v 5
|
||||
vi 6
|
||||
vii 7
|
||||
viii 8
|
||||
ix 9
|
||||
|
@@ -0,0 +1,9 @@
|
||||
c 1
|
||||
cc 2
|
||||
ccc 3
|
||||
cd 4
|
||||
d 5
|
||||
dc 6
|
||||
dcc 7
|
||||
dccc 8
|
||||
cm 9
|
||||
|
@@ -0,0 +1,9 @@
|
||||
x 1
|
||||
xx 2
|
||||
xxx 3
|
||||
xl 4
|
||||
l 5
|
||||
lx 6
|
||||
lxx 7
|
||||
lxxx 8
|
||||
xc 9
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
una
|
||||
dos
|
||||
tres
|
||||
cuatro
|
||||
|
@@ -0,0 +1,2 @@
|
||||
quince cuarto
|
||||
treinta media
|
||||
|
@@ -0,0 +1,7 @@
|
||||
cinco
|
||||
seis
|
||||
siete
|
||||
ocho
|
||||
nueve
|
||||
diez
|
||||
once
|
||||
|
@@ -0,0 +1,12 @@
|
||||
1 12
|
||||
2 1
|
||||
3 2
|
||||
4 3
|
||||
5 4
|
||||
6 5
|
||||
7 6
|
||||
8 7
|
||||
9 8
|
||||
10 9
|
||||
11 10
|
||||
12 11
|
||||
|
@@ -0,0 +1,25 @@
|
||||
1 0
|
||||
2 1
|
||||
3 2
|
||||
4 3
|
||||
5 4
|
||||
6 5
|
||||
7 6
|
||||
8 7
|
||||
9 8
|
||||
10 9
|
||||
11 10
|
||||
12 11
|
||||
13 12
|
||||
14 13
|
||||
15 14
|
||||
16 15
|
||||
17 16
|
||||
18 17
|
||||
19 18
|
||||
20 19
|
||||
21 20
|
||||
22 21
|
||||
23 22
|
||||
0 23
|
||||
1 24
|
||||
|
@@ -0,0 +1,13 @@
|
||||
12 0
|
||||
1 13
|
||||
2 14
|
||||
3 15
|
||||
4 16
|
||||
5 17
|
||||
6 18
|
||||
7 19
|
||||
8 20
|
||||
9 21
|
||||
10 22
|
||||
11 23
|
||||
12 24
|
||||
|
@@ -0,0 +1,59 @@
|
||||
1 59
|
||||
2 58
|
||||
3 57
|
||||
4 56
|
||||
5 55
|
||||
6 54
|
||||
7 53
|
||||
8 52
|
||||
9 51
|
||||
10 50
|
||||
11 49
|
||||
12 48
|
||||
13 47
|
||||
14 46
|
||||
15 45
|
||||
16 44
|
||||
17 43
|
||||
18 42
|
||||
19 41
|
||||
20 40
|
||||
21 39
|
||||
22 38
|
||||
23 37
|
||||
24 36
|
||||
25 35
|
||||
26 34
|
||||
27 33
|
||||
28 32
|
||||
29 31
|
||||
30 30
|
||||
31 29
|
||||
32 28
|
||||
33 27
|
||||
34 26
|
||||
35 25
|
||||
36 24
|
||||
37 23
|
||||
38 22
|
||||
39 21
|
||||
40 20
|
||||
41 19
|
||||
42 18
|
||||
43 17
|
||||
44 16
|
||||
45 15
|
||||
46 14
|
||||
47 13
|
||||
48 12
|
||||
49 11
|
||||
50 10
|
||||
51 9
|
||||
52 8
|
||||
53 7
|
||||
54 6
|
||||
55 5
|
||||
56 4
|
||||
57 3
|
||||
58 2
|
||||
59 1
|
||||
|
@@ -0,0 +1,12 @@
|
||||
un
|
||||
una
|
||||
dos
|
||||
tres
|
||||
cuatro
|
||||
cinco
|
||||
seis
|
||||
siete
|
||||
ocho
|
||||
nueve
|
||||
diez
|
||||
once
|
||||
|
@@ -0,0 +1,20 @@
|
||||
p.m. pm
|
||||
p. m. pm
|
||||
p.m pm
|
||||
p m. pm
|
||||
pm pm
|
||||
P.M. pm
|
||||
P.M pm
|
||||
P M. pm
|
||||
P. M. pm
|
||||
PM pm
|
||||
a.m. am
|
||||
a.m. am
|
||||
a.m am
|
||||
a m. am
|
||||
am am
|
||||
A.M. am
|
||||
A.M am
|
||||
A M. am
|
||||
A. M. am
|
||||
AM am
|
||||
|
@@ -0,0 +1,7 @@
|
||||
cst c s t
|
||||
cet c e t
|
||||
pst p s t
|
||||
est e s t
|
||||
pt p t
|
||||
et e t
|
||||
gmt g m t
|
||||
|
@@ -0,0 +1,42 @@
|
||||
ud. usted
|
||||
Ud. usted
|
||||
uds. ustedes
|
||||
Uds. ustedes
|
||||
Vd. vosotros
|
||||
Vds. vosotros
|
||||
vd. vosotros
|
||||
vds. vosotros
|
||||
D. Don
|
||||
Da. Donna
|
||||
Dr. Doctor
|
||||
Dra. Doctora
|
||||
dr. doctor
|
||||
dra. doctora
|
||||
d. don
|
||||
da. donna
|
||||
E este
|
||||
EE. Estados Unidos
|
||||
ee. estados unidos
|
||||
Gob. gobierno
|
||||
gob. gobierno
|
||||
esq. esquina
|
||||
Av. avenida
|
||||
Avda. avenida
|
||||
av. avenida
|
||||
avda. avenida
|
||||
O oeste
|
||||
pág. página
|
||||
p.ej. por ejemplo
|
||||
Prof. profesor
|
||||
Profa. profesora
|
||||
prof. profesor
|
||||
profa. profesora
|
||||
S sur
|
||||
N norte
|
||||
Sr. señor
|
||||
sr. señor
|
||||
Sra. señora
|
||||
sra. señora
|
||||
Srta. señorita
|
||||
srta. señorita
|
||||
tel. teléfono
|
||||
|
@@ -0,0 +1,169 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, DAMO_SPACE
|
||||
from fun_text_processing.text_normalization.es import LOCALIZATION
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path, load_labels
|
||||
from pynini.lib import pynutil
|
||||
|
||||
digits = pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "input")
|
||||
tens = pynini.project(pynini.string_file(get_abs_path("data/numbers/ties.tsv")), "input")
|
||||
teens = pynini.project(pynini.string_file(get_abs_path("data/numbers/teen.tsv")), "input")
|
||||
twenties = pynini.project(pynini.string_file(get_abs_path("data/numbers/twenties.tsv")), "input")
|
||||
hundreds = pynini.project(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")), "input")
|
||||
|
||||
accents = pynini.string_map([("á", "a"), ("é", "e"), ("í", "i"), ("ó", "o"), ("ú", "u")])
|
||||
|
||||
if LOCALIZATION == "am": # Setting localization for central and northern america formatting
|
||||
cardinal_separator = pynini.string_map([",", DAMO_SPACE])
|
||||
decimal_separator = pynini.accep(".")
|
||||
else:
|
||||
cardinal_separator = pynini.string_map([".", DAMO_SPACE])
|
||||
decimal_separator = pynini.accep(",")
|
||||
|
||||
ones = pynini.union("un", "ún")
|
||||
fem_ones = pynini.union(
|
||||
pynini.cross("un", "una"), pynini.cross("ún", "una"), pynini.cross("uno", "una")
|
||||
)
|
||||
one_to_one_hundred = pynini.union(
|
||||
digits, "uno", tens, teens, twenties, tens + pynini.accep(" y ") + digits
|
||||
)
|
||||
fem_hundreds = hundreds @ pynini.cdrewrite(pynini.cross("ientos", "ientas"), "", "", DAMO_SIGMA)
|
||||
|
||||
|
||||
def strip_accent(fst: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Converts all accented vowels to non-accented equivalents
|
||||
|
||||
Args:
|
||||
fst: Any fst. Composes vowel conversion onto fst's output strings
|
||||
"""
|
||||
return fst @ pynini.cdrewrite(accents, "", "", DAMO_SIGMA)
|
||||
|
||||
|
||||
def shift_cardinal_gender(fst: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Applies gender conversion rules to a cardinal string. These include: rendering all masculine forms of "uno" (including apocopated forms) as "una" and
|
||||
Converting all gendered numbers in the hundreds series (200,300,400...) to feminine equivalent (e.g. "doscientos" -> "doscientas"). Conversion only applies
|
||||
to value place for <1000 and multiple of 1000. (e.g. "doscientos mil doscientos" -> "doscientas mil doscientas".) For place values greater than the thousands, there
|
||||
is no gender shift as the higher powers of ten ("millones", "billones") are masculine nouns and any conversion would be formally
|
||||
ungrammatical.
|
||||
e.g.
|
||||
"doscientos" -> "doscientas"
|
||||
"doscientos mil" -> "doscientas mil"
|
||||
"doscientos millones" -> "doscientos millones"
|
||||
"doscientos mil millones" -> "doscientos mil millones"
|
||||
"doscientos millones doscientos mil doscientos" -> "doscientos millones doscientas mil doscientas"
|
||||
|
||||
Args:
|
||||
fst: Any fst. Composes conversion onto fst's output strings
|
||||
"""
|
||||
before_mil = (
|
||||
DAMO_SPACE
|
||||
+ (pynini.accep("mil") | pynini.accep("milésimo"))
|
||||
+ pynini.closure(DAMO_SPACE + hundreds, 0, 1)
|
||||
+ pynini.closure(DAMO_SPACE + one_to_one_hundred, 0, 1)
|
||||
+ pynini.union(pynini.accep("[EOS]"), pynini.accep('"'), decimal_separator)
|
||||
)
|
||||
before_double_digits = pynini.closure(DAMO_SPACE + one_to_one_hundred, 0, 1) + pynini.union(
|
||||
pynini.accep("[EOS]"), pynini.accep('"')
|
||||
)
|
||||
|
||||
fem_allign = pynini.cdrewrite(
|
||||
fem_hundreds, "", before_mil, DAMO_SIGMA
|
||||
) # doscientas mil dosciento
|
||||
fem_allign @= pynini.cdrewrite(
|
||||
fem_hundreds, "", before_double_digits, DAMO_SIGMA
|
||||
) # doscientas mil doscienta
|
||||
|
||||
fem_allign @= pynini.cdrewrite(
|
||||
fem_ones, "", pynini.union("[EOS]", '"', decimal_separator), DAMO_SIGMA
|
||||
) # If before a quote or EOS, we know it's the end of a string
|
||||
|
||||
return fst @ fem_allign
|
||||
|
||||
|
||||
def shift_number_gender(fst: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Performs gender conversion on all verbalized numbers in output. All values in the hundreds series (200,300,400) are changed to
|
||||
feminine gender (e.g. "doscientos" -> "doscientas") and all forms of "uno" (including apocopated forms) are converted to "una".
|
||||
This has no boundary restriction and will perform shift across all values in output string.
|
||||
e.g.
|
||||
"doscientos" -> "doscientas"
|
||||
"doscientos millones" -> "doscientas millones"
|
||||
"doscientos millones doscientos" -> "doscientas millones doscientas"
|
||||
|
||||
Args:
|
||||
fst: Any fst. Composes conversion onto fst's output strings
|
||||
"""
|
||||
fem_allign = pynini.cdrewrite(fem_hundreds, "", "", DAMO_SIGMA)
|
||||
fem_allign @= pynini.cdrewrite(
|
||||
fem_ones, "", pynini.union(DAMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), DAMO_SIGMA
|
||||
) # If before a quote or EOS, we know it's the end of a string
|
||||
|
||||
return fst @ fem_allign
|
||||
|
||||
|
||||
def strip_cardinal_apocope(fst: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Reverts apocope on cardinal strings in line with formation rules. e.g. "un" -> "uno". Due to cardinal formation rules, this in effect only
|
||||
affects strings where the final value is a variation of "un".
|
||||
e.g.
|
||||
"un" -> "uno"
|
||||
"veintiún" -> "veintiuno"
|
||||
|
||||
Args:
|
||||
fst: Any fst. Composes conversion onto fst's output strings
|
||||
"""
|
||||
# Since cardinals use apocope by default for large values (e.g. "millón"), this only needs to act on the last instance of one
|
||||
strip = pynini.cross("un", "uno") | pynini.cross("ún", "uno")
|
||||
strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), DAMO_SIGMA)
|
||||
return fst @ strip
|
||||
|
||||
|
||||
def add_cardinal_apocope_fem(fst: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Adds apocope on cardinal strings in line with stressing rules. e.g. "una" -> "un". This only occurs when "una" precedes a stressed "a" sound in formal speech. This is not predictable
|
||||
with text string, so is included for non-deterministic cases.
|
||||
e.g.
|
||||
"una" -> "un"
|
||||
"veintiuna" -> "veintiun"
|
||||
|
||||
Args:
|
||||
fst: Any fst. Composes conversion onto fst's output strings
|
||||
"""
|
||||
# Since the stress trigger follows the cardinal string and only affects the preceding sound, this only needs to act on the last instance of one
|
||||
strip = pynini.cross("una", "un") | pynini.cross("veintiuna", "veintiún")
|
||||
strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), DAMO_SIGMA)
|
||||
return fst @ strip
|
||||
|
||||
|
||||
def roman_to_int(fst: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000.
|
||||
e.g.
|
||||
"V" -> "5"
|
||||
"i" -> "1"
|
||||
|
||||
Args:
|
||||
fst: Any fst. Composes fst onto Roman conversion outputs.
|
||||
"""
|
||||
|
||||
def _load_roman(file: str):
|
||||
roman = load_labels(get_abs_path(file))
|
||||
roman_numerals = [(x, y) for x, y in roman] + [(x.upper(), y) for x, y in roman]
|
||||
return pynini.string_map(roman_numerals)
|
||||
|
||||
digit = _load_roman("data/roman/digit.tsv")
|
||||
ties = _load_roman("data/roman/ties.tsv")
|
||||
hundreds = _load_roman("data/roman/hundreds.tsv")
|
||||
|
||||
graph = (
|
||||
digit
|
||||
| ties + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01))
|
||||
| (
|
||||
hundreds
|
||||
+ (ties | pynutil.add_weight(pynutil.insert("0"), 0.01))
|
||||
+ (digit | pynutil.add_weight(pynutil.insert("0"), 0.01))
|
||||
)
|
||||
).optimize()
|
||||
|
||||
return graph @ fst
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
DAMO_WHITE_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import cardinal_separator
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv")))
|
||||
digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv")))
|
||||
teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv")))
|
||||
ties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv")))
|
||||
twenties = pynini.invert(pynini.string_file(get_abs_path("data/numbers/twenties.tsv")))
|
||||
hundreds = pynini.invert(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")))
|
||||
|
||||
|
||||
def filter_punctuation(fst: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Helper function for parsing number strings. Converts common cardinal strings (groups of three digits delineated by 'cardinal_separator' - see graph_utils)
|
||||
and converts to a string of digits:
|
||||
"1 000" -> "1000"
|
||||
"1.000.000" -> "1000000"
|
||||
Args:
|
||||
fst: Any pynini.FstLike object. Function composes fst onto string parser fst
|
||||
|
||||
Returns:
|
||||
fst: A pynini.FstLike object
|
||||
"""
|
||||
exactly_three_digits = DAMO_DIGIT**3 # for blocks of three
|
||||
up_to_three_digits = pynini.closure(DAMO_DIGIT, 1, 3) # for start of string
|
||||
|
||||
cardinal_string = pynini.closure(
|
||||
DAMO_DIGIT, 1
|
||||
) # For string w/o punctuation (used for page numbers, thousand series)
|
||||
|
||||
cardinal_string |= (
|
||||
up_to_three_digits
|
||||
+ pynutil.delete(cardinal_separator)
|
||||
+ pynini.closure(exactly_three_digits + pynutil.delete(cardinal_separator))
|
||||
+ exactly_three_digits
|
||||
)
|
||||
|
||||
return cardinal_string @ fst
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying cardinals, e.g.
|
||||
"1000" -> cardinal { integer: "mil" }
|
||||
"2.000.000" -> cardinal { integer: "dos millones" }
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="cardinal", kind="classify", deterministic=deterministic)
|
||||
|
||||
# Any single digit
|
||||
graph_digit = digit
|
||||
digits_no_one = (DAMO_DIGIT - "1") @ graph_digit
|
||||
|
||||
# Any double digit
|
||||
graph_tens = teen
|
||||
graph_tens |= ties + (pynutil.delete("0") | (pynutil.insert(" y ") + graph_digit))
|
||||
graph_tens |= twenties
|
||||
|
||||
self.tens = graph_tens.optimize()
|
||||
|
||||
self.two_digit_non_zero = pynini.union(
|
||||
graph_digit, graph_tens, (pynini.cross("0", DAMO_SPACE) + graph_digit)
|
||||
).optimize()
|
||||
|
||||
# Three digit strings
|
||||
graph_hundreds = hundreds + pynini.union(
|
||||
pynutil.delete("00"),
|
||||
(insert_space + graph_tens),
|
||||
(pynini.cross("0", DAMO_SPACE) + graph_digit),
|
||||
)
|
||||
graph_hundreds |= pynini.cross("100", "cien")
|
||||
graph_hundreds |= (
|
||||
pynini.cross("1", "ciento")
|
||||
+ insert_space
|
||||
+ pynini.union(graph_tens, pynutil.delete("0") + graph_digit)
|
||||
)
|
||||
|
||||
self.hundreds = graph_hundreds.optimize()
|
||||
|
||||
# For all three digit strings with leading zeroes (graph appends '0's to manage place in string)
|
||||
graph_hundreds_component = pynini.union(graph_hundreds, pynutil.delete("0") + graph_tens)
|
||||
|
||||
graph_hundreds_component_at_least_one_none_zero_digit = graph_hundreds_component | (
|
||||
pynutil.delete("00") + graph_digit
|
||||
)
|
||||
graph_hundreds_component_at_least_one_none_zero_digit_no_one = graph_hundreds_component | (
|
||||
pynutil.delete("00") + digits_no_one
|
||||
)
|
||||
|
||||
graph_thousands_component_at_least_one_none_zero_digit = pynini.union(
|
||||
pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit,
|
||||
graph_hundreds_component_at_least_one_none_zero_digit_no_one
|
||||
+ pynutil.insert(" mil")
|
||||
+ (
|
||||
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
||||
| pynutil.delete("000")
|
||||
),
|
||||
pynini.cross("001", "mil")
|
||||
+ (
|
||||
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
||||
| pynutil.delete("000")
|
||||
),
|
||||
)
|
||||
|
||||
graph_thousands_component_at_least_one_none_zero_digit_no_one = pynini.union(
|
||||
pynutil.delete("000") + graph_hundreds_component_at_least_one_none_zero_digit_no_one,
|
||||
graph_hundreds_component_at_least_one_none_zero_digit_no_one
|
||||
+ pynutil.insert(" mil")
|
||||
+ (
|
||||
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
||||
| pynutil.delete("000")
|
||||
),
|
||||
pynini.cross("001", "mil")
|
||||
+ (
|
||||
(insert_space + graph_hundreds_component_at_least_one_none_zero_digit)
|
||||
| pynutil.delete("000")
|
||||
),
|
||||
)
|
||||
|
||||
graph_million = pynutil.add_weight(pynini.cross("000001", "un millón"), -0.001)
|
||||
graph_million |= (
|
||||
graph_thousands_component_at_least_one_none_zero_digit_no_one
|
||||
+ pynutil.insert(" millones")
|
||||
)
|
||||
graph_million |= pynutil.delete("000000")
|
||||
graph_million += insert_space
|
||||
|
||||
graph_billion = pynutil.add_weight(pynini.cross("000001", "un billón"), -0.001)
|
||||
graph_billion |= (
|
||||
graph_thousands_component_at_least_one_none_zero_digit_no_one
|
||||
+ pynutil.insert(" billones")
|
||||
)
|
||||
graph_billion |= pynutil.delete("000000")
|
||||
graph_billion += insert_space
|
||||
|
||||
graph_trillion = pynutil.add_weight(pynini.cross("000001", "un trillón"), -0.001)
|
||||
graph_trillion |= (
|
||||
graph_thousands_component_at_least_one_none_zero_digit_no_one
|
||||
+ pynutil.insert(" trillones")
|
||||
)
|
||||
graph_trillion |= pynutil.delete("000000")
|
||||
graph_trillion += insert_space
|
||||
|
||||
graph = (
|
||||
graph_trillion
|
||||
+ graph_billion
|
||||
+ graph_million
|
||||
+ (graph_thousands_component_at_least_one_none_zero_digit | pynutil.delete("000000"))
|
||||
)
|
||||
|
||||
self.graph = (
|
||||
((DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT, 0))
|
||||
@ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", DAMO_SIGMA)
|
||||
@ DAMO_DIGIT**24
|
||||
@ graph
|
||||
@ pynini.cdrewrite(delete_space, "[BOS]", "", DAMO_SIGMA)
|
||||
@ pynini.cdrewrite(delete_space, "", "[EOS]", DAMO_SIGMA)
|
||||
@ pynini.cdrewrite(
|
||||
pynini.cross(pynini.closure(DAMO_WHITE_SPACE, 2), DAMO_SPACE),
|
||||
DAMO_ALPHA,
|
||||
DAMO_ALPHA,
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
)
|
||||
self.graph |= zero
|
||||
|
||||
self.graph = filter_punctuation(self.graph).optimize()
|
||||
|
||||
optional_minus_graph = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,98 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
articles = pynini.union("de", "del", "el", "del año")
|
||||
delete_leading_zero = (pynutil.delete("0") | (DAMO_DIGIT - "0")) + DAMO_DIGIT
|
||||
month_numbers = pynini.string_file(get_abs_path("data/dates/months.tsv"))
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying date, e.g.
|
||||
"01.04.2010" -> date { day: "un" month: "enero" year: "dos mil diez" preserve_order: true }
|
||||
"marzo 4 2000" -> date { month: "marzo" day: "cuatro" year: "dos mil" }
|
||||
"1990-20-01" -> date { year: "mil novecientos noventa" day: "veinte" month: "enero" }
|
||||
|
||||
Args:
|
||||
cardinal: cardinal GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic: bool):
|
||||
super().__init__(name="date", kind="classify", deterministic=deterministic)
|
||||
|
||||
number_to_month = month_numbers.optimize()
|
||||
month_graph = pynini.project(number_to_month, "output")
|
||||
|
||||
numbers = cardinal.graph
|
||||
optional_leading_zero = delete_leading_zero | DAMO_DIGIT
|
||||
|
||||
# 01, 31, 1
|
||||
digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ numbers
|
||||
day = (pynutil.insert('day: "') + digit_day + pynutil.insert('"')).optimize()
|
||||
|
||||
digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)])
|
||||
number_to_month = digit_month @ number_to_month
|
||||
|
||||
month_name = (pynutil.insert('month: "') + month_graph + pynutil.insert('"')).optimize()
|
||||
month_number = (
|
||||
pynutil.insert('month: "') + number_to_month + pynutil.insert('"')
|
||||
).optimize()
|
||||
|
||||
# prefer cardinal over year
|
||||
year = (DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT, 1, 3) # 90, 990, 1990
|
||||
year @= numbers
|
||||
self.year = year
|
||||
|
||||
year_only = pynutil.insert('year: "') + year + pynutil.insert('"')
|
||||
year_with_articles = (
|
||||
pynutil.insert('year: "')
|
||||
+ pynini.closure(articles + DAMO_SPACE, 0, 1)
|
||||
+ year
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_dmy = (
|
||||
day
|
||||
+ pynini.closure(pynutil.delete(" de"))
|
||||
+ DAMO_SPACE
|
||||
+ month_name
|
||||
+ pynini.closure(DAMO_SPACE + year_with_articles, 0, 1)
|
||||
)
|
||||
|
||||
graph_mdy = ( # English influences on language
|
||||
month_name
|
||||
+ delete_extra_space
|
||||
+ day
|
||||
+ pynini.closure(DAMO_SPACE + year_with_articles, 0, 1)
|
||||
)
|
||||
|
||||
separators = [".", "-", "/"]
|
||||
for sep in separators:
|
||||
year_optional = pynini.closure(pynini.cross(sep, DAMO_SPACE) + year_only, 0, 1)
|
||||
new_graph = day + pynini.cross(sep, DAMO_SPACE) + month_number + year_optional
|
||||
graph_dmy |= new_graph
|
||||
if not deterministic:
|
||||
new_graph = month_number + pynini.cross(sep, DAMO_SPACE) + day + year_optional
|
||||
graph_mdy |= new_graph
|
||||
|
||||
dash = "-"
|
||||
day_optional = pynini.closure(pynini.cross(dash, DAMO_SPACE) + day, 0, 1)
|
||||
graph_ymd = (
|
||||
DAMO_DIGIT**4 @ year_only + pynini.cross(dash, DAMO_SPACE) + month_number + day_optional
|
||||
)
|
||||
|
||||
final_graph = graph_dmy + pynutil.insert(" preserve_order: true")
|
||||
final_graph |= graph_ymd
|
||||
final_graph |= graph_mdy
|
||||
|
||||
self.final_graph = final_graph.optimize()
|
||||
self.fst = self.add_tokens(self.final_graph).optimize()
|
||||
@@ -0,0 +1,128 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
cardinal_separator,
|
||||
decimal_separator,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv"))
|
||||
digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv")))
|
||||
zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv")))
|
||||
|
||||
|
||||
def get_quantity(
|
||||
decimal_graph: "pynini.FstLike", cardinal_graph: "pynini.FstLike"
|
||||
) -> "pynini.FstLike":
|
||||
"""
|
||||
Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
|
||||
e.g. 2 millones -> integer_part: "dos" quantity: "millones"
|
||||
e.g. 2,4 millones -> integer_part: "dos" fractional_part: "quatro" quantity: "millones"
|
||||
e.g. 2,400 millones -> integer_part: "dos mil cuatrocientos" fractional_part: "quatro" quantity: "millones"
|
||||
|
||||
Args:
|
||||
decimal_graph: DecimalFST
|
||||
cardinal_graph: CardinalFST
|
||||
"""
|
||||
numbers = pynini.closure(DAMO_DIGIT, 1, 6) @ cardinal_graph
|
||||
numbers = pynini.cdrewrite(pynutil.delete(cardinal_separator), "", "", DAMO_SIGMA) @ numbers
|
||||
|
||||
res = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ numbers # The cardinal we're passing only produces 'un' for one, so gender agreement is safe (all quantities are masculine). Limit to 10^6 power.
|
||||
+ pynutil.insert('"')
|
||||
+ DAMO_SPACE
|
||||
+ pynutil.insert('quantity: "')
|
||||
+ quantities
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
res |= (
|
||||
decimal_graph
|
||||
+ DAMO_SPACE
|
||||
+ pynutil.insert('quantity: "')
|
||||
+ quantities
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal, e.g.
|
||||
-11,4006 billones -> decimal { negative: "true" integer_part: "once" fractional_part: "cuatro cero cero seis" quantity: "billones" preserve_order: true }
|
||||
1 billón -> decimal { integer_part: "un" quantity: "billón" preserve_order: true }
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
||||
graph_digit = digit | zero
|
||||
|
||||
if not deterministic:
|
||||
graph = pynini.union(graph_digit, cardinal.hundreds, cardinal.tens)
|
||||
graph += pynini.closure(insert_space + graph)
|
||||
|
||||
else:
|
||||
# General pattern seems to be 1-3 digits: map as cardinal, default to digits otherwise \
|
||||
graph = pynini.union(
|
||||
graph_digit,
|
||||
cardinal.tens,
|
||||
cardinal.hundreds,
|
||||
graph_digit + pynini.closure(insert_space + graph_digit, 3),
|
||||
zero
|
||||
+ pynini.closure(insert_space + zero)
|
||||
+ pynini.closure(insert_space + graph_digit), # For cases such as "1,010"
|
||||
)
|
||||
|
||||
# Need to strip apocope everywhere BUT end of string
|
||||
reverse_apocope = pynini.string_map([("un", "uno"), ("ún", "uno")])
|
||||
apply_reverse_apocope = pynini.cdrewrite(reverse_apocope, "", DAMO_SPACE, DAMO_SIGMA)
|
||||
graph @= apply_reverse_apocope
|
||||
|
||||
# Technically decimals should be space delineated groups of three, e.g. (1,333 333). This removes any possible spaces
|
||||
strip_formatting = pynini.cdrewrite(delete_space, "", "", DAMO_SIGMA)
|
||||
graph = strip_formatting @ graph
|
||||
|
||||
self.graph = graph.optimize()
|
||||
|
||||
graph_separator = pynutil.delete(decimal_separator)
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
|
||||
)
|
||||
|
||||
self.graph_fractional = (
|
||||
pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
# Integer graph maintains apocope except for ones place
|
||||
graph_integer = (
|
||||
strip_cardinal_apocope(cardinal.graph)
|
||||
if deterministic
|
||||
else pynini.union(cardinal.graph, strip_cardinal_apocope(cardinal.graph))
|
||||
) # Gives us forms w/ and w/o apocope
|
||||
self.graph_integer = pynutil.insert('integer_part: "') + graph_integer + pynutil.insert('"')
|
||||
final_graph_wo_sign = (
|
||||
self.graph_integer + graph_separator + insert_space + self.graph_fractional
|
||||
)
|
||||
|
||||
self.final_graph_wo_negative = (
|
||||
final_graph_wo_sign | get_quantity(final_graph_wo_sign, cardinal.graph).optimize()
|
||||
)
|
||||
final_graph = optional_graph_negative + self.final_graph_wo_negative
|
||||
|
||||
final_graph += pynutil.insert(" preserve_order: true")
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,70 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path, load_labels
|
||||
from pynini.lib import pynutil
|
||||
|
||||
common_domains = [x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))]
|
||||
symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))]
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying electronic: email addresses
|
||||
e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true }
|
||||
e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true }
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="electronic", kind="classify", deterministic=deterministic)
|
||||
|
||||
dot = pynini.accep(".")
|
||||
accepted_common_domains = pynini.union(*common_domains)
|
||||
accepted_symbols = pynini.union(*symbols) - dot
|
||||
accepted_characters = pynini.closure(DAMO_ALPHA | DAMO_DIGIT | accepted_symbols)
|
||||
acceepted_characters_with_dot = pynini.closure(
|
||||
DAMO_ALPHA | DAMO_DIGIT | accepted_symbols | dot
|
||||
)
|
||||
|
||||
# email
|
||||
username = (
|
||||
pynutil.insert('username: "')
|
||||
+ acceepted_characters_with_dot
|
||||
+ pynutil.insert('"')
|
||||
+ pynini.cross("@", " ")
|
||||
)
|
||||
domain_graph = accepted_characters + dot + accepted_characters
|
||||
domain_graph = pynutil.insert('domain: "') + domain_graph + pynutil.insert('"')
|
||||
domain_common_graph = (
|
||||
pynutil.insert('domain: "')
|
||||
+ accepted_characters
|
||||
+ accepted_common_domains
|
||||
+ pynini.closure(
|
||||
(accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1
|
||||
)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
graph = (username + domain_graph) | domain_common_graph
|
||||
|
||||
# url
|
||||
protocol_start = pynini.accep("https://") | pynini.accep("http://")
|
||||
protocol_end = (
|
||||
pynini.accep("www.")
|
||||
if deterministic
|
||||
else pynini.accep("www.") | pynini.cross("www.", "doble ve doble ve doble ve.")
|
||||
)
|
||||
protocol = protocol_start | protocol_end | (protocol_start + protocol_end)
|
||||
protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"')
|
||||
graph |= protocol + insert_space + (domain_graph | domain_common_graph)
|
||||
self.graph = graph
|
||||
|
||||
final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true"))
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,148 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
ordinal_exceptions = pynini.string_file(get_abs_path("data/fractions/ordinal_exceptions.tsv"))
|
||||
higher_powers_of_ten = pynini.string_file(get_abs_path("data/fractions/powers_of_ten.tsv"))
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying fraction
|
||||
"23 4/5" ->
|
||||
tokens { fraction { integer: "veintitrés" numerator: "cuatro" denominator: "quinto" mophosyntactic_features: "ordinal" } }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
ordinal: OrdinalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, ordinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="fraction", kind="classify", deterministic=deterministic)
|
||||
cardinal_graph = cardinal.graph
|
||||
ordinal_graph = ordinal.graph
|
||||
|
||||
# 2-10 are all ordinals
|
||||
three_to_ten = pynini.string_map(
|
||||
[
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"6",
|
||||
"7",
|
||||
"8",
|
||||
"9",
|
||||
"10",
|
||||
]
|
||||
)
|
||||
block_three_to_ten = pynutil.delete(three_to_ten) # To block cardinal productions
|
||||
if not deterministic: # Multiples of tens are sometimes rendered as ordinals
|
||||
three_to_ten |= pynini.string_map(
|
||||
[
|
||||
"20",
|
||||
"30",
|
||||
"40",
|
||||
"50",
|
||||
"60",
|
||||
"70",
|
||||
"80",
|
||||
"90",
|
||||
]
|
||||
)
|
||||
graph_three_to_ten = three_to_ten @ ordinal_graph
|
||||
graph_three_to_ten @= pynini.cdrewrite(ordinal_exceptions, "", "", DAMO_SIGMA)
|
||||
|
||||
# Higher powers of tens (and multiples) are converted to ordinals.
|
||||
hundreds = pynini.string_map(
|
||||
[
|
||||
"100",
|
||||
"200",
|
||||
"300",
|
||||
"400",
|
||||
"500",
|
||||
"600",
|
||||
"700",
|
||||
"800",
|
||||
"900",
|
||||
]
|
||||
)
|
||||
graph_hundreds = hundreds @ ordinal_graph
|
||||
|
||||
multiples_of_thousand = ordinal.multiples_of_thousand # So we can have X milésimos
|
||||
|
||||
graph_higher_powers_of_ten = (
|
||||
pynini.closure(ordinal.one_to_one_thousand + DAMO_SPACE, 0, 1)
|
||||
+ pynini.closure("mil ", 0, 1)
|
||||
+ pynini.closure(ordinal.one_to_one_thousand + DAMO_SPACE, 0, 1)
|
||||
) # x millones / x mil millones / x mil z millones
|
||||
graph_higher_powers_of_ten += higher_powers_of_ten
|
||||
graph_higher_powers_of_ten = cardinal_graph @ graph_higher_powers_of_ten
|
||||
graph_higher_powers_of_ten @= pynini.cdrewrite(
|
||||
pynutil.delete("un "),
|
||||
pynini.accep("[BOS]"),
|
||||
pynini.project(higher_powers_of_ten, "output"),
|
||||
DAMO_SIGMA,
|
||||
) # we drop 'un' from these ordinals (millionths, not one-millionths)
|
||||
|
||||
graph_higher_powers_of_ten = (
|
||||
multiples_of_thousand | graph_hundreds | graph_higher_powers_of_ten
|
||||
)
|
||||
block_higher_powers_of_ten = pynutil.delete(
|
||||
pynini.project(graph_higher_powers_of_ten, "input")
|
||||
) # For cardinal graph
|
||||
|
||||
graph_fractions_ordinals = graph_higher_powers_of_ten | graph_three_to_ten
|
||||
graph_fractions_ordinals += pynutil.insert(
|
||||
'" morphosyntactic_features: "ordinal"'
|
||||
) # We note the root for processing later
|
||||
|
||||
# Blocking the digits and hundreds from Cardinal graph
|
||||
graph_fractions_cardinals = pynini.cdrewrite(
|
||||
block_three_to_ten | block_higher_powers_of_ten,
|
||||
pynini.accep("[BOS]"),
|
||||
pynini.accep("[EOS]"),
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
graph_fractions_cardinals @= DAMO_CHAR.plus @ pynini.cdrewrite(
|
||||
pynutil.delete("0"), pynini.accep("[BOS]"), pynini.accep("[EOS]"), DAMO_SIGMA
|
||||
) # Empty characters become '0' for DAMO_CHAR fst, so need to block
|
||||
graph_fractions_cardinals @= cardinal_graph
|
||||
graph_fractions_cardinals += pynutil.insert(
|
||||
'" morphosyntactic_features: "add_root"'
|
||||
) # blocking these entries to reduce erroneous possibilities in debugging
|
||||
|
||||
if deterministic:
|
||||
graph_fractions_cardinals = (
|
||||
pynini.closure(DAMO_DIGIT, 1, 2) @ graph_fractions_cardinals
|
||||
) # Past hundreds the conventional scheme can be hard to read. For determinism we stop here
|
||||
|
||||
graph_denominator = pynini.union(
|
||||
graph_fractions_ordinals,
|
||||
graph_fractions_cardinals,
|
||||
pynutil.add_weight(cardinal_graph + pynutil.insert('"'), 0.001),
|
||||
) # Last form is simply recording the cardinal. Weighting so last resort
|
||||
|
||||
integer = (
|
||||
pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"') + DAMO_SPACE
|
||||
)
|
||||
numerator = (
|
||||
pynutil.insert('numerator: "')
|
||||
+ cardinal_graph
|
||||
+ (pynini.cross("/", '" ') | pynini.cross(" / ", '" '))
|
||||
)
|
||||
denominator = pynutil.insert('denominator: "') + graph_denominator
|
||||
|
||||
self.graph = pynini.closure(integer, 0, 1) + numerator + denominator
|
||||
|
||||
final_graph = self.add_tokens(self.graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,168 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_NON_BREAKING_SPACE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
unit = pynini.string_file(get_abs_path("data/measures/measurements.tsv"))
|
||||
unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv"))
|
||||
unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv"))
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying measure, e.g.
|
||||
"2,4 g" -> measure { cardinal { integer_part: "dos" fractional_part: "cuatro" units: "gramos" preserve_order: true } }
|
||||
"1 g" -> measure { cardinal { integer: "un" units: "gramo" preserve_order: true } }
|
||||
"1 millón g" -> measure { cardinal { integer: "un quantity: "millón" units: "gramos" preserve_order: true } }
|
||||
e.g. "a-8" —> "a ocho"
|
||||
e.g. "1,2-a" —> "uno coma dos a"
|
||||
This class also converts words containing numbers and letters
|
||||
e.g. "a-8" —> "a ocho"
|
||||
e.g. "1,2-a" —> "uno coma dos a"
|
||||
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
fraction: FractionFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True
|
||||
):
|
||||
super().__init__(name="measure", kind="classify", deterministic=deterministic)
|
||||
cardinal_graph = cardinal.graph
|
||||
|
||||
unit_singular = unit
|
||||
unit_plural = unit_singular @ (unit_plural_fem | unit_plural_masc)
|
||||
|
||||
graph_unit_singular = convert_space(unit_singular)
|
||||
graph_unit_plural = convert_space(unit_plural)
|
||||
|
||||
optional_graph_negative = pynini.closure("-", 0, 1)
|
||||
|
||||
graph_unit_denominator = (
|
||||
pynini.cross("/", "por") + pynutil.insert(DAMO_NON_BREAKING_SPACE) + graph_unit_singular
|
||||
)
|
||||
|
||||
optional_unit_denominator = pynini.closure(
|
||||
pynutil.insert(DAMO_NON_BREAKING_SPACE) + graph_unit_denominator,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
unit_plural = (
|
||||
pynutil.insert('units: "')
|
||||
+ ((graph_unit_plural + optional_unit_denominator) | graph_unit_denominator)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
unit_singular_graph = (
|
||||
pynutil.insert('units: "')
|
||||
+ ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
subgraph_decimal = (
|
||||
decimal.fst + insert_space + pynini.closure(DAMO_SPACE, 0, 1) + unit_plural
|
||||
)
|
||||
|
||||
subgraph_cardinal = (
|
||||
(optional_graph_negative + (DAMO_SIGMA - "1")) @ cardinal.fst
|
||||
+ insert_space
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ unit_plural
|
||||
)
|
||||
|
||||
subgraph_cardinal |= (
|
||||
(optional_graph_negative + pynini.accep("1")) @ cardinal.fst
|
||||
+ insert_space
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ unit_singular_graph
|
||||
)
|
||||
|
||||
subgraph_fraction = (
|
||||
fraction.fst + insert_space + pynini.closure(delete_space, 0, 1) + unit_singular_graph
|
||||
)
|
||||
|
||||
decimal_times = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.insert(' } units: "')
|
||||
+ pynini.union("x", "X")
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
cardinal_times = (
|
||||
pynutil.insert('cardinal { integer: "')
|
||||
+ strip_cardinal_apocope(cardinal_graph)
|
||||
+ pynutil.insert('" } units: "')
|
||||
+ pynini.union("x", "X")
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
cardinal_dash_alpha = (
|
||||
pynutil.insert('cardinal { integer: "')
|
||||
+ strip_cardinal_apocope(cardinal_graph)
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert('" } units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
decimal_dash_alpha = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert(' } units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
alpha_dash_cardinal = (
|
||||
pynutil.insert('units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(' cardinal { integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('" } preserve_order: true')
|
||||
)
|
||||
|
||||
alpha_dash_decimal = (
|
||||
pynutil.insert('units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(" decimal { ")
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.insert(" } preserve_order: true")
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
subgraph_decimal
|
||||
| subgraph_cardinal
|
||||
| cardinal_dash_alpha
|
||||
| alpha_dash_cardinal
|
||||
| decimal_dash_alpha
|
||||
| subgraph_fraction
|
||||
| decimal_times
|
||||
| cardinal_times
|
||||
| alpha_dash_decimal
|
||||
)
|
||||
final_graph += pynutil.insert(" preserve_order: true")
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,209 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import decimal_separator
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path, load_labels
|
||||
from pynini.lib import pynutil
|
||||
|
||||
maj_singular_labels = load_labels(get_abs_path("data/money/currency_major.tsv"))
|
||||
maj_singular = pynini.string_file((get_abs_path("data/money/currency_major.tsv")))
|
||||
min_singular = pynini.string_file(get_abs_path("data/money/currency_minor.tsv"))
|
||||
fem_plural = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv")))
|
||||
masc_plural = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv")))
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying money, e.g.
|
||||
"€1" -> money { currency_maj: "euro" integer_part: "un"}
|
||||
"€1,000" -> money { currency_maj: "euro" integer_part: "un" }
|
||||
"€1,001" -> money { currency_maj: "euro" integer_part: "un" fractional_part: "cero cero un" }
|
||||
"£1,4" -> money { integer_part: "una" currency_maj: "libra" fractional_part: "cuarenta" preserve_order: true }
|
||||
-> money { integer_part: "una" currency_maj: "libra" fractional_part: "cuarenta" currency_min: "penique" preserve_order: true }
|
||||
"0,01 £" -> money { fractional_part: "un" currency_min: "penique" preserve_order: true }
|
||||
"0,02 £" -> money { fractional_part: "dos" currency_min: "peniques" preserve_order: true }
|
||||
"£0,01 million" -> money { currency_maj: "libra" integer_part: "cero" fractional_part: "cero un" quantity: "million" }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="money", kind="classify", deterministic=deterministic)
|
||||
cardinal_graph = cardinal.graph
|
||||
graph_decimal_final = decimal.final_graph_wo_negative
|
||||
|
||||
maj_singular_graph = maj_singular
|
||||
min_singular_graph = min_singular
|
||||
maj_plural_graph = maj_singular @ (fem_plural | masc_plural)
|
||||
min_plural_graph = min_singular @ (fem_plural | masc_plural)
|
||||
|
||||
graph_maj_singular = (
|
||||
pynutil.insert('currency_maj: "') + maj_singular_graph + pynutil.insert('"')
|
||||
)
|
||||
graph_maj_plural = (
|
||||
pynutil.insert('currency_maj: "') + maj_plural_graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_integer_one = (
|
||||
pynutil.insert('integer_part: "') + pynini.cross("1", "un") + pynutil.insert('"')
|
||||
)
|
||||
|
||||
decimal_with_quantity = (DAMO_SIGMA + DAMO_ALPHA) @ graph_decimal_final
|
||||
|
||||
graph_decimal_plural = pynini.union(
|
||||
graph_maj_plural
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ graph_decimal_final, # $1,05
|
||||
graph_decimal_final
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ graph_maj_plural, # 1,05 $
|
||||
)
|
||||
graph_decimal_plural = (
|
||||
(DAMO_SIGMA - "1") + decimal_separator + DAMO_SIGMA
|
||||
) @ graph_decimal_plural # Can't have "un euros"
|
||||
|
||||
graph_decimal_singular = pynini.union(
|
||||
graph_maj_singular
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ graph_decimal_final, # $1,05
|
||||
graph_decimal_final
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ graph_maj_singular, # 1,05 $
|
||||
)
|
||||
graph_decimal_singular = (
|
||||
pynini.accep("1") + decimal_separator + DAMO_SIGMA
|
||||
) @ graph_decimal_singular
|
||||
|
||||
graph_decimal = pynini.union(
|
||||
graph_decimal_singular,
|
||||
graph_decimal_plural,
|
||||
graph_maj_plural
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ decimal_with_quantity,
|
||||
)
|
||||
|
||||
graph_integer = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ ((DAMO_SIGMA - "1") @ cardinal_graph)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_integer_only = pynini.union(
|
||||
graph_maj_singular
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ graph_integer_one,
|
||||
graph_integer_one
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ insert_space
|
||||
+ graph_maj_singular,
|
||||
)
|
||||
graph_integer_only |= pynini.union(
|
||||
graph_maj_plural + pynini.closure(delete_space, 0, 1) + insert_space + graph_integer,
|
||||
graph_integer + pynini.closure(delete_space, 0, 1) + insert_space + graph_maj_plural,
|
||||
)
|
||||
|
||||
graph = graph_integer_only | graph_decimal
|
||||
|
||||
# remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits
|
||||
# e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10
|
||||
# not accepted: 002, 00, 0,
|
||||
two_digits_fractional_part = (
|
||||
pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(pynutil.delete("0"))
|
||||
) @ (
|
||||
(pynutil.delete("0") + (DAMO_DIGIT - "0"))
|
||||
| ((DAMO_DIGIT - "0") + pynutil.insert("0"))
|
||||
| ((DAMO_DIGIT - "0") + DAMO_DIGIT)
|
||||
)
|
||||
|
||||
graph_min_singular = (
|
||||
pynutil.insert('currency_min: "') + min_singular_graph + pynutil.insert('"')
|
||||
)
|
||||
graph_min_plural = (
|
||||
pynutil.insert('currency_min: "') + min_plural_graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
# format ** euro ** cent
|
||||
decimal_graph_with_minor = None
|
||||
for curr_symbol, _ in maj_singular_labels:
|
||||
preserve_order = pynutil.insert(" preserve_order: true")
|
||||
|
||||
integer_plus_maj = pynini.union(
|
||||
graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural,
|
||||
graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular,
|
||||
)
|
||||
# non zero integer part
|
||||
integer_plus_maj = (pynini.closure(DAMO_DIGIT) - "0") @ integer_plus_maj
|
||||
|
||||
graph_fractional_one = (
|
||||
pynutil.insert('fractional_part: "')
|
||||
+ two_digits_fractional_part @ pynini.cross("1", "un")
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_fractional = (
|
||||
two_digits_fractional_part
|
||||
@ (pynini.closure(DAMO_DIGIT, 1, 2) - "1")
|
||||
@ cardinal.two_digit_non_zero
|
||||
)
|
||||
graph_fractional = (
|
||||
pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"')
|
||||
)
|
||||
|
||||
fractional_plus_min = pynini.union(
|
||||
graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural,
|
||||
graph_fractional_one
|
||||
+ insert_space
|
||||
+ pynutil.insert(curr_symbol) @ graph_min_singular,
|
||||
)
|
||||
|
||||
decimal_graph_with_minor_curr = (
|
||||
integer_plus_maj + pynini.cross(decimal_separator, DAMO_SPACE) + fractional_plus_min
|
||||
)
|
||||
decimal_graph_with_minor_curr |= pynutil.add_weight(
|
||||
integer_plus_maj
|
||||
+ pynini.cross(decimal_separator, DAMO_SPACE)
|
||||
+ pynutil.insert('fractional_part: "')
|
||||
+ two_digits_fractional_part @ cardinal.two_digit_non_zero
|
||||
+ pynutil.insert('"'),
|
||||
weight=0.0001,
|
||||
)
|
||||
|
||||
decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min
|
||||
decimal_graph_with_minor_curr = pynini.union(
|
||||
pynutil.delete(curr_symbol)
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ decimal_graph_with_minor_curr
|
||||
+ preserve_order,
|
||||
decimal_graph_with_minor_curr
|
||||
+ preserve_order
|
||||
+ pynini.closure(delete_space, 0, 1)
|
||||
+ pynutil.delete(curr_symbol),
|
||||
)
|
||||
|
||||
decimal_graph_with_minor = (
|
||||
decimal_graph_with_minor_curr
|
||||
if decimal_graph_with_minor is None
|
||||
else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr)
|
||||
)
|
||||
|
||||
final_graph = graph | pynutil.add_weight(decimal_graph_with_minor, -0.001)
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,173 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import roman_to_int, strip_accent
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
digit = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/digit.tsv")))
|
||||
teens = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/teen.tsv")))
|
||||
twenties = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/twenties.tsv")))
|
||||
ties = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/ties.tsv")))
|
||||
hundreds = pynini.invert(pynini.string_file(get_abs_path("data/ordinals/hundreds.tsv")))
|
||||
|
||||
|
||||
def get_one_to_one_thousand(cardinal: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Produces an acceptor for verbalizations of all numbers from 1 to 1000. Needed for ordinals and fractions.
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
|
||||
Returns:
|
||||
fst: A pynini.FstLike object
|
||||
"""
|
||||
numbers = pynini.string_map([str(_) for _ in range(1, 1000)]) @ cardinal
|
||||
return pynini.project(numbers, "output").optimize()
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying ordinal
|
||||
"21.º" -> ordinal { integer: "vigésimo primero" morphosyntactic_features: "gender_masc" }
|
||||
This class converts ordinal up to the millionth (millonésimo) order (exclusive).
|
||||
|
||||
This FST also records the ending of the ordinal (called "morphosyntactic_features"):
|
||||
either as gender_masc, gender_fem, or apocope. Also introduces plural feature for non-deterministic graphs.
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="ordinal", kind="classify")
|
||||
cardinal_graph = cardinal.graph
|
||||
|
||||
graph_digit = digit.optimize()
|
||||
graph_teens = teens.optimize()
|
||||
graph_ties = ties.optimize()
|
||||
graph_twenties = twenties.optimize()
|
||||
graph_hundreds = hundreds.optimize()
|
||||
|
||||
if not deterministic:
|
||||
# Some alternative derivations
|
||||
graph_ties = graph_ties | pynini.cross("sesenta", "setuagésimo")
|
||||
|
||||
graph_teens = graph_teens | pynini.cross("once", "decimoprimero")
|
||||
graph_teens |= pynini.cross("doce", "decimosegundo")
|
||||
|
||||
graph_digit = graph_digit | pynini.cross("nueve", "nono")
|
||||
graph_digit |= pynini.cross("siete", "sétimo")
|
||||
|
||||
graph_tens_component = (
|
||||
graph_teens
|
||||
| (graph_ties + pynini.closure(pynini.cross(" y ", DAMO_SPACE) + graph_digit, 0, 1))
|
||||
| graph_twenties
|
||||
)
|
||||
|
||||
graph_hundred_component = pynini.union(
|
||||
graph_hundreds
|
||||
+ pynini.closure(DAMO_SPACE + pynini.union(graph_tens_component, graph_digit), 0, 1),
|
||||
graph_tens_component,
|
||||
graph_digit,
|
||||
)
|
||||
|
||||
# Need to go up to thousands for fractions
|
||||
self.one_to_one_thousand = get_one_to_one_thousand(cardinal_graph)
|
||||
|
||||
thousands = pynini.cross("mil", "milésimo")
|
||||
|
||||
graph_thousands = (
|
||||
strip_accent(self.one_to_one_thousand) + DAMO_SPACE + thousands
|
||||
) # Cardinals become prefix for thousands series. Snce accent on the powers of ten we strip accent from leading words
|
||||
graph_thousands @= pynini.cdrewrite(
|
||||
delete_space, "", "milésimo", DAMO_SIGMA
|
||||
) # merge as a prefix
|
||||
graph_thousands |= thousands
|
||||
|
||||
self.multiples_of_thousand = (cardinal_graph @ graph_thousands).optimize()
|
||||
|
||||
if (
|
||||
not deterministic
|
||||
): # Formally the words preceding the power of ten should be a prefix, but some maintain word boundaries.
|
||||
graph_thousands |= (
|
||||
(self.one_to_one_thousand @ graph_hundred_component) + DAMO_SPACE + thousands
|
||||
)
|
||||
|
||||
graph_thousands += pynini.closure(DAMO_SPACE + graph_hundred_component, 0, 1)
|
||||
|
||||
ordinal_graph = graph_thousands | graph_hundred_component
|
||||
ordinal_graph = cardinal_graph @ ordinal_graph
|
||||
|
||||
if not deterministic:
|
||||
# The 10's and 20's series can also be two words
|
||||
split_words = pynini.cross("decimo", "décimo ") | pynini.cross("vigesimo", "vigésimo ")
|
||||
split_words = pynini.cdrewrite(split_words, "", DAMO_CHAR, DAMO_SIGMA)
|
||||
ordinal_graph |= ordinal_graph @ split_words
|
||||
|
||||
# If "octavo" is preceeded by the "o" within string, it needs deletion
|
||||
ordinal_graph @= pynini.cdrewrite(pynutil.delete("o"), "", "octavo", DAMO_SIGMA)
|
||||
|
||||
self.graph = ordinal_graph.optimize()
|
||||
|
||||
masc = pynini.accep("gender_masc")
|
||||
fem = pynini.accep("gender_fem")
|
||||
apocope = pynini.accep("apocope")
|
||||
|
||||
delete_period = pynini.closure(
|
||||
pynutil.delete("."), 0, 1
|
||||
) # Sometimes the period is omitted f
|
||||
|
||||
accept_masc = delete_period + pynini.cross("º", masc)
|
||||
accep_fem = delete_period + pynini.cross("ª", fem)
|
||||
accep_apocope = delete_period + pynini.cross("ᵉʳ", apocope)
|
||||
|
||||
# Managing Romanization
|
||||
graph_roman = (
|
||||
pynutil.insert('integer: "') + roman_to_int(ordinal_graph) + pynutil.insert('"')
|
||||
)
|
||||
if not deterministic:
|
||||
# Introduce plural
|
||||
plural = pynini.closure(pynutil.insert("/plural"), 0, 1)
|
||||
accept_masc += plural
|
||||
accep_fem += plural
|
||||
|
||||
# Romanizations have no morphology marker, so in non-deterministic case we provide option for all
|
||||
insert_morphology = pynutil.insert(pynini.union(masc, fem)) + plural
|
||||
insert_morphology |= pynutil.insert(apocope)
|
||||
insert_morphology = (
|
||||
pynutil.insert(' morphosyntactic_features: "')
|
||||
+ insert_morphology
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_roman += insert_morphology
|
||||
|
||||
else:
|
||||
# We insert both genders as default
|
||||
graph_roman += pynutil.insert(
|
||||
' morphosyntactic_features: "gender_masc"'
|
||||
) | pynutil.insert(' morphosyntactic_features: "gender_fem"')
|
||||
|
||||
# Rest of graph
|
||||
convert_abbreviation = accept_masc | accep_fem | accep_apocope
|
||||
|
||||
graph = (
|
||||
pynutil.insert('integer: "')
|
||||
+ ordinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(' morphosyntactic_features: "')
|
||||
+ convert_abbreviation
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
graph = pynini.union(graph, graph_roman)
|
||||
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,131 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, GraphFst, insert_space
|
||||
from fun_text_processing.text_normalization.es.graph_utils import ones
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv"))
|
||||
graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv"))
|
||||
graph_twenties = pynini.string_file(get_abs_path("data/numbers/twenties.tsv"))
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying telephone numbers, e.g.
|
||||
123-123-5678 -> { number_part: "uno dos tres uno dos tres cinco seis siete ocho" }.
|
||||
In Spanish, digits are generally read individually, or as 2-digit numbers,
|
||||
eg. "123" = "uno dos tres",
|
||||
"1234" = "doce treinta y cuatro".
|
||||
This will verbalize sequences of 10 (3+3+4 e.g. 123-456-7890).
|
||||
9 (3+3+3 e.g. 123-456-789) and 8 (4+4 e.g. 1234-5678) digits.
|
||||
|
||||
(we ignore more complicated cases such as "doscientos y dos" or "tres nueves").
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="telephone", kind="classify")
|
||||
|
||||
# create `single_digits` and `double_digits` graphs as these will be
|
||||
# the building blocks of possible telephone numbers
|
||||
single_digits = pynini.invert(graph_digit).optimize() | pynini.cross("0", "cero")
|
||||
|
||||
double_digits = pynini.union(
|
||||
graph_twenties,
|
||||
graph_teen,
|
||||
(graph_ties + pynutil.delete("0")),
|
||||
(graph_ties + insert_space + pynutil.insert("y") + insert_space + graph_digit),
|
||||
)
|
||||
double_digits = pynini.invert(double_digits)
|
||||
|
||||
# define `ten_digit_graph`, `nine_digit_graph`, `eight_digit_graph`
|
||||
# which produces telephone numbers spoken (1) only with single digits,
|
||||
# or (2) spoken with double digits (and sometimes single digits)
|
||||
|
||||
# 10-digit option (1): all single digits
|
||||
ten_digit_graph = (
|
||||
pynini.closure(single_digits + insert_space, 3, 3)
|
||||
+ pynutil.delete("-")
|
||||
+ pynini.closure(single_digits + insert_space, 3, 3)
|
||||
+ pynutil.delete("-")
|
||||
+ pynini.closure(single_digits + insert_space, 3, 3)
|
||||
+ single_digits
|
||||
)
|
||||
|
||||
# 9-digit option (1): all single digits
|
||||
nine_digit_graph = (
|
||||
pynini.closure(single_digits + insert_space, 3, 3)
|
||||
+ pynutil.delete("-")
|
||||
+ pynini.closure(single_digits + insert_space, 3, 3)
|
||||
+ pynutil.delete("-")
|
||||
+ pynini.closure(single_digits + insert_space, 2, 2)
|
||||
+ single_digits
|
||||
)
|
||||
|
||||
# 8-digit option (1): all single digits
|
||||
eight_digit_graph = (
|
||||
pynini.closure(single_digits + insert_space, 4, 4)
|
||||
+ pynutil.delete("-")
|
||||
+ pynini.closure(single_digits + insert_space, 3, 3)
|
||||
+ single_digits
|
||||
)
|
||||
|
||||
if not deterministic:
|
||||
# 10-digit option (2): (1+2) + (1+2) + (2+2) digits
|
||||
ten_digit_graph |= (
|
||||
single_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
+ insert_space
|
||||
+ pynutil.delete("-")
|
||||
+ single_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
+ insert_space
|
||||
+ pynutil.delete("-")
|
||||
+ double_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
)
|
||||
|
||||
# 9-digit option (2): (1+2) + (1+2) + (1+2) digits
|
||||
nine_digit_graph |= (
|
||||
single_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
+ insert_space
|
||||
+ pynutil.delete("-")
|
||||
+ single_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
+ insert_space
|
||||
+ pynutil.delete("-")
|
||||
+ single_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
)
|
||||
|
||||
# 8-digit option (2): (2+2) + (2+2) digits
|
||||
eight_digit_graph |= (
|
||||
double_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
+ insert_space
|
||||
+ pynutil.delete("-")
|
||||
+ double_digits
|
||||
+ insert_space
|
||||
+ double_digits
|
||||
)
|
||||
|
||||
number_part = pynini.union(ten_digit_graph, nine_digit_graph, eight_digit_graph)
|
||||
number_part @= pynini.cdrewrite(pynini.cross(ones, "uno"), "", "", DAMO_SIGMA)
|
||||
|
||||
number_part = pynutil.insert('number_part: "') + number_part + pynutil.insert('"')
|
||||
|
||||
graph = number_part
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,226 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
time_zone_graph = pynini.string_file(get_abs_path("data/time/time_zone.tsv"))
|
||||
suffix = pynini.string_file(get_abs_path("data/time/time_suffix.tsv"))
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying time, e.g.
|
||||
"02:15 est" -> time { hours: "dos" minutes: "quince" zone: "e s t"}
|
||||
"2 h" -> time { hours: "dos" }
|
||||
"9 h" -> time { hours: "nueve" }
|
||||
"02:15:10 h" -> time { hours: "dos" minutes: "quince" seconds: "diez"}
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="classify", deterministic=deterministic)
|
||||
|
||||
delete_time_delimiter = pynutil.delete(pynini.union(".", ":"))
|
||||
|
||||
one = pynini.string_map([("un", "una"), ("ún", "una")])
|
||||
change_one = pynini.cdrewrite(one, "", "", DAMO_SIGMA)
|
||||
cardinal_graph = cardinal.graph @ change_one
|
||||
|
||||
day_suffix = pynutil.insert('suffix: "') + suffix + pynutil.insert('"')
|
||||
day_suffix = delete_space + insert_space + day_suffix
|
||||
|
||||
delete_hora_suffix = delete_space + insert_space + pynutil.delete("h")
|
||||
delete_minute_suffix = delete_space + insert_space + pynutil.delete("min")
|
||||
delete_second_suffix = delete_space + insert_space + pynutil.delete("s")
|
||||
|
||||
labels_hour_24 = [
|
||||
str(x) for x in range(0, 25)
|
||||
] # Can see both systems. Twelve hour requires am/pm for ambiguity resolution
|
||||
labels_hour_12 = [str(x) for x in range(1, 13)]
|
||||
labels_minute_single = [str(x) for x in range(1, 10)]
|
||||
labels_minute_double = [str(x) for x in range(10, 60)]
|
||||
|
||||
delete_leading_zero_to_double_digit = (
|
||||
pynini.closure(pynutil.delete("0") | (DAMO_DIGIT - "0"), 0, 1) + DAMO_DIGIT
|
||||
)
|
||||
|
||||
graph_24 = (
|
||||
pynini.closure(DAMO_DIGIT, 1, 2)
|
||||
@ delete_leading_zero_to_double_digit
|
||||
@ pynini.union(*labels_hour_24)
|
||||
)
|
||||
graph_12 = (
|
||||
pynini.closure(DAMO_DIGIT, 1, 2)
|
||||
@ delete_leading_zero_to_double_digit
|
||||
@ pynini.union(*labels_hour_12)
|
||||
)
|
||||
|
||||
graph_hour_24 = graph_24 @ cardinal_graph
|
||||
graph_hour_12 = graph_12 @ cardinal_graph
|
||||
|
||||
graph_minute_single = delete_leading_zero_to_double_digit @ pynini.union(
|
||||
*labels_minute_single
|
||||
)
|
||||
graph_minute_double = pynini.union(*labels_minute_double)
|
||||
|
||||
graph_minute = pynini.union(graph_minute_single, graph_minute_double) @ cardinal_graph
|
||||
|
||||
final_graph_hour_only_24 = (
|
||||
pynutil.insert('hours: "') + graph_hour_24 + pynutil.insert('"') + delete_hora_suffix
|
||||
)
|
||||
final_graph_hour_only_12 = (
|
||||
pynutil.insert('hours: "') + graph_hour_12 + pynutil.insert('"') + day_suffix
|
||||
)
|
||||
|
||||
final_graph_hour_24 = pynutil.insert('hours: "') + graph_hour_24 + pynutil.insert('"')
|
||||
final_graph_hour_12 = pynutil.insert('hours: "') + graph_hour_12 + pynutil.insert('"')
|
||||
|
||||
final_graph_minute = pynutil.insert('minutes: "') + graph_minute + pynutil.insert('"')
|
||||
final_graph_second = pynutil.insert('seconds: "') + graph_minute + pynutil.insert('"')
|
||||
final_time_zone_optional = pynini.closure(
|
||||
delete_space
|
||||
+ insert_space
|
||||
+ pynutil.insert('zone: "')
|
||||
+ time_zone_graph
|
||||
+ pynutil.insert('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
# 02.30 h
|
||||
graph_hm = (
|
||||
final_graph_hour_24
|
||||
+ delete_time_delimiter
|
||||
+ (pynutil.delete("00") | (insert_space + final_graph_minute))
|
||||
+ pynini.closure(
|
||||
delete_time_delimiter
|
||||
+ (pynini.cross("00", ' seconds: "0"') | (insert_space + final_graph_second)),
|
||||
0,
|
||||
1,
|
||||
) # For seconds 2.30.35 h
|
||||
+ pynini.closure(delete_hora_suffix, 0, 1) # 2.30 is valid if unambiguous
|
||||
+ final_time_zone_optional
|
||||
)
|
||||
|
||||
# 2 h 30 min
|
||||
graph_hm |= (
|
||||
final_graph_hour_24
|
||||
+ delete_hora_suffix
|
||||
+ delete_space
|
||||
+ (pynutil.delete("00") | (insert_space + final_graph_minute))
|
||||
+ delete_minute_suffix
|
||||
+ pynini.closure(
|
||||
delete_space
|
||||
+ (pynini.cross("00", ' seconds: "0"') | (insert_space + final_graph_second))
|
||||
+ delete_second_suffix,
|
||||
0,
|
||||
1,
|
||||
) # For seconds
|
||||
+ final_time_zone_optional
|
||||
)
|
||||
|
||||
# 2.30 a. m. (Only for 12 hour clock)
|
||||
graph_hm |= (
|
||||
final_graph_hour_12
|
||||
+ delete_time_delimiter
|
||||
+ (pynutil.delete("00") | (insert_space + final_graph_minute))
|
||||
+ pynini.closure(
|
||||
delete_time_delimiter
|
||||
+ (pynini.cross("00", ' seconds: "0"') | (insert_space + final_graph_second)),
|
||||
0,
|
||||
1,
|
||||
) # For seconds 2.30.35 a. m.
|
||||
+ day_suffix
|
||||
+ final_time_zone_optional
|
||||
)
|
||||
|
||||
graph_h = (
|
||||
pynini.union(final_graph_hour_only_24, final_graph_hour_only_12)
|
||||
+ final_time_zone_optional
|
||||
) # Should always have a time indicator, else we'll pass to cardinals
|
||||
|
||||
if not deterministic:
|
||||
# This includes alternate vocalization (hour menos min, min para hour), here we shift the times and indicate a `style` tag
|
||||
hour_shift_24 = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/hour_to_24.tsv"))
|
||||
)
|
||||
hour_shift_12 = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/hour_to_12.tsv"))
|
||||
)
|
||||
minute_shift = pynini.string_file(get_abs_path("data/time/minute_to.tsv"))
|
||||
|
||||
graph_hour_to_24 = graph_24 @ hour_shift_24 @ cardinal_graph
|
||||
graph_hour_to_12 = graph_12 @ hour_shift_12 @ cardinal_graph
|
||||
|
||||
graph_minute_to = (
|
||||
pynini.union(graph_minute_single, graph_minute_double)
|
||||
@ minute_shift
|
||||
@ cardinal_graph
|
||||
)
|
||||
|
||||
final_graph_hour_to_24 = (
|
||||
pynutil.insert('hours: "') + graph_hour_to_24 + pynutil.insert('"')
|
||||
)
|
||||
final_graph_hour_to_12 = (
|
||||
pynutil.insert('hours: "') + graph_hour_to_12 + pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph_minute_to = (
|
||||
pynutil.insert('minutes: "') + graph_minute_to + pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_menos = pynutil.insert(' style: "1"')
|
||||
graph_para = pynutil.insert(' style: "2"')
|
||||
|
||||
final_graph_style = graph_menos | graph_para
|
||||
|
||||
# 02.30 h (omitting seconds since a bit awkward)
|
||||
graph_hm |= (
|
||||
final_graph_hour_to_24
|
||||
+ delete_time_delimiter
|
||||
+ insert_space
|
||||
+ final_graph_minute_to
|
||||
+ pynini.closure(delete_hora_suffix, 0, 1) # 2.30 is valid if unambiguous
|
||||
+ final_time_zone_optional
|
||||
+ final_graph_style
|
||||
)
|
||||
|
||||
# 2 h 30 min
|
||||
graph_hm |= (
|
||||
final_graph_hour_to_24
|
||||
+ delete_hora_suffix
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ final_graph_minute_to
|
||||
+ delete_minute_suffix
|
||||
+ final_time_zone_optional
|
||||
+ final_graph_style
|
||||
)
|
||||
|
||||
# 2.30 a. m. (Only for 12 hour clock)
|
||||
graph_hm |= (
|
||||
final_graph_hour_to_12
|
||||
+ delete_time_delimiter
|
||||
+ insert_space
|
||||
+ final_graph_minute_to
|
||||
+ day_suffix
|
||||
+ final_time_zone_optional
|
||||
+ final_graph_style
|
||||
)
|
||||
|
||||
final_graph = graph_hm | graph_h
|
||||
if deterministic:
|
||||
final_graph = final_graph + pynutil.insert(" preserve_order: true")
|
||||
final_graph = final_graph.optimize()
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,151 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_WHITE_SPACE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
|
||||
from fun_text_processing.text_normalization.es.taggers.cardinal import CardinalFst
|
||||
from fun_text_processing.text_normalization.es.taggers.date import DateFst
|
||||
from fun_text_processing.text_normalization.es.taggers.decimals import DecimalFst
|
||||
from fun_text_processing.text_normalization.es.taggers.electronic import ElectronicFst
|
||||
from fun_text_processing.text_normalization.es.taggers.fraction import FractionFst
|
||||
from fun_text_processing.text_normalization.es.taggers.measure import MeasureFst
|
||||
from fun_text_processing.text_normalization.es.taggers.money import MoneyFst
|
||||
from fun_text_processing.text_normalization.es.taggers.ordinal import OrdinalFst
|
||||
from fun_text_processing.text_normalization.es.taggers.telephone import TelephoneFst
|
||||
from fun_text_processing.text_normalization.es.taggers.time import TimeFst
|
||||
from fun_text_processing.text_normalization.es.taggers.whitelist import WhiteListFst
|
||||
from fun_text_processing.text_normalization.es.taggers.word import WordFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class ClassifyFst(GraphFst):
|
||||
"""
|
||||
Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State aRchive (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
input_case: accepting either "lower_cased" or "cased" input.
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
whitelist: path to a file with whitelist replacements
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_case: str,
|
||||
deterministic: bool = False,
|
||||
cache_dir: str = None,
|
||||
overwrite_cache: bool = False,
|
||||
whitelist: str = None,
|
||||
):
|
||||
super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic)
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
whitelist_file = os.path.basename(whitelist) if whitelist else ""
|
||||
far_file = os.path.join(
|
||||
cache_dir, f"_{input_case}_es_tn_{deterministic}_deterministic{whitelist_file}.far"
|
||||
)
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
|
||||
logging.info(f"ClassifyFst.fst was restored from {far_file}.")
|
||||
else:
|
||||
logging.info(f"Creating ClassifyFst grammars. This might take some time...")
|
||||
|
||||
self.cardinal = CardinalFst(deterministic=deterministic)
|
||||
cardinal_graph = self.cardinal.fst
|
||||
|
||||
self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
ordinal_graph = self.ordinal.fst
|
||||
|
||||
self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
decimal_graph = self.decimal.fst
|
||||
|
||||
self.fraction = FractionFst(
|
||||
cardinal=self.cardinal, ordinal=self.ordinal, deterministic=deterministic
|
||||
)
|
||||
fraction_graph = self.fraction.fst
|
||||
self.measure = MeasureFst(
|
||||
cardinal=self.cardinal,
|
||||
decimal=self.decimal,
|
||||
fraction=self.fraction,
|
||||
deterministic=deterministic,
|
||||
)
|
||||
measure_graph = self.measure.fst
|
||||
self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
date_graph = self.date.fst
|
||||
word_graph = WordFst(deterministic=deterministic).fst
|
||||
self.time = TimeFst(self.cardinal, deterministic=deterministic)
|
||||
time_graph = self.time.fst
|
||||
self.telephone = TelephoneFst(deterministic=deterministic)
|
||||
telephone_graph = self.telephone.fst
|
||||
self.electronic = ElectronicFst(deterministic=deterministic)
|
||||
electronic_graph = self.electronic.fst
|
||||
self.money = MoneyFst(
|
||||
cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic
|
||||
)
|
||||
money_graph = self.money.fst
|
||||
self.whitelist = WhiteListFst(
|
||||
input_case=input_case, deterministic=deterministic, input_file=whitelist
|
||||
)
|
||||
whitelist_graph = self.whitelist.fst
|
||||
punct_graph = PunctuationFst(deterministic=deterministic).fst
|
||||
|
||||
classify = (
|
||||
pynutil.add_weight(whitelist_graph, 1.01)
|
||||
| pynutil.add_weight(time_graph, 1.09)
|
||||
| pynutil.add_weight(measure_graph, 1.08)
|
||||
| pynutil.add_weight(cardinal_graph, 1.1)
|
||||
| pynutil.add_weight(fraction_graph, 1.09)
|
||||
| pynutil.add_weight(date_graph, 1.1)
|
||||
| pynutil.add_weight(ordinal_graph, 1.1)
|
||||
| pynutil.add_weight(decimal_graph, 1.1)
|
||||
| pynutil.add_weight(money_graph, 1.1)
|
||||
| pynutil.add_weight(telephone_graph, 1.1)
|
||||
| pynutil.add_weight(electronic_graph, 1.1)
|
||||
| pynutil.add_weight(word_graph, 200)
|
||||
)
|
||||
punct = (
|
||||
pynutil.insert("tokens { ")
|
||||
+ pynutil.add_weight(punct_graph, weight=2.1)
|
||||
+ pynutil.insert(" }")
|
||||
)
|
||||
punct = pynini.closure(
|
||||
pynini.compose(pynini.closure(DAMO_WHITE_SPACE, 1), delete_extra_space)
|
||||
| (pynutil.insert(" ") + punct),
|
||||
1,
|
||||
)
|
||||
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
|
||||
token_plus_punct = (
|
||||
pynini.closure(punct + pynutil.insert(" "))
|
||||
+ token
|
||||
+ pynini.closure(pynutil.insert(" ") + punct)
|
||||
)
|
||||
|
||||
graph = token_plus_punct + pynini.closure(
|
||||
(
|
||||
pynini.compose(pynini.closure(DAMO_WHITE_SPACE, 1), delete_extra_space)
|
||||
| (pynutil.insert(" ") + punct + pynutil.insert(" "))
|
||||
)
|
||||
+ token_plus_punct
|
||||
)
|
||||
|
||||
graph = delete_space + graph + delete_space
|
||||
graph |= punct
|
||||
|
||||
self.fst = graph.optimize()
|
||||
|
||||
if far_file:
|
||||
generator_main(far_file, {"tokenize_and_classify": self.fst})
|
||||
logging.info(f"ClassifyFst grammars are saved to {far_file}.")
|
||||
@@ -0,0 +1,52 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path, load_labels
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying whitelist, e.g.
|
||||
"sr." -> tokens { name: "señor" }
|
||||
This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
|
||||
|
||||
Args:
|
||||
input_case: accepting either "lower_cased" or "cased" input.
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
input_file: path to a file with whitelist replacements
|
||||
"""
|
||||
|
||||
def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None):
|
||||
super().__init__(name="whitelist", kind="classify", deterministic=deterministic)
|
||||
|
||||
def _get_whitelist_graph(input_case, file):
|
||||
whitelist = load_labels(file)
|
||||
if input_case == "lower_cased":
|
||||
whitelist = [[x[0].lower()] + x[1:] for x in whitelist]
|
||||
graph = pynini.string_map(whitelist)
|
||||
return graph
|
||||
|
||||
graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv"))
|
||||
if not deterministic and input_case != "lower_cased":
|
||||
graph |= pynutil.add_weight(
|
||||
_get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")),
|
||||
weight=0.0001,
|
||||
)
|
||||
|
||||
if input_file:
|
||||
whitelist_provided = _get_whitelist_graph(input_case, input_file)
|
||||
if not deterministic:
|
||||
graph |= whitelist_provided
|
||||
else:
|
||||
graph = whitelist_provided
|
||||
|
||||
if not deterministic:
|
||||
units_graph = _get_whitelist_graph(
|
||||
input_case, file=get_abs_path("data/measures/measurements.tsv")
|
||||
)
|
||||
graph |= units_graph
|
||||
|
||||
self.graph = graph
|
||||
self.final_graph = convert_space(self.graph).optimize()
|
||||
self.fst = (pynutil.insert('name: "') + self.final_graph + pynutil.insert('"')).optimize()
|
||||
@@ -0,0 +1,19 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying word.
|
||||
e.g. dormir -> tokens { name: "dormir" }
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="word", kind="classify")
|
||||
word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
|
||||
self.fst = word.optimize()
|
||||
@@ -0,0 +1,28 @@
|
||||
import csv
|
||||
import os
|
||||
|
||||
|
||||
def get_abs_path(rel_path):
|
||||
"""
|
||||
Get absolute path
|
||||
|
||||
Args:
|
||||
rel_path: relative path to this file
|
||||
|
||||
Returns absolute path
|
||||
"""
|
||||
return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path
|
||||
|
||||
|
||||
def load_labels(abs_path):
|
||||
"""
|
||||
loads relative path file as dictionary
|
||||
|
||||
Args:
|
||||
abs_path: absolute path
|
||||
|
||||
Returns dictionary of mappings
|
||||
"""
|
||||
label_tsv = open(abs_path)
|
||||
labels = list(csv.reader(label_tsv, delimiter="\t"))
|
||||
return labels
|
||||
@@ -0,0 +1,47 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
add_cardinal_apocope_fem,
|
||||
shift_cardinal_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinals
|
||||
e.g. cardinal { integer: "dos" } -> "dos"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true" ', "menos "), 0, 1)
|
||||
self.optional_sign = optional_sign
|
||||
|
||||
integer = pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
self.integer = pynutil.delete(' "') + integer + pynutil.delete('"')
|
||||
|
||||
integer = pynutil.delete("integer:") + self.integer
|
||||
|
||||
graph_masc = optional_sign + integer
|
||||
graph_fem = shift_cardinal_gender(graph_masc)
|
||||
|
||||
self.graph_masc = pynini.optimize(graph_masc)
|
||||
self.graph_fem = pynini.optimize(graph_fem)
|
||||
|
||||
# Adding adjustment for fem gender (choice of gender will be random)
|
||||
graph = graph_masc | graph_fem
|
||||
|
||||
if not deterministic:
|
||||
# For alternate renderings when apocope is omitted (i.e. cardinal stands alone)
|
||||
graph |= strip_cardinal_apocope(graph_masc)
|
||||
# "una" will drop to "un" in unique contexts
|
||||
graph |= add_cardinal_apocope_fem(graph_fem)
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,75 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import strip_cardinal_apocope
|
||||
from fun_text_processing.text_normalization.es.taggers.date import articles
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing date, e.g.
|
||||
date { day: "treinta y uno" month: "marzo" year: "dos mil" } -> "treinta y uno de marzo de dos mil"
|
||||
date { day: "uno" month: "mayo" year: "del mil novecientos noventa" } -> "primero de mayo del mil novecientos noventa"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="date", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
day_cardinal = (
|
||||
pynutil.delete('day: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
day = strip_cardinal_apocope(day_cardinal)
|
||||
|
||||
primero = pynini.cdrewrite(pynini.cross("uno", "primero"), "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
day = (
|
||||
(day @ primero) if deterministic else pynini.union(day, day @ primero)
|
||||
) # Primero for first day is traditional, but will vary depending on region
|
||||
|
||||
month = pynutil.delete('month: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
|
||||
year = (
|
||||
pynutil.delete('year: "')
|
||||
+ articles
|
||||
+ DAMO_SPACE
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
# Insert preposition if wasn't originally with the year. This would mean a space was present
|
||||
year = pynutil.add_weight(year, -0.001)
|
||||
year |= (
|
||||
pynutil.delete('year: "')
|
||||
+ pynutil.insert("de ")
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
# day month year
|
||||
graph_dmy = (
|
||||
day
|
||||
+ pynini.cross(DAMO_SPACE, " de ")
|
||||
+ month
|
||||
+ pynini.closure(pynini.accep(" ") + year, 0, 1)
|
||||
)
|
||||
|
||||
graph_mdy = month + DAMO_SPACE + day + pynini.closure(DAMO_SPACE + year, 0, 1)
|
||||
if deterministic:
|
||||
graph_mdy += pynutil.delete(
|
||||
" preserve_order: true"
|
||||
) # Only accepts this if was explicitly passed
|
||||
|
||||
self.graph = graph_dmy | graph_mdy
|
||||
final_graph = self.graph + delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(final_graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,89 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es import LOCALIZATION
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
add_cardinal_apocope_fem,
|
||||
shift_cardinal_gender,
|
||||
shift_number_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "dos" fractional_part: "cuatro cero" quantity: "billones" } -> menos dos coma quatro cero billones
|
||||
decimal { integer_part: "un" quantity: "billón" } -> un billón
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
||||
|
||||
optional_sign = pynini.closure(
|
||||
pynini.cross('negative: "true"', "menos ") + delete_space, 0, 1
|
||||
)
|
||||
integer = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
fractional_default = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
conjunction = (
|
||||
pynutil.insert(" punto ") if LOCALIZATION == "am" else pynutil.insert(" coma ")
|
||||
)
|
||||
if not deterministic:
|
||||
conjunction |= pynutil.insert(pynini.union(" con ", " y "))
|
||||
fractional_default |= strip_cardinal_apocope(fractional_default)
|
||||
fractional = conjunction + fractional_default
|
||||
|
||||
quantity = (
|
||||
delete_space
|
||||
+ insert_space
|
||||
+ pynutil.delete('quantity: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_quantity = pynini.closure(quantity, 0, 1)
|
||||
|
||||
graph_masc = optional_sign + pynini.union(
|
||||
(integer + quantity), (integer + delete_space + fractional + optional_quantity)
|
||||
)
|
||||
|
||||
# Allowing permutation for fem gender, don't include quantity since "million","billion", etc.. are masculine
|
||||
graph_fem = optional_sign + (
|
||||
shift_cardinal_gender(integer) + delete_space + shift_number_gender(fractional)
|
||||
)
|
||||
if not deterministic: # "una" will drop to "un" in certain cases
|
||||
graph_fem |= add_cardinal_apocope_fem(graph_fem)
|
||||
|
||||
self.numbers_only_quantity = (
|
||||
optional_sign
|
||||
+ pynini.union(
|
||||
(integer + quantity), (integer + delete_space + fractional + quantity)
|
||||
).optimize()
|
||||
)
|
||||
|
||||
self.graph_masc = (graph_masc + delete_preserve_order).optimize()
|
||||
self.graph_fem = (graph_fem + delete_preserve_order).optimize()
|
||||
|
||||
graph = graph_masc | graph_fem
|
||||
|
||||
graph += delete_preserve_order
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,67 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
digit_no_zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv")))
|
||||
zero = pynini.invert(pynini.string_file(get_abs_path("data/numbers/zero.tsv")))
|
||||
|
||||
graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv"))
|
||||
server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv"))
|
||||
domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic
|
||||
e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c arroba hotmail punto com"
|
||||
-> "a b c arroba h o t m a i l punto c o m"
|
||||
-> "a b c arroba hotmail punto c o m"
|
||||
-> "a b c at h o t m a i l punto com"
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
graph_digit_no_zero = (
|
||||
digit_no_zero
|
||||
@ pynini.cdrewrite(pynini.cross("un", "uno"), "", "", DAMO_SIGMA).optimize()
|
||||
)
|
||||
graph_digit = graph_digit_no_zero | zero
|
||||
|
||||
def add_space_after_char():
|
||||
return pynini.closure(DAMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + (
|
||||
DAMO_NOT_QUOTE - pynini.accep(" ")
|
||||
)
|
||||
|
||||
verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", DAMO_SIGMA)
|
||||
|
||||
user_name = pynutil.delete('username: "') + add_space_after_char() + pynutil.delete('"')
|
||||
user_name @= verbalize_characters
|
||||
|
||||
convert_defaults = (
|
||||
pynutil.add_weight(DAMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common
|
||||
)
|
||||
domain = convert_defaults + pynini.closure(insert_space + convert_defaults)
|
||||
domain @= verbalize_characters
|
||||
|
||||
domain = pynutil.delete('domain: "') + domain + pynutil.delete('"')
|
||||
protocol = (
|
||||
pynutil.delete('protocol: "')
|
||||
+ add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", DAMO_SIGMA)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | (
|
||||
user_name + pynini.accep(" ") + pynutil.insert("arroba ") + domain
|
||||
)
|
||||
delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,195 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_NOT_SPACE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
accents,
|
||||
shift_cardinal_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing fraction
|
||||
e.g. tokens { fraction { integer: "treinta y tres" numerator: "cuatro" denominator: "quinto" } } ->
|
||||
treinta y tres y cuatro quintos
|
||||
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="fraction", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
# Derivational strings append 'avo' as a suffix. Adding space for processing aid
|
||||
fraction_stem = pynutil.insert(" avo")
|
||||
plural = pynutil.insert("s")
|
||||
conjunction = pynutil.insert(" y ")
|
||||
|
||||
integer = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ strip_cardinal_apocope(pynini.closure(DAMO_NOT_QUOTE))
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
numerator_one = pynutil.delete('numerator: "') + pynini.accep("un") + pynutil.delete('" ')
|
||||
numerator = (
|
||||
pynutil.delete('numerator: "')
|
||||
+ pynini.difference(pynini.closure(DAMO_NOT_QUOTE), "un")
|
||||
+ pynutil.delete('" ')
|
||||
)
|
||||
|
||||
denominator_add_stem = pynutil.delete('denominator: "') + (
|
||||
pynini.closure(DAMO_NOT_QUOTE)
|
||||
+ fraction_stem
|
||||
+ pynutil.delete('" morphosyntactic_features: "add_root"')
|
||||
)
|
||||
denominator_ordinal = pynutil.delete('denominator: "') + (
|
||||
pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('" morphosyntactic_features: "ordinal"')
|
||||
)
|
||||
denominator_cardinal = pynutil.delete('denominator: "') + (
|
||||
pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
denominator_singular = pynini.union(denominator_add_stem, denominator_ordinal)
|
||||
if not deterministic:
|
||||
# Occasional exceptions
|
||||
denominator_singular |= denominator_add_stem @ pynini.string_map(
|
||||
[("once avo", "undécimo"), ("doce avo", "duodécimo")]
|
||||
)
|
||||
denominator_plural = denominator_singular + plural
|
||||
|
||||
# Merging operations
|
||||
merge = pynini.cdrewrite(
|
||||
pynini.cross(" y ", "i"), "", "", DAMO_SIGMA
|
||||
) # The denominator must be a single word, with the conjunction "y" replaced by i
|
||||
merge @= pynini.cdrewrite(
|
||||
delete_space, "", pynini.difference(DAMO_CHAR, "parte"), DAMO_SIGMA
|
||||
)
|
||||
|
||||
# The merger can produce duplicate vowels. This is not allowed in orthography
|
||||
delete_duplicates = pynini.string_map([("aa", "a"), ("oo", "o")]) # Removes vowels
|
||||
delete_duplicates = pynini.cdrewrite(delete_duplicates, "", "", DAMO_SIGMA)
|
||||
|
||||
remove_accents = pynini.cdrewrite(
|
||||
accents,
|
||||
pynini.union(DAMO_SPACE, pynini.accep("[BOS]")) + pynini.closure(DAMO_NOT_SPACE),
|
||||
pynini.closure(DAMO_NOT_SPACE) + pynini.union("avo", "ava", "ésimo", "ésima"),
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
merge_into_single_word = merge @ remove_accents @ delete_duplicates
|
||||
|
||||
fraction_default = (
|
||||
numerator + delete_space + insert_space + (denominator_plural @ merge_into_single_word)
|
||||
)
|
||||
|
||||
fraction_with_one = (
|
||||
numerator_one
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (denominator_singular @ merge_into_single_word)
|
||||
)
|
||||
|
||||
fraction_with_cardinal = strip_cardinal_apocope(numerator | numerator_one)
|
||||
fraction_with_cardinal += (
|
||||
delete_space + pynutil.insert(" sobre ") + strip_cardinal_apocope(denominator_cardinal)
|
||||
)
|
||||
|
||||
if not deterministic:
|
||||
# There is an alternative rendering where ordinals act as adjectives for 'parte'. This requires use of the feminine
|
||||
# Other rules will manage use of "un" at end, so just worry about endings
|
||||
exceptions = pynini.string_map([("tercia", "tercera")])
|
||||
apply_exceptions = pynini.cdrewrite(exceptions, "", "", DAMO_SIGMA)
|
||||
vowel_change = pynini.cdrewrite(
|
||||
pynini.cross("o", "a"), "", pynini.accep("[EOS]"), DAMO_SIGMA
|
||||
)
|
||||
|
||||
denominator_singular_fem = (
|
||||
shift_cardinal_gender(denominator_singular) @ vowel_change @ apply_exceptions
|
||||
)
|
||||
denominator_plural_fem = denominator_singular_fem + plural
|
||||
|
||||
numerator_one_fem = shift_cardinal_gender(numerator_one)
|
||||
numerator_fem = shift_cardinal_gender(numerator)
|
||||
|
||||
fraction_with_cardinal |= (
|
||||
(numerator_one_fem | numerator_fem)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" sobre ")
|
||||
+ shift_cardinal_gender(denominator_cardinal)
|
||||
)
|
||||
|
||||
# Still need to manage stems
|
||||
merge_stem = pynini.cdrewrite(
|
||||
delete_space, "", pynini.union("avo", "ava", "avos", "avas"), DAMO_SIGMA
|
||||
) # For managing alternative spacing
|
||||
merge_stem @= remove_accents @ delete_duplicates
|
||||
|
||||
fraction_with_one_fem = numerator_one_fem + delete_space + insert_space
|
||||
fraction_with_one_fem += pynini.union(
|
||||
denominator_singular_fem @ merge_stem,
|
||||
denominator_singular_fem @ merge_into_single_word,
|
||||
) # Both forms exists
|
||||
fraction_with_one_fem += pynutil.insert(" parte")
|
||||
fraction_with_one_fem @= pynini.cdrewrite(
|
||||
pynini.cross("una media", "media"), "", "", DAMO_SIGMA
|
||||
) # "media" not "una media"
|
||||
|
||||
fraction_default_fem = numerator_fem + delete_space + insert_space
|
||||
fraction_default_fem += pynini.union(
|
||||
denominator_plural_fem @ merge_stem, denominator_plural_fem @ merge_into_single_word
|
||||
)
|
||||
fraction_default_fem += pynutil.insert(" partes")
|
||||
|
||||
fraction_default |= (
|
||||
numerator + delete_space + insert_space + denominator_plural @ merge_stem
|
||||
) # Case of no merger
|
||||
fraction_default |= fraction_default_fem
|
||||
|
||||
fraction_with_one |= (
|
||||
numerator_one + delete_space + insert_space + denominator_singular @ merge_stem
|
||||
)
|
||||
fraction_with_one |= fraction_with_one_fem
|
||||
|
||||
fraction_with_one @= pynini.cdrewrite(
|
||||
pynini.cross("un medio", "medio"), "", "", DAMO_SIGMA
|
||||
) # "medio" not "un medio"
|
||||
|
||||
fraction = fraction_with_one | fraction_default | fraction_with_cardinal
|
||||
graph_masc = pynini.closure(integer + delete_space + conjunction, 0, 1) + fraction
|
||||
|
||||
# Manage cases of fem gender (only shows on integer except for "medio")
|
||||
integer_fem = shift_cardinal_gender(integer)
|
||||
fraction_default |= (
|
||||
shift_cardinal_gender(numerator)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (denominator_plural @ pynini.cross("medios", "medias"))
|
||||
)
|
||||
fraction_with_one |= (
|
||||
pynutil.delete(numerator_one)
|
||||
+ delete_space
|
||||
+ (denominator_singular @ pynini.cross("medio", "media"))
|
||||
)
|
||||
|
||||
fraction_fem = fraction_with_one | fraction_default | fraction_with_cardinal
|
||||
graph_fem = pynini.closure(integer_fem + delete_space + conjunction, 0, 1) + fraction_fem
|
||||
|
||||
self.graph_masc = pynini.optimize(graph_masc)
|
||||
self.graph_fem = pynini.optimize(graph_fem)
|
||||
|
||||
self.graph = graph_masc | graph_fem
|
||||
|
||||
delete_tokens = self.delete_tokens(self.graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,118 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
DAMO_WHITE_SPACE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import ones
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
unit_plural_fem = pynini.string_file(get_abs_path("data/measures/measurements_plural_fem.tsv"))
|
||||
unit_plural_masc = pynini.string_file(get_abs_path("data/measures/measurements_plural_masc.tsv"))
|
||||
|
||||
unit_singular_fem = pynini.project(unit_plural_fem, "input")
|
||||
unit_singular_masc = pynini.project(unit_plural_masc, "input")
|
||||
|
||||
unit_plural_fem = pynini.project(unit_plural_fem, "output")
|
||||
unit_plural_masc = pynini.project(unit_plural_masc, "output")
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { cardinal { integer: "dos" units: "gramos" } } -> "dos gramos"
|
||||
measure { decimal { integer_part: "dos" quantity: "millones" units: "gramos" } } -> "dos millones de gramos"
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
cardinal: CardinalFst
|
||||
fraction: FractionFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool
|
||||
):
|
||||
super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
graph_decimal_masc = decimal.delete_tokens(decimal.graph_masc)
|
||||
graph_decimal_fem = decimal.delete_tokens(decimal.graph_fem)
|
||||
graph_cardinal_masc = cardinal.delete_tokens(cardinal.graph_masc)
|
||||
graph_cardinal_fem = cardinal.delete_tokens(cardinal.graph_fem)
|
||||
graph_fraction_fem = fraction.delete_tokens(fraction.graph_fem)
|
||||
graph_fraction_masc = fraction.delete_tokens(fraction.graph_masc)
|
||||
|
||||
unit_masc = (unit_plural_masc | unit_singular_masc) + pynini.closure(
|
||||
DAMO_WHITE_SPACE + "por" + pynini.closure(DAMO_NOT_QUOTE, 1), 0, 1
|
||||
)
|
||||
unit_masc |= "por" + pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
unit_masc = (
|
||||
pynutil.delete('units: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE) @ unit_masc)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
unit_fem = (unit_plural_fem | unit_singular_fem) + pynini.closure(
|
||||
DAMO_WHITE_SPACE + "por" + pynini.closure(DAMO_NOT_QUOTE, 1), 0, 1
|
||||
)
|
||||
unit_fem = (
|
||||
pynutil.delete('units: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE) @ unit_fem)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph_masc = (graph_cardinal_masc | graph_decimal_masc) + DAMO_WHITE_SPACE + unit_masc
|
||||
graph_masc |= graph_fraction_masc + DAMO_WHITE_SPACE + pynutil.insert("de ") + unit_masc
|
||||
graph_masc |= pynutil.add_weight(
|
||||
graph_fraction_masc @ (DAMO_SIGMA + pynini.union("medio", "medios"))
|
||||
+ DAMO_WHITE_SPACE
|
||||
+ unit_masc,
|
||||
-0.001,
|
||||
) # "medio litro" not "medio de litro"
|
||||
|
||||
graph_fem = (graph_cardinal_fem | graph_decimal_fem) + DAMO_WHITE_SPACE + unit_fem
|
||||
graph_fem |= graph_fraction_fem + DAMO_WHITE_SPACE + pynutil.insert("de ") + unit_fem
|
||||
graph_fem |= pynutil.add_weight(
|
||||
graph_fraction_fem @ (DAMO_SIGMA + pynini.union("media", "medias"))
|
||||
+ DAMO_WHITE_SPACE
|
||||
+ unit_fem,
|
||||
-0.001,
|
||||
)
|
||||
|
||||
graph = graph_masc | graph_fem
|
||||
|
||||
graph = (
|
||||
pynini.cdrewrite(
|
||||
pynutil.insert(" de"),
|
||||
'quantity: "' + pynini.closure(DAMO_NOT_QUOTE, 1),
|
||||
'"',
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
@ graph
|
||||
) # billones de xyz
|
||||
|
||||
graph @= pynini.cdrewrite(
|
||||
pynini.cross(ones, "uno"), "", DAMO_WHITE_SPACE + "por", DAMO_SIGMA
|
||||
)
|
||||
|
||||
# To manage alphanumeric combonations ("a-8, 5x"), we let them use a weighted default path.
|
||||
alpha_num_unit = (
|
||||
pynutil.delete('units: "') + pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('"')
|
||||
)
|
||||
graph_alpha_num = pynini.union(
|
||||
(graph_cardinal_masc | graph_decimal_masc) + DAMO_SPACE + alpha_num_unit,
|
||||
alpha_num_unit + delete_extra_space + (graph_cardinal_masc | graph_decimal_masc),
|
||||
)
|
||||
|
||||
graph |= pynutil.add_weight(graph_alpha_num, 0.01)
|
||||
|
||||
graph += delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,169 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import (
|
||||
shift_cardinal_gender,
|
||||
strip_cardinal_apocope,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
fem = pynini.string_file((get_abs_path("data/money/currency_plural_fem.tsv")))
|
||||
masc = pynini.string_file((get_abs_path("data/money/currency_plural_masc.tsv")))
|
||||
|
||||
fem_singular = pynini.project(fem, "input")
|
||||
masc_singular = pynini.project(masc, "input")
|
||||
|
||||
fem_plural = pynini.project(fem, "output")
|
||||
masc_plural = pynini.project(masc, "output")
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { currency_maj: "euro" integer_part: "un"} -> "un euro"
|
||||
money { currency_maj: "euro" integer_part: "un" fractional_part: "cero cero un"} -> "uno coma cero cero uno euros"
|
||||
money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" preserve_order: true} -> "una libra cuarenta"
|
||||
money { integer_part: "un" currency_maj: "libra" fractional_part: "cuarenta" currency_min: "peniques" preserve_order: true} -> "una libra con cuarenta peniques"
|
||||
money { fractional_part: "un" currency_min: "penique" preserve_order: true} -> "un penique"
|
||||
|
||||
Args:
|
||||
decimal: GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="money", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
maj_singular_masc = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
maj_singular_fem = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
maj_plural_masc = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
maj_plural_fem = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
maj_masc = maj_plural_masc | maj_singular_masc # Tagger kept quantity resolution stable
|
||||
maj_fem = maj_plural_fem | maj_singular_fem
|
||||
|
||||
min_singular_masc = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
min_singular_fem = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_singular)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
min_plural_masc = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ masc_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
min_plural_fem = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ (pynini.closure(DAMO_NOT_QUOTE, 1) @ fem_plural)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
min_masc = min_plural_masc | min_singular_masc
|
||||
min_fem = min_plural_fem | min_singular_fem
|
||||
|
||||
fractional_part = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
integer_part = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_add_and = pynini.closure(pynutil.insert(pynini.union("con ", "y ")), 0, 1)
|
||||
|
||||
# *** currency_maj
|
||||
graph_integer_masc = integer_part + DAMO_SPACE + maj_masc
|
||||
graph_integer_fem = shift_cardinal_gender(integer_part) + DAMO_SPACE + maj_fem
|
||||
|
||||
graph_integer = graph_integer_fem | graph_integer_masc
|
||||
|
||||
# *** currency_maj + (***) | ((con) *** current_min)
|
||||
graph_integer_with_minor_masc = (
|
||||
graph_integer_masc
|
||||
+ DAMO_SPACE
|
||||
+ pynini.union(
|
||||
optional_add_and + strip_cardinal_apocope(fractional_part),
|
||||
(optional_add_and + fractional_part + DAMO_SPACE + min_masc),
|
||||
(optional_add_and + shift_cardinal_gender(fractional_part) + DAMO_SPACE + min_fem),
|
||||
) # Could be minor currency that is different gender
|
||||
+ delete_preserve_order
|
||||
)
|
||||
|
||||
graph_integer_with_minor_fem = (
|
||||
graph_integer_fem
|
||||
+ DAMO_SPACE
|
||||
+ pynini.union(
|
||||
optional_add_and + shift_cardinal_gender(fractional_part),
|
||||
(optional_add_and + fractional_part + DAMO_SPACE + min_masc),
|
||||
(optional_add_and + shift_cardinal_gender(fractional_part) + DAMO_SPACE + min_fem),
|
||||
) # Could be minor currency that is different gender
|
||||
+ delete_preserve_order
|
||||
)
|
||||
|
||||
graph_integer_with_minor = graph_integer_with_minor_fem | graph_integer_with_minor_masc
|
||||
|
||||
## *** coma *** currency_maj
|
||||
graph_decimal_masc = decimal.graph_masc + DAMO_SPACE + maj_masc
|
||||
|
||||
graph_decimal_fem = decimal.graph_fem
|
||||
graph_decimal_fem |= (
|
||||
decimal.numbers_only_quantity
|
||||
) # can still have "x billions" with fem currency
|
||||
graph_decimal_fem += DAMO_SPACE + maj_fem
|
||||
|
||||
graph_decimal = graph_decimal_fem | graph_decimal_masc
|
||||
graph_decimal = (
|
||||
pynini.cdrewrite(
|
||||
pynutil.insert(" de"),
|
||||
'quantity: "' + pynini.closure(DAMO_NOT_QUOTE, 1),
|
||||
'"',
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
@ graph_decimal
|
||||
) # formally it's millones/billones de ***
|
||||
|
||||
# *** current_min
|
||||
graph_minor_masc = fractional_part + DAMO_SPACE + min_masc + delete_preserve_order
|
||||
graph_minor_fem = (
|
||||
shift_cardinal_gender(fractional_part) + DAMO_SPACE + min_fem + delete_preserve_order
|
||||
)
|
||||
|
||||
graph_minor = graph_minor_fem | graph_minor_masc
|
||||
|
||||
graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,68 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.graph_utils import shift_number_gender
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing ordinals
|
||||
e.g. ordinal { integer: "tercer" } } -> "tercero"
|
||||
-> "tercera"
|
||||
-> "tercer"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
graph = (
|
||||
pynutil.delete('integer: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
# masculne gender we leave as is
|
||||
graph_masc = graph + pynutil.delete(' morphosyntactic_features: "gender_masc')
|
||||
|
||||
# shift gender
|
||||
graph_fem_ending = graph @ pynini.cdrewrite(
|
||||
pynini.cross("o", "a"), "", DAMO_SPACE | pynini.accep("[EOS]"), DAMO_SIGMA
|
||||
)
|
||||
graph_fem = shift_number_gender(graph_fem_ending) + pynutil.delete(
|
||||
' morphosyntactic_features: "gender_fem'
|
||||
)
|
||||
|
||||
# Apocope just changes tercero and primero. May occur if someone wrote 11.er (uncommon)
|
||||
graph_apocope = (
|
||||
pynini.cross("tercero", "tercer")
|
||||
| pynini.cross("primero", "primer")
|
||||
| pynini.cross("undécimo", "decimoprimer")
|
||||
) # In case someone wrote 11.er with deterministic
|
||||
graph_apocope = (
|
||||
graph @ pynini.cdrewrite(graph_apocope, "", "", DAMO_SIGMA)
|
||||
) + pynutil.delete(' morphosyntactic_features: "apocope')
|
||||
|
||||
graph = graph_apocope | graph_masc | graph_fem
|
||||
|
||||
if not deterministic:
|
||||
# Plural graph
|
||||
graph_plural = pynini.cdrewrite(
|
||||
pynutil.insert("s"),
|
||||
pynini.union("o", "a"),
|
||||
DAMO_SPACE | pynini.accep("[EOS]"),
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
|
||||
graph |= (graph @ graph_plural) + pynutil.delete("/plural")
|
||||
|
||||
self.graph = graph + pynutil.delete('"')
|
||||
|
||||
delete_tokens = self.delete_tokens(self.graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,26 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing telephone, e.g.
|
||||
telephone { number_part: "uno dos tres uno dos tres cinco seis siete ocho" }
|
||||
-> uno dos tres uno dos tres cinco seis siete ocho
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="telephone", kind="verbalize")
|
||||
|
||||
number_part = (
|
||||
pynutil.delete('number_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
delete_tokens = self.delete_tokens(number_part)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,249 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.text_normalization.es.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
alt_minutes = pynini.string_file(get_abs_path("data/time/alt_minutes.tsv"))
|
||||
|
||||
morning_times = pynini.string_file(get_abs_path("data/time/morning_times.tsv"))
|
||||
afternoon_times = pynini.string_file(get_abs_path("data/time/afternoon_times.tsv"))
|
||||
evening_times = pynini.string_file(get_abs_path("data/time/evening_times.tsv"))
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing time, e.g.
|
||||
time { hours: "doce" minutes: "media" suffix: "a m" } -> doce y media de la noche
|
||||
time { hours: "doce" } -> twelve o'clock
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
change_minutes = pynini.cdrewrite(
|
||||
alt_minutes, pynini.accep("[BOS]"), pynini.accep("[EOS]"), DAMO_SIGMA
|
||||
)
|
||||
|
||||
morning_phrases = pynini.cross("am", "de la mañana")
|
||||
afternoon_phrases = pynini.cross("pm", "de la tarde")
|
||||
evening_phrases = pynini.cross("pm", "de la noche")
|
||||
|
||||
# For the 12's
|
||||
mid_times = pynini.accep("doce")
|
||||
mid_phrases = (
|
||||
pynini.string_map([("pm", "del mediodía"), ("am", "de la noche")])
|
||||
if deterministic
|
||||
else pynini.string_map(
|
||||
[
|
||||
("pm", "de la mañana"),
|
||||
("pm", "del día"),
|
||||
("pm", "del mediodía"),
|
||||
("am", "de la noche"),
|
||||
("am", "de la medianoche"),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
hour = (
|
||||
pynutil.delete("hours:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
minute = (
|
||||
pynutil.delete("minutes:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
minute = (
|
||||
(minute @ change_minutes)
|
||||
if deterministic
|
||||
else pynini.union(minute, minute @ change_minutes)
|
||||
)
|
||||
|
||||
suffix = (
|
||||
pynutil.delete("suffix:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
zone = (
|
||||
pynutil.delete("zone:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_zone = pynini.closure(delete_space + insert_space + zone, 0, 1)
|
||||
second = (
|
||||
pynutil.delete("seconds:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph_hms = (
|
||||
hour
|
||||
+ pynutil.insert(" horas ")
|
||||
+ delete_space
|
||||
+ minute
|
||||
+ pynutil.insert(" minutos y ")
|
||||
+ delete_space
|
||||
+ second
|
||||
+ pynutil.insert(" segundos")
|
||||
)
|
||||
|
||||
graph_hm = hour + delete_space + pynutil.insert(" y ") + minute
|
||||
graph_hm |= pynini.union(
|
||||
(hour @ morning_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ morning_phrases),
|
||||
(hour @ afternoon_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ afternoon_phrases),
|
||||
(hour @ evening_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ evening_phrases),
|
||||
(hour @ mid_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" y ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ mid_phrases),
|
||||
)
|
||||
|
||||
graph_h = pynini.union(
|
||||
hour,
|
||||
(hour @ morning_times) + delete_space + insert_space + (suffix @ morning_phrases),
|
||||
(hour @ afternoon_times) + delete_space + insert_space + (suffix @ afternoon_phrases),
|
||||
(hour @ evening_times) + delete_space + insert_space + (suffix @ evening_phrases),
|
||||
(hour @ mid_times) + delete_space + insert_space + (suffix @ mid_phrases),
|
||||
)
|
||||
|
||||
graph = (graph_hms | graph_hm | graph_h) + optional_zone
|
||||
|
||||
if not deterministic:
|
||||
graph_style_1 = pynutil.delete(' style: "1"')
|
||||
graph_style_2 = pynutil.delete(' style: "2"')
|
||||
|
||||
graph_menos = hour + delete_space + pynutil.insert(" menos ") + minute + graph_style_1
|
||||
graph_menos |= (
|
||||
(hour @ morning_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ morning_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos |= (
|
||||
(hour @ afternoon_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ afternoon_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos |= (
|
||||
(hour @ evening_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ evening_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos |= (
|
||||
(hour @ mid_times)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" menos ")
|
||||
+ minute
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ mid_phrases)
|
||||
+ graph_style_1
|
||||
)
|
||||
graph_menos += optional_zone
|
||||
|
||||
graph_para = minute + pynutil.insert(" para las ") + delete_space + hour + graph_style_2
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ morning_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ morning_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ afternoon_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ afternoon_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ evening_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ evening_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para |= (
|
||||
minute
|
||||
+ pynutil.insert(" para las ")
|
||||
+ delete_space
|
||||
+ (hour @ mid_times)
|
||||
+ delete_space
|
||||
+ insert_space
|
||||
+ (suffix @ mid_phrases)
|
||||
+ graph_style_2
|
||||
)
|
||||
graph_para += optional_zone
|
||||
graph_para @= pynini.cdrewrite(
|
||||
pynini.cross(" las ", " la "), "para", "una", DAMO_SIGMA
|
||||
) # Need agreement with one
|
||||
|
||||
graph |= graph_menos | graph_para
|
||||
delete_tokens = self.delete_tokens(graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,62 @@
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
|
||||
from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.date import DateFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.decimals import DecimalFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.electronic import ElectronicFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.fraction import FractionFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.ordinal import OrdinalFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.telephone import TelephoneFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.time import TimeFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
|
||||
cardinal = CardinalFst(deterministic=deterministic)
|
||||
cardinal_graph = cardinal.fst
|
||||
ordinal = OrdinalFst(deterministic=deterministic)
|
||||
ordinal_graph = ordinal.fst
|
||||
decimal = DecimalFst(deterministic=deterministic)
|
||||
decimal_graph = decimal.fst
|
||||
fraction = FractionFst(deterministic=deterministic)
|
||||
fraction_graph = fraction.fst
|
||||
date = DateFst(deterministic=deterministic)
|
||||
date_graph = date.fst
|
||||
measure = MeasureFst(
|
||||
cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic
|
||||
)
|
||||
measure_graph = measure.fst
|
||||
electronic = ElectronicFst(deterministic=deterministic)
|
||||
electronic_graph = electronic.fst
|
||||
whitelist_graph = WhiteListFst(deterministic=deterministic).fst
|
||||
money_graph = MoneyFst(decimal=decimal, deterministic=deterministic).fst
|
||||
telephone_graph = TelephoneFst(deterministic=deterministic).fst
|
||||
time_graph = TimeFst(deterministic=deterministic).fst
|
||||
|
||||
graph = (
|
||||
cardinal_graph
|
||||
| measure_graph
|
||||
| decimal_graph
|
||||
| ordinal_graph
|
||||
| date_graph
|
||||
| electronic_graph
|
||||
| money_graph
|
||||
| fraction_graph
|
||||
| whitelist_graph
|
||||
| telephone_graph
|
||||
| time_graph
|
||||
)
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.verbalizers.word import WordFst
|
||||
from fun_text_processing.text_normalization.es.verbalizers.verbalize import VerbalizeFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False
|
||||
):
|
||||
super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(
|
||||
cache_dir, f"es_tn_{deterministic}_deterministic_verbalizer.far"
|
||||
)
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["verbalize"]
|
||||
logging.info(f"VerbalizeFinalFst graph was restored from {far_file}.")
|
||||
else:
|
||||
|
||||
verbalize = VerbalizeFst(deterministic=deterministic).fst
|
||||
word = WordFst(deterministic=deterministic).fst
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
if far_file:
|
||||
generator_main(far_file, {"verbalize": self.fst})
|
||||
logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.")
|
||||
Reference in New Issue
Block a user