chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
.com punkt com
|
||||
.uk punkt uk
|
||||
.fr punkt fr
|
||||
.net dot net
|
||||
.br punkt br
|
||||
.in punkt in
|
||||
.ru punkt ru
|
||||
.de punkt de
|
||||
.it punkt it
|
||||
|
@@ -0,0 +1,12 @@
|
||||
gmail g mail
|
||||
nvidia
|
||||
outlook
|
||||
hotmail
|
||||
yahoo
|
||||
live
|
||||
yandex
|
||||
orange
|
||||
wanadoo
|
||||
web
|
||||
comcast
|
||||
aol
|
||||
|
@@ -0,0 +1,20 @@
|
||||
. punkt
|
||||
: doppelpunkt
|
||||
- bindestrich
|
||||
_ unterstrich
|
||||
! ausrufezeichen
|
||||
# raute
|
||||
$ dollar zeichen
|
||||
% prozent zeichen
|
||||
& und
|
||||
' apostroph
|
||||
* asterisk
|
||||
+ plus
|
||||
/ slash
|
||||
= gleichheitszeichen
|
||||
? fragezeichen
|
||||
^ zirkumflex
|
||||
{ linke klammer auf
|
||||
} rechte klammer zu
|
||||
~ tilde
|
||||
, komma
|
||||
|
@@ -0,0 +1,30 @@
|
||||
halb zwei
|
||||
drittel drei
|
||||
viertel vier
|
||||
fünftel fünf
|
||||
sechstel sechs
|
||||
siebtel sieben
|
||||
achtel acht
|
||||
neuntel neun
|
||||
zehntel zehn
|
||||
elftel elf
|
||||
zwölftel zwölf
|
||||
dreizehntel dreizehn
|
||||
vierzehntel vierzehn
|
||||
fünfzehntel fünfzehn
|
||||
sechzehntel sechzehn
|
||||
siebzehntel siebzehn
|
||||
achtzehntel achtzehn
|
||||
neunzehntel neunzehn
|
||||
zwanzigstel zwanzig
|
||||
dreißigstel dreißig
|
||||
vierzigstel vierzig
|
||||
fünfzigstel fünfzig
|
||||
sechzigstel sechzig
|
||||
siebzigstel siebzig
|
||||
achtzigstel achtzig
|
||||
neunzigstel neunzig
|
||||
hundertstel hundert
|
||||
tausendstel tausend
|
||||
millionstel million
|
||||
milliardstel milliarde
|
||||
|
@@ -0,0 +1,82 @@
|
||||
% prozent
|
||||
f fahrenheit
|
||||
c celsius
|
||||
°C grad celsius
|
||||
°F grad fahrenheit
|
||||
K kelvin
|
||||
km kilometer
|
||||
m meter
|
||||
cm zentimeter
|
||||
mm millimeter
|
||||
μm mikrometer
|
||||
nm nanometer
|
||||
dm dezimeter
|
||||
pm pikometer
|
||||
hm hektometer
|
||||
ha hektar
|
||||
mi meile
|
||||
m² quadrat meter -0.0001
|
||||
km² quadrat kilometer -0.0001
|
||||
mm² quadrat millimeter -0.0001
|
||||
cm² quadrat zentimeter -0.0001
|
||||
m³ kubik meter -0.0001
|
||||
km³ kubik kilometer -0.0001
|
||||
mm³ kubik millimeter -0.0001
|
||||
cm³ kubik zentimeter -0.0001
|
||||
m2 quadrat meter
|
||||
km2 quadrat kilometer
|
||||
mm2 quadrat millimeter
|
||||
cm2 quadrat zentimeter
|
||||
m3 kubik meter
|
||||
km3 kubik kilometer
|
||||
mm3 kubik millimeter
|
||||
cm3 kubik zentimeter
|
||||
ft fuß
|
||||
g gramm
|
||||
µg mikrogramm
|
||||
mg milligramm
|
||||
kg kilogramm
|
||||
lb pfund
|
||||
oz unze
|
||||
cwt zentner
|
||||
gr korn
|
||||
dr drachne
|
||||
μg mikrogramm
|
||||
pg petagramm
|
||||
h stunde
|
||||
s sekunde
|
||||
min minute
|
||||
ds decisekunde
|
||||
ms millisekunde
|
||||
μs mikrosekunde
|
||||
hz hertz
|
||||
kw kilowatt
|
||||
kwh kilowattstunde
|
||||
ghz gigahertz
|
||||
khz kilohertz
|
||||
mhz megahertz
|
||||
v volt
|
||||
mc megacoulomb
|
||||
mA milliampere
|
||||
A ampere
|
||||
tw terawatt
|
||||
mv millivolt
|
||||
mw megawatt
|
||||
gw gigawatt
|
||||
ω ohm
|
||||
db dezibel
|
||||
gb gigabyte
|
||||
kb kilobit
|
||||
pb petabit
|
||||
mb megabyte
|
||||
kb kilobyte
|
||||
tb terabyte
|
||||
kv kilovolt
|
||||
mv megavolt
|
||||
kn kilonewton
|
||||
ml milliliter
|
||||
l liter
|
||||
ma megaampere
|
||||
bar bar
|
||||
kcal kilokalorie
|
||||
cal kalorie
|
||||
|
@@ -0,0 +1,10 @@
|
||||
meile meilen
|
||||
unze unzen
|
||||
drachne drachnen
|
||||
stunde stunden
|
||||
sekunde sekunden
|
||||
minute minuten
|
||||
minute minuten
|
||||
bit bits
|
||||
byte bytes
|
||||
kalorie kalorien
|
||||
|
@@ -0,0 +1,25 @@
|
||||
€ euro
|
||||
$ dollar
|
||||
$ us dollar
|
||||
£ pfund
|
||||
₩ won
|
||||
nzd neuseeland dollar
|
||||
rs rupie
|
||||
chf schweizer franken
|
||||
dkk dänische krone
|
||||
fim finnische mark
|
||||
aed dirham
|
||||
¥ yen
|
||||
czk tschechische krone
|
||||
mro ouguiya
|
||||
pkr pakistanische rupie
|
||||
crc colon
|
||||
hkd hong kong dollar
|
||||
npr nepalesische rupee
|
||||
awg aruba florin
|
||||
nok norwegische krone
|
||||
tzs tansania schilling
|
||||
sek schwedisch krone
|
||||
cyp zypern pfund
|
||||
dm d-mark
|
||||
dm deutsche mark
|
||||
|
@@ -0,0 +1,3 @@
|
||||
$ cent
|
||||
€ cent
|
||||
£ pence
|
||||
|
@@ -0,0 +1,3 @@
|
||||
$ cent
|
||||
€ cent
|
||||
£ penny
|
||||
|
@@ -0,0 +1,13 @@
|
||||
jan januar
|
||||
feb februar
|
||||
mär märz
|
||||
apr april
|
||||
mai mai
|
||||
jun juni
|
||||
jul juli
|
||||
aug august
|
||||
sep september
|
||||
sept september
|
||||
okt oktober
|
||||
nov november
|
||||
dez dezember
|
||||
|
@@ -0,0 +1,24 @@
|
||||
1 januar
|
||||
2 februar
|
||||
3 märz
|
||||
4 april
|
||||
5 mai
|
||||
6 juni
|
||||
7 juli
|
||||
8 august
|
||||
9 september
|
||||
10 oktober
|
||||
11 november
|
||||
12 dezember
|
||||
01 januar
|
||||
02 februar
|
||||
03 märz
|
||||
04 april
|
||||
05 mai
|
||||
06 juni
|
||||
07 juli
|
||||
08 august
|
||||
09 september
|
||||
10 oktober
|
||||
11 november
|
||||
12 dezember
|
||||
|
@@ -0,0 +1,8 @@
|
||||
zwei 2
|
||||
drei 3
|
||||
vier 4
|
||||
fünf 5
|
||||
sechs 6
|
||||
sieben 7
|
||||
acht 8
|
||||
neun 9
|
||||
|
@@ -0,0 +1,3 @@
|
||||
eine 1
|
||||
ein 1
|
||||
eins 1 -0.0001
|
||||
|
@@ -0,0 +1,12 @@
|
||||
million
|
||||
millionen
|
||||
milliarde
|
||||
milliarden
|
||||
billion
|
||||
billionen
|
||||
billiarde
|
||||
billiarden
|
||||
trillion
|
||||
trillionen
|
||||
trilliarde
|
||||
trilliarde
|
||||
|
@@ -0,0 +1,10 @@
|
||||
zehn 10
|
||||
elf 11
|
||||
zwölf 12
|
||||
dreizehn 13
|
||||
vierzehn 14
|
||||
fünfzehn 15
|
||||
sechzehn 16
|
||||
siebzehn 17
|
||||
achtzehn 18
|
||||
neunzehn 19
|
||||
|
@@ -0,0 +1,8 @@
|
||||
zwanzig 2
|
||||
dreißig 3
|
||||
vierzig 4
|
||||
fünfzig 5
|
||||
sechzig 6
|
||||
siebzig 7
|
||||
achtzig 8
|
||||
neunzig 9
|
||||
|
@@ -0,0 +1 @@
|
||||
null 0
|
||||
|
@@ -0,0 +1,9 @@
|
||||
ers eins
|
||||
zwei zwei
|
||||
drit drei
|
||||
vier vier
|
||||
fünf fünf
|
||||
sechs sechs
|
||||
sieb sieben
|
||||
ach acht
|
||||
neun neun
|
||||
|
@@ -0,0 +1,4 @@
|
||||
hunderts hundert
|
||||
tausends tausend
|
||||
millions million
|
||||
milliards milliarde
|
||||
|
@@ -0,0 +1,8 @@
|
||||
zwanzigs zwanzig
|
||||
dreißigs dreißig
|
||||
vierzigs vierzig
|
||||
fünfzigs fünfzig
|
||||
sechzigs sechzig
|
||||
siebzigs siebzig
|
||||
achtzigs achtzig
|
||||
neunzigs neunzig
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
1 12
|
||||
2 1
|
||||
3 2
|
||||
4 3
|
||||
5 4
|
||||
6 5
|
||||
7 6
|
||||
8 7
|
||||
9 8
|
||||
10 9
|
||||
11 10
|
||||
12 11
|
||||
|
@@ -0,0 +1,13 @@
|
||||
12 0
|
||||
1 13
|
||||
2 14
|
||||
3 15
|
||||
4 16
|
||||
5 17
|
||||
6 18
|
||||
7 19
|
||||
8 20
|
||||
9 21
|
||||
10 22
|
||||
11 23
|
||||
12 24
|
||||
|
@@ -0,0 +1,59 @@
|
||||
1 59
|
||||
2 58
|
||||
3 57
|
||||
4 56
|
||||
5 55
|
||||
6 54
|
||||
7 53
|
||||
8 52
|
||||
9 51
|
||||
10 50
|
||||
11 49
|
||||
12 48
|
||||
13 47
|
||||
14 46
|
||||
15 45
|
||||
16 44
|
||||
17 43
|
||||
18 42
|
||||
19 41
|
||||
20 40
|
||||
21 39
|
||||
22 38
|
||||
23 37
|
||||
24 36
|
||||
25 35
|
||||
26 34
|
||||
27 33
|
||||
28 32
|
||||
29 31
|
||||
30 30
|
||||
31 29
|
||||
32 28
|
||||
33 27
|
||||
34 26
|
||||
35 25
|
||||
36 24
|
||||
37 23
|
||||
38 22
|
||||
39 21
|
||||
40 20
|
||||
41 19
|
||||
42 18
|
||||
43 17
|
||||
44 16
|
||||
45 15
|
||||
46 14
|
||||
47 13
|
||||
48 12
|
||||
49 11
|
||||
50 10
|
||||
51 9
|
||||
52 8
|
||||
53 7
|
||||
54 6
|
||||
55 5
|
||||
56 4
|
||||
57 3
|
||||
58 2
|
||||
59 1
|
||||
|
@@ -0,0 +1,7 @@
|
||||
cst c s t
|
||||
cet c e t
|
||||
pst p s t
|
||||
est e s t
|
||||
pt p t
|
||||
et e t
|
||||
gmt g m t
|
||||
|
@@ -0,0 +1,7 @@
|
||||
z.B. zum beispiel
|
||||
d.h. dass heißt
|
||||
Dr. doktor
|
||||
Mr. mister
|
||||
Mrs. misses
|
||||
Ms. miss
|
||||
Nr. nummer
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
from collections import defaultdict
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
AND = "und"
|
||||
|
||||
|
||||
def get_ties_digit(digit_path: str, tie_path: str) -> "pynini.FstLike":
|
||||
"""
|
||||
getting all inverse normalizations for numbers between 21 - 100
|
||||
|
||||
Args:
|
||||
digit_path: file to digit tsv
|
||||
tie_path: file to tie tsv, e.g. 20, 30, etc.
|
||||
Returns:
|
||||
res: fst that converts numbers to their verbalization
|
||||
"""
|
||||
|
||||
digits = defaultdict(list)
|
||||
ties = defaultdict(list)
|
||||
for k, v in load_labels(digit_path):
|
||||
digits[v].append(k)
|
||||
digits["1"] = ["ein"]
|
||||
|
||||
for k, v in load_labels(tie_path):
|
||||
ties[v].append(k)
|
||||
|
||||
d = []
|
||||
for i in range(21, 100):
|
||||
s = str(i)
|
||||
if s[1] == "0":
|
||||
continue
|
||||
|
||||
for di in digits[s[1]]:
|
||||
for ti in ties[s[0]]:
|
||||
word = di + f" {AND} " + ti
|
||||
d.append((word, s))
|
||||
|
||||
res = pynini.string_map(d)
|
||||
return res
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying cardinals, e.g.
|
||||
"101" -> cardinal { integer: "ein hundert und zehn" }
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = False):
|
||||
super().__init__(name="cardinal", kind="classify", deterministic=deterministic)
|
||||
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert()
|
||||
graph_digit_no_one = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert()
|
||||
graph_one = pynini.string_file(get_abs_path("data/numbers/ones.tsv")).invert()
|
||||
graph_digit = graph_digit_no_one | graph_one
|
||||
self.digit = (graph_digit | graph_zero).optimize()
|
||||
graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).invert()
|
||||
|
||||
graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).invert()
|
||||
# separator = "."
|
||||
|
||||
def tens_no_zero():
|
||||
return (
|
||||
pynutil.delete("0") + graph_digit
|
||||
| get_ties_digit(
|
||||
get_abs_path("data/numbers/digit.tsv"), get_abs_path("data/numbers/ties.tsv")
|
||||
).invert()
|
||||
| graph_teen
|
||||
| (graph_ties + pynutil.delete("0"))
|
||||
)
|
||||
|
||||
def hundred_non_zero():
|
||||
return (graph_digit_no_one + insert_space | pynini.cross("1", "ein ")) + pynutil.insert(
|
||||
"hundert"
|
||||
) + (
|
||||
pynini.closure(insert_space + pynutil.insert(AND, weight=0.0001), 0, 1)
|
||||
+ insert_space
|
||||
+ tens_no_zero()
|
||||
| pynutil.delete("00")
|
||||
) | pynutil.delete(
|
||||
"0"
|
||||
) + tens_no_zero()
|
||||
|
||||
def thousand():
|
||||
return (
|
||||
hundred_non_zero() + insert_space + pynutil.insert("tausend")
|
||||
| pynutil.delete("000")
|
||||
) + (insert_space + hundred_non_zero() | pynutil.delete("000"))
|
||||
|
||||
optional_plural_quantity_en = pynini.closure(pynutil.insert("en", weight=-0.0001), 0, 1)
|
||||
optional_plural_quantity_n = pynini.closure(pynutil.insert("n", weight=-0.0001), 0, 1)
|
||||
graph_million = pynini.union(
|
||||
hundred_non_zero()
|
||||
+ insert_space
|
||||
+ pynutil.insert("million")
|
||||
+ optional_plural_quantity_en,
|
||||
pynutil.delete("000"),
|
||||
)
|
||||
|
||||
graph_billion = pynini.union(
|
||||
hundred_non_zero()
|
||||
+ insert_space
|
||||
+ pynutil.insert("milliarde")
|
||||
+ optional_plural_quantity_n,
|
||||
pynutil.delete("000"),
|
||||
)
|
||||
|
||||
graph_trillion = pynini.union(
|
||||
hundred_non_zero()
|
||||
+ insert_space
|
||||
+ pynutil.insert("billion")
|
||||
+ optional_plural_quantity_en,
|
||||
pynutil.delete("000"),
|
||||
)
|
||||
|
||||
graph_quadrillion = pynini.union(
|
||||
hundred_non_zero()
|
||||
+ insert_space
|
||||
+ pynutil.insert("billiarde")
|
||||
+ optional_plural_quantity_n,
|
||||
pynutil.delete("000"),
|
||||
)
|
||||
|
||||
graph_quintillion = pynini.union(
|
||||
hundred_non_zero()
|
||||
+ insert_space
|
||||
+ pynutil.insert("trillion")
|
||||
+ optional_plural_quantity_en,
|
||||
pynutil.delete("000"),
|
||||
)
|
||||
|
||||
graph_sextillion = pynini.union(
|
||||
hundred_non_zero()
|
||||
+ insert_space
|
||||
+ pynutil.insert("trilliarde")
|
||||
+ optional_plural_quantity_n,
|
||||
pynutil.delete("000"),
|
||||
)
|
||||
graph = pynini.union(
|
||||
graph_sextillion
|
||||
+ insert_space
|
||||
+ graph_quintillion
|
||||
+ insert_space
|
||||
+ graph_quadrillion
|
||||
+ insert_space
|
||||
+ graph_trillion
|
||||
+ insert_space
|
||||
+ graph_billion
|
||||
+ insert_space
|
||||
+ graph_million
|
||||
+ insert_space
|
||||
+ thousand()
|
||||
)
|
||||
|
||||
fix_syntax = [
|
||||
("eins tausend", "ein tausend"),
|
||||
("eins millionen", "eine million"),
|
||||
("eins milliarden", "eine milliarde"),
|
||||
("eins billionen", "eine billion"),
|
||||
("eins billiarden", "eine billiarde"),
|
||||
]
|
||||
fix_syntax = pynini.union(*[pynini.cross(*x) for x in fix_syntax])
|
||||
self.graph = (
|
||||
((DAMO_DIGIT - "0" + pynini.closure(DAMO_DIGIT, 0)) - "0" - "1")
|
||||
@ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", DAMO_SIGMA)
|
||||
@ DAMO_DIGIT**24
|
||||
@ graph
|
||||
@ pynini.cdrewrite(delete_space, "[BOS]", "", DAMO_SIGMA)
|
||||
@ pynini.cdrewrite(delete_space, "", "[EOS]", DAMO_SIGMA)
|
||||
@ pynini.cdrewrite(pynini.cross(" ", " "), "", "", DAMO_SIGMA)
|
||||
@ pynini.cdrewrite(fix_syntax, "[BOS]", "", DAMO_SIGMA)
|
||||
)
|
||||
self.graph |= graph_zero | pynini.cross("1", "eins")
|
||||
|
||||
# self.graph = pynini.cdrewrite(pynutil.delete(separator), "", "", DAMO_SIGMA) @ self.graph
|
||||
self.graph = self.graph.optimize()
|
||||
|
||||
self.graph_hundred_component_at_least_one_none_zero_digit = (
|
||||
((DAMO_DIGIT - "0" + pynini.closure(DAMO_DIGIT, 0)) - "0" - "1")
|
||||
@ pynini.cdrewrite(pynini.closure(pynutil.insert("0")), "[BOS]", "", DAMO_SIGMA)
|
||||
@ DAMO_DIGIT**3
|
||||
@ hundred_non_zero()
|
||||
) | pynini.cross("1", "eins")
|
||||
|
||||
self.graph_hundred_component_at_least_one_none_zero_digit = (
|
||||
self.graph_hundred_component_at_least_one_none_zero_digit.optimize()
|
||||
)
|
||||
|
||||
self.two_digit_non_zero = (
|
||||
pynini.closure(DAMO_DIGIT, 1, 2)
|
||||
@ self.graph_hundred_component_at_least_one_none_zero_digit
|
||||
)
|
||||
|
||||
optional_minus_graph = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
|
||||
)
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,130 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_DIGIT,
|
||||
TO_LOWER,
|
||||
GraphFst,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
graph_teen = pynini.invert(pynini.string_file(get_abs_path("data/numbers/teen.tsv"))).optimize()
|
||||
graph_digit = pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv"))).optimize()
|
||||
ties_graph = pynini.invert(pynini.string_file(get_abs_path("data/numbers/ties.tsv"))).optimize()
|
||||
delete_leading_zero = (pynutil.delete("0") | (DAMO_DIGIT - "0")) + DAMO_DIGIT
|
||||
|
||||
|
||||
def get_year_graph(cardinal: GraphFst) -> "pynini.FstLike":
|
||||
"""
|
||||
Returns year verbalizations as fst
|
||||
|
||||
< 2000 neunzehn (hundert) (vier und zwanzig), >= 2000 regular cardinal
|
||||
**00 ** hundert
|
||||
|
||||
Args:
|
||||
delete_leading_zero: removed leading zero
|
||||
cardinal: cardinal GraphFst
|
||||
"""
|
||||
|
||||
year_gt_2000 = (pynini.union("21", "20") + DAMO_DIGIT**2) @ cardinal.graph
|
||||
|
||||
graph_two_digit = delete_leading_zero @ cardinal.two_digit_non_zero
|
||||
hundred = pynutil.insert("hundert")
|
||||
graph_double_double = (
|
||||
(pynini.accep("1") + DAMO_DIGIT) @ graph_two_digit
|
||||
+ insert_space
|
||||
+ pynini.closure(hundred + insert_space, 0, 1)
|
||||
+ graph_two_digit
|
||||
)
|
||||
# for 20**
|
||||
graph_double_double |= pynini.accep("20") @ graph_two_digit + insert_space + graph_two_digit
|
||||
graph = (
|
||||
graph_double_double
|
||||
| (pynini.accep("1") + DAMO_DIGIT) @ graph_two_digit
|
||||
+ insert_space
|
||||
+ pynutil.delete("00")
|
||||
+ hundred
|
||||
| year_gt_2000
|
||||
)
|
||||
return graph
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying date, e.g.
|
||||
"01.04.2010" -> date { day: "erster" month: "april" year: "zwei tausend zehn" preserve_order: true }
|
||||
"1994" -> date { year: "neunzehn vier und neuzig" }
|
||||
"1900" -> date { year: "neunzehn hundert" }
|
||||
|
||||
Args:
|
||||
cardinal: cardinal GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic: bool):
|
||||
super().__init__(name="date", kind="classify", deterministic=deterministic)
|
||||
|
||||
month_abbr_graph = load_labels(get_abs_path("data/months/abbr_to_name.tsv"))
|
||||
number_to_month = pynini.string_file(get_abs_path("data/months/numbers.tsv")).optimize()
|
||||
month_graph = pynini.union(*[x[1] for x in month_abbr_graph]).optimize()
|
||||
month_abbr_graph = pynini.string_map(month_abbr_graph)
|
||||
month_abbr_graph = (
|
||||
pynutil.add_weight(month_abbr_graph, weight=0.0001)
|
||||
| ((TO_LOWER + pynini.closure(DAMO_CHAR)) @ month_abbr_graph)
|
||||
) + pynini.closure(pynutil.delete(".", weight=-0.0001), 0, 1)
|
||||
|
||||
self.month_abbr = month_abbr_graph
|
||||
month_graph |= (TO_LOWER + pynini.closure(DAMO_CHAR)) @ month_graph
|
||||
# jan.-> januar, Jan-> januar, januar-> januar
|
||||
month_graph |= month_abbr_graph
|
||||
|
||||
numbers = cardinal.graph_hundred_component_at_least_one_none_zero_digit
|
||||
optional_leading_zero = delete_leading_zero | DAMO_DIGIT
|
||||
# 01, 31, 1
|
||||
digit_day = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 32)]) @ numbers
|
||||
day = (pynutil.insert('day: "') + digit_day + pynutil.insert('"')).optimize()
|
||||
|
||||
digit_month = optional_leading_zero @ pynini.union(*[str(x) for x in range(1, 13)])
|
||||
number_to_month = digit_month @ number_to_month
|
||||
digit_month @= numbers
|
||||
|
||||
month_name = (pynutil.insert('month: "') + month_graph + pynutil.insert('"')).optimize()
|
||||
month_number = (
|
||||
pynutil.insert('month: "')
|
||||
+ (pynutil.add_weight(digit_month, weight=0.0001) | number_to_month)
|
||||
+ pynutil.insert('"')
|
||||
).optimize()
|
||||
|
||||
# prefer cardinal over year
|
||||
year = pynutil.add_weight(get_year_graph(cardinal=cardinal), weight=0.001)
|
||||
self.year = year
|
||||
|
||||
year_only = pynutil.insert('year: "') + year + pynutil.insert('"')
|
||||
|
||||
graph_dmy = (
|
||||
day
|
||||
+ pynutil.delete(".")
|
||||
+ pynini.closure(pynutil.delete(" "), 0, 1)
|
||||
+ insert_space
|
||||
+ month_name
|
||||
+ pynini.closure(pynini.accep(" ") + year_only, 0, 1)
|
||||
)
|
||||
|
||||
separators = ["."]
|
||||
for sep in separators:
|
||||
year_optional = pynini.closure(pynini.cross(sep, " ") + year_only, 0, 1)
|
||||
new_graph = day + pynini.cross(sep, " ") + month_number + year_optional
|
||||
graph_dmy |= new_graph
|
||||
|
||||
dash = "-"
|
||||
day_optional = pynini.closure(pynini.cross(dash, " ") + day, 0, 1)
|
||||
graph_ymd = year_only + pynini.cross(dash, " ") + month_number + day_optional
|
||||
|
||||
final_graph = graph_dmy + pynutil.insert(" preserve_order: true")
|
||||
final_graph |= year_only
|
||||
final_graph |= graph_ymd
|
||||
|
||||
self.final_graph = final_graph.optimize()
|
||||
self.fst = self.add_tokens(self.final_graph).optimize()
|
||||
@@ -0,0 +1,82 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, insert_space
|
||||
from pynini.lib import pynutil
|
||||
|
||||
quantities = pynini.string_file(get_abs_path("data/numbers/quantities.tsv"))
|
||||
|
||||
|
||||
def get_quantity(
|
||||
decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike"
|
||||
) -> "pynini.FstLike":
|
||||
"""
|
||||
Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
|
||||
e.g. 1 million -> integer_part: "eine" quantity: "million"
|
||||
e.g. 1.4 million -> integer_part: "eins" fractional_part: "vier" quantity: "million"
|
||||
|
||||
Args:
|
||||
decimal: decimal FST
|
||||
cardinal_up_to_hundred: cardinal FST
|
||||
"""
|
||||
numbers = cardinal_up_to_hundred
|
||||
|
||||
res = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ numbers
|
||||
+ pynutil.insert('"')
|
||||
+ pynini.accep(" ")
|
||||
+ pynutil.insert('quantity: "')
|
||||
+ quantities
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
res |= (
|
||||
decimal
|
||||
+ pynini.accep(" ")
|
||||
+ pynutil.insert('quantity: "')
|
||||
+ quantities
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal, e.g.
|
||||
-11,4006 billion -> decimal { negative: "true" integer_part: "elf" fractional_part: "vier null null sechs" quantity: "billion" preserve_order: true }
|
||||
1 billion -> decimal { integer_part: "eins" quantity: "billion" preserve_order: true }
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert()
|
||||
graph_digit |= pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert()
|
||||
graph_digit |= pynini.cross("1", "eins")
|
||||
self.graph = graph_digit + pynini.closure(insert_space + graph_digit).optimize()
|
||||
|
||||
point = pynutil.delete(",")
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
|
||||
)
|
||||
|
||||
self.graph_fractional = (
|
||||
pynutil.insert('fractional_part: "') + self.graph + pynutil.insert('"')
|
||||
)
|
||||
self.graph_integer = (
|
||||
pynutil.insert('integer_part: "') + cardinal.graph + pynutil.insert('"')
|
||||
)
|
||||
final_graph_wo_sign = self.graph_integer + point + insert_space + self.graph_fractional
|
||||
|
||||
self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(
|
||||
final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit
|
||||
)
|
||||
final_graph = optional_graph_negative + self.final_graph_wo_negative
|
||||
final_graph += pynutil.insert(" preserve_order: true")
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,64 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying electronic: email addresses
|
||||
e.g. "abc@hotmail.com" -> electronic { username: "abc" domain: "hotmail.com" preserve_order: true }
|
||||
e.g. "www.abc.com/123" -> electronic { protocol: "www." domain: "abc.com/123" preserve_order: true }
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="electronic", kind="classify", deterministic=deterministic)
|
||||
|
||||
dot = pynini.accep(".")
|
||||
accepted_common_domains = [
|
||||
x[0] for x in load_labels(get_abs_path("data/electronic/domain.tsv"))
|
||||
]
|
||||
accepted_common_domains = pynini.union(*accepted_common_domains)
|
||||
accepted_symbols = [x[0] for x in load_labels(get_abs_path("data/electronic/symbols.tsv"))]
|
||||
accepted_symbols = pynini.union(*accepted_symbols) - dot
|
||||
accepted_characters = pynini.closure(DAMO_ALPHA | DAMO_DIGIT | accepted_symbols)
|
||||
|
||||
# email
|
||||
username = (
|
||||
pynutil.insert('username: "')
|
||||
+ accepted_characters
|
||||
+ pynutil.insert('"')
|
||||
+ pynini.cross("@", " ")
|
||||
)
|
||||
domain_graph = accepted_characters + dot + accepted_characters
|
||||
domain_graph = pynutil.insert('domain: "') + domain_graph + pynutil.insert('"')
|
||||
domain_common_graph = (
|
||||
pynutil.insert('domain: "')
|
||||
+ accepted_characters
|
||||
+ accepted_common_domains
|
||||
+ pynini.closure(
|
||||
(accepted_symbols | dot) + pynini.closure(accepted_characters, 1), 0, 1
|
||||
)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
graph = (username + domain_graph) | domain_common_graph
|
||||
|
||||
# url
|
||||
protocol_start = pynini.accep("https://") | pynini.accep("http://")
|
||||
protocol_end = pynini.accep("www.")
|
||||
protocol = protocol_start | protocol_end | (protocol_start + protocol_end)
|
||||
protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"')
|
||||
graph |= protocol + insert_space + (domain_graph | domain_common_graph)
|
||||
self.graph = graph
|
||||
|
||||
final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true"))
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,42 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying fraction
|
||||
"23 4/6" ->
|
||||
fraction { integer: "drei und zwanzig" numerator: "vier" denominator: "sechs" preserve_order: true }
|
||||
|
||||
Args:
|
||||
cardinal: cardinal GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal, deterministic: bool = True):
|
||||
super().__init__(name="fraction", kind="classify", deterministic=deterministic)
|
||||
cardinal_graph = cardinal.graph
|
||||
|
||||
self.optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("-", '"true" '), 0, 1
|
||||
)
|
||||
self.integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
|
||||
self.numerator = (
|
||||
pynutil.insert('numerator: "')
|
||||
+ cardinal_graph
|
||||
+ pynini.cross(pynini.union("/", " / "), '" ')
|
||||
)
|
||||
self.denominator = pynutil.insert('denominator: "') + cardinal_graph + pynutil.insert('"')
|
||||
|
||||
self.graph = (
|
||||
self.optional_graph_negative
|
||||
+ pynini.closure(self.integer + pynini.accep(" "), 0, 1)
|
||||
+ self.numerator
|
||||
+ self.denominator
|
||||
)
|
||||
|
||||
graph = self.graph + pynutil.insert(" preserve_order: true")
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,185 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
DAMO_NON_BREAKING_SPACE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.examples import plurals
|
||||
from pynini.lib import pynutil
|
||||
|
||||
unit_singular = pynini.string_file(get_abs_path("data/measure/measurements.tsv"))
|
||||
suppletive = pynini.string_file(get_abs_path("data/measure/suppletive.tsv"))
|
||||
|
||||
|
||||
def singular_to_plural():
|
||||
# plural endung n/en maskuline Nomen mit den Endungen e, ent, and, ant, ist, or
|
||||
_n = DAMO_SIGMA + pynini.union("e") + pynutil.insert("n")
|
||||
_en = (
|
||||
DAMO_SIGMA
|
||||
+ pynini.union(
|
||||
"ent", "and", "ant", "ist", "or", "ion", "ik", "heit", "keit", "schaft", "tät", "ung"
|
||||
)
|
||||
+ pynutil.insert("en")
|
||||
)
|
||||
_nen = DAMO_SIGMA + pynini.union("in") + (pynutil.insert("e") | pynutil.insert("nen"))
|
||||
_fremd = DAMO_SIGMA + pynini.union("ma", "um", "us") + pynutil.insert("en")
|
||||
# maskuline Nomen mit den Endungen eur, ich, ier, ig, ling, ör
|
||||
_e = DAMO_SIGMA + pynini.union("eur", "ich", "ier", "ig", "ling", "ör") + pynutil.insert("e")
|
||||
_s = DAMO_SIGMA + pynini.union("a", "i", "o", "u", "y") + pynutil.insert("s")
|
||||
|
||||
graph_plural = plurals._priority_union(
|
||||
suppletive, pynini.union(_n, _en, _nen, _fremd, _e, _s), DAMO_SIGMA
|
||||
).optimize()
|
||||
|
||||
return graph_plural
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying measure, e.g.
|
||||
"2,4 oz" -> measure { cardinal { integer_part: "zwei" fractional_part: "vier" units: "unzen" preserve_order: true } }
|
||||
"1 oz" -> measure { cardinal { integer: "zwei" units: "unze" preserve_order: true } }
|
||||
"1 million oz" -> measure { cardinal { integer: "eins" quantity: "million" units: "unze" preserve_order: true } }
|
||||
This class also converts words containing numbers and letters
|
||||
e.g. "a-8" —> "a acht"
|
||||
e.g. "1,2-a" —> "ein komma zwei a"
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst, deterministic: bool = True
|
||||
):
|
||||
super().__init__(name="measure", kind="classify", deterministic=deterministic)
|
||||
cardinal_graph = cardinal.graph
|
||||
|
||||
graph_unit_singular = convert_space(unit_singular)
|
||||
graph_unit_plural = graph_unit_singular @ pynini.cdrewrite(
|
||||
convert_space(suppletive), "", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
optional_graph_negative = pynini.closure("-", 0, 1)
|
||||
|
||||
graph_unit_denominator = (
|
||||
pynini.cross("/", "pro") + pynutil.insert(DAMO_NON_BREAKING_SPACE) + graph_unit_singular
|
||||
)
|
||||
|
||||
optional_unit_denominator = pynini.closure(
|
||||
pynutil.insert(DAMO_NON_BREAKING_SPACE) + graph_unit_denominator,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
unit_plural = (
|
||||
pynutil.insert('units: "')
|
||||
+ (graph_unit_plural + (optional_unit_denominator) | graph_unit_denominator)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
unit_singular_graph = (
|
||||
pynutil.insert('units: "')
|
||||
+ ((graph_unit_singular + optional_unit_denominator) | graph_unit_denominator)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
subgraph_decimal = (
|
||||
decimal.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural
|
||||
)
|
||||
|
||||
subgraph_cardinal = (
|
||||
(optional_graph_negative + (pynini.closure(DAMO_DIGIT) - "1")) @ cardinal.fst
|
||||
+ insert_space
|
||||
+ pynini.closure(pynutil.delete(" "), 0, 1)
|
||||
+ unit_plural
|
||||
)
|
||||
|
||||
subgraph_cardinal |= (
|
||||
(optional_graph_negative + pynini.accep("1"))
|
||||
@ cardinal.fst
|
||||
@ pynini.cdrewrite(pynini.cross("eins", "ein"), "", "", DAMO_SIGMA)
|
||||
+ insert_space
|
||||
+ pynini.closure(pynutil.delete(" "), 0, 1)
|
||||
+ unit_singular_graph
|
||||
)
|
||||
|
||||
subgraph_fraction = (
|
||||
fraction.fst + insert_space + pynini.closure(pynutil.delete(" "), 0, 1) + unit_plural
|
||||
)
|
||||
|
||||
cardinal_dash_alpha = (
|
||||
pynutil.insert('cardinal { integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert('" } units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
alpha_dash_cardinal = (
|
||||
pynutil.insert('units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(' cardinal { integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('" }')
|
||||
)
|
||||
|
||||
decimal_dash_alpha = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert(' } units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
decimal_times = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.insert(' } units: "')
|
||||
+ pynini.union("x", "X")
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
cardinal_times = (
|
||||
pynutil.insert('cardinal { integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('" } units: "')
|
||||
+ pynini.union("x", "X")
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
alpha_dash_decimal = (
|
||||
pynutil.insert('units: "')
|
||||
+ pynini.closure(DAMO_ALPHA, 1)
|
||||
+ pynutil.delete("-")
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(" decimal { ")
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.insert(" }")
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
subgraph_decimal
|
||||
| subgraph_cardinal
|
||||
| cardinal_dash_alpha
|
||||
| alpha_dash_cardinal
|
||||
| decimal_dash_alpha
|
||||
| decimal_times
|
||||
| alpha_dash_decimal
|
||||
| subgraph_fraction
|
||||
| cardinal_times
|
||||
)
|
||||
final_graph += pynutil.insert(" preserve_order: true")
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,160 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
min_singular = pynini.string_file(get_abs_path("data/money/currency_minor_singular.tsv"))
|
||||
min_plural = pynini.string_file(get_abs_path("data/money/currency_minor_plural.tsv"))
|
||||
maj_singular = pynini.string_file((get_abs_path("data/money/currency.tsv")))
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying money, e.g.
|
||||
"€1" -> money { currency_maj: "euro" integer_part: "ein"}
|
||||
"€1,000" -> money { currency_maj: "euro" integer_part: "ein" }
|
||||
"€1,001" -> money { currency_maj: "euro" integer_part: "eins" fractional_part: "null null eins"}
|
||||
"£1,4" -> money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" preserve_order: true}
|
||||
-> money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" currency_min: "pence" preserve_order: true}
|
||||
"£0,01" -> money { fractional_part: "ein" currency_min: "penny" preserve_order: true}
|
||||
"£0,01 million" -> money { currency_maj: "pfund" integer_part: "null" fractional_part: "null eins" quantity: "million"}
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="money", kind="classify", deterministic=deterministic)
|
||||
cardinal_graph = cardinal.graph
|
||||
graph_decimal_final = decimal.fst
|
||||
|
||||
maj_singular_labels = load_labels(get_abs_path("data/money/currency.tsv"))
|
||||
maj_singular_graph = convert_space(maj_singular)
|
||||
maj_plural_graph = maj_singular_graph
|
||||
|
||||
graph_maj_singular = (
|
||||
pynutil.insert('currency_maj: "') + maj_singular_graph + pynutil.insert('"')
|
||||
)
|
||||
graph_maj_plural = (
|
||||
pynutil.insert('currency_maj: "') + maj_plural_graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
optional_delete_fractional_zeros = pynini.closure(
|
||||
pynutil.delete(",") + pynini.closure(pynutil.delete("0"), 1), 0, 1
|
||||
)
|
||||
graph_integer_one = (
|
||||
pynutil.insert('integer_part: "') + pynini.cross("1", "ein") + pynutil.insert('"')
|
||||
)
|
||||
|
||||
# only for decimals where third decimal after comma is non-zero or with quantity
|
||||
decimal_delete_last_zeros = (
|
||||
pynini.closure(DAMO_DIGIT, 1)
|
||||
+ pynini.accep(",")
|
||||
+ pynini.closure(DAMO_DIGIT, 2)
|
||||
+ (DAMO_DIGIT - "0")
|
||||
+ pynini.closure(pynutil.delete("0"))
|
||||
)
|
||||
decimal_with_quantity = DAMO_SIGMA + DAMO_ALPHA
|
||||
graph_decimal = (
|
||||
graph_maj_plural
|
||||
+ insert_space
|
||||
+ (decimal_delete_last_zeros | decimal_with_quantity) @ graph_decimal_final
|
||||
)
|
||||
|
||||
graph_integer = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ ((DAMO_SIGMA - "1") @ cardinal_graph)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_integer_only = graph_maj_singular + insert_space + graph_integer_one
|
||||
graph_integer_only |= graph_maj_plural + insert_space + graph_integer
|
||||
|
||||
graph = (graph_integer_only + optional_delete_fractional_zeros) | graph_decimal
|
||||
|
||||
# remove trailing zeros of non zero number in the first 2 digits and fill up to 2 digits
|
||||
# e.g. 2000 -> 20, 0200->02, 01 -> 01, 10 -> 10
|
||||
# not accepted: 002, 00, 0,
|
||||
two_digits_fractional_part = (
|
||||
pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(pynutil.delete("0"))
|
||||
) @ (
|
||||
(pynutil.delete("0") + (DAMO_DIGIT - "0"))
|
||||
| ((DAMO_DIGIT - "0") + pynutil.insert("0"))
|
||||
| ((DAMO_DIGIT - "0") + DAMO_DIGIT)
|
||||
)
|
||||
|
||||
graph_min_singular = pynutil.insert(' currency_min: "') + min_singular + pynutil.insert('"')
|
||||
graph_min_plural = pynutil.insert(' currency_min: "') + min_plural + pynutil.insert('"')
|
||||
|
||||
# format ** euro ** cent
|
||||
decimal_graph_with_minor = None
|
||||
for curr_symbol, _ in maj_singular_labels:
|
||||
preserve_order = pynutil.insert(" preserve_order: true")
|
||||
integer_plus_maj = (
|
||||
graph_integer + insert_space + pynutil.insert(curr_symbol) @ graph_maj_plural
|
||||
)
|
||||
integer_plus_maj |= (
|
||||
graph_integer_one + insert_space + pynutil.insert(curr_symbol) @ graph_maj_singular
|
||||
)
|
||||
# non zero integer part
|
||||
integer_plus_maj = (pynini.closure(DAMO_DIGIT) - "0") @ integer_plus_maj
|
||||
|
||||
graph_fractional_one = two_digits_fractional_part @ pynini.cross("1", "ein")
|
||||
graph_fractional_one = (
|
||||
pynutil.insert('fractional_part: "') + graph_fractional_one + pynutil.insert('"')
|
||||
)
|
||||
graph_fractional = (
|
||||
two_digits_fractional_part
|
||||
@ (pynini.closure(DAMO_DIGIT, 1, 2) - "1")
|
||||
@ cardinal.two_digit_non_zero
|
||||
)
|
||||
graph_fractional = (
|
||||
pynutil.insert('fractional_part: "') + graph_fractional + pynutil.insert('"')
|
||||
)
|
||||
|
||||
fractional_plus_min = (
|
||||
graph_fractional + insert_space + pynutil.insert(curr_symbol) @ graph_min_plural
|
||||
)
|
||||
fractional_plus_min |= (
|
||||
graph_fractional_one
|
||||
+ insert_space
|
||||
+ pynutil.insert(curr_symbol) @ graph_min_singular
|
||||
)
|
||||
|
||||
decimal_graph_with_minor_curr = (
|
||||
integer_plus_maj + pynini.cross(",", " ") + fractional_plus_min
|
||||
)
|
||||
decimal_graph_with_minor_curr |= pynutil.add_weight(
|
||||
integer_plus_maj
|
||||
+ pynini.cross(",", " ")
|
||||
+ pynutil.insert('fractional_part: "')
|
||||
+ two_digits_fractional_part @ cardinal.two_digit_non_zero
|
||||
+ pynutil.insert('"'),
|
||||
weight=0.0001,
|
||||
)
|
||||
|
||||
decimal_graph_with_minor_curr |= pynutil.delete("0,") + fractional_plus_min
|
||||
decimal_graph_with_minor_curr = (
|
||||
pynutil.delete(curr_symbol) + decimal_graph_with_minor_curr + preserve_order
|
||||
)
|
||||
|
||||
decimal_graph_with_minor = (
|
||||
decimal_graph_with_minor_curr
|
||||
if decimal_graph_with_minor is None
|
||||
else pynini.union(decimal_graph_with_minor, decimal_graph_with_minor_curr)
|
||||
)
|
||||
|
||||
final_graph = graph | decimal_graph_with_minor
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,37 @@
|
||||
# Adapted from https://github.com/google/TextNormalizationCoveringGrammars
|
||||
# Russian minimally supervised number grammar.
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying cardinals, e.g.
|
||||
"2." -> ordinal { integer: "zwei" } }
|
||||
"2tes" -> ordinal { integer: "zwei" } }
|
||||
|
||||
Args:
|
||||
cardinal: cardinal GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic=False):
|
||||
super().__init__(name="ordinal", kind="classify", deterministic=deterministic)
|
||||
|
||||
cardinal_graph = cardinal.graph
|
||||
endings = ["ter", "tes", "tem", "te", "ten"]
|
||||
self.graph = (
|
||||
(
|
||||
pynini.closure(DAMO_DIGIT | pynini.accep("."))
|
||||
+ pynutil.delete(
|
||||
pynutil.add_weight(pynini.union(*endings), weight=0.0001) | pynini.accep(".")
|
||||
)
|
||||
)
|
||||
@ cardinal_graph
|
||||
).optimize()
|
||||
final_graph = pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,66 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_DIGIT, GraphFst, insert_space
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying telephone, which includes country code, number part and extension
|
||||
|
||||
E.g
|
||||
"+49 1234-1233" -> telephone { country_code: "plus neun und vierzig" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true }
|
||||
"(012) 1234-1233" -> telephone { country_code: "null eins zwei" number_part: "eins zwei drei vier eins zwei drei drei" preserve_order: true }
|
||||
(0**)
|
||||
|
||||
Args:
|
||||
cardinal: cardinal GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="telephone", kind="classify", deterministic=deterministic)
|
||||
|
||||
graph_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
).optimize()
|
||||
graph_digit_no_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
).optimize() | pynini.cross("1", "eins")
|
||||
graph_digit = graph_digit_no_zero | graph_zero
|
||||
|
||||
numbers_with_single_digits = pynini.closure(graph_digit + insert_space) + graph_digit
|
||||
|
||||
two_digit_and_zero = (DAMO_DIGIT**2 @ cardinal.two_digit_non_zero) | graph_zero
|
||||
# def add_space_after_two_digit():
|
||||
# return pynini.closure(two_digit_and_zero + insert_space) + (
|
||||
# two_digit_and_zero
|
||||
# )
|
||||
|
||||
country_code = pynini.closure(pynini.cross("+", "plus "), 0, 1) + two_digit_and_zero
|
||||
country_code |= (
|
||||
pynutil.delete("(")
|
||||
+ graph_zero
|
||||
+ insert_space
|
||||
+ numbers_with_single_digits
|
||||
+ pynutil.delete(")")
|
||||
)
|
||||
country_code |= graph_zero + insert_space + numbers_with_single_digits
|
||||
|
||||
country_code = pynutil.insert('country_code: "') + country_code + pynutil.insert('"')
|
||||
|
||||
del_separator = pynini.cross(pynini.union("-", " "), " ")
|
||||
# numbers_with_two_digits = pynini.closure(graph_digit + insert_space) + add_space_after_two_digit() + pynini.closure(insert_space + graph_digit)
|
||||
# numbers = numbers_with_two_digits + pynini.closure(del_separator + numbers_with_two_digits, 0, 1)
|
||||
numbers = numbers_with_single_digits + pynini.closure(
|
||||
del_separator + numbers_with_single_digits, 0, 1
|
||||
)
|
||||
number_length = pynini.closure((DAMO_DIGIT | pynini.union("-", " ", ")", "(")), 7)
|
||||
number_part = pynini.compose(number_length, numbers)
|
||||
number = pynutil.insert('number_part: "') + number_part + pynutil.insert('"')
|
||||
|
||||
graph = country_code + pynini.accep(" ") + number
|
||||
self.graph = graph
|
||||
final_graph = self.add_tokens(self.graph + pynutil.insert(" preserve_order: true"))
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,94 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying time, e.g.
|
||||
"02:15 Uhr est" -> time { hours: "2" minutes: "15" zone: "e s t"}
|
||||
"2 Uhr" -> time { hours: "2" }
|
||||
"09:00 Uhr" -> time { hours: "2" }
|
||||
"02:15:10 Uhr" -> time { hours: "2" minutes: "15" seconds: "10"}
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="classify", deterministic=deterministic)
|
||||
|
||||
final_suffix = pynutil.delete(" ") + pynutil.delete("Uhr") | pynutil.delete("uhr")
|
||||
time_zone_graph = pynini.string_file(get_abs_path("data/time/time_zone.tsv"))
|
||||
|
||||
labels_hour = [str(x) for x in range(0, 25)]
|
||||
labels_minute_single = [str(x) for x in range(1, 10)]
|
||||
labels_minute_double = [str(x) for x in range(10, 60)]
|
||||
|
||||
delete_leading_zero_to_double_digit = (
|
||||
pynutil.delete("0") | (DAMO_DIGIT - "0")
|
||||
) + DAMO_DIGIT
|
||||
|
||||
graph_hour = pynini.union(*labels_hour)
|
||||
|
||||
graph_minute_single = pynini.union(*labels_minute_single)
|
||||
graph_minute_double = pynini.union(*labels_minute_double)
|
||||
|
||||
final_graph_hour_only = pynutil.insert('hours: "') + graph_hour + pynutil.insert('"')
|
||||
final_graph_hour = (
|
||||
pynutil.insert('hours: "')
|
||||
+ delete_leading_zero_to_double_digit @ graph_hour
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
final_graph_minute = (
|
||||
pynutil.insert('minutes: "')
|
||||
+ (pynutil.delete("0") + graph_minute_single | graph_minute_double)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
final_graph_second = (
|
||||
pynutil.insert('seconds: "')
|
||||
+ (pynutil.delete("0") + graph_minute_single | graph_minute_double)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
final_time_zone_optional = pynini.closure(
|
||||
pynini.accep(" ")
|
||||
+ pynutil.insert('zone: "')
|
||||
+ convert_space(time_zone_graph)
|
||||
+ pynutil.insert('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
# 02:30 Uhr
|
||||
graph_hm = (
|
||||
final_graph_hour
|
||||
+ pynutil.delete(":")
|
||||
+ (pynutil.delete("00") | (insert_space + final_graph_minute))
|
||||
+ final_suffix
|
||||
+ final_time_zone_optional
|
||||
)
|
||||
|
||||
# 10:30:05 Uhr,
|
||||
graph_hms = (
|
||||
final_graph_hour
|
||||
+ pynutil.delete(":")
|
||||
+ (pynini.cross("00", ' minutes: "0"') | (insert_space + final_graph_minute))
|
||||
+ pynutil.delete(":")
|
||||
+ (pynini.cross("00", ' seconds: "0"') | (insert_space + final_graph_second))
|
||||
+ final_suffix
|
||||
+ final_time_zone_optional
|
||||
+ pynutil.insert(" preserve_order: true")
|
||||
)
|
||||
|
||||
# 2 Uhr est
|
||||
graph_h = final_graph_hour_only + final_suffix + final_time_zone_optional
|
||||
final_graph = (graph_hm | graph_h | graph_hms).optimize()
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,145 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.taggers.cardinal import CardinalFst
|
||||
from fun_text_processing.text_normalization.de.taggers.date import DateFst
|
||||
from fun_text_processing.text_normalization.de.taggers.decimal import DecimalFst
|
||||
from fun_text_processing.text_normalization.de.taggers.electronic import ElectronicFst
|
||||
from fun_text_processing.text_normalization.de.taggers.fraction import FractionFst
|
||||
from fun_text_processing.text_normalization.de.taggers.measure import MeasureFst
|
||||
from fun_text_processing.text_normalization.de.taggers.money import MoneyFst
|
||||
from fun_text_processing.text_normalization.de.taggers.ordinal import OrdinalFst
|
||||
from fun_text_processing.text_normalization.de.taggers.telephone import TelephoneFst
|
||||
from fun_text_processing.text_normalization.de.taggers.time import TimeFst
|
||||
from fun_text_processing.text_normalization.de.taggers.whitelist import WhiteListFst
|
||||
from fun_text_processing.text_normalization.de.taggers.word import WordFst
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class ClassifyFst(GraphFst):
|
||||
"""
|
||||
Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
input_case: accepting either "lower_cased" or "cased" input.
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
whitelist: path to a file with whitelist replacements
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_case: str,
|
||||
deterministic: bool = False,
|
||||
cache_dir: str = None,
|
||||
overwrite_cache: bool = False,
|
||||
whitelist: str = None,
|
||||
):
|
||||
super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic)
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
whitelist_file = os.path.basename(whitelist) if whitelist else ""
|
||||
far_file = os.path.join(
|
||||
cache_dir, f"_{input_case}_de_tn_{deterministic}_deterministic{whitelist_file}.far"
|
||||
)
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
|
||||
no_digits = pynini.closure(pynini.difference(DAMO_CHAR, DAMO_DIGIT))
|
||||
self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize()
|
||||
logging.info(f"ClassifyFst.fst was restored from {far_file}.")
|
||||
else:
|
||||
logging.info(f"Creating ClassifyFst grammars. This might take some time...")
|
||||
|
||||
self.cardinal = CardinalFst(deterministic=deterministic)
|
||||
cardinal_graph = self.cardinal.fst
|
||||
|
||||
self.ordinal = OrdinalFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
ordinal_graph = self.ordinal.fst
|
||||
|
||||
self.decimal = DecimalFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
decimal_graph = self.decimal.fst
|
||||
|
||||
self.fraction = FractionFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
fraction_graph = self.fraction.fst
|
||||
self.measure = MeasureFst(
|
||||
cardinal=self.cardinal,
|
||||
decimal=self.decimal,
|
||||
fraction=self.fraction,
|
||||
deterministic=deterministic,
|
||||
)
|
||||
measure_graph = self.measure.fst
|
||||
self.date = DateFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
date_graph = self.date.fst
|
||||
word_graph = WordFst(deterministic=deterministic).fst
|
||||
self.time = TimeFst(deterministic=deterministic)
|
||||
time_graph = self.time.fst
|
||||
self.telephone = TelephoneFst(cardinal=self.cardinal, deterministic=deterministic)
|
||||
telephone_graph = self.telephone.fst
|
||||
self.electronic = ElectronicFst(deterministic=deterministic)
|
||||
electronic_graph = self.electronic.fst
|
||||
self.money = MoneyFst(
|
||||
cardinal=self.cardinal, decimal=self.decimal, deterministic=deterministic
|
||||
)
|
||||
money_graph = self.money.fst
|
||||
self.whitelist = WhiteListFst(
|
||||
input_case=input_case, deterministic=deterministic, input_file=whitelist
|
||||
)
|
||||
whitelist_graph = self.whitelist.fst
|
||||
punct_graph = PunctuationFst(deterministic=deterministic).fst
|
||||
|
||||
classify = (
|
||||
pynutil.add_weight(whitelist_graph, 1.01)
|
||||
| pynutil.add_weight(time_graph, 1.1)
|
||||
| pynutil.add_weight(measure_graph, 1.1)
|
||||
| pynutil.add_weight(cardinal_graph, 1.1)
|
||||
| pynutil.add_weight(fraction_graph, 1.1)
|
||||
| pynutil.add_weight(date_graph, 1.1)
|
||||
| pynutil.add_weight(ordinal_graph, 1.1)
|
||||
| pynutil.add_weight(decimal_graph, 1.1)
|
||||
| pynutil.add_weight(money_graph, 1.1)
|
||||
| pynutil.add_weight(telephone_graph, 1.1)
|
||||
| pynutil.add_weight(electronic_graph, 1.1)
|
||||
)
|
||||
|
||||
classify |= pynutil.add_weight(word_graph, 100)
|
||||
|
||||
punct = (
|
||||
pynutil.insert("tokens { ")
|
||||
+ pynutil.add_weight(punct_graph, weight=1.1)
|
||||
+ pynutil.insert(" }")
|
||||
)
|
||||
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
|
||||
token_plus_punct = (
|
||||
pynini.closure(punct + pynutil.insert(" "))
|
||||
+ token
|
||||
+ pynini.closure(pynutil.insert(" ") + punct)
|
||||
)
|
||||
|
||||
graph = token_plus_punct + pynini.closure(
|
||||
pynutil.add_weight(delete_extra_space, 1.1) + token_plus_punct
|
||||
)
|
||||
graph = delete_space + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
no_digits = pynini.closure(pynini.difference(DAMO_CHAR, DAMO_DIGIT))
|
||||
self.fst_no_digits = pynini.compose(self.fst, no_digits).optimize()
|
||||
|
||||
if far_file:
|
||||
generator_main(far_file, {"tokenize_and_classify": self.fst})
|
||||
logging.info(f"ClassifyFst grammars are saved to {far_file}.")
|
||||
@@ -0,0 +1,52 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst, convert_space
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying whitelist, e.g.
|
||||
"Mr." -> tokens { name: "mister" }
|
||||
This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
|
||||
|
||||
Args:
|
||||
input_case: accepting either "lower_cased" or "cased" input.
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
input_file: path to a file with whitelist replacements
|
||||
"""
|
||||
|
||||
def __init__(self, input_case: str, deterministic: bool = True, input_file: str = None):
|
||||
super().__init__(name="whitelist", kind="classify", deterministic=deterministic)
|
||||
|
||||
def _get_whitelist_graph(input_case, file):
|
||||
whitelist = load_labels(file)
|
||||
if input_case == "lower_cased":
|
||||
whitelist = [[x[0].lower()] + x[1:] for x in whitelist]
|
||||
graph = pynini.string_map(whitelist)
|
||||
return graph
|
||||
|
||||
graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist.tsv"))
|
||||
if not deterministic and input_case != "lower_cased":
|
||||
graph |= pynutil.add_weight(
|
||||
_get_whitelist_graph("lower_cased", get_abs_path("data/whitelist.tsv")),
|
||||
weight=0.0001,
|
||||
)
|
||||
|
||||
if input_file:
|
||||
whitelist_provided = _get_whitelist_graph(input_case, input_file)
|
||||
if not deterministic:
|
||||
graph |= whitelist_provided
|
||||
else:
|
||||
graph = whitelist_provided
|
||||
|
||||
if not deterministic:
|
||||
units_graph = _get_whitelist_graph(
|
||||
input_case, file=get_abs_path("data/measure/measurements.tsv")
|
||||
)
|
||||
graph |= units_graph
|
||||
|
||||
self.graph = graph
|
||||
self.final_graph = convert_space(self.graph).optimize()
|
||||
self.fst = (pynutil.insert('name: "') + self.final_graph + pynutil.insert('"')).optimize()
|
||||
@@ -0,0 +1,19 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_SPACE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying word.
|
||||
e.g. sleep -> tokens { name: "sleep" }
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="word", kind="classify")
|
||||
word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
|
||||
self.fst = word.optimize()
|
||||
@@ -0,0 +1,34 @@
|
||||
import csv
|
||||
import os
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
def get_abs_path(rel_path):
|
||||
"""
|
||||
Get absolute path
|
||||
|
||||
Args:
|
||||
rel_path: relative path to this file
|
||||
|
||||
Returns absolute path
|
||||
"""
|
||||
abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path
|
||||
|
||||
if not os.path.exists(abs_path):
|
||||
logging.warning(f"{abs_path} does not exist")
|
||||
return abs_path
|
||||
|
||||
|
||||
def load_labels(abs_path):
|
||||
"""
|
||||
loads relative path file as dictionary
|
||||
|
||||
Args:
|
||||
abs_path: absolute path
|
||||
|
||||
Returns dictionary of mappings
|
||||
"""
|
||||
label_tsv = open(abs_path, encoding="utf-8")
|
||||
labels = list(csv.reader(label_tsv, delimiter="\t"))
|
||||
return labels
|
||||
@@ -0,0 +1,28 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinals
|
||||
e.g. cardinal { integer: "zwei" } -> "zwei"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic)
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true" ', "minus "), 0, 1)
|
||||
self.optional_sign = optional_sign
|
||||
integer = pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
|
||||
self.integer = pynutil.delete(' "') + integer + pynutil.delete('"')
|
||||
|
||||
integer = pynutil.delete("integer:") + self.integer
|
||||
self.numbers = integer
|
||||
graph = optional_sign + self.numbers
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,55 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing date, e.g.
|
||||
date { day: "vier" month: "april" year: "zwei tausend zwei" } -> "vierter april zwei tausend zwei"
|
||||
date { day: "vier" month: "mai" year: "zwei tausend zwei" } -> "vierter mai zwei tausend zwei"
|
||||
|
||||
Args:
|
||||
ordinal: ordinal verbalizer GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, ordinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="date", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
day_cardinal = (
|
||||
pynutil.delete('day: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
day = day_cardinal @ pynini.cdrewrite(
|
||||
ordinal.ordinal_stem, "", "[EOS]", DAMO_SIGMA
|
||||
) + pynutil.insert("ter")
|
||||
|
||||
months_names = pynini.union(
|
||||
*[x[1] for x in load_labels(get_abs_path("data/months/abbr_to_name.tsv"))]
|
||||
)
|
||||
month = pynutil.delete('month: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
final_month = month @ months_names
|
||||
final_month |= month @ pynini.difference(DAMO_SIGMA, months_names) @ pynini.cdrewrite(
|
||||
ordinal.ordinal_stem, "", "[EOS]", DAMO_SIGMA
|
||||
) + pynutil.insert("ter")
|
||||
|
||||
year = pynutil.delete('year: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
|
||||
# day month year
|
||||
graph_dmy = (
|
||||
day + pynini.accep(" ") + final_month + pynini.closure(pynini.accep(" ") + year, 0, 1)
|
||||
)
|
||||
graph_dmy |= final_month + pynini.accep(" ") + year
|
||||
|
||||
self.graph = graph_dmy | year
|
||||
final_graph = self.graph + delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(final_graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,57 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.taggers.decimal import quantities
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "elf" fractional_part: "vier null sechs" quantity: "billionen" } -> minus elf komma vier null sechs billionen
|
||||
decimal { integer_part: "eins" quantity: "billion" } -> eins billion
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="decimal", kind="classify", deterministic=deterministic)
|
||||
|
||||
delete_space = pynutil.delete(" ")
|
||||
self.optional_sign = pynini.closure(
|
||||
pynini.cross('negative: "true"', "minus ") + delete_space, 0, 1
|
||||
)
|
||||
self.integer = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.fractional_default = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
self.fractional = pynutil.insert(" komma ") + self.fractional_default
|
||||
|
||||
self.quantity = (
|
||||
delete_space
|
||||
+ insert_space
|
||||
+ pynutil.delete('quantity: "')
|
||||
+ quantities
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.optional_quantity = pynini.closure(self.quantity, 0, 1)
|
||||
|
||||
graph = self.optional_sign + (
|
||||
self.integer + self.quantity
|
||||
| self.integer + delete_space + self.fractional + self.optional_quantity
|
||||
)
|
||||
|
||||
self.numbers = graph
|
||||
graph += delete_preserve_order
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,64 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic
|
||||
e.g. electronic { username: "abc" domain: "hotmail.com" } -> "a b c at hotmail punkt com"
|
||||
-> "a b c at h o t m a i l punkt c o m"
|
||||
-> "a b c at hotmail punkt c o m"
|
||||
-> "a b c at h o t m a i l punkt com"
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
|
||||
graph_digit_no_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
).optimize() | pynini.cross("1", "eins")
|
||||
graph_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
).optimize()
|
||||
graph_digit = graph_digit_no_zero | graph_zero
|
||||
graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).optimize()
|
||||
server_common = pynini.string_file(get_abs_path("data/electronic/server_name.tsv"))
|
||||
domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
||||
|
||||
def add_space_after_char():
|
||||
return pynini.closure(DAMO_NOT_QUOTE - pynini.accep(" ") + insert_space) + (
|
||||
DAMO_NOT_QUOTE - pynini.accep(" ")
|
||||
)
|
||||
|
||||
verbalize_characters = pynini.cdrewrite(graph_symbols | graph_digit, "", "", DAMO_SIGMA)
|
||||
|
||||
user_name = pynutil.delete('username: "') + add_space_after_char() + pynutil.delete('"')
|
||||
user_name @= verbalize_characters
|
||||
|
||||
convert_defaults = (
|
||||
pynutil.add_weight(DAMO_NOT_QUOTE, weight=0.0001) | domain_common | server_common
|
||||
)
|
||||
domain = convert_defaults + pynini.closure(insert_space + convert_defaults)
|
||||
domain @= verbalize_characters
|
||||
|
||||
domain = pynutil.delete('domain: "') + domain + pynutil.delete('"')
|
||||
protocol = (
|
||||
pynutil.delete('protocol: "')
|
||||
+ add_space_after_char() @ pynini.cdrewrite(graph_symbols, "", "", DAMO_SIGMA)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.graph = (pynini.closure(protocol + pynini.accep(" "), 0, 1) + domain) | (
|
||||
user_name + pynini.accep(" ") + pynutil.insert("at ") + domain
|
||||
)
|
||||
delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,68 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing fraction
|
||||
e.g. fraction { integer: "drei" numerator: "eins" denominator: "zwei" }-> drei ein halb
|
||||
e.g. fraction { numerator: "vier" denominator: "zwei" } -> vier halbe
|
||||
e.g. fraction { numerator: "drei" denominator: "vier" } -> drei viertel
|
||||
|
||||
Args:
|
||||
ordinal: ordinal GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, ordinal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="fraction", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
optional_sign = pynini.closure(
|
||||
pynini.cross('negative: "true"', "minus ") + pynutil.delete(" "), 0, 1
|
||||
)
|
||||
change_one = pynini.cdrewrite(
|
||||
pynutil.add_weight(pynini.cross("eins", "ein"), weight=-0.0001),
|
||||
"[BOS]",
|
||||
"[EOS]",
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
change_numerator_two = pynini.cdrewrite(
|
||||
pynini.cross("zweitel", "halbe"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
integer = pynutil.delete('integer_part: "') + change_one + pynutil.delete('" ')
|
||||
numerator = pynutil.delete('numerator: "') + change_one + pynutil.delete('" ')
|
||||
denominator = (
|
||||
pynutil.delete('denominator: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE)
|
||||
@ (
|
||||
pynini.cdrewrite(
|
||||
pynini.closure(ordinal.ordinal_stem, 0, 1), "", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
+ pynutil.insert("tel")
|
||||
)
|
||||
@ change_numerator_two
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
integer += insert_space + pynini.closure(pynutil.insert("und ", weight=0.001), 0, 1)
|
||||
|
||||
denominator_one_half = pynini.cdrewrite(
|
||||
pynini.cross("ein halbe", "ein halb"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
)
|
||||
|
||||
fraction_default = (numerator + insert_space + denominator) @ denominator_one_half
|
||||
|
||||
self.graph = optional_sign + pynini.closure(integer, 0, 1) + fraction_default
|
||||
|
||||
graph = self.graph + delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,41 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { cardinal { integer: "zwei" units: "unzen" } } -> "zwei unzen"
|
||||
measure { cardinal { integer_part: "zwei" quantity: "millionen" units: "unzen" } } -> "zwei millionen unzen"
|
||||
|
||||
Args:
|
||||
decimal: decimal GraphFst
|
||||
cardinal: cardinal GraphFst
|
||||
fraction: fraction GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst, deterministic: bool
|
||||
):
|
||||
super().__init__(name="measure", kind="verbalize", deterministic=deterministic)
|
||||
unit = pynutil.delete('units: "') + pynini.closure(DAMO_NOT_QUOTE) + pynutil.delete('"')
|
||||
|
||||
graph_decimal = decimal.fst
|
||||
graph_cardinal = cardinal.fst
|
||||
graph_fraction = fraction.fst
|
||||
|
||||
graph = (graph_cardinal | graph_decimal | graph_fraction) + pynini.accep(" ") + unit
|
||||
|
||||
graph |= unit + delete_extra_space + (graph_cardinal | graph_decimal)
|
||||
graph += delete_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,77 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { currency_maj: "euro" integer_part: "ein"} -> "ein euro"
|
||||
money { currency_maj: "euro" integer_part: "eins" fractional_part: "null null eins"} -> "eins komma null null eins euro"
|
||||
money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" preserve_order: true} -> "ein pfund vierzig"
|
||||
money { integer_part: "ein" currency_maj: "pfund" fractional_part: "vierzig" currency_min: "pence" preserve_order: true} -> "ein pfund vierzig pence"
|
||||
money { fractional_part: "ein" currency_min: "penny" preserve_order: true} -> "ein penny"
|
||||
money { currency_maj: "pfund" integer_part: "null" fractional_part: "null eins" quantity: "million"} -> "null komma null eins million pfund"
|
||||
|
||||
Args:
|
||||
decimal: GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="money", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
keep_space = pynini.accep(" ")
|
||||
|
||||
maj = (
|
||||
pynutil.delete('currency_maj: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
min = (
|
||||
pynutil.delete('currency_min: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
fractional_part = (
|
||||
pynutil.delete('fractional_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
integer_part = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_add_and = pynini.closure(pynutil.insert("und "), 0, 1)
|
||||
|
||||
# *** currency_maj
|
||||
graph_integer = integer_part + keep_space + maj
|
||||
|
||||
# *** currency_maj + (***) | ((und) *** current_min)
|
||||
graph_integer_with_minor = (
|
||||
integer_part
|
||||
+ keep_space
|
||||
+ maj
|
||||
+ keep_space
|
||||
+ (fractional_part | (optional_add_and + fractional_part + keep_space + min))
|
||||
+ delete_preserve_order
|
||||
)
|
||||
|
||||
# *** komma *** currency_maj
|
||||
graph_decimal = decimal.fst + keep_space + maj
|
||||
|
||||
# *** current_min
|
||||
graph_minor = fractional_part + keep_space + min + delete_preserve_order
|
||||
|
||||
graph = graph_integer | graph_integer_with_minor | graph_decimal | graph_minor
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,45 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing roman numerals
|
||||
e.g. ordinal { integer: "vier" } } -> "vierter"
|
||||
-> "viertes" ...
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="ordinal", kind="verbalize", deterministic=deterministic)
|
||||
graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv")).invert()
|
||||
graph_ties = pynini.string_file(get_abs_path("data/ordinals/ties.tsv")).invert()
|
||||
graph_thousands = pynini.string_file(get_abs_path("data/ordinals/thousands.tsv")).invert()
|
||||
|
||||
graph = (
|
||||
pynutil.delete('integer: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
suffixes = pynini.union("ten", "tem", "ter", "tes", "te")
|
||||
convert_rest = pynutil.insert(suffixes, weight=0.01)
|
||||
self.ordinal_stem = graph_digit | graph_ties | graph_thousands
|
||||
|
||||
suffix = pynini.cdrewrite(
|
||||
pynini.closure(self.ordinal_stem, 0, 1) + convert_rest,
|
||||
"",
|
||||
"[EOS]",
|
||||
DAMO_SIGMA,
|
||||
).optimize()
|
||||
self.graph = pynini.compose(graph, suffix)
|
||||
self.suffix = suffix
|
||||
delete_tokens = self.delete_tokens(self.graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,40 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing telephone, e.g.
|
||||
telephone { country_code: "plus neun und vierzig" number_part: "null eins eins eins null null null" }
|
||||
-> "plus neun und vierzig null eins eins eins null null null"
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="telephone", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
country_code = (
|
||||
pynutil.delete('country_code: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
number_part = (
|
||||
pynutil.delete('number_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
self.graph = country_code + pynini.accep(" ") + number_part
|
||||
|
||||
graph = self.graph + delete_preserve_order
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,126 @@
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.utils import get_abs_path, load_labels
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_preserve_order,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic, e.g.
|
||||
time { hours: "2" minutes: "15"} -> "zwei uhr fünfzehn"
|
||||
time { minutes: "15" hours: "2" } -> "viertel nach zwei"
|
||||
time { minutes: "15" hours: "2" } -> "fünfzehn nach zwei"
|
||||
time { hours: "14" minutes: "15"} -> "vierzehn uhr fünfzehn"
|
||||
time { minutes: "15" hours: "14" } -> "viertel nach zwei"
|
||||
time { minutes: "15" hours: "14" } -> "fünfzehn nach drei"
|
||||
time { minutes: "45" hours: "14" } -> "viertel vor drei"
|
||||
|
||||
Args:
|
||||
cardinal_tagger: cardinal_tagger tagger GraphFst
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal_tagger: GraphFst, deterministic: bool = True):
|
||||
super().__init__(name="time", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
# add weight so when using inverse text normalization this conversion is depriotized
|
||||
night_to_early = pynutil.add_weight(
|
||||
pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/hour_to_night.tsv"))
|
||||
).optimize(),
|
||||
weight=0.0001,
|
||||
)
|
||||
hour_to = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/hour_to.tsv"))
|
||||
).optimize()
|
||||
minute_to = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/time/minute_to.tsv"))
|
||||
).optimize()
|
||||
time_zone_graph = pynini.invert(
|
||||
convert_space(
|
||||
pynini.union(*[x[1] for x in load_labels(get_abs_path("data/time/time_zone.tsv"))])
|
||||
)
|
||||
)
|
||||
|
||||
graph_zero = pynini.invert(
|
||||
pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
).optimize()
|
||||
number_verbalization = graph_zero | cardinal_tagger.two_digit_non_zero
|
||||
hour = pynutil.delete('hours: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
hour_verbalized = hour @ number_verbalization @ pynini.cdrewrite(
|
||||
pynini.cross("eins", "ein"), "[BOS]", "[EOS]", DAMO_SIGMA
|
||||
) + pynutil.insert(" uhr")
|
||||
minute = pynutil.delete('minutes: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
zone = pynutil.delete('zone: "') + time_zone_graph + pynutil.delete('"')
|
||||
optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1)
|
||||
second = pynutil.delete('seconds: "') + pynini.closure(DAMO_DIGIT, 1) + pynutil.delete('"')
|
||||
graph_hms = (
|
||||
hour_verbalized
|
||||
+ pynini.accep(" ")
|
||||
+ minute @ number_verbalization
|
||||
+ pynutil.insert(" minuten")
|
||||
+ pynini.accep(" ")
|
||||
+ second @ number_verbalization
|
||||
+ pynutil.insert(" sekunden")
|
||||
+ optional_zone
|
||||
)
|
||||
graph_hms @= pynini.cdrewrite(
|
||||
pynini.cross("eins minuten", "eine minute")
|
||||
| pynini.cross("eins sekunden", "eine sekunde"),
|
||||
pynini.union(" ", "[BOS]"),
|
||||
"",
|
||||
DAMO_SIGMA,
|
||||
)
|
||||
|
||||
min_30 = [str(x) for x in range(1, 31)]
|
||||
min_30 = pynini.union(*min_30)
|
||||
min_29 = [str(x) for x in range(1, 30)]
|
||||
min_29 = pynini.union(*min_29)
|
||||
|
||||
graph_h = hour_verbalized
|
||||
graph_hm = hour_verbalized + pynini.accep(" ") + minute @ number_verbalization
|
||||
|
||||
graph_m_past_h = (
|
||||
minute @ min_30 @ (number_verbalization | pynini.cross("15", "viertel"))
|
||||
+ pynini.accep(" ")
|
||||
+ pynutil.insert("nach ")
|
||||
# + hour @ number_verbalization
|
||||
+ hour
|
||||
@ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
@ number_verbalization
|
||||
)
|
||||
graph_m30_h = (
|
||||
minute @ pynini.cross("30", "halb")
|
||||
+ pynini.accep(" ")
|
||||
+ hour
|
||||
@ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
@ hour_to
|
||||
@ number_verbalization
|
||||
)
|
||||
graph_m_to_h = (
|
||||
minute @ minute_to @ min_29 @ (number_verbalization | pynini.cross("15", "viertel"))
|
||||
+ pynini.accep(" ")
|
||||
+ pynutil.insert("vor ")
|
||||
+ hour
|
||||
@ pynini.cdrewrite(night_to_early, "[BOS]", "[EOS]", DAMO_SIGMA)
|
||||
@ hour_to
|
||||
@ number_verbalization
|
||||
)
|
||||
|
||||
self.graph = (
|
||||
graph_hms
|
||||
| graph_h
|
||||
| graph_hm
|
||||
| pynutil.add_weight(graph_m_past_h, weight=0.0001)
|
||||
| pynutil.add_weight(graph_m30_h, weight=0.0001)
|
||||
| pynutil.add_weight(graph_m_to_h, weight=0.0001)
|
||||
) + optional_zone
|
||||
delete_tokens = self.delete_tokens(self.graph + delete_preserve_order)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,64 @@
|
||||
from fun_text_processing.text_normalization.de.taggers.cardinal import CardinalFst as CardinalTagger
|
||||
from fun_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.date import DateFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.decimal import DecimalFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.electronic import ElectronicFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.fraction import FractionFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.ordinal import OrdinalFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.telephone import TelephoneFst
|
||||
from fun_text_processing.text_normalization.de.verbalizers.time import TimeFst
|
||||
from fun_text_processing.text_normalization.en.graph_utils import GraphFst
|
||||
from fun_text_processing.text_normalization.en.verbalizers.whitelist import WhiteListFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, deterministic: bool = True):
|
||||
super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic)
|
||||
cardinal_tagger = CardinalTagger(deterministic=deterministic)
|
||||
cardinal = CardinalFst(deterministic=deterministic)
|
||||
cardinal_graph = cardinal.fst
|
||||
ordinal = OrdinalFst(deterministic=deterministic)
|
||||
ordinal_graph = ordinal.fst
|
||||
decimal = DecimalFst(deterministic=deterministic)
|
||||
decimal_graph = decimal.fst
|
||||
fraction = FractionFst(ordinal=ordinal, deterministic=deterministic)
|
||||
fraction_graph = fraction.fst
|
||||
date = DateFst(ordinal=ordinal)
|
||||
date_graph = date.fst
|
||||
measure = MeasureFst(
|
||||
cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic
|
||||
)
|
||||
measure_graph = measure.fst
|
||||
electronic = ElectronicFst(deterministic=deterministic)
|
||||
electronic_graph = electronic.fst
|
||||
whitelist_graph = WhiteListFst(deterministic=deterministic).fst
|
||||
money_graph = MoneyFst(decimal=decimal).fst
|
||||
telephone_graph = TelephoneFst(deterministic=deterministic).fst
|
||||
time_graph = TimeFst(cardinal_tagger=cardinal_tagger, deterministic=deterministic).fst
|
||||
|
||||
graph = (
|
||||
cardinal_graph
|
||||
| measure_graph
|
||||
| decimal_graph
|
||||
| ordinal_graph
|
||||
| date_graph
|
||||
| electronic_graph
|
||||
| money_graph
|
||||
| fraction_graph
|
||||
| whitelist_graph
|
||||
| telephone_graph
|
||||
| time_graph
|
||||
)
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.text_normalization.de.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.text_normalization.en.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.text_normalization.en.verbalizers.word import WordFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence
|
||||
|
||||
Args:
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple options (used for audio-based normalization)
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False
|
||||
):
|
||||
super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic)
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(
|
||||
cache_dir, f"de_tn_{deterministic}_deterministic_verbalizer.far"
|
||||
)
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["verbalize"]
|
||||
logging.info(f"VerbalizeFinalFst graph was restored from {far_file}.")
|
||||
else:
|
||||
verbalize = VerbalizeFst(deterministic=deterministic).fst
|
||||
word = WordFst(deterministic=deterministic).fst
|
||||
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
if far_file:
|
||||
generator_main(far_file, {"verbalize": self.fst})
|
||||
logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.")
|
||||
Reference in New Issue
Block a user