chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,30 @@
|
||||
# Note on French spelling
|
||||
|
||||
Due to a 1990 orthographic reform, there are currently two conventions for written French numbers:
|
||||
|
||||
1. **Reformed** All composite words are joined by a hyphen:
|
||||
e.g. `1122 -> mille-cent-vingt-deux`
|
||||
|
||||
2. **Traditional** Hyphenation only occurs (with exception) for numbers from 17 to 99 (inclusive):
|
||||
e.g. `1122 -> mille cent vingt-deux`
|
||||
|
||||
As available training data for upstream ASR will vary in use of convention, NeMo's French ITN accomodates either style for normalization e.g.
|
||||
|
||||
```
|
||||
python inverse_normalize.py "mille-cent-vingt-deux" --language="fr" --> 1122
|
||||
python inverse_normalize.py "mille cent vingt-deux" --language="fr" --> 1122
|
||||
```
|
||||
|
||||
As a result, there exists some ambiguity in the case of currency conversions, namely minor denominations of the dollar e.g.
|
||||
|
||||
```
|
||||
300 -> "trois-cents" # Reformed spelling
|
||||
300 -> "trois cents" # Traditional spelling
|
||||
3 ¢ -> "trois cents" # Valid for both
|
||||
```
|
||||
|
||||
Cardinals take priority in such cases.
|
||||
|
||||
```
|
||||
python inverse_normalize.py "trois cents" --language="fr" -> 300
|
||||
```
|
||||
@@ -0,0 +1,7 @@
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.tokenize_and_classify import (
|
||||
ClassifyFst,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.verbalize_final import (
|
||||
VerbalizeFinalFst,
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
com
|
||||
es
|
||||
uk
|
||||
fr
|
||||
net
|
||||
br
|
||||
in
|
||||
ru
|
||||
de
|
||||
it
|
||||
edu
|
||||
co
|
||||
ar
|
||||
bo
|
||||
cl
|
||||
co
|
||||
ec
|
||||
fk
|
||||
gf
|
||||
fy
|
||||
pe
|
||||
py
|
||||
sr
|
||||
ve
|
||||
uy
|
||||
|
@@ -0,0 +1,17 @@
|
||||
g mail gmail
|
||||
gmail
|
||||
n vidia nvidia
|
||||
nvidia
|
||||
outlook
|
||||
hotmail
|
||||
yahoo
|
||||
aol
|
||||
gmx
|
||||
msn
|
||||
live
|
||||
yandex
|
||||
orange
|
||||
wanadoo
|
||||
web
|
||||
google
|
||||
comcast
|
||||
|
@@ -0,0 +1,12 @@
|
||||
chez @
|
||||
at @
|
||||
à @
|
||||
arobase @
|
||||
point .
|
||||
barre oblique /
|
||||
tiret -
|
||||
tiret bas _
|
||||
souligné _
|
||||
sous-tiret _
|
||||
blanc souligné _
|
||||
underscore _
|
||||
|
@@ -0,0 +1,41 @@
|
||||
demie deux
|
||||
demies deux
|
||||
demi deux
|
||||
demis deux
|
||||
tiers trois
|
||||
quart quatre
|
||||
quarts quatre
|
||||
quatrièmes quatre
|
||||
quatrième quatre
|
||||
cinquième cinq
|
||||
cinquièmes cinq
|
||||
neuvième neuf
|
||||
neuvièmes neuf
|
||||
onzième onze
|
||||
onzièmes onze
|
||||
douzième douze
|
||||
douzièmes douze
|
||||
treizième treize
|
||||
treizièmes treize
|
||||
quatorzième quatorze
|
||||
quatorzièmes quatorze
|
||||
quinzième quinze
|
||||
quinzièmes quinze
|
||||
seizième seize
|
||||
seizièmes seize
|
||||
trentième trente
|
||||
trentièmes trente
|
||||
quarantième quarante
|
||||
quarantièmes quarante
|
||||
cinquantième cinquante
|
||||
cinquantièmes cinquante
|
||||
soixantième soixante
|
||||
soixantièmes soixante
|
||||
septantième septante
|
||||
septantièmes septante
|
||||
huitantième huitante
|
||||
huitantièmes huitante
|
||||
nonantième nonante
|
||||
nonantièmes nonante
|
||||
millième mille
|
||||
millièmes mille
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
exa E
|
||||
péta P
|
||||
téra T
|
||||
giga G
|
||||
méga M
|
||||
kilo k
|
||||
hecto h
|
||||
déca da
|
||||
déci d
|
||||
centi c
|
||||
milli m
|
||||
micro µ
|
||||
nano n
|
||||
pico p
|
||||
femto f
|
||||
atto a
|
||||
|
@@ -0,0 +1,27 @@
|
||||
mètre m
|
||||
mètre carré m²
|
||||
mètres carrés m²
|
||||
mètre cube m³
|
||||
mètres cubes m³
|
||||
seconde s
|
||||
minute min
|
||||
heure h
|
||||
degré °
|
||||
degrés °
|
||||
degré celsius °C
|
||||
degrés celsius °C
|
||||
gramme g
|
||||
litre l
|
||||
kilo kg
|
||||
pouce ''
|
||||
livre lb
|
||||
poid lb
|
||||
mile ml
|
||||
pour cent %
|
||||
pour mille ‰
|
||||
mètre heure km/h
|
||||
mètres heure m/h
|
||||
mètre à l’heure m/h
|
||||
mètres à l’heure m/h
|
||||
mètre par l’heure m/h
|
||||
mètres par l’heure m/h
|
||||
|
@@ -0,0 +1,40 @@
|
||||
dollar $
|
||||
dollars $
|
||||
dollar américain $ US
|
||||
dollars américains $ US
|
||||
dollar des États-Unis $ US
|
||||
dollars des États-Unis $ US
|
||||
dollar canadien $ CA
|
||||
dollars canadiens $ CA
|
||||
dollar australien $ AU
|
||||
dollars australien $ AU
|
||||
dollar néo-zélandais $ NZ
|
||||
dollars néo-zélandais $ NZ
|
||||
dollar de Hong Kong $ HK
|
||||
dollars de Hong Kong $ HK
|
||||
euro €
|
||||
euros €
|
||||
livre sterling £
|
||||
livre £
|
||||
livres £
|
||||
livre britannique £
|
||||
livres britanniques £
|
||||
sterling £
|
||||
won ₩
|
||||
won sud-coréen ₩
|
||||
yen japonais ¥
|
||||
yens japonais ¥
|
||||
yen ¥
|
||||
yens ¥
|
||||
yuan ¥
|
||||
yuans ¥
|
||||
franc CHF
|
||||
franc suisse CHF
|
||||
dinar algérien DA
|
||||
dinars algériens DA
|
||||
dinar DA
|
||||
dinars DA
|
||||
dirham marocain DH
|
||||
dirhams marocains DH
|
||||
dirham DH
|
||||
dirhams DH
|
||||
|
@@ -0,0 +1,5 @@
|
||||
centime €
|
||||
centimes €
|
||||
eurocent €
|
||||
eurocents €
|
||||
pence £
|
||||
|
@@ -0,0 +1,12 @@
|
||||
janvier
|
||||
février
|
||||
mars
|
||||
avril
|
||||
mai
|
||||
juin
|
||||
juillet
|
||||
août
|
||||
septembre
|
||||
octobre
|
||||
novembre
|
||||
décembre
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
un 1
|
||||
une 1
|
||||
deux 2
|
||||
trois 3
|
||||
quatre 4
|
||||
cinq 5
|
||||
six 6
|
||||
sept 7
|
||||
huit 8
|
||||
neuf 9
|
||||
|
@@ -0,0 +1,2 @@
|
||||
cent
|
||||
cents
|
||||
|
@@ -0,0 +1,6 @@
|
||||
onze 11
|
||||
douze 12
|
||||
treize 13
|
||||
quatorze 14
|
||||
quinze 15
|
||||
seize 16
|
||||
|
@@ -0,0 +1,23 @@
|
||||
mille
|
||||
million
|
||||
millions
|
||||
milliard
|
||||
milliards
|
||||
billion
|
||||
billions
|
||||
trillion
|
||||
trillions
|
||||
quadrillion
|
||||
quadrillions
|
||||
quintillion
|
||||
quintillions
|
||||
sextillion
|
||||
sextillions
|
||||
septillion
|
||||
septillions
|
||||
octillion
|
||||
octillions
|
||||
nonillion
|
||||
nonillions
|
||||
decillion
|
||||
decillions
|
||||
|
@@ -0,0 +1,13 @@
|
||||
dix 1
|
||||
vingt 2
|
||||
vingts 2
|
||||
trente 3
|
||||
quarante 4
|
||||
cinquante 5
|
||||
soixante 6
|
||||
soixante-dix 7
|
||||
septante 7
|
||||
huitante 8
|
||||
quatre-vingt 8
|
||||
quatre-vingt-dix 9
|
||||
nonante 9
|
||||
|
@@ -0,0 +1,25 @@
|
||||
vingts 20
|
||||
vingt-et-un 21
|
||||
vingt-et-une 21
|
||||
trente-et-un 31
|
||||
trente-et-une 31
|
||||
quarante-et-un 41
|
||||
quarante-et-une 41
|
||||
cinquante-et-un 51
|
||||
cinquante-et-une 51
|
||||
soixante-et-un 61
|
||||
soixante-et-une 61
|
||||
soixante-et-onze 71
|
||||
soixante-douze 72
|
||||
soixante-treize 73
|
||||
soixante-quatorze 74
|
||||
soixante-quinze 75
|
||||
soixante-seize 76
|
||||
quatre-vingts 80
|
||||
quatre-vingt 80
|
||||
quatre-vingt-onze 91
|
||||
quatre-vingt-douze 92
|
||||
quatre-vingt-treize 93
|
||||
quatre-vingt-quatorze 94
|
||||
quatre-vingt-quinze 95
|
||||
quatre-vingt-seize 96
|
||||
|
@@ -0,0 +1 @@
|
||||
zéro 0
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
quatrième quatre
|
||||
cinquième cinq
|
||||
neuvième neuf
|
||||
onzième onze
|
||||
douzième douze
|
||||
treizième treize
|
||||
quatorzième quatorze
|
||||
quinzième quinze
|
||||
seizième seize
|
||||
trentième trente
|
||||
quarantième quarante
|
||||
cinquantième cinquante
|
||||
soixantième soixante
|
||||
septantième septante
|
||||
huitantième huitante
|
||||
nonantième nonante
|
||||
millième mille
|
||||
|
@@ -0,0 +1,4 @@
|
||||
premier er
|
||||
premiers ers
|
||||
première re
|
||||
premières res
|
||||
|
@@ -0,0 +1 @@
|
||||
siècle
|
||||
|
@@ -0,0 +1,4 @@
|
||||
second d
|
||||
seconde de
|
||||
seconds ds
|
||||
secondes des
|
||||
|
@@ -0,0 +1,9 @@
|
||||
I 1
|
||||
II 2
|
||||
III 3
|
||||
IV 4
|
||||
V 5
|
||||
VI 6
|
||||
VII 7
|
||||
VIII 8
|
||||
IX 9
|
||||
|
@@ -0,0 +1,9 @@
|
||||
C 1
|
||||
CC 2
|
||||
CCC 3
|
||||
CD 4
|
||||
D 5
|
||||
DC 6
|
||||
DCC 7
|
||||
DCCC 8
|
||||
CM 9
|
||||
|
@@ -0,0 +1,9 @@
|
||||
X 1
|
||||
XX 2
|
||||
XXX 3
|
||||
XL 4
|
||||
L 5
|
||||
LX 6
|
||||
LXX 7
|
||||
LXXX 8
|
||||
XC 9
|
||||
|
@@ -0,0 +1,12 @@
|
||||
1 13
|
||||
2 14
|
||||
3 15
|
||||
4 16
|
||||
5 17
|
||||
6 18
|
||||
7 19
|
||||
8 20
|
||||
9 21
|
||||
10 22
|
||||
11 23
|
||||
12 0
|
||||
|
@@ -0,0 +1,26 @@
|
||||
zéro 0
|
||||
une 1
|
||||
deux 2
|
||||
trois 3
|
||||
quatre 4
|
||||
cinq 5
|
||||
six 6
|
||||
sept 7
|
||||
huit 8
|
||||
neuf 9
|
||||
dix 10
|
||||
onze 11
|
||||
douze 12
|
||||
treize 13
|
||||
quatorze 14
|
||||
quinze 15
|
||||
seize 16
|
||||
dix-sept 17
|
||||
dix-huit 18
|
||||
dix-neuf 19
|
||||
vingt 20
|
||||
vingt-et-une 21
|
||||
vingt et une
|
||||
vingt-deux 22
|
||||
vingt-trois 23
|
||||
vingt-quatre 24
|
||||
|
@@ -0,0 +1,25 @@
|
||||
1 0
|
||||
2 1
|
||||
3 2
|
||||
4 3
|
||||
5 4
|
||||
6 5
|
||||
7 6
|
||||
8 7
|
||||
9 8
|
||||
10 9
|
||||
11 10
|
||||
12 11
|
||||
13 12
|
||||
14 13
|
||||
15 14
|
||||
16 15
|
||||
17 16
|
||||
18 17
|
||||
19 18
|
||||
20 19
|
||||
21 20
|
||||
22 21
|
||||
23 22
|
||||
24 23
|
||||
0 23
|
||||
|
@@ -0,0 +1,63 @@
|
||||
une 01
|
||||
deux 02
|
||||
trois 03
|
||||
quatre 04
|
||||
cinq 05
|
||||
six 06
|
||||
sept 07
|
||||
huit 08
|
||||
neuf 09
|
||||
dix 10
|
||||
onze 11
|
||||
douze 12
|
||||
treize 13
|
||||
quatorze 14
|
||||
quinze 15
|
||||
seize 16
|
||||
dix-sept 17
|
||||
dix-huit 18
|
||||
dix-neuf 19
|
||||
vingt 20
|
||||
vingt-et-une 21
|
||||
vingt et une 21
|
||||
vingt-deux 22
|
||||
vingt-trois 23
|
||||
vingt-quatre 27
|
||||
vingt-cinq 25
|
||||
vingt-six 26
|
||||
vingt-sept 27
|
||||
vingt-huit 28
|
||||
vingt-neuf 29
|
||||
trente 30
|
||||
trente-et-une 31
|
||||
trente et une 31
|
||||
trente-deux 32
|
||||
trente-trois 33
|
||||
trente-quatre 34
|
||||
trente-cinq 35
|
||||
trente-six 36
|
||||
trente-sept 37
|
||||
trente-huit 38
|
||||
trente-neuf 39
|
||||
quarante 40
|
||||
quarante-et-une 41
|
||||
quarante et une 41
|
||||
quarante-deux 42
|
||||
quarante-trois 43
|
||||
quarante-quatre 44
|
||||
quarante-cinq 45
|
||||
quarante-six 46
|
||||
quarante-sept 47
|
||||
quarante-huit 48
|
||||
quarante-neuf 49
|
||||
cinquante 50
|
||||
cinquante-et-une 51
|
||||
cinquante et une 51
|
||||
cinquante-deux 52
|
||||
cinquante-trois 53
|
||||
cinquante-quatre 54
|
||||
cinquante-cinq 55
|
||||
cinquante-six 56
|
||||
cinquante-sept 57
|
||||
cinquante-huit 58
|
||||
cinquante-neuf 59
|
||||
|
@@ -0,0 +1,59 @@
|
||||
01 59
|
||||
02 58
|
||||
03 57
|
||||
04 56
|
||||
05 55
|
||||
06 54
|
||||
07 53
|
||||
08 52
|
||||
09 51
|
||||
10 50
|
||||
11 49
|
||||
12 48
|
||||
13 47
|
||||
14 46
|
||||
15 45
|
||||
16 44
|
||||
17 43
|
||||
18 42
|
||||
19 41
|
||||
20 40
|
||||
21 39
|
||||
22 38
|
||||
23 37
|
||||
24 36
|
||||
25 35
|
||||
26 34
|
||||
27 33
|
||||
28 32
|
||||
29 31
|
||||
30 30
|
||||
31 29
|
||||
32 28
|
||||
33 27
|
||||
34 26
|
||||
35 25
|
||||
36 24
|
||||
37 23
|
||||
38 22
|
||||
39 21
|
||||
40 20
|
||||
41 19
|
||||
42 18
|
||||
43 17
|
||||
44 16
|
||||
45 15
|
||||
46 14
|
||||
47 13
|
||||
48 12
|
||||
49 11
|
||||
50 10
|
||||
51 09
|
||||
52 08
|
||||
53 07
|
||||
54 06
|
||||
55 05
|
||||
56 04
|
||||
57 03
|
||||
58 02
|
||||
59 01
|
||||
|
@@ -0,0 +1 @@
|
||||
du matin
|
||||
|
@@ -0,0 +1,2 @@
|
||||
de l'après-midi
|
||||
du soir
|
||||
|
@@ -0,0 +1,16 @@
|
||||
monsieur M.
|
||||
messieurs MM.
|
||||
madame Mᵐᵉ
|
||||
mesdames Mᵐᵉˢ
|
||||
mademoiselle Mˡˡᵉ
|
||||
mademoiselles Mˡˡᵉˢ
|
||||
docteur Dʳ
|
||||
docteurs Dʳˢ
|
||||
docteure Dʳᵉ
|
||||
docteures Dʳᵉˢ
|
||||
après jésus-christ apr. J.-C.
|
||||
avant Jésus-Christ av. J.-C.
|
||||
ca v.
|
||||
vers v.
|
||||
l’honorable le hon.
|
||||
le très hononrable le très hon.
|
||||
|
@@ -0,0 +1,180 @@
|
||||
import os
|
||||
import string
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini import Far
|
||||
from pynini.examples import plurals
|
||||
from pynini.export import export
|
||||
from pynini.lib import byte, pynutil, utf8
|
||||
|
||||
DAMO_CHAR = utf8.VALID_UTF8_CHAR
|
||||
|
||||
DAMO_DIGIT = byte.DIGIT
|
||||
DAMO_LOWER = pynini.union(*string.ascii_lowercase).optimize()
|
||||
DAMO_UPPER = pynini.union(*string.ascii_uppercase).optimize()
|
||||
DAMO_ALPHA = pynini.union(DAMO_LOWER, DAMO_UPPER).optimize()
|
||||
DAMO_ALNUM = pynini.union(DAMO_DIGIT, DAMO_ALPHA).optimize()
|
||||
DAMO_HEX = pynini.union(*string.hexdigits).optimize()
|
||||
DAMO_NON_BREAKING_SPACE = "\u00A0"
|
||||
DAMO_SPACE = " "
|
||||
DAMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize()
|
||||
DAMO_NOT_SPACE = pynini.difference(DAMO_CHAR, DAMO_WHITE_SPACE).optimize()
|
||||
DAMO_NOT_QUOTE = pynini.difference(DAMO_CHAR, r'"').optimize()
|
||||
|
||||
DAMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize()
|
||||
DAMO_GRAPH = pynini.union(DAMO_ALNUM, DAMO_PUNCT).optimize()
|
||||
|
||||
DAMO_SIGMA = pynini.closure(DAMO_CHAR)
|
||||
|
||||
delete_space = pynutil.delete(pynini.closure(DAMO_WHITE_SPACE))
|
||||
insert_space = pynutil.insert(" ")
|
||||
delete_extra_space = pynini.cross(pynini.closure(DAMO_WHITE_SPACE, 1), " ")
|
||||
|
||||
# French frequently compounds numbers with hyphen.
|
||||
delete_hyphen = pynutil.delete(pynini.closure("-", 0, 1))
|
||||
insert_hyphen = pynutil.insert("-")
|
||||
suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv"))
|
||||
|
||||
_s = DAMO_SIGMA + pynutil.insert("s")
|
||||
_x = DAMO_SIGMA + pynini.string_map([("eau"), ("eu"), ("ou")]) + pynutil.insert("x")
|
||||
_aux = DAMO_SIGMA + pynini.string_map([("al", "aux"), ("ail", "aux")])
|
||||
|
||||
graph_plural = plurals._priority_union(
|
||||
suppletive, plurals._priority_union(_s, pynini.union(_x, _aux), DAMO_SIGMA), DAMO_SIGMA
|
||||
).optimize()
|
||||
|
||||
SINGULAR_TO_PLURAL = graph_plural
|
||||
PLURAL_TO_SINGULAR = pynini.invert(graph_plural)
|
||||
TO_LOWER = pynini.union(
|
||||
*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]
|
||||
)
|
||||
TO_UPPER = pynini.invert(TO_LOWER)
|
||||
|
||||
|
||||
def generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]):
|
||||
"""
|
||||
Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name.
|
||||
|
||||
Args:
|
||||
file_name: exported file name
|
||||
graphs: Mapping of a rule name and Pynini WFST graph to be exported
|
||||
"""
|
||||
exporter = export.Exporter(file_name)
|
||||
for rule, graph in graphs.items():
|
||||
exporter[rule] = graph.optimize()
|
||||
exporter.close()
|
||||
print(f"Created {file_name}")
|
||||
|
||||
|
||||
def get_plurals(fst):
|
||||
"""
|
||||
Given singular returns plurals
|
||||
|
||||
Args:
|
||||
fst: Fst
|
||||
|
||||
Returns plurals to given singular forms
|
||||
"""
|
||||
return SINGULAR_TO_PLURAL @ fst
|
||||
|
||||
|
||||
def get_singulars(fst):
|
||||
"""
|
||||
Given plural returns singulars
|
||||
|
||||
Args:
|
||||
fst: Fst
|
||||
|
||||
Returns singulars to given plural forms
|
||||
"""
|
||||
return PLURAL_TO_SINGULAR @ fst
|
||||
|
||||
|
||||
def convert_space(fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Converts space to nonbreaking space.
|
||||
Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty"
|
||||
This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it.
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns output fst where breaking spaces are converted to non breaking spaces
|
||||
"""
|
||||
return fst @ pynini.cdrewrite(
|
||||
pynini.cross(DAMO_SPACE, DAMO_NON_BREAKING_SPACE), "", "", DAMO_SIGMA
|
||||
)
|
||||
|
||||
|
||||
class GraphFst:
|
||||
"""
|
||||
Base class for all grammar fsts.
|
||||
|
||||
Args:
|
||||
name: name of grammar class
|
||||
kind: either 'classify' or 'verbalize'
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, kind: str, deterministic: bool = True):
|
||||
self.name = name
|
||||
self.kind = kind
|
||||
self._fst = None
|
||||
self.deterministic = deterministic
|
||||
|
||||
self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far")
|
||||
if self.far_exist():
|
||||
self._fst = Far(
|
||||
self.far_path, mode="r", arc_type="standard", far_type="default"
|
||||
).get_fst()
|
||||
|
||||
def far_exist(self) -> bool:
|
||||
"""
|
||||
Returns true if FAR can be loaded
|
||||
"""
|
||||
return self.far_path.exists()
|
||||
|
||||
@property
|
||||
def fst(self) -> "pynini.FstLike":
|
||||
return self._fst
|
||||
|
||||
@fst.setter
|
||||
def fst(self, fst):
|
||||
self._fst = fst
|
||||
|
||||
def add_tokens(self, fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Wraps class name around to given fst
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns:
|
||||
Fst: fst
|
||||
"""
|
||||
return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }")
|
||||
|
||||
def delete_tokens(self, fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Deletes class name wrap around output of given fst
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns:
|
||||
Fst: fst
|
||||
"""
|
||||
res = (
|
||||
pynutil.delete(f"{self.name}")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ fst
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
@@ -0,0 +1,287 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_DIGIT,
|
||||
DAMO_NOT_SPACE,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_hyphen,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
def rewrite(cardinal: "pynini.FstLike") -> "pynini.FstLike":
|
||||
"""
|
||||
Function to rewrite cardinals written in traditional orthograph (no '-' for numbers >100)
|
||||
to current orthography ('-' between all words in number string)
|
||||
e.g. deux mille cent vingt-trois -> deux-mille-cent-vingt-trois.
|
||||
In cases where original orthography is current, or string is mixture of two orthographies,
|
||||
will render invalid form that will not pass through CardinalFst
|
||||
e.g. deux-mille cent-vingt-trois -> "deux##vingt-trois" ('#' is not accepted in cardinal FST and will fail to convert.)
|
||||
e.g. deux
|
||||
|
||||
Args:
|
||||
cardinal: cardinal FST
|
||||
"""
|
||||
|
||||
# Traditional orthography does not hyphenate numbers > 100, this will insert hyphens in
|
||||
# those contexts.
|
||||
targets = pynini.string_map(
|
||||
[
|
||||
"et", # for 'et un/onze'
|
||||
"cent",
|
||||
"mille",
|
||||
"million",
|
||||
"milliard",
|
||||
"billion",
|
||||
"billiard",
|
||||
"trillion",
|
||||
"trilliard",
|
||||
]
|
||||
)
|
||||
targets += pynini.accep("s").ques
|
||||
|
||||
no_spaces = pynini.closure(DAMO_NOT_SPACE)
|
||||
|
||||
# Valid numbers in reformed orthography will have no spaces.
|
||||
new_orthography_sigma = no_spaces
|
||||
|
||||
# Old orthography will not have these strings. Replacing with character to mark.
|
||||
targets_for_filtering = ("-" + targets) | ("-" + targets + "-") | (targets + "-")
|
||||
|
||||
filter = pynini.cdrewrite(
|
||||
pynini.cross(targets_for_filtering, "#"), "", "", DAMO_SIGMA
|
||||
) # Invalid for cardinal
|
||||
|
||||
old_orthography_sigma = pynini.difference(
|
||||
DAMO_CHAR, "#"
|
||||
) # Marked character removed from sigma_star.
|
||||
old_orthography_sigma.closure()
|
||||
|
||||
# Only accept strings that occur in old orthography. (This avoids tying two non-related numbers together.)
|
||||
# e.g. mille cent-une -> mille-cent-une
|
||||
filter @= old_orthography_sigma
|
||||
|
||||
# Now know replacements will only work around targets
|
||||
replace_left = pynini.cdrewrite(pynini.cross(" ", "-"), "", targets, DAMO_SIGMA)
|
||||
|
||||
replace_right = pynini.cdrewrite(pynini.cross(" ", "-"), targets, "", DAMO_SIGMA)
|
||||
|
||||
replace = replace_left @ replace_right
|
||||
|
||||
graph = new_orthography_sigma | (filter @ replace)
|
||||
|
||||
return graph @ cardinal
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying cardinals
|
||||
e.g. mois vingt-trois -> cardinal { negative: "-" integer: "23"}
|
||||
This class converts cardinals up to (but not including) "un-quatrillion",
|
||||
i.e up to "one septillion" in English (10^{24}).
|
||||
Cardinals below nine are not converted (in order to avoid
|
||||
"j'ai un pomme." --> "j'ai 1 pomme" and any other odd conversions.)
|
||||
This transducer accomodates both traditional hyphenation of numbers ('-' for most numbers <100)
|
||||
and current hyphenation (all elements of number are hyphenated), prioritizing the latter.
|
||||
e.g cent cinquante et un -> cardinal { integer: "151"}
|
||||
cent-cinquante-et-un -> cardinal { integer: "151"}
|
||||
This is done through a context dependent rewrite that attempts to map old spelling to new.
|
||||
e.g. cent cinquante et un -> cent-cinquante-et-un
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="cardinal", kind="classify")
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_teens = pynini.string_file(get_abs_path("data/numbers/teen.tsv"))
|
||||
graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv"))
|
||||
graph_ties_unique = pynini.string_file(get_abs_path("data/numbers/ties_unique.tsv"))
|
||||
|
||||
# Tens components
|
||||
graph_tens_component = graph_ties + ((delete_hyphen + graph_digit) | pynutil.insert("0"))
|
||||
graph_tens_component = pynini.union(graph_tens_component, graph_teens, graph_ties_unique)
|
||||
|
||||
graph_tens_component_with_leading_zeros = pynini.union(
|
||||
graph_tens_component,
|
||||
(pynutil.insert("0") + (graph_digit | pynutil.insert("0", weight=0.01))),
|
||||
)
|
||||
|
||||
# Hundreds components
|
||||
graph_cent_singular = pynutil.delete("cent") # Used in hundreds place
|
||||
graph_cent_plural = pynini.cross(
|
||||
"cents", "00"
|
||||
) # Only used as terminus of hundred sequence. deux cents -> 200, deux cent un -> 201
|
||||
|
||||
graph_digit_no_one = pynini.project(pynini.union("un", "une"), "input")
|
||||
graph_digit_no_one = (
|
||||
pynini.project(graph_digit, "input") - graph_digit_no_one.arcsort()
|
||||
) @ graph_digit
|
||||
|
||||
graph_hundreds_component_singular = (
|
||||
graph_digit_no_one + delete_hyphen + graph_cent_singular
|
||||
) # Regular way: [1-9] * 100
|
||||
|
||||
graph_hundreds_component_singular = pynini.union(
|
||||
graph_hundreds_component_singular, pynini.cross("cent", "1")
|
||||
)
|
||||
graph_hundreds_component_singular += delete_hyphen
|
||||
graph_hundreds_component_singular += graph_tens_component_with_leading_zeros
|
||||
|
||||
graph_hundreds_component_plural = graph_digit_no_one + delete_hyphen + graph_cent_plural
|
||||
|
||||
graph_hundreds_component = pynini.union(
|
||||
graph_hundreds_component_singular,
|
||||
graph_hundreds_component_plural,
|
||||
pynutil.insert("0") + graph_tens_component_with_leading_zeros,
|
||||
)
|
||||
|
||||
graph_hundreds_component_at_least_one_none_zero_digit = graph_hundreds_component @ (
|
||||
pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT)
|
||||
)
|
||||
self.graph_hundreds_component_at_least_one_none_zero_digit = rewrite(
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
).optimize()
|
||||
|
||||
# Graph thousands (we'll need this for cases of mille millions, mille milliards...)
|
||||
graph_tens_of_hundreds_component_singular = (
|
||||
graph_tens_component + delete_hyphen + graph_cent_singular
|
||||
) # Tens of hundreds. e.g. 1900 = nineteen hundred/ 'dix neuf cents"
|
||||
graph_tens_of_hundreds_component_singular += (
|
||||
delete_hyphen + graph_tens_component_with_leading_zeros
|
||||
)
|
||||
graph_tens_of_hundreds_component_plural = (
|
||||
graph_tens_component + delete_hyphen + graph_cent_plural
|
||||
)
|
||||
graph_tens_of_hundred_component = (
|
||||
graph_tens_of_hundreds_component_plural | graph_tens_of_hundreds_component_singular
|
||||
)
|
||||
|
||||
graph_thousands = pynini.union(
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ pynutil.delete("mille"),
|
||||
pynutil.insert("001") + pynutil.delete("mille"), # because 'mille', not 'un mille'
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
# All other large amounts
|
||||
graph_millions = pynini.union(
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ (pynutil.delete("million") | pynutil.delete("millions")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph_milliards = pynini.union( # French for English 'billion'
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ (pynutil.delete("milliard") | pynutil.delete("milliards")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph_billions = pynini.union( # NOTE: this is English 'trillion.'
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ (pynutil.delete("billions") | pynutil.delete("billion")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph_mille_billion = pynini.union(
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ pynutil.delete("mille"),
|
||||
pynutil.insert("001")
|
||||
+ pynutil.delete("mille"), # because we say 'mille', not 'un mille'
|
||||
)
|
||||
graph_mille_billion += delete_hyphen + (
|
||||
graph_millions | pynutil.insert("000") + pynutil.delete("billions")
|
||||
) # allow for 'mil millones'
|
||||
graph_mille_billion |= pynutil.insert("000000", weight=0.1)
|
||||
|
||||
graph_billiards = pynini.union(
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ (pynutil.delete("billiards") | pynutil.delete("billiard")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph_trillions = pynini.union( # One thousand English trillions.
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ (pynutil.delete("trillions") | pynutil.delete("trillion")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph_trilliards = pynini.union(
|
||||
graph_hundreds_component_at_least_one_none_zero_digit
|
||||
+ delete_hyphen
|
||||
+ (pynutil.delete("trilliards") | pynutil.delete("trilliard")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph = pynini.union(
|
||||
graph_trilliards
|
||||
+ delete_hyphen
|
||||
+ graph_trillions
|
||||
+ delete_hyphen
|
||||
+ graph_billiards
|
||||
+ delete_hyphen
|
||||
+ graph_billions
|
||||
+ delete_hyphen
|
||||
+ graph_milliards
|
||||
+ delete_hyphen
|
||||
+ graph_millions
|
||||
+ delete_hyphen
|
||||
+ graph_thousands
|
||||
+ delete_hyphen
|
||||
+ graph_hundreds_component,
|
||||
graph_tens_of_hundred_component,
|
||||
graph_zero,
|
||||
)
|
||||
|
||||
graph = graph @ pynini.union(
|
||||
pynutil.delete(pynini.closure("0"))
|
||||
+ pynini.difference(DAMO_DIGIT, "0")
|
||||
+ pynini.closure(DAMO_DIGIT),
|
||||
"0",
|
||||
)
|
||||
|
||||
graph = rewrite(graph)
|
||||
|
||||
self.graph_no_exception = graph.optimize()
|
||||
|
||||
# save self.numbers_up_to_thousand for use in DecimalFst
|
||||
digits_up_to_thousand = DAMO_DIGIT | (DAMO_DIGIT**2) | (DAMO_DIGIT**3)
|
||||
numbers_up_to_thousand = pynini.compose(graph, digits_up_to_thousand).optimize()
|
||||
self.numbers_up_to_thousand = numbers_up_to_thousand
|
||||
|
||||
# save self.numbers_up_to_million for use in DecimalFst
|
||||
digits_up_to_million = (
|
||||
DAMO_DIGIT
|
||||
| (DAMO_DIGIT**2)
|
||||
| (DAMO_DIGIT**3)
|
||||
| (DAMO_DIGIT**4)
|
||||
| (DAMO_DIGIT**5)
|
||||
| (DAMO_DIGIT**6)
|
||||
)
|
||||
numbers_up_to_million = pynini.compose(graph, digits_up_to_million).optimize()
|
||||
self.numbers_up_to_million = numbers_up_to_million
|
||||
|
||||
# don't convert cardinals from zero to nine inclusive
|
||||
graph_exception = pynini.project(pynini.union(graph_digit, graph_zero), "input")
|
||||
|
||||
self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph
|
||||
|
||||
optional_minus_graph = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("moins", '"-"') + DAMO_SPACE, 0, 1
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,48 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying date, in the form of (day) month (year) or year
|
||||
e.g. le vingt-quatre juillet deux-mille-treize -> date { day: "24" month: "juli" year: "2013" preserve_order: true }
|
||||
e.g. le vingt-quatre juillet deux-mille-treize -> date { day: "24" month: "juli" year: "2013" preserve_order: true }
|
||||
e.g. le premier janvier -> date { day: "1" month: "janvier" preserve_order: true }
|
||||
|
||||
Also will convert colloquialism of spelling in which tens of hundreds are used to express date. (e.g. nineteen hundred and four)
|
||||
e.g. le vingt mais dix-neuf-cent-quatre -> date { day: "20" month: "mais" year: "1904" preserve_order: true }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="date", kind="classify")
|
||||
|
||||
self.cardinal = cardinal.graph_no_exception
|
||||
|
||||
year_graph = self.cardinal
|
||||
|
||||
month_graph = pynini.string_file(get_abs_path("data/months.tsv"))
|
||||
month_graph = pynutil.insert('month: "') + month_graph + pynutil.insert('"')
|
||||
|
||||
day_graph = self.cardinal | pynini.cross(
|
||||
"premier", "1"
|
||||
) # Premier is only ordinal used for dates
|
||||
day_graph = pynutil.insert('day: "') + day_graph + pynutil.insert('"')
|
||||
optional_graph_year = pynini.closure(
|
||||
delete_extra_space + pynutil.insert('year: "') + year_graph + pynutil.insert('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
graph_dmy = day_graph + delete_extra_space + month_graph + optional_graph_year
|
||||
|
||||
final_graph = graph_dmy
|
||||
final_graph += pynutil.insert(" preserve_order: true")
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,126 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_hyphen,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
def get_quantity(
|
||||
decimal: "pynini.FstLike", cardinal_up_to_thousand: "pynini.FstLike"
|
||||
) -> "pynini.FstLike":
|
||||
"""
|
||||
Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
|
||||
e.g. one million -> integer_part: "1" quantity: "million"
|
||||
e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million"
|
||||
|
||||
Will tag cases up to denominations of tens of hundreds of thousand. 'douze cent mille millions' -> 1 200 000 millions
|
||||
|
||||
Args:
|
||||
decimal: decimal FST
|
||||
cardinal_up_to_million: cardinal FST
|
||||
"""
|
||||
numbers = cardinal_up_to_thousand @ (
|
||||
pynutil.delete(pynini.closure("0"))
|
||||
+ pynini.difference(DAMO_DIGIT, "0")
|
||||
+ pynini.closure(DAMO_DIGIT)
|
||||
)
|
||||
|
||||
suffix = pynini.union(
|
||||
"million",
|
||||
"millions",
|
||||
"milliard",
|
||||
"milliards",
|
||||
"billion",
|
||||
"billions",
|
||||
"billiard",
|
||||
"billiards",
|
||||
"trillion",
|
||||
"trillions",
|
||||
"trilliard",
|
||||
"trilliards",
|
||||
)
|
||||
res = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ numbers
|
||||
+ pynutil.insert('"')
|
||||
+ (
|
||||
pynini.union(delete_hyphen, delete_extra_space)
|
||||
) # Can be written either as 'deux-millions' or 'deux millions' depending on whether it registers as a noun or part of cardinal.
|
||||
+ pynutil.insert(' quantity: "')
|
||||
+ suffix
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
res |= (
|
||||
decimal + delete_extra_space + pynutil.insert(' quantity: "') + suffix + pynutil.insert('"')
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal
|
||||
Decimal point is "," (virgule).
|
||||
e.g. moins un virgule deux six -> decimal { negative: "true" integer_part: "1" fractional_part: "26" }
|
||||
|
||||
This decimal rule assumes that decimals can be pronounced as:
|
||||
(a cardinal) + ('virgule') plus (any sequence of cardinals <1 million, including 'zero')
|
||||
|
||||
Also writes large numbers in shortened form, e.g.
|
||||
e.g. un virgule deux-six-million -> decimal { negative: "false" integer_part: "1" fractional_part: "26" quantity: "million" }
|
||||
e.g. deux-million -> decimal { negative: "false" integer_part: "2" quantity: "millions" }
|
||||
e.g. moins cent-vingt-quatre-millions -> decimal { negative: "true" integer_part: "124" quantity: "millions" }
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="decimal", kind="classify")
|
||||
|
||||
# number after decimal point can be any series of cardinals <1 million, including 'zero'
|
||||
graph_decimal = cardinal.numbers_up_to_million
|
||||
graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal
|
||||
self.graph = graph_decimal
|
||||
|
||||
# decimal point is denote by virgule
|
||||
graph_fractional_separator = pynutil.delete("virgule")
|
||||
|
||||
# Possible negatives
|
||||
optional_graph_negative = (
|
||||
pynutil.insert("negative: ") + pynini.cross("moins", '"true"') + delete_extra_space
|
||||
)
|
||||
optional_graph_negative = optional_graph_negative.ques
|
||||
|
||||
# Fractional portion
|
||||
graph_fractional = (
|
||||
pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"')
|
||||
)
|
||||
|
||||
# Integers
|
||||
cardinal_graph = cardinal.graph_no_exception | pynini.string_file(
|
||||
get_abs_path("data/numbers/zero.tsv")
|
||||
)
|
||||
graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
|
||||
|
||||
# Final graphs
|
||||
final_graph_wo_sign = (
|
||||
pynini.closure(graph_integer + delete_extra_space, 0, 1)
|
||||
+ graph_fractional_separator
|
||||
+ delete_extra_space
|
||||
+ graph_fractional
|
||||
)
|
||||
final_graph = optional_graph_negative + final_graph_wo_sign
|
||||
|
||||
self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(
|
||||
final_graph_wo_sign, cardinal.graph_hundreds_component_at_least_one_none_zero_digit
|
||||
)
|
||||
final_graph |= optional_graph_negative + get_quantity(
|
||||
final_graph_wo_sign, cardinal.graph_hundreds_component_at_least_one_none_zero_digit
|
||||
)
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,106 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
GraphFst,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying 'electronic' semiotic classes, i.e.
|
||||
email address (which get converted to "username" and "domain" fields),
|
||||
and URLS (which get converted to a "protocol" field).
|
||||
e.g. c d f une arobase a b c point e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } }
|
||||
e.g. double vé double vé double vé a b c point e d u -> tokens { electronic { protocol: "www.abc.edu" } }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="electronic", kind="classify")
|
||||
|
||||
delete_extra_space = pynutil.delete(" ")
|
||||
alpha_num = (
|
||||
DAMO_ALPHA
|
||||
| pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
| pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
)
|
||||
|
||||
symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv"))
|
||||
ampersand = pynini.string_map([("arobase"), ("chez"), ("at"), ("à")])
|
||||
|
||||
accepted_username = alpha_num | symbols
|
||||
process_dot = pynini.cross("point", ".")
|
||||
username = (
|
||||
pynutil.insert('username: "')
|
||||
+ alpha_num
|
||||
+ delete_extra_space
|
||||
+ pynini.closure(accepted_username + delete_extra_space)
|
||||
+ alpha_num
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num
|
||||
server = single_alphanum | pynini.string_file(
|
||||
get_abs_path("data/electronic/server_name.tsv")
|
||||
)
|
||||
domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
||||
domain_graph = (
|
||||
pynutil.insert('domain: "')
|
||||
+ server
|
||||
+ delete_extra_space
|
||||
+ process_dot
|
||||
+ delete_extra_space
|
||||
+ domain
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
graph = (
|
||||
username
|
||||
+ delete_extra_space
|
||||
+ pynutil.delete(ampersand)
|
||||
+ insert_space
|
||||
+ delete_extra_space
|
||||
+ domain_graph
|
||||
)
|
||||
|
||||
############# url ###
|
||||
protocol_end = pynini.cross(
|
||||
pynini.union("www", "w w w", "double vé double vé double vé"), "www"
|
||||
)
|
||||
protocol_start = pynini.cross(pynini.union("http", "h t t p", "ache té té pé"), "http")
|
||||
protocol_start |= pynini.cross(
|
||||
pynini.union("https", "h t t p s", "ache té té pé esse"), "https"
|
||||
)
|
||||
protocol_start += pynini.cross(
|
||||
pynini.union(
|
||||
" deux-points barre oblique barre oblique ",
|
||||
" deux-points barre barre ",
|
||||
" deux-points double barre ",
|
||||
" deux-points slash slash ",
|
||||
),
|
||||
"://",
|
||||
)
|
||||
|
||||
# e.g. .com, .es
|
||||
ending = (
|
||||
delete_extra_space
|
||||
+ symbols
|
||||
+ delete_extra_space
|
||||
+ (domain | pynini.closure(accepted_username + delete_extra_space) + accepted_username)
|
||||
)
|
||||
|
||||
protocol = (
|
||||
pynini.closure(protocol_start, 0, 1)
|
||||
+ protocol_end
|
||||
+ delete_extra_space
|
||||
+ process_dot
|
||||
+ delete_extra_space
|
||||
+ (pynini.closure(delete_extra_space + accepted_username, 1) | server)
|
||||
+ pynini.closure(ending, 1)
|
||||
)
|
||||
protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"')
|
||||
graph |= protocol
|
||||
########
|
||||
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,67 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying fraction
|
||||
e.g. demi -> tokens { fraction { numerator: "1" denominator: "2" } }
|
||||
e.g. un et demi -> tokens { fraction { integer_part: "1" numerator: "1" denominator: "2" } }
|
||||
e.g. trois et deux centième -> tokens { fraction { integer_part: "3" numerator: "2" denominator: "100" } }
|
||||
|
||||
Args:
|
||||
cardinal: OrdinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="fraction", kind="classify")
|
||||
# integer_part # numerator # denominator
|
||||
|
||||
graph_cardinal = cardinal.graph_no_exception
|
||||
graph_strip_undo_root_change = pynini.string_file(
|
||||
get_abs_path("data/fractions.tsv")
|
||||
) # add in absolute path
|
||||
|
||||
graph_strip_no_root_change = pynutil.delete("ième") # For no change to root
|
||||
graph_strip_no_root_change += pynutil.delete("s").ques # for plurals
|
||||
|
||||
graph_strip = graph_strip_no_root_change | graph_strip_undo_root_change
|
||||
|
||||
self.fractional = ((pynini.closure(DAMO_CHAR) + graph_strip) @ graph_cardinal).optimize()
|
||||
|
||||
integer = pynutil.insert('integer_part: "') + graph_cardinal + pynutil.insert('" ')
|
||||
integer += delete_space
|
||||
integer += pynutil.delete("et") # used to demarcate integer and fractional parts
|
||||
|
||||
numerator = pynutil.insert('numerator: "') + graph_cardinal + pynutil.insert('"')
|
||||
denominator = pynutil.insert(' denominator: "') + self.fractional + pynutil.insert('"')
|
||||
|
||||
# Demi (half) can occur alone without explicit numerator.
|
||||
graph_demi_component = (
|
||||
pynutil.delete("demi") + pynutil.delete("e").ques + pynutil.delete("s").ques
|
||||
)
|
||||
graph_demi_component += pynutil.insert('numerator: "1" denominator: "2"')
|
||||
|
||||
graph_fraction_component = numerator + delete_space + denominator
|
||||
graph_fraction_component |= graph_demi_component
|
||||
self.graph_fraction_component = graph_fraction_component
|
||||
|
||||
graph = pynini.closure(integer + delete_space, 0, 1) + graph_fraction_component
|
||||
graph = graph.optimize()
|
||||
self.final_graph_wo_negative = graph
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("moins", '"true"') + delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
graph = optional_graph_negative + graph
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,85 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
get_singulars,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying measure. Allows for plural form for unit.
|
||||
e.g. moins onze kilogramme -> measure { negative: "true" cardinal { integer: "11" } units: "kg" }
|
||||
e.g. trois heures -> measure { cardinal { integer: "3" } units: "h" }
|
||||
e.g. demi gramme -> measure { fraction { numerator: "1" denominator: "2" } units: "g" }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
fraction: FractionFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst, fraction: GraphFst):
|
||||
super().__init__(name="measure", kind="classify")
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
|
||||
graph_prefix = pynini.string_file(get_abs_path("data/measurements/magnitudes.tsv"))
|
||||
graph_unit_singular = pynini.string_file(get_abs_path("data/measurements/measurements.tsv"))
|
||||
|
||||
unit = get_singulars(graph_unit_singular) | graph_unit_singular
|
||||
unit = graph_prefix.ques + unit
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("moins", '"true"') + delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
unit_misc = (
|
||||
pynutil.insert("/")
|
||||
+ (pynutil.delete("par") | pynutil.delete("à"))
|
||||
+ delete_space
|
||||
+ unit
|
||||
)
|
||||
|
||||
unit = (
|
||||
pynutil.insert('units: "')
|
||||
+ (unit | unit_misc | pynutil.add_weight(unit + delete_space + unit_misc, 0.01))
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
subgraph_decimal = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ optional_graph_negative
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.insert(" }")
|
||||
+ delete_extra_space
|
||||
+ unit
|
||||
)
|
||||
|
||||
subgraph_fraction = (
|
||||
pynutil.insert("fraction { ")
|
||||
+ optional_graph_negative
|
||||
+ fraction.final_graph_wo_negative
|
||||
+ pynutil.insert(" }")
|
||||
+ delete_extra_space
|
||||
+ unit
|
||||
)
|
||||
|
||||
subgraph_cardinal = (
|
||||
pynutil.insert("cardinal { ")
|
||||
+ optional_graph_negative
|
||||
+ pynutil.insert('integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(" }")
|
||||
+ delete_extra_space
|
||||
+ unit
|
||||
)
|
||||
final_graph = subgraph_decimal | subgraph_cardinal | subgraph_fraction
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,130 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying money
|
||||
e.g. douze euro cinq -> money { integer_part: "12" currency: "€" fractional_part: 05}
|
||||
e.g. zéro euro cinq -> money { integer_part: "0" currency: "€" fractional_part: 05}
|
||||
e.g. cinq centimes -> money { integer_part: "0" currency: "€" fractional_part: 05}
|
||||
|
||||
Note, the currency symbol seems more common for exact amounts and quantities less than 'un million'
|
||||
For 'round' quantities of >=million (milliard, billion), the symbol is dropped. This allows
|
||||
use of the 'de' preposition.
|
||||
e.g. cinq millions d'euros -> money { integer_part: "5" currency: "d'euros" fractional_part: 00}
|
||||
e.g. un milliard d'euro -> money { integer_part: "5" currency: "d'euro" fractional_part: 00}
|
||||
e.g. trois virgule trois millions d'euros -> money { integer_part: "3" currency: "d'euros" fractional_part: 3}
|
||||
|
||||
Currency is included for uniform tagging.
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst):
|
||||
super().__init__(name="money", kind="classify")
|
||||
# quantity, integer_part, fractional_part, currency
|
||||
|
||||
# quantities
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
graph_decimal = decimal.final_graph_wo_negative
|
||||
|
||||
# Converts currency names to symbols
|
||||
convert_currency_major = pynini.string_file(
|
||||
get_abs_path("data/money/currency_major.tsv")
|
||||
) # major denominations
|
||||
convert_currency_minor = pynini.string_file(
|
||||
get_abs_path("data/money/currency_minor.tsv")
|
||||
) # minor denominations to major symbol. (e.g. 5 cents -> 0.05 $ )
|
||||
|
||||
accept_all_currency = (convert_currency_major | convert_currency_minor).project(
|
||||
"input"
|
||||
) # recognizes all currencies
|
||||
|
||||
# Graphs for large round amounts ('deux billiards d'euros', 'un milliard de dollars')
|
||||
graph_de = pynini.union(
|
||||
"de ", "des ", "d'"
|
||||
) # the use of de/d'only occurs with round amounts
|
||||
graph_currency_component_large_round_amounts = graph_de + accept_all_currency
|
||||
graph_currency_component_large_round_amounts = (
|
||||
pynutil.insert(' currency: "')
|
||||
+ graph_currency_component_large_round_amounts
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_money_large_round_amounts = (
|
||||
graph_decimal + delete_space
|
||||
) # graph_decimal includes tags and quantities already
|
||||
graph_money_large_round_amounts += graph_currency_component_large_round_amounts
|
||||
|
||||
# For standard currency
|
||||
add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
|
||||
pynutil.insert("0") + DAMO_DIGIT
|
||||
)
|
||||
|
||||
# Graphs integer denomination for large denominations (e.g. $)
|
||||
graph_integer_component_major = (
|
||||
pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
|
||||
)
|
||||
graph_integer_component_major += delete_space
|
||||
|
||||
graph_currency_component_major = (
|
||||
pynutil.insert(' currency: "') + convert_currency_major + pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_decimal_component_major = (
|
||||
delete_space
|
||||
+ pynutil.insert(' fractional_part: "')
|
||||
+ (cardinal_graph @ add_leading_zero_to_double_digit)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
# Rare cases where 'et' will separate major and minor denominations.
|
||||
delete_minor_currency = pynini.project(convert_currency_minor, "input")
|
||||
delete_minor_currency = delete_extra_space + pynutil.delete(delete_minor_currency)
|
||||
|
||||
delete_et = delete_extra_space + pynutil.delete("et")
|
||||
|
||||
graph_money_major = (
|
||||
graph_integer_component_major
|
||||
+ graph_currency_component_major
|
||||
+ delete_et.ques
|
||||
+ graph_decimal_component_major.ques
|
||||
+ delete_minor_currency.ques
|
||||
)
|
||||
|
||||
# For cases when only small denominations are used.
|
||||
graph_integer_component_minor = pynutil.insert('integer_part: "0"')
|
||||
|
||||
graph_decimal_component_minor = (
|
||||
pynutil.insert(' fractional_part: "')
|
||||
+ (cardinal_graph @ add_leading_zero_to_double_digit)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
graph_decimal_component_minor += delete_extra_space
|
||||
|
||||
graph_currency_component_minor = (
|
||||
pynutil.insert(' currency: "') + convert_currency_minor + pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_money_minor = (
|
||||
graph_integer_component_minor
|
||||
+ graph_decimal_component_minor
|
||||
+ graph_currency_component_minor
|
||||
)
|
||||
|
||||
graph_money_standard_amounts = graph_money_major | graph_money_minor
|
||||
|
||||
final_graph = graph_money_large_round_amounts | graph_money_standard_amounts
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,67 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying ordinal
|
||||
vingt-deuxième -> ordinal { integer: "22" morphosyntactic_features: "e" }
|
||||
|
||||
Also notes specific nouns that have unique normalization conventions.
|
||||
For instance, 'siècles' are rendered in roman numerals when given an ordinal adjective.
|
||||
e.g. dix-neuvième siècle -> XIXe
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="ordinal", kind="classify")
|
||||
|
||||
graph_cardinal = cardinal.graph_no_exception
|
||||
graph_undo_root_change = pynini.string_file(
|
||||
get_abs_path("data/ordinals/digits_root_change.tsv")
|
||||
) # Returns base number to normal after root change.
|
||||
graph_firsts = pynini.string_file(get_abs_path("data/ordinals/firsts.tsv"))
|
||||
graph_second = pynini.string_file(get_abs_path("data/ordinals/second.tsv"))
|
||||
graph_special_ordinals = pynini.string_file(get_abs_path("data/ordinals/key_nouns.tsv"))
|
||||
|
||||
# Removes morpheme
|
||||
graph_no_root_change = pynutil.delete("ième") # For no change to root
|
||||
|
||||
graph_strip_morpheme = pynini.union(graph_no_root_change, graph_undo_root_change)
|
||||
graph_strip_morpheme = DAMO_SIGMA + graph_strip_morpheme
|
||||
|
||||
graph_integer_component = graph_strip_morpheme @ graph_cardinal
|
||||
|
||||
graph_morpheme_component = pynutil.insert("e") # Put the superscript in.
|
||||
graph_morpheme_component += pynini.accep("s").ques # In case of plurals.
|
||||
|
||||
# Concatenate with cardinal graph.
|
||||
graph_ordinal = pynutil.insert('integer: "') + graph_integer_component + pynutil.insert('"')
|
||||
graph_ordinal += (
|
||||
pynutil.insert(' morphosyntactic_features: "') + graph_morpheme_component
|
||||
) # Leave open in case further morphems occur
|
||||
|
||||
# Primer has a different subscript depending on gender, need to take note if
|
||||
# 'premier' or 'première'
|
||||
graph_firsts = pynutil.insert('integer: "1" morphosyntactic_features: "') + graph_firsts
|
||||
|
||||
# Second used 'd' as a superscript.
|
||||
graph_second = pynutil.insert('integer: "2" morphosyntactic_features: "') + graph_second
|
||||
|
||||
graph = graph_firsts | graph_second | graph_ordinal
|
||||
|
||||
# For roman numerals. Carries over designation to verbalizer
|
||||
graph_special_ordinals = pynutil.insert("/") + delete_space + graph_special_ordinals
|
||||
|
||||
graph += graph_special_ordinals.ques + pynutil.insert('"')
|
||||
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,22 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class PunctuationFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying punctuation
|
||||
e.g. a, -> tokens { name: "a" } tokens { name: "," }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="punctuation", kind="classify")
|
||||
|
||||
s = "!#$%&'()*+,-./:;<=>?@^_`{|}~"
|
||||
guillemets = "\u00AB" + "\u00BB" # quotation marks in French.
|
||||
s += guillemets
|
||||
punct = pynini.union(*s)
|
||||
|
||||
graph = pynutil.insert('name: "') + punct + pynutil.insert('"')
|
||||
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,72 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
GraphFst,
|
||||
delete_hyphen,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying telephone numbers. Assumes conventional grouping for Metropolitan France (and overseas departments)
|
||||
(two number sequences are grouped as individual cardinals) or digit by digit (chiffre-par-chiffre) e.g.
|
||||
"zero un quatre-vingt-deux zero deux vingt-deux cinquante" -> { number_part: "01 42 02 22 50" }
|
||||
"zero un quatre deux zero deux deux deux cinq zero" -> { number_part: "01 42 02 22 50" }
|
||||
|
||||
In cases where only one digit of the first pairing is admitted, assumes that the 0 was skipped.
|
||||
"une vingt-trois quatre-vingt zero six dix-sept" -> { number_part: "01 23 40 06 17" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="telephone", kind="classify")
|
||||
|
||||
# create `single_digits` and `double_digits` graphs as these will be
|
||||
# the building blocks of possible telephone numbers
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv"))
|
||||
graph_ties_unique = pynini.string_file((get_abs_path("data/numbers/ties_unique.tsv")))
|
||||
graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv"))
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
|
||||
double_digits = pynini.union(
|
||||
graph_teen,
|
||||
graph_ties_unique,
|
||||
(graph_ties + pynutil.insert("0")),
|
||||
(graph_ties + delete_hyphen + graph_digit),
|
||||
)
|
||||
|
||||
graph_first_pair = graph_zero + delete_space + graph_digit
|
||||
graph_first_pair |= pynutil.insert("0") + graph_digit # if zero is omitted
|
||||
graph_first_pair += (
|
||||
delete_space + insert_space
|
||||
) # delete_space since closure allows possible gaps to be removed
|
||||
|
||||
# All digits
|
||||
single_digits = graph_digit | graph_zero
|
||||
|
||||
graph_pair_all_digits = single_digits + delete_space
|
||||
graph_pair_all_digits += single_digits
|
||||
|
||||
graph_all_digits = pynini.closure(graph_pair_all_digits + delete_space + insert_space, 3, 3)
|
||||
graph_all_digits = graph_first_pair + graph_all_digits + graph_pair_all_digits
|
||||
|
||||
# Paired digits
|
||||
graph_pair_digits_and_ties = double_digits | graph_pair_all_digits
|
||||
|
||||
graph_digits_and_ties = pynini.closure(
|
||||
graph_pair_digits_and_ties + delete_space + insert_space, 3, 3
|
||||
)
|
||||
graph_digits_and_ties = (
|
||||
graph_first_pair + graph_digits_and_ties + graph_pair_digits_and_ties
|
||||
)
|
||||
|
||||
number_part = pynini.union(graph_all_digits, graph_digits_and_ties)
|
||||
|
||||
number_part = pynutil.insert('number_part: "') + number_part + pynutil.insert('"')
|
||||
|
||||
graph = number_part
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,109 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, delete_space
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying time
|
||||
e.g. huit heures -> time { hours: "8" minutes: "00" }
|
||||
e.g. treize heures -> time { hours: "13" minutes: "00" }
|
||||
e.g. treize heures dix -> time { hours: "13" minutes: "10" }
|
||||
e.g. huit heures du matin -> time { hours: "8" minutes: "00" suffix: "avant mid"}
|
||||
e.g. huite heures du après midi -> time { hours: "8" minutes: "00" suffix: "après-midi"}
|
||||
e.g. douze heures moins qart -> time { hours: "11" minutes: "45" }
|
||||
e.g. douze heures et qart -> time { hours: "12" minutes: "15" }
|
||||
e.g. midi et qart -> time { hours: "12" minutes: "15" }
|
||||
e.g. minuit et medi -> time { hours: "0" minutes: "30" }
|
||||
e.g. douze heures moins medi -> time { hours: "11" minutes: "30" }
|
||||
e.g. douze heures moins trois -> time { hours: "11" minutes: "57" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="time", kind="classify")
|
||||
# hours, minutes, seconds, suffix, zone, style, speak_period
|
||||
|
||||
# time_zone = pynini.invert(pynini.string_file(get_abs_path("data/time/time_zone.tsv")))
|
||||
graph_hours_to = pynini.string_file(get_abs_path("data/time/hours_to.tsv"))
|
||||
graph_minutes_to = pynini.string_file(get_abs_path("data/time/minutes_to.tsv"))
|
||||
graph_hours = pynini.string_file(get_abs_path("data/time/hours.tsv"))
|
||||
graph_minutes = pynini.string_file(get_abs_path("data/time/minutes.tsv"))
|
||||
graph_suffix_am = pynini.string_file(get_abs_path("data/time/time_suffix_am.tsv"))
|
||||
graph_suffix_pm = pynini.string_file(get_abs_path("data/time/time_suffix_pm.tsv"))
|
||||
|
||||
graph_suffix = pynini.cross(graph_suffix_am, "am") | pynini.cross(graph_suffix_pm, "pm")
|
||||
|
||||
# Mapping 'heures'
|
||||
graph_heures = pynini.accep("heure") + pynini.accep("s").ques
|
||||
graph_heures = pynutil.delete(graph_heures)
|
||||
|
||||
graph_hours += delete_space + graph_heures
|
||||
|
||||
# Midi and minuit
|
||||
graph_midi = pynini.cross("midi", "12")
|
||||
graph_minuit = pynini.cross("minuit", "0")
|
||||
|
||||
# Mapping 'et demi' and 'et qart'
|
||||
graph_et = pynutil.delete("et") + delete_space
|
||||
|
||||
graph_demi = pynini.accep("demi")
|
||||
graph_demi += pynini.accep("e").ques # people vary on feminine or masculine form
|
||||
graph_demi = pynini.cross(graph_demi, "30")
|
||||
|
||||
graph_quart = pynini.accep("quart")
|
||||
graph_quart = pynini.accep("le ").ques + graph_quart # sometimes used
|
||||
graph_quart = pynini.cross(graph_quart, "15")
|
||||
graph_trois_quart = pynini.cross("trois quarts", "45")
|
||||
|
||||
graph_fractions = pynini.union(graph_demi, graph_quart, graph_trois_quart)
|
||||
|
||||
graph_et_fractions = graph_et + graph_fractions
|
||||
|
||||
# Hours component is usually just a cardinal + 'heures' (ignored in case of 'midi/minuit').
|
||||
graph_hours_component = pynini.union(graph_hours, graph_midi, graph_minuit)
|
||||
graph_hours_component = (
|
||||
pynutil.insert('hours: "') + graph_hours_component + pynutil.insert('"')
|
||||
)
|
||||
graph_hours_component += delete_space
|
||||
|
||||
# Minutes component
|
||||
graph_minutes_component = (
|
||||
pynutil.insert(' minutes: "')
|
||||
+ pynini.union(graph_minutes, graph_et_fractions)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
# Hour and minutes together. For 'demi' and 'qart', 'et' is used as a conjunction.
|
||||
graph_time_standard = graph_hours_component + graph_minutes_component.ques
|
||||
|
||||
# For time until hour. "quatre heures moins qart" -> 4 h 00 - 0 h 15 = 3 h 45
|
||||
graph_moins = pynutil.delete("moins")
|
||||
graph_moins += delete_space
|
||||
|
||||
graph_hours_to_component = graph_hours | graph_midi | graph_minuit
|
||||
graph_hours_to_component @= graph_hours_to
|
||||
graph_hours_to_component = (
|
||||
pynutil.insert('hours: "') + graph_hours_to_component + pynutil.insert('"')
|
||||
)
|
||||
graph_hours_to_component += delete_space
|
||||
|
||||
graph_minutes_to_component = pynini.union(graph_minutes, graph_fractions)
|
||||
graph_minutes_to_component @= graph_minutes_to
|
||||
graph_minutes_to_component = (
|
||||
pynutil.insert(' minutes: "') + graph_minutes_to_component + pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_time_to = graph_hours_to_component + graph_moins + graph_minutes_to_component
|
||||
|
||||
graph_time_no_suffix = graph_time_standard | graph_time_to
|
||||
|
||||
graph_suffix_component = pynutil.insert(' suffix: "') + graph_suffix + pynutil.insert('"')
|
||||
graph_suffix_component = delete_space + graph_suffix_component
|
||||
graph_suffix_component = graph_suffix_component.ques
|
||||
|
||||
final_graph = graph_time_no_suffix + graph_suffix_component
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,108 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.date import DateFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.electronic import ElectronicFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.fraction import FractionFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.ordinal import OrdinalFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.punctuation import PunctuationFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.telephone import TelephoneFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.time import TimeFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.whitelist import WhiteListFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.taggers.word import WordFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class ClassifyFst(GraphFst):
|
||||
"""
|
||||
Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
|
||||
super().__init__(name="tokenize_and_classify", kind="classify")
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(cache_dir, "_fr_itn.far")
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
|
||||
logging.info(f"ClassifyFst.fst was restored from {far_file}.")
|
||||
else:
|
||||
logging.info(f"Creating ClassifyFst grammars.")
|
||||
|
||||
cardinal = CardinalFst()
|
||||
cardinal_graph = cardinal.fst
|
||||
|
||||
fraction = FractionFst(cardinal)
|
||||
fraction_graph = fraction.fst
|
||||
|
||||
ordinal = OrdinalFst(cardinal)
|
||||
ordinal_graph = ordinal.fst
|
||||
|
||||
decimal = DecimalFst(cardinal)
|
||||
decimal_graph = decimal.fst
|
||||
|
||||
measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction).fst
|
||||
date_graph = DateFst(cardinal).fst
|
||||
word_graph = WordFst().fst
|
||||
time_graph = TimeFst().fst
|
||||
money_graph = MoneyFst(cardinal, decimal).fst
|
||||
whitelist_graph = WhiteListFst().fst
|
||||
punct_graph = PunctuationFst().fst
|
||||
electronic_graph = ElectronicFst().fst
|
||||
telephone_graph = TelephoneFst().fst
|
||||
|
||||
classify = (
|
||||
pynutil.add_weight(whitelist_graph, 1.01)
|
||||
| pynutil.add_weight(time_graph, 1.05)
|
||||
| pynutil.add_weight(date_graph, 1.09)
|
||||
| pynutil.add_weight(decimal_graph, 1.08)
|
||||
| pynutil.add_weight(measure_graph, 1.1)
|
||||
| pynutil.add_weight(cardinal_graph, 1.1)
|
||||
| pynutil.add_weight(ordinal_graph, 1.1)
|
||||
| pynutil.add_weight(fraction_graph, 1.09)
|
||||
| pynutil.add_weight(money_graph, 1.07)
|
||||
| pynutil.add_weight(telephone_graph, 1.1)
|
||||
| pynutil.add_weight(electronic_graph, 1.1)
|
||||
| pynutil.add_weight(word_graph, 100)
|
||||
)
|
||||
|
||||
punct = (
|
||||
pynutil.insert("tokens { ")
|
||||
+ pynutil.add_weight(punct_graph, weight=1.1)
|
||||
+ pynutil.insert(" }")
|
||||
)
|
||||
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
|
||||
token_plus_punct = (
|
||||
pynini.closure(punct + pynutil.insert(" "))
|
||||
+ token
|
||||
+ pynini.closure(pynutil.insert(" ") + punct)
|
||||
)
|
||||
|
||||
graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
|
||||
graph = delete_space + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
|
||||
if far_file:
|
||||
generator_main(far_file, {"tokenize_and_classify": self.fst})
|
||||
logging.info(f"ClassifyFst grammars are saved to {far_file}.")
|
||||
@@ -0,0 +1,19 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst, convert_space
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying whitelisted tokens
|
||||
e.g. misses -> tokens { name: "mrs." }
|
||||
This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="whitelist", kind="classify")
|
||||
|
||||
whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv"))
|
||||
graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,15 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_NOT_SPACE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class.
|
||||
e.g. sleep -> tokens { name: "sleep" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="word", kind="classify")
|
||||
word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
|
||||
self.fst = word.optimize()
|
||||
@@ -0,0 +1,13 @@
|
||||
import os
|
||||
|
||||
|
||||
def get_abs_path(rel_path):
|
||||
"""
|
||||
Get absolute path
|
||||
|
||||
Args:
|
||||
rel_path: relative path to this file
|
||||
|
||||
Returns absolute path
|
||||
"""
|
||||
return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinal
|
||||
e.g. cardinal { negative: "-" integer: "23" } -> -23
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="cardinal", kind="verbalize")
|
||||
optional_sign = pynini.closure(
|
||||
pynutil.delete("negative:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ DAMO_NOT_QUOTE
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
graph = (
|
||||
pynutil.delete("integer:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.numbers = graph
|
||||
graph = optional_sign + graph
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,62 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing date, e.g.
|
||||
date { day: "1" month: "janvier" preserve_order: true } -> 1 de enero
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="date", kind="verbalize")
|
||||
|
||||
convert_primer = pynini.cross("1", "1ᵉʳ")
|
||||
day = (
|
||||
pynutil.delete("day:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ (
|
||||
pynini.closure(DAMO_NOT_QUOTE, 1) | pynutil.add_weight(convert_primer, -1)
|
||||
) # first of the month is ordinal
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
month = (
|
||||
pynutil.delete("month:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
year = (
|
||||
pynutil.delete("year:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
# day month
|
||||
graph_dm = day + delete_extra_space + month
|
||||
graph_dmy = graph_dm + delete_extra_space + year
|
||||
|
||||
optional_preserve_order = pynini.closure(
|
||||
pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
|
||||
| pynutil.delete("field_order:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ DAMO_NOT_QUOTE
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space
|
||||
)
|
||||
|
||||
final_graph = (graph_dm | graph_dmy) + delete_space + optional_preserve_order
|
||||
|
||||
delete_tokens = self.delete_tokens(final_graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,81 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_NON_BREAKING_SPACE,
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class NumberParser(GraphFst):
|
||||
"""
|
||||
Finite state transducer for parsing strings of digis. Breaks up digit strings into groups of three for
|
||||
strings of digits of four or more (inclusive). Groupings are separated by non-breaking space.
|
||||
e.g. '1000' -> '1 000'
|
||||
e.g. '1000,33333' -> '1 000,333 33
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="parser", kind="verbalize")
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "12" fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="decimal", kind="verbalize")
|
||||
|
||||
# Need parser to group digits by threes
|
||||
exactly_three_digits = DAMO_DIGIT**3
|
||||
at_most_three_digits = pynini.closure(DAMO_DIGIT, 1, 3)
|
||||
|
||||
space_every_three_integer = (
|
||||
at_most_three_digits
|
||||
+ (pynutil.insert(DAMO_NON_BREAKING_SPACE) + exactly_three_digits).closure()
|
||||
)
|
||||
space_every_three_decimal = (
|
||||
pynini.accep(",")
|
||||
+ (exactly_three_digits + pynutil.insert(DAMO_NON_BREAKING_SPACE)).closure()
|
||||
+ at_most_three_digits
|
||||
)
|
||||
group_by_threes = space_every_three_integer | space_every_three_decimal
|
||||
self.group_by_threes = group_by_threes
|
||||
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
|
||||
integer = (
|
||||
pynutil.delete("integer_part:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
integer = integer @ group_by_threes
|
||||
optional_integer = pynini.closure(integer + delete_space, 0, 1)
|
||||
fractional = (
|
||||
pynutil.insert(",")
|
||||
+ pynutil.delete("fractional_part:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
fractional = fractional @ group_by_threes
|
||||
optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
|
||||
quantity = (
|
||||
pynutil.delete("quantity:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1)
|
||||
graph = (optional_integer + optional_fractional + optional_quantity).optimize()
|
||||
self.numbers = graph
|
||||
graph = optional_sign + graph
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,35 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic
|
||||
e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="electronic", kind="verbalize")
|
||||
user_name = (
|
||||
pynutil.delete("username:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
domain = (
|
||||
pynutil.delete("domain:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = user_name + delete_space + pynutil.insert("@") + domain
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,43 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing fraction
|
||||
e.g. fraction { integer_part: "1" numerator: "2" denominator: "3" } } -> 1 2/3
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="fraction", kind="verbalize")
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
|
||||
integer = (
|
||||
pynutil.delete('integer_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
+ insert_space
|
||||
)
|
||||
numerator = (
|
||||
pynutil.delete('numerator: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
denominator = (
|
||||
pynutil.insert("/")
|
||||
+ pynutil.delete('denominator: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = (
|
||||
pynini.closure(integer + delete_space, 0, 1) + numerator + delete_space + denominator
|
||||
).optimize()
|
||||
self.numbers = graph
|
||||
delete_tokens = self.delete_tokens(optional_sign + graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,68 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
cardinal: CardinalFst
|
||||
fraction: FractionFst
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, cardinal: GraphFst, fraction: GraphFst):
|
||||
super().__init__(name="measure", kind="verbalize")
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
|
||||
unit = (
|
||||
pynutil.delete("units:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space
|
||||
)
|
||||
graph_decimal = (
|
||||
pynutil.delete("decimal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ decimal.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph_cardinal = (
|
||||
pynutil.delete("cardinal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ cardinal.numbers
|
||||
@ decimal.group_by_threes # measurements most obey three by three spacing
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph_fraction = (
|
||||
pynutil.delete("fraction {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ fraction.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
|
||||
graph = (
|
||||
(graph_cardinal | graph_decimal | graph_fraction)
|
||||
+ delete_space
|
||||
+ pynutil.insert(" ")
|
||||
+ unit
|
||||
)
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,31 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { integer_part: "12" fractional_part: "05" currency: "$" } -> 12.05 $
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst):
|
||||
super().__init__(name="money", kind="verbalize")
|
||||
unit = (
|
||||
pynutil.delete("currency:")
|
||||
+ delete_extra_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = decimal.numbers + delete_space + unit
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,71 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing ordinal, e.g.
|
||||
ordinal { integer: "13" morphosyntactic_features: "e" } -> 13ᵉ
|
||||
|
||||
Given 'special' terms for ordinals (e.g. siècle), renders
|
||||
amount in conventional format. e.g.
|
||||
|
||||
ordinal { integer: "13" morphosyntactic_features: "e/siècle" } -> XIIIᵉ
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="ordinal", kind="verbalize")
|
||||
graph_integer = (
|
||||
pynutil.delete("integer:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
replace_suffix = pynini.union(
|
||||
pynini.cross("e", "ᵉ"), # only delete first quote since there may be more features
|
||||
pynini.cross("d", "ᵈ"),
|
||||
pynini.cross("r", "ʳ"),
|
||||
pynini.cross("s", "ˢ"),
|
||||
)
|
||||
replace_suffix = pynutil.delete(' morphosyntactic_features: "') + replace_suffix.plus
|
||||
|
||||
graph_arabic = graph_integer + replace_suffix.plus
|
||||
|
||||
# For roman.
|
||||
graph_roman_digits = pynini.string_file(
|
||||
get_abs_path("data/roman/digits_large.tsv")
|
||||
).invert()
|
||||
graph_roman_ties = pynini.string_file(get_abs_path("data/roman/ties_large.tsv")).invert()
|
||||
graph_roman_hundreds = pynini.string_file(
|
||||
get_abs_path("data/roman/hundreds_large.tsv")
|
||||
).invert()
|
||||
graph_roman_zero_digit = pynutil.delete("0")
|
||||
|
||||
graph_roman_hundreds = DAMO_DIGIT**3 @ (
|
||||
graph_roman_hundreds
|
||||
+ pynini.union(graph_roman_ties, graph_roman_zero_digit)
|
||||
+ pynini.union(graph_roman_digits, graph_roman_zero_digit)
|
||||
)
|
||||
graph_roman_ties = DAMO_DIGIT**2 @ (
|
||||
graph_roman_ties + pynini.union(graph_roman_digits, graph_roman_zero_digit)
|
||||
)
|
||||
graph_roman_digits = DAMO_DIGIT @ graph_roman_digits
|
||||
|
||||
graph_roman_integers = graph_roman_hundreds | graph_roman_ties | graph_roman_digits
|
||||
|
||||
graph_roman = (graph_integer @ graph_roman_integers) + replace_suffix
|
||||
graph_roman += pynini.cross("/", " ") + "siècle"
|
||||
|
||||
graph = (graph_roman | graph_arabic) + pynutil.delete('"')
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,22 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing telephone, e.g.
|
||||
telephone { number_part: "02 33 43 53 22" }
|
||||
-> 02 33 43 53 22
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="telephone", kind="verbalize")
|
||||
|
||||
number_part = (
|
||||
pynutil.delete('number_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
delete_tokens = self.delete_tokens(number_part)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,61 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing time, e.g.
|
||||
time { hours: "8" minutes: "30" suffix: "du matin"} -> 8 h 30
|
||||
time { hours: "8" minutes: "30" } -> 8 h 30
|
||||
time { hours: "8" minutes: "30" suffix: "du soir"} -> 20 h 30
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="time", kind="verbalize")
|
||||
|
||||
hour_to_night = pynini.string_file(get_abs_path("data/time/hour_to_night.tsv"))
|
||||
|
||||
day_suffixes = pynutil.delete('suffix: "am"')
|
||||
night_suffixes = pynutil.delete('suffix: "pm"')
|
||||
|
||||
hour = (
|
||||
pynutil.delete("hours:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1, 2)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
minute = (
|
||||
pynutil.delete("minutes:")
|
||||
+ delete_extra_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1, 2)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = (
|
||||
hour
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert("h")
|
||||
+ minute.ques
|
||||
+ delete_space
|
||||
+ day_suffixes.ques
|
||||
)
|
||||
|
||||
graph |= (
|
||||
hour @ hour_to_night
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert("h")
|
||||
+ minute.ques
|
||||
+ delete_space
|
||||
+ night_suffixes
|
||||
)
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,51 @@
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import GraphFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.date import DateFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.electronic import ElectronicFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.fraction import FractionFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.ordinal import OrdinalFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.telephone import TelephoneFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.time import TimeFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.whitelist import WhiteListFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="verbalize", kind="verbalize")
|
||||
cardinal = CardinalFst()
|
||||
cardinal_graph = cardinal.fst
|
||||
ordinal_graph = OrdinalFst().fst
|
||||
decimal = DecimalFst()
|
||||
decimal_graph = decimal.fst
|
||||
fraction = FractionFst()
|
||||
fraction_graph = fraction.fst
|
||||
measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal, fraction=fraction).fst
|
||||
money_graph = MoneyFst(decimal=decimal).fst
|
||||
time_graph = TimeFst().fst
|
||||
date_graph = DateFst().fst
|
||||
whitelist_graph = WhiteListFst().fst
|
||||
telephone_graph = TelephoneFst().fst
|
||||
electronic_graph = ElectronicFst().fst
|
||||
graph = (
|
||||
time_graph
|
||||
| date_graph
|
||||
| money_graph
|
||||
| measure_graph
|
||||
| fraction_graph
|
||||
| ordinal_graph
|
||||
| decimal_graph
|
||||
| cardinal_graph
|
||||
| whitelist_graph
|
||||
| telephone_graph
|
||||
| electronic_graph
|
||||
)
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,33 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.fr.verbalizers.word import WordFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence, e.g.
|
||||
tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="verbalize_final", kind="verbalize")
|
||||
verbalize = VerbalizeFst().fst
|
||||
word = WordFst().fst
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,27 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing whitelist
|
||||
e.g. tokens { name: "mrs." } -> mrs.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="whitelist", kind="verbalize")
|
||||
graph = (
|
||||
pynutil.delete("name:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,29 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.fr.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing plain tokens
|
||||
e.g. tokens { name: "sleep" } -> sleep
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="word", kind="verbalize")
|
||||
chars = pynini.closure(DAMO_CHAR - " ", 1)
|
||||
char = (
|
||||
pynutil.delete("name:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ chars
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = char @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
|
||||
self.fst = graph.optimize()
|
||||
Reference in New Issue
Block a user