chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,7 @@
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.tokenize_and_classify import (
|
||||
ClassifyFst,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.verbalize_final import (
|
||||
VerbalizeFinalFst,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
$ đô
|
||||
$ usd
|
||||
$ đô la
|
||||
$ đô la mỹ
|
||||
€ euro
|
||||
€ ơ rô
|
||||
¥ yên
|
||||
₩ won
|
||||
₩ uôn
|
||||
RM ringgit
|
||||
₫ đồng
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
com
|
||||
uk
|
||||
fr
|
||||
net
|
||||
br
|
||||
in
|
||||
ru
|
||||
de
|
||||
it
|
||||
vn
|
||||
edu
|
||||
|
@@ -0,0 +1,16 @@
|
||||
g mail gmail
|
||||
gmail
|
||||
n vidia nvidia
|
||||
nvidia
|
||||
outlook
|
||||
hotmail
|
||||
yahoo
|
||||
aol
|
||||
gmx
|
||||
msn
|
||||
live
|
||||
yandex
|
||||
orange
|
||||
wanadoo
|
||||
web
|
||||
comcast
|
||||
|
@@ -0,0 +1,20 @@
|
||||
. chấm
|
||||
- gạch
|
||||
_ gạch dưới
|
||||
_ shift gạch
|
||||
_ síp gạch
|
||||
! chấm than
|
||||
# thăng
|
||||
$ đô
|
||||
% phần trăm
|
||||
& và
|
||||
* sao
|
||||
: hai chấm
|
||||
+ cộng
|
||||
/ sẹc
|
||||
= bằng
|
||||
? chấm hỏi
|
||||
? hỏi chấm
|
||||
^ mũ
|
||||
| hoặc
|
||||
, phẩy
|
||||
|
@@ -0,0 +1,5 @@
|
||||
k nghìn
|
||||
k ngàn
|
||||
m triệu
|
||||
b tỉ
|
||||
b tỷ
|
||||
|
@@ -0,0 +1,10 @@
|
||||
% phần trăm
|
||||
& và
|
||||
* nhân
|
||||
* sao
|
||||
+ cộng
|
||||
- trừ
|
||||
/ chia
|
||||
= bằng
|
||||
^ mũ
|
||||
| hoặc
|
||||
|
@@ -0,0 +1,185 @@
|
||||
°f fahrenheit
|
||||
°f độ f
|
||||
°c celsius
|
||||
°c độ c
|
||||
km kilomet
|
||||
km ki lô met
|
||||
km ki lô mét
|
||||
km kilô mét
|
||||
km kilo mét
|
||||
m met
|
||||
m mét
|
||||
cm centimet
|
||||
cm cen ti mét
|
||||
cm xen ti mét
|
||||
cm xăng ti mét
|
||||
mm millimet
|
||||
mm mi li mét
|
||||
mm mili mét
|
||||
ha hecta
|
||||
ha héc ta
|
||||
mi mile
|
||||
m² met vuông
|
||||
m² mét vuông
|
||||
km² kilomet vuông
|
||||
km² ki lô met vuông
|
||||
km² kilo mét vuông
|
||||
km² ki lô mét vuông
|
||||
ft foot
|
||||
% phần trăm
|
||||
hz héc
|
||||
hz hẹc
|
||||
kw kilowatt
|
||||
kw kilo watt
|
||||
kw ki lô watt
|
||||
kw ki lô oát
|
||||
kw ki lô goát
|
||||
hp mã lực
|
||||
mg milligram
|
||||
mg milli gram
|
||||
mg mi li gram
|
||||
mg mi li gam
|
||||
mg mili gam
|
||||
kg kilogram
|
||||
kg kilo gram
|
||||
kg ki lô gram
|
||||
kg kilo gam
|
||||
kg ki lô gam
|
||||
kg kí lô
|
||||
kg kí
|
||||
ghz giga hẹc
|
||||
ghz giga héc
|
||||
ghz gi ga héc
|
||||
ghz gi ga hẹc
|
||||
khz kilo hẹc
|
||||
khz kilo héc
|
||||
khz ki lô héc
|
||||
khz ki lô hẹc
|
||||
mhz mega héc
|
||||
mhz mega hẹc
|
||||
mhz mê ga hẹc
|
||||
mhz mê ga héc
|
||||
v vôn
|
||||
h giờ
|
||||
phút phút
|
||||
s giây
|
||||
nm nanomet
|
||||
nm nano mét
|
||||
nm na nô mét
|
||||
mA milli ampe
|
||||
mA mi li ampe
|
||||
mA mi li am pe
|
||||
kwh kilowatt giờ
|
||||
kwh kilo watt giờ
|
||||
kwh ki lô oát giờ
|
||||
kwh ki lô goát giờ
|
||||
m³ met khối
|
||||
m³ mét khối
|
||||
cm³ centimet khối
|
||||
cm³ cen ti mét khối
|
||||
cm³ xen ti mét khối
|
||||
cm³ xăng ti mét khối
|
||||
tw tera watt
|
||||
tw terawatt
|
||||
tw te ra watt
|
||||
tw tê ra watt
|
||||
tw tê ra oát
|
||||
tw tê ra goát
|
||||
mv milli vôn
|
||||
mv mi li vôn
|
||||
mw megawatt
|
||||
mw mega watt
|
||||
mw mê ga watt
|
||||
mw mê ga oát
|
||||
mw mê ga goát
|
||||
μm micromet
|
||||
μm micro mét
|
||||
μm muy crô mét
|
||||
μm mi crô mét
|
||||
μm muy cờ rô mét
|
||||
μm mi cờ rô mét
|
||||
inch inch
|
||||
tb terabyte
|
||||
tb tera byte
|
||||
tb te ra byte
|
||||
tb te ra bai
|
||||
tb tê ra byte
|
||||
tb tê ra bai
|
||||
g gram
|
||||
g gam
|
||||
ω ohm
|
||||
ω ôm
|
||||
db decibel
|
||||
db deci ben
|
||||
db đề xi ben
|
||||
μg microgram
|
||||
μg micro gram
|
||||
μg muy crô gram
|
||||
μg micro gam
|
||||
μg muy crô gam
|
||||
μg muy cờ rô gam
|
||||
pg petagram
|
||||
pg peta gam
|
||||
pg pê ta gam
|
||||
gb gigabyte
|
||||
gb giga byte
|
||||
gb giga bai
|
||||
gb gi ga byte
|
||||
gb gi ga bai
|
||||
mb megabyte
|
||||
mb mega byte
|
||||
mb mega bai
|
||||
mb me ga byte
|
||||
mb me ga bai
|
||||
mb mê ga byte
|
||||
mb mê ga bai
|
||||
kb kilobyte
|
||||
kb kilo byte
|
||||
kb kilo bai
|
||||
kb ki lô byte
|
||||
kb ki lô bai
|
||||
kbps kilobit trên giây
|
||||
mbps megabit trên giây
|
||||
kv kilovolt
|
||||
kv kilo vôn
|
||||
kv ki lô vôn
|
||||
mv megavolt
|
||||
mv mega vôn
|
||||
mv me ga vôn
|
||||
mv mê ga vôn
|
||||
yd yard
|
||||
rad radian
|
||||
gpa giga pascal
|
||||
gpa gi ga pascal
|
||||
gw gigawatt
|
||||
gw giga watt
|
||||
gw gi ga watt
|
||||
gw gi ga oát
|
||||
ns nano giây
|
||||
ns na nô giây
|
||||
μs micro giây
|
||||
μs muy crô giây
|
||||
μs muy cờ rô giây
|
||||
pa pascal
|
||||
ms milli giây
|
||||
ms mi li giây
|
||||
dm decimet
|
||||
dm đề xi mét
|
||||
dm³ đề xi mét khối
|
||||
dm³ decimet khối
|
||||
mm² millimet vuông
|
||||
mm² mi li mét vuông
|
||||
cm² centimet vuông
|
||||
cm² centi mét vuông
|
||||
cm² xăng ti mét vuông
|
||||
cm² xen ti mét vuông
|
||||
pb peta byte
|
||||
pb petabyte
|
||||
pb pê ta byte
|
||||
kcal kilocalories
|
||||
kcal kilocalo
|
||||
kcal ki lô calo
|
||||
l lít
|
||||
ml millilit
|
||||
ml mili lít
|
||||
ml mi li lít
|
||||
|
@@ -0,0 +1,14 @@
|
||||
một 1
|
||||
hai 2
|
||||
ba 3
|
||||
tư 4
|
||||
bốn 4
|
||||
năm 5
|
||||
sáu 6
|
||||
bảy 7
|
||||
bẩy 7
|
||||
tám 8
|
||||
chín 9
|
||||
mười 10
|
||||
mười một 11
|
||||
mười hai 12
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
một 1
|
||||
hai 2
|
||||
ba 3
|
||||
bốn 4
|
||||
năm 5
|
||||
sáu 6
|
||||
bảy 7
|
||||
bẩy 7
|
||||
tám 8
|
||||
chín 9
|
||||
|
@@ -0,0 +1 @@
|
||||
trăm
|
||||
|
@@ -0,0 +1,11 @@
|
||||
mười 10
|
||||
mười một 11
|
||||
mười hai 12
|
||||
mười ba 13
|
||||
mười bốn 14
|
||||
mười lăm 15
|
||||
mười sáu 16
|
||||
mười bảy 17
|
||||
mười bẩy 17
|
||||
mười tám 18
|
||||
mười chín 19
|
||||
|
@@ -0,0 +1,6 @@
|
||||
nghìn
|
||||
triệu
|
||||
tỉ
|
||||
nghìn tỉ
|
||||
triệu tỉ
|
||||
tỉ tỉ
|
||||
|
@@ -0,0 +1,9 @@
|
||||
hai 2
|
||||
ba 3
|
||||
bốn 4
|
||||
năm 5
|
||||
sáu 6
|
||||
bảy 7
|
||||
bẩy 7
|
||||
tám 8
|
||||
chín 9
|
||||
|
@@ -0,0 +1 @@
|
||||
không 0
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
nhất 1
|
||||
nhì 2
|
||||
hai 2
|
||||
ba 3
|
||||
bốn 4
|
||||
tư 4
|
||||
năm 5
|
||||
sáu 6
|
||||
bảy 7
|
||||
bẩy 7
|
||||
tám 8
|
||||
chín 9
|
||||
|
@@ -0,0 +1,29 @@
|
||||
không 0
|
||||
một 1
|
||||
hai 2
|
||||
ba 3
|
||||
bốn 4
|
||||
năm 5
|
||||
sáu 6
|
||||
bảy 7
|
||||
bẩy 7
|
||||
tám 8
|
||||
chín 9
|
||||
mười 10
|
||||
mười một 11
|
||||
mười hai 12
|
||||
mười ba 13
|
||||
mười bốn 14
|
||||
mười lăm 15
|
||||
mười sáu 16
|
||||
mười bảy 17
|
||||
mười bẩy 17
|
||||
mười tám 18
|
||||
mười chín 19
|
||||
hai mươi 20
|
||||
hai mươi mốt 21
|
||||
hai mốt 21
|
||||
hai mươi hai 22
|
||||
hai hai 22
|
||||
hai mươi ba 23
|
||||
hai ba 23
|
||||
|
@@ -0,0 +1,25 @@
|
||||
1 0
|
||||
2 1
|
||||
3 2
|
||||
4 3
|
||||
5 4
|
||||
6 5
|
||||
7 6
|
||||
8 7
|
||||
9 8
|
||||
10 9
|
||||
11 10
|
||||
12 11
|
||||
13 12
|
||||
14 13
|
||||
15 14
|
||||
16 15
|
||||
17 16
|
||||
18 17
|
||||
19 18
|
||||
20 19
|
||||
21 20
|
||||
22 21
|
||||
23 22
|
||||
24 23
|
||||
0 23
|
||||
|
@@ -0,0 +1,12 @@
|
||||
1 13
|
||||
2 14
|
||||
3 15
|
||||
4 16
|
||||
5 17
|
||||
6 18
|
||||
7 19
|
||||
8 20
|
||||
9 21
|
||||
10 22
|
||||
11 23
|
||||
12 0
|
||||
|
@@ -0,0 +1,118 @@
|
||||
không 0
|
||||
một 1
|
||||
hai 2
|
||||
ba 3
|
||||
bốn 4
|
||||
năm 5
|
||||
sáu 6
|
||||
bảy 7
|
||||
bẩy 7
|
||||
tám 8
|
||||
chín 9
|
||||
mười 10
|
||||
mười một 11
|
||||
mười hai 12
|
||||
mười ba 13
|
||||
mười bốn 14
|
||||
mười lăm 15
|
||||
mười sáu 16
|
||||
mười bảy 17
|
||||
mười bẩy 17
|
||||
mười tám 18
|
||||
mười chín 19
|
||||
hai mươi 20
|
||||
hai mươi mốt 21
|
||||
hai mươi hai 22
|
||||
hai mươi ba 23
|
||||
hai mươi tư 24
|
||||
hai mươi bốn 24
|
||||
hai mươi lăm 25
|
||||
hai mươi sáu 26
|
||||
hai mươi bảy 27
|
||||
hai mươi bẩy 27
|
||||
hai mươi tám 28
|
||||
hai mươi chín 29
|
||||
ba mươi 30
|
||||
ba mươi mốt 21
|
||||
ba mươi hai 22
|
||||
ba mươi ba 33
|
||||
ba mươi tư 34
|
||||
ba mươi bốn 34
|
||||
ba mươi lăm 35
|
||||
ba mươi sáu 36
|
||||
ba mươi bảy 37
|
||||
ba mươi bẩy 37
|
||||
ba mươi tám 38
|
||||
ba mươi chín 39
|
||||
bốn mươi 40
|
||||
bốn mươi mốt 41
|
||||
bốn mươi hai 42
|
||||
bốn mươi ba 43
|
||||
bốn mươi tư 44
|
||||
bốn mươi bốn 44
|
||||
bốn mươi lăm 45
|
||||
bốn mươi sáu 46
|
||||
bốn mươi bảy 47
|
||||
bốn mươi bẩy 47
|
||||
bốn mươi tám 48
|
||||
bốn mươi chín 49
|
||||
năm mươi 50
|
||||
năm mươi mốt 51
|
||||
năm mươi hai 52
|
||||
năm mươi ba 53
|
||||
năm mươi tư 54
|
||||
năm mươi bốn 54
|
||||
năm mươi lăm 55
|
||||
năm mươi sáu 56
|
||||
năm mươi bảy 57
|
||||
năm mươi bẩy 57
|
||||
năm mươi tám 58
|
||||
năm mươi chín 59
|
||||
hai mốt 21
|
||||
hai hai 22
|
||||
hai ba 23
|
||||
hai tư 24
|
||||
hai bốn 24
|
||||
hai lăm 25
|
||||
hai năm 25
|
||||
hai sáu 26
|
||||
hai bảy 27
|
||||
hai bẩy 27
|
||||
hai tám 28
|
||||
hai chín 29
|
||||
ba mốt 21
|
||||
ba hai 22
|
||||
ba ba 33
|
||||
ba tư 34
|
||||
ba bốn 34
|
||||
ba lăm 35
|
||||
ba năm 35
|
||||
ba sáu 36
|
||||
ba bảy 37
|
||||
ba bẩy 37
|
||||
ba tám 38
|
||||
ba chín 39
|
||||
bốn mốt 41
|
||||
bốn hai 42
|
||||
bốn ba 43
|
||||
bốn tư 44
|
||||
bốn bốn 44
|
||||
bốn lăm 45
|
||||
bốn năm 45
|
||||
bốn sáu 46
|
||||
bốn bảy 47
|
||||
bốn bẩy 47
|
||||
bốn tám 48
|
||||
bốn chín 49
|
||||
năm mốt 51
|
||||
năm hai 52
|
||||
năm ba 53
|
||||
năm tư 54
|
||||
năm bốn 54
|
||||
năm lăm 55
|
||||
năm năm 55
|
||||
năm sáu 56
|
||||
năm bảy 57
|
||||
năm bẩy 57
|
||||
năm tám 58
|
||||
năm chín 59
|
||||
|
@@ -0,0 +1,59 @@
|
||||
1 59
|
||||
2 58
|
||||
3 57
|
||||
4 56
|
||||
5 55
|
||||
6 54
|
||||
7 53
|
||||
8 52
|
||||
9 51
|
||||
10 50
|
||||
11 49
|
||||
12 48
|
||||
13 47
|
||||
14 46
|
||||
15 45
|
||||
16 44
|
||||
17 43
|
||||
18 42
|
||||
19 41
|
||||
20 40
|
||||
21 39
|
||||
22 38
|
||||
23 37
|
||||
24 36
|
||||
25 35
|
||||
26 34
|
||||
27 33
|
||||
28 32
|
||||
29 31
|
||||
30 30
|
||||
31 29
|
||||
32 28
|
||||
33 27
|
||||
34 26
|
||||
35 25
|
||||
36 24
|
||||
37 23
|
||||
38 22
|
||||
39 21
|
||||
40 20
|
||||
41 19
|
||||
42 18
|
||||
43 17
|
||||
44 16
|
||||
45 15
|
||||
46 14
|
||||
47 13
|
||||
48 12
|
||||
49 11
|
||||
50 10
|
||||
51 9
|
||||
52 8
|
||||
53 7
|
||||
54 6
|
||||
55 5
|
||||
56 4
|
||||
57 3
|
||||
58 2
|
||||
59 1
|
||||
|
@@ -0,0 +1,8 @@
|
||||
p m p.m.
|
||||
pm p.m.
|
||||
p.m.
|
||||
p.m p.m.
|
||||
am a.m.
|
||||
a.m.
|
||||
a.m a.m.
|
||||
a m a.m.
|
||||
|
@@ -0,0 +1,7 @@
|
||||
cst c s t
|
||||
cet c e t
|
||||
pst p s t
|
||||
est e s t
|
||||
pt p t
|
||||
et e t
|
||||
gmt g m t
|
||||
|
@@ -0,0 +1,143 @@
|
||||
import os
|
||||
import string
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import pynini
|
||||
from pynini import Far
|
||||
from pynini.export import export
|
||||
from pynini.lib import byte, pynutil, utf8
|
||||
|
||||
DAMO_CHAR = utf8.VALID_UTF8_CHAR
|
||||
|
||||
DAMO_DIGIT = byte.DIGIT
|
||||
DAMO_LOWER = pynini.union(*string.ascii_lowercase).optimize()
|
||||
DAMO_UPPER = pynini.union(*string.ascii_uppercase).optimize()
|
||||
DAMO_ALPHA = pynini.union(DAMO_LOWER, DAMO_UPPER).optimize()
|
||||
DAMO_ALNUM = pynini.union(DAMO_DIGIT, DAMO_ALPHA).optimize()
|
||||
DAMO_HEX = pynini.union(*string.hexdigits).optimize()
|
||||
DAMO_NON_BREAKING_SPACE = "\u00A0"
|
||||
DAMO_SPACE = " "
|
||||
DAMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize()
|
||||
DAMO_NOT_SPACE = pynini.difference(DAMO_CHAR, DAMO_WHITE_SPACE).optimize()
|
||||
DAMO_NOT_QUOTE = pynini.difference(DAMO_CHAR, r'"').optimize()
|
||||
|
||||
DAMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize()
|
||||
DAMO_GRAPH = pynini.union(DAMO_ALNUM, DAMO_PUNCT).optimize()
|
||||
|
||||
DAMO_SIGMA = pynini.closure(DAMO_CHAR)
|
||||
|
||||
delete_space = pynutil.delete(pynini.closure(DAMO_WHITE_SPACE))
|
||||
insert_space = pynutil.insert(" ")
|
||||
delete_extra_space = pynini.cross(pynini.closure(DAMO_WHITE_SPACE, 1), " ")
|
||||
|
||||
# French frequently compounds numbers with hyphen.
|
||||
delete_hyphen = pynutil.delete(pynini.closure("-", 0, 1))
|
||||
insert_hyphen = pynutil.insert("-")
|
||||
|
||||
TO_LOWER = pynini.union(
|
||||
*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]
|
||||
)
|
||||
TO_UPPER = pynini.invert(TO_LOWER)
|
||||
|
||||
|
||||
def generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]):
|
||||
"""
|
||||
Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name.
|
||||
|
||||
Args:
|
||||
file_name: exported file name
|
||||
graphs: Mapping of a rule name and Pynini WFST graph to be exported
|
||||
"""
|
||||
exporter = export.Exporter(file_name)
|
||||
for rule, graph in graphs.items():
|
||||
exporter[rule] = graph.optimize()
|
||||
exporter.close()
|
||||
print(f"Created {file_name}")
|
||||
|
||||
|
||||
def convert_space(fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Converts space to nonbreaking space.
|
||||
Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty"
|
||||
This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it.
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns output fst where breaking spaces are converted to non breaking spaces
|
||||
"""
|
||||
return fst @ pynini.cdrewrite(
|
||||
pynini.cross(DAMO_SPACE, DAMO_NON_BREAKING_SPACE), "", "", DAMO_SIGMA
|
||||
)
|
||||
|
||||
|
||||
class GraphFst:
|
||||
"""
|
||||
Base class for all grammar fsts.
|
||||
|
||||
Args:
|
||||
name: name of grammar class
|
||||
kind: either 'classify' or 'verbalize'
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, kind: str, deterministic: bool = True):
|
||||
self.name = name
|
||||
self.kind = kind
|
||||
self._fst = None
|
||||
self.deterministic = deterministic
|
||||
|
||||
self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far")
|
||||
if self.far_exist():
|
||||
self._fst = Far(
|
||||
self.far_path, mode="r", arc_type="standard", far_type="default"
|
||||
).get_fst()
|
||||
|
||||
def far_exist(self) -> bool:
|
||||
"""
|
||||
Returns true if FAR can be loaded
|
||||
"""
|
||||
return self.far_path.exists()
|
||||
|
||||
@property
|
||||
def fst(self) -> "pynini.FstLike":
|
||||
return self._fst
|
||||
|
||||
@fst.setter
|
||||
def fst(self, fst):
|
||||
self._fst = fst
|
||||
|
||||
def add_tokens(self, fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Wraps class name around to given fst
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns:
|
||||
Fst: fst
|
||||
"""
|
||||
return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }")
|
||||
|
||||
def delete_tokens(self, fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Deletes class name wrap around output of given fst
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns:
|
||||
Fst: fst
|
||||
"""
|
||||
res = (
|
||||
pynutil.delete(f"{self.name}")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ fst
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
@@ -0,0 +1,156 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying cardinals
|
||||
e.g. trừ hai mươi ba -> cardinal { integer: "23" negative: "-" } }
|
||||
e.g. hai nghìn lẻ chín -> cardinal { integer: "2009"} }
|
||||
Numbers below ten are not converted.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="cardinal", kind="classify")
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv"))
|
||||
graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv"))
|
||||
|
||||
graph_one = pynini.cross("mốt", "1")
|
||||
graph_four = pynini.cross("tư", "4")
|
||||
graph_five = pynini.cross("lăm", "5")
|
||||
graph_half = pynini.cross("rưỡi", "5")
|
||||
graph_hundred = pynini.cross("trăm", "")
|
||||
graph_ten = pynini.cross("mươi", "")
|
||||
zero = pynini.cross(pynini.union("linh", "lẻ"), "0")
|
||||
|
||||
optional_ten = pynini.closure(delete_space + graph_ten, 0, 1)
|
||||
last_digit_exception = pynini.project(pynini.cross("năm", "5"), "input")
|
||||
last_digit = pynini.union(
|
||||
(pynini.project(graph_digit, "input") - last_digit_exception.arcsort()) @ graph_digit,
|
||||
graph_one,
|
||||
graph_four,
|
||||
graph_five,
|
||||
)
|
||||
|
||||
graph_hundred_ties_component = (graph_digit | graph_zero) + delete_space + graph_hundred
|
||||
graph_hundred_ties_component += delete_space
|
||||
graph_hundred_ties_component += pynini.union(
|
||||
graph_teen,
|
||||
(graph_half | graph_four | graph_one) + pynutil.insert("0"),
|
||||
graph_ties + optional_ten + ((delete_space + last_digit) | pynutil.insert("0")),
|
||||
zero + delete_space + (graph_digit | graph_four),
|
||||
pynutil.insert("00"),
|
||||
)
|
||||
graph_hundred_ties_component |= (
|
||||
pynutil.insert("0")
|
||||
+ delete_space
|
||||
+ pynini.union(
|
||||
graph_teen,
|
||||
graph_ties + optional_ten + delete_space + last_digit,
|
||||
graph_ties + delete_space + graph_ten + pynutil.insert("0"),
|
||||
zero + delete_space + (graph_digit | graph_four),
|
||||
)
|
||||
)
|
||||
graph_hundred_component = graph_hundred_ties_component | (
|
||||
pynutil.insert("00") + delete_space + graph_digit
|
||||
)
|
||||
|
||||
graph_hundred_component_at_least_one_none_zero_digit = graph_hundred_component @ (
|
||||
pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT)
|
||||
)
|
||||
self.graph_hundred_component_at_least_one_none_zero_digit = (
|
||||
graph_hundred_component_at_least_one_none_zero_digit
|
||||
)
|
||||
graph_hundred_ties_zero = graph_hundred_ties_component | pynutil.insert("000")
|
||||
|
||||
graph_thousands = pynini.union(
|
||||
graph_hundred_component_at_least_one_none_zero_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete(pynini.union("nghìn", "ngàn")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph_ten_thousand = pynini.union(
|
||||
graph_hundred_component_at_least_one_none_zero_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete("vạn"),
|
||||
pynutil.insert("0000", weight=0.1),
|
||||
)
|
||||
|
||||
graph_ten_thousand_suffix = pynini.union(
|
||||
graph_digit + delete_space + pynutil.delete(pynini.union("nghìn", "ngàn")),
|
||||
pynutil.insert("0", weight=0.1),
|
||||
)
|
||||
|
||||
graph_million = pynini.union(
|
||||
graph_hundred_component_at_least_one_none_zero_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete("triệu"),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
graph_billion = pynini.union(
|
||||
graph_hundred_component_at_least_one_none_zero_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete(pynini.union("tỉ", "tỷ")),
|
||||
pynutil.insert("000", weight=0.1),
|
||||
)
|
||||
|
||||
graph = pynini.union(
|
||||
graph_billion
|
||||
+ delete_space
|
||||
+ graph_million
|
||||
+ delete_space
|
||||
+ graph_thousands
|
||||
+ delete_space
|
||||
+ graph_hundred_ties_zero,
|
||||
graph_ten_thousand
|
||||
+ delete_space
|
||||
+ graph_ten_thousand_suffix
|
||||
+ delete_space
|
||||
+ graph_hundred_ties_zero,
|
||||
graph_hundred_component_at_least_one_none_zero_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete(pynini.union("nghìn", "ngàn"))
|
||||
+ delete_space
|
||||
+ (((last_digit | graph_half) + pynutil.insert("00")) | graph_hundred_ties_zero),
|
||||
graph_digit,
|
||||
graph_zero,
|
||||
)
|
||||
|
||||
graph = graph @ pynini.union(
|
||||
pynutil.delete(pynini.closure("0"))
|
||||
+ pynini.difference(DAMO_DIGIT, "0")
|
||||
+ pynini.closure(DAMO_DIGIT),
|
||||
"0",
|
||||
)
|
||||
|
||||
# don't convert cardinals from zero to nine inclusive
|
||||
graph_exception = pynini.project(pynini.union(graph_digit, graph_zero), "input")
|
||||
|
||||
self.graph_no_exception = graph
|
||||
|
||||
self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph
|
||||
|
||||
optional_minus_graph = pynini.closure(
|
||||
pynutil.insert("negative: ")
|
||||
+ pynini.cross(pynini.union("âm", "trừ"), '"-"')
|
||||
+ DAMO_SPACE,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,170 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).optimize()
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize()
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize()
|
||||
ties_graph = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).optimize()
|
||||
|
||||
|
||||
def _get_month_graph():
|
||||
"""
|
||||
Transducer for month, e.g. march -> march
|
||||
"""
|
||||
month_graph = pynini.string_file(get_abs_path("data/months.tsv")).optimize()
|
||||
return month_graph
|
||||
|
||||
|
||||
def _get_ties_graph():
|
||||
"""
|
||||
Transducer for 20-99 e.g
|
||||
hai ba -> 23
|
||||
"""
|
||||
graph_one = pynini.cross("mốt", "1")
|
||||
graph_four = pynini.cross("tư", "4")
|
||||
graph_five = pynini.cross("lăm", "5")
|
||||
graph_ten = pynini.cross("mươi", "")
|
||||
optional_ten = pynini.closure(delete_space + graph_ten, 0, 1)
|
||||
|
||||
graph = pynini.union(
|
||||
ties_graph
|
||||
+ optional_ten
|
||||
+ delete_space
|
||||
+ (graph_digit | graph_one | graph_four | graph_five),
|
||||
ties_graph + delete_space + graph_ten + pynutil.insert("0"),
|
||||
)
|
||||
return graph
|
||||
|
||||
|
||||
def _get_year_graph():
|
||||
"""
|
||||
Transducer for year, e.g. hai không hai mươi -> 2020
|
||||
"""
|
||||
|
||||
def _get_digits_graph():
|
||||
zero = pynini.cross((pynini.union("linh", "lẻ")), "0")
|
||||
four = pynini.cross("tư", "4")
|
||||
graph = pynini.union(
|
||||
zero + delete_space + (graph_digit | four),
|
||||
graph_zero + delete_space + graph_digit,
|
||||
)
|
||||
graph.optimize()
|
||||
return graph
|
||||
|
||||
def _get_hundreds_graph(graph_ties, graph_digits):
|
||||
graph = (
|
||||
graph_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete("trăm")
|
||||
+ delete_space
|
||||
+ (graph_teen | graph_ties | graph_digits)
|
||||
)
|
||||
return graph
|
||||
|
||||
def _get_thousands_graph(graph_ties, graph_digits):
|
||||
graph_hundred_component = (
|
||||
(graph_digit | graph_zero) + delete_space + pynutil.delete("trăm")
|
||||
) | pynutil.insert("0")
|
||||
graph = (
|
||||
graph_digit
|
||||
+ delete_space
|
||||
+ pynutil.delete(pynini.union("nghìn", "ngàn"))
|
||||
+ delete_space
|
||||
+ graph_hundred_component
|
||||
+ delete_space
|
||||
+ (graph_teen | graph_ties | graph_digits)
|
||||
)
|
||||
return graph
|
||||
|
||||
graph_ties = _get_ties_graph()
|
||||
graph_digits = _get_digits_graph()
|
||||
graph_hundreds = _get_hundreds_graph(graph_ties, graph_digits)
|
||||
graph_thousands = _get_thousands_graph(graph_ties, graph_digits)
|
||||
year_graph = (
|
||||
# 20 19, 40 12, 2012, 2 0 0 5, 2 0 17, 938 - assuming no limit on the year
|
||||
graph_digit
|
||||
+ delete_space
|
||||
+ (graph_digit | graph_zero)
|
||||
+ delete_space
|
||||
+ (graph_teen | graph_ties | graph_digits)
|
||||
| graph_thousands
|
||||
| graph_hundreds
|
||||
| (
|
||||
graph_digit
|
||||
+ pynutil.insert("0")
|
||||
+ delete_space
|
||||
+ (graph_ties | graph_digits | graph_teen)
|
||||
)
|
||||
)
|
||||
year_graph.optimize()
|
||||
return year_graph
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying date,
|
||||
e.g. mười lăm tháng một năm hai nghìn mười hai -> date { day: "15" month: "1" year: "2012" preserve_order: true }
|
||||
e.g. ngày ba mốt tháng mười hai năm một chín chín chín -> date { day: "31" month: "12" year: "2012" preserve_order: true }
|
||||
e.g. năm hai không hai mốt -> date { year: "2021" preserve_order: true }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="date", kind="classify")
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
year_graph = _get_year_graph()
|
||||
YEAR_WEIGHT = 0.001
|
||||
year_graph = pynutil.add_weight(year_graph, YEAR_WEIGHT)
|
||||
month_graph = _get_month_graph()
|
||||
|
||||
month_graph = pynutil.insert('month: "') + month_graph + pynutil.insert('"')
|
||||
month_exception = pynini.project(pynini.cross("năm", "5"), "input")
|
||||
month_graph_exception = (
|
||||
pynini.project(month_graph, "input") - month_exception.arcsort()
|
||||
) @ month_graph
|
||||
|
||||
day_graph = pynutil.insert('day: "') + cardinal_graph + pynutil.insert('"')
|
||||
# day_suffix = pynini.union("ngày", "mùng")
|
||||
# optional_day = pynini.closure(day_suffix + delete_space, 0, 1)
|
||||
|
||||
graph_month = pynutil.delete("tháng") + delete_space + month_graph_exception
|
||||
graph_year = (
|
||||
delete_extra_space
|
||||
+ pynutil.delete("năm")
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('year: "')
|
||||
+ pynutil.add_weight(year_graph, -YEAR_WEIGHT)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
optional_graph_year = pynini.closure(graph_year, 0, 1)
|
||||
graph_my = pynutil.delete("tháng") + delete_space + month_graph + graph_year
|
||||
graph_dmy = (
|
||||
day_graph
|
||||
+ delete_space
|
||||
+ pynutil.delete("tháng")
|
||||
+ delete_extra_space
|
||||
+ month_graph
|
||||
+ optional_graph_year
|
||||
)
|
||||
graph_year = (
|
||||
pynutil.delete("năm")
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('year: "')
|
||||
+ year_graph
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph = (graph_dmy | graph_my | graph_month | graph_year) + pynutil.insert(
|
||||
" preserve_order: true"
|
||||
)
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,130 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
|
||||
|
||||
def get_quantity(
|
||||
decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike"
|
||||
) -> "pynini.FstLike":
|
||||
"""
|
||||
Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
|
||||
e.g. một triệu -> integer_part: "1" quantity: "triệu"
|
||||
e.g. một tỷ rưỡi -> integer_part: "1" fractional_part: "5" quantity: "tỷ"
|
||||
|
||||
Args:
|
||||
decimal: decimal FST
|
||||
cardinal_up_to_hundred: cardinal FST
|
||||
"""
|
||||
numbers = cardinal_up_to_hundred @ (
|
||||
pynutil.delete(pynini.closure("0"))
|
||||
+ pynini.difference(DAMO_DIGIT, "0")
|
||||
+ pynini.closure(DAMO_DIGIT)
|
||||
)
|
||||
suffix = pynini.union("triệu", "tỉ", "tỷ", "vạn")
|
||||
graph_four = pynini.cross("tư", "4")
|
||||
graph_one = pynini.cross("mốt", "1")
|
||||
graph_half = pynini.cross("rưỡi", "5")
|
||||
last_digit_exception = pynini.project(pynini.cross("năm", "5"), "input")
|
||||
last_digit = pynini.union(
|
||||
(pynini.project(graph_digit, "input") - last_digit_exception.arcsort()) @ graph_digit,
|
||||
graph_one,
|
||||
graph_four,
|
||||
graph_half,
|
||||
)
|
||||
optional_fraction_graph = pynini.closure(
|
||||
delete_extra_space
|
||||
+ pynutil.insert('fractional_part: "')
|
||||
+ (last_digit | graph_half | graph_one | graph_four)
|
||||
+ pynutil.insert('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
res = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ numbers
|
||||
+ pynutil.insert('"')
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('quantity: "')
|
||||
+ suffix
|
||||
+ pynutil.insert('"')
|
||||
+ optional_fraction_graph
|
||||
)
|
||||
res |= (
|
||||
decimal
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('quantity: "')
|
||||
+ (suffix | "ngàn" | "nghìn")
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
return res
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal
|
||||
e.g. âm hai hai phẩy không năm tư năm tỉ -> decimal { negative: "true" integer_part: "22" fractional_part: "054" quantity: "tỉ" }
|
||||
e.g. không chấm ba lăm -> decimal { integer_part: "0" fractional_part: "35" }
|
||||
e.g. một triệu rưỡi -> decimal { integer_part: "1" quantity: "triệu" fractional_part: "5" }
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="decimal", kind="classify")
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
|
||||
graph_decimal = graph_digit | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
graph_one = pynini.cross("mốt", "1")
|
||||
graph_four = pynini.cross("tư", "4")
|
||||
graph_five = pynini.cross("lăm", "5")
|
||||
|
||||
graph_decimal = pynini.union(
|
||||
graph_decimal,
|
||||
graph_four,
|
||||
pynini.closure(graph_decimal + delete_space, 1)
|
||||
+ (graph_decimal | graph_four | graph_five | graph_one),
|
||||
)
|
||||
self.graph = graph_decimal
|
||||
|
||||
point = pynutil.delete("chấm") | pynutil.delete("phẩy")
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ")
|
||||
+ pynini.cross(pynini.union("âm", "trừ"), '"true"')
|
||||
+ delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
graph_fractional = (
|
||||
pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"')
|
||||
)
|
||||
graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
|
||||
final_graph_wo_sign = (
|
||||
pynini.closure(graph_integer + delete_extra_space, 0, 1)
|
||||
+ point
|
||||
+ delete_extra_space
|
||||
+ graph_fractional
|
||||
)
|
||||
final_graph = optional_graph_negative + final_graph_wo_sign
|
||||
|
||||
self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(
|
||||
final_graph_wo_sign,
|
||||
cardinal.graph_hundred_component_at_least_one_none_zero_digit,
|
||||
)
|
||||
final_graph |= optional_graph_negative + get_quantity(
|
||||
final_graph_wo_sign,
|
||||
cardinal.graph_hundred_component_at_least_one_none_zero_digit,
|
||||
)
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,90 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
GraphFst,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying electronic: as URLs, email addresses, etc.
|
||||
e.g. c d f một a còng a b c dot e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="electronic", kind="classify")
|
||||
|
||||
delete_extra_space = pynutil.delete(" ")
|
||||
alpha_num = (
|
||||
DAMO_ALPHA
|
||||
| pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
| pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
)
|
||||
|
||||
symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert()
|
||||
|
||||
accepted_username = alpha_num | symbols
|
||||
process_dot = pynini.cross("chấm", ".")
|
||||
username = (
|
||||
pynutil.insert('username: "')
|
||||
+ alpha_num
|
||||
+ pynini.closure(delete_extra_space + accepted_username)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num
|
||||
server = single_alphanum | pynini.string_file(
|
||||
get_abs_path("data/electronic/server_name.tsv")
|
||||
)
|
||||
domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
||||
multi_domain = (
|
||||
pynini.closure(process_dot + delete_extra_space + domain + delete_extra_space)
|
||||
+ process_dot
|
||||
+ delete_extra_space
|
||||
+ domain
|
||||
)
|
||||
domain_graph = (
|
||||
pynutil.insert('domain: "')
|
||||
+ server
|
||||
+ delete_extra_space
|
||||
+ multi_domain
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
graph = (
|
||||
username
|
||||
+ delete_extra_space
|
||||
+ pynutil.delete(pynini.union("a còng", "a móc", "a vòng"))
|
||||
+ insert_space
|
||||
+ delete_extra_space
|
||||
+ domain_graph
|
||||
)
|
||||
|
||||
############# url ###
|
||||
protocol_end = pynini.cross(pynini.union("w w w", "www"), "www")
|
||||
protocol_start = (
|
||||
pynini.cross("h t t p", "http") | pynini.cross("h t t p s", "https")
|
||||
) + pynini.cross(" hai chấm sẹc sẹc ", "://")
|
||||
# .com,
|
||||
ending = (
|
||||
delete_extra_space
|
||||
+ symbols
|
||||
+ delete_extra_space
|
||||
+ (domain | pynini.closure(accepted_username + delete_extra_space) + accepted_username)
|
||||
)
|
||||
|
||||
protocol = (
|
||||
pynini.closure(protocol_start, 0, 1)
|
||||
+ protocol_end
|
||||
+ delete_extra_space
|
||||
+ process_dot
|
||||
+ pynini.closure(delete_extra_space + accepted_username, 1)
|
||||
+ pynini.closure(ending, 1, 2)
|
||||
)
|
||||
protocol = pynutil.insert('protocol: "') + protocol + pynutil.insert('"')
|
||||
graph |= protocol
|
||||
########
|
||||
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,52 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying fraction
|
||||
e.g. 2 phần 3 -> tokens { fraction { numerator: "2" denominator: "3" } }
|
||||
e.g. 2 trên 3 -> tokens { fraction { numerator: "2" denominator: "3" } }
|
||||
e.g. 2 chia 3 -> tokens { fraction { numerator: "2" denominator: "3" } }
|
||||
|
||||
Args:
|
||||
cardinal: OrdinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="fraction", kind="classify")
|
||||
# integer_part # numerator # denominator
|
||||
|
||||
graph_cardinal = cardinal.graph_no_exception
|
||||
graph_four = pynini.cross("tư", "4")
|
||||
|
||||
numerator = pynutil.insert('numerator: "') + graph_cardinal + pynutil.insert('"')
|
||||
fraction_component = pynutil.delete(pynini.union("phần", "trên", "chia"))
|
||||
denominator = (
|
||||
pynutil.insert('denominator: "') + (graph_cardinal | graph_four) + pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_fraction_component = (
|
||||
numerator + delete_space + fraction_component + delete_extra_space + denominator
|
||||
)
|
||||
self.graph_fraction_component = graph_fraction_component
|
||||
|
||||
graph = graph_fraction_component
|
||||
graph = graph.optimize()
|
||||
self.final_graph_wo_negative = graph
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ")
|
||||
+ pynini.cross(pynini.union("âm", "trừ"), '"true"')
|
||||
+ delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
graph = optional_graph_negative + graph
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,99 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying measure
|
||||
e.g. trừ mười hai ki lô gam -> measure { negative: "true" cardinal { integer: "12" } units: "kg" }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst):
|
||||
super().__init__(name="measure", kind="classify")
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_four = pynini.cross("tư", "4")
|
||||
graph_one = pynini.cross("mốt", "1")
|
||||
graph_half = pynini.cross("rưỡi", "5")
|
||||
|
||||
graph_unit = pynini.string_file(get_abs_path("data/measurements.tsv"))
|
||||
graph_unit_singular = pynini.invert(graph_unit) # singular -> abbr
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ")
|
||||
+ pynini.cross(pynini.union("âm", "trừ"), '"true"')
|
||||
+ delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
unit_singular = convert_space(graph_unit_singular)
|
||||
unit_misc = (
|
||||
pynutil.insert("/")
|
||||
+ pynutil.delete("trên")
|
||||
+ delete_space
|
||||
+ convert_space(graph_unit_singular)
|
||||
)
|
||||
|
||||
unit_singular = (
|
||||
pynutil.insert('units: "')
|
||||
+ (
|
||||
unit_singular
|
||||
| unit_misc
|
||||
| pynutil.add_weight(unit_singular + delete_space + unit_misc, 0.01)
|
||||
)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
subgraph_decimal = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ optional_graph_negative
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.insert(" }")
|
||||
+ delete_extra_space
|
||||
+ unit_singular
|
||||
)
|
||||
|
||||
subgraph_cardinal = (
|
||||
pynutil.insert("cardinal { ")
|
||||
+ optional_graph_negative
|
||||
+ pynutil.insert('integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(" }")
|
||||
+ delete_extra_space
|
||||
+ unit_singular
|
||||
)
|
||||
fraction_graph = (
|
||||
delete_extra_space
|
||||
+ pynutil.insert('fractional_part: "')
|
||||
+ (graph_digit | graph_half | graph_one | graph_four)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
subgraph_cardinal |= (
|
||||
pynutil.insert("cardinal { ")
|
||||
+ optional_graph_negative
|
||||
+ pynutil.insert('integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('" }')
|
||||
+ delete_extra_space
|
||||
+ unit_singular
|
||||
+ fraction_graph
|
||||
)
|
||||
final_graph = subgraph_decimal | subgraph_cardinal
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,69 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying money
|
||||
e.g. mười hai đô la mỹ -> money { integer_part: "12" currency: "$" }
|
||||
e.g. mười phẩy chín đồng -> money { integer_part: "10.9" currency: "đ" }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst):
|
||||
super().__init__(name="money", kind="classify")
|
||||
# quantity, integer_part, fractional_part, currency
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
graph_decimal_final = decimal.final_graph_wo_negative
|
||||
graph_half = pynini.cross("rưỡi", "5")
|
||||
|
||||
unit = pynini.string_file(get_abs_path("data/currency.tsv"))
|
||||
unit_singular = pynini.invert(unit)
|
||||
|
||||
graph_unit_singular = (
|
||||
pynutil.insert('currency: "') + convert_space(unit_singular) + pynutil.insert('"')
|
||||
)
|
||||
|
||||
add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
|
||||
pynutil.insert("0") + DAMO_DIGIT
|
||||
)
|
||||
|
||||
# twelve dollars fifty, only after integer
|
||||
optional_cents_suffix = pynini.closure(
|
||||
delete_extra_space
|
||||
+ pynutil.insert('fractional_part: "')
|
||||
+ (
|
||||
pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7)
|
||||
| graph_half
|
||||
)
|
||||
+ pynutil.insert('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
graph_integer = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ delete_extra_space
|
||||
+ graph_unit_singular
|
||||
+ optional_cents_suffix
|
||||
)
|
||||
|
||||
graph_decimal = (
|
||||
graph_decimal_final + delete_extra_space + graph_unit_singular + optional_cents_suffix
|
||||
)
|
||||
final_graph = graph_integer | graph_decimal
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,29 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying ordinal
|
||||
e.g. thứ nhất -> ordinal { integer: "1" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="ordinal", kind="classify")
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv"))
|
||||
graph_ordinal = pynini.cross("thứ", "")
|
||||
graph = graph_digit
|
||||
|
||||
self.graph = graph
|
||||
final_graph = (
|
||||
pynutil.insert('integer: "')
|
||||
+ graph_ordinal
|
||||
+ delete_space
|
||||
+ self.graph
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,20 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class PunctuationFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying punctuation
|
||||
e.g. a, -> tokens { name: "a" } tokens { name: "," }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="punctuation", kind="classify")
|
||||
|
||||
s = "!#$%&'()*+,-./:;<=>?@^_`{|}~"
|
||||
punct = pynini.union(*s)
|
||||
|
||||
graph = pynutil.insert('name: "') + punct + pynutil.insert('"')
|
||||
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,27 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, delete_space
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying telephone numbers, e.g.
|
||||
một hai ba một hai ba năm sáu bảy tám -> { number_part: "1231235678" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="telephone", kind="classify")
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
digit = graph_digit | graph_zero
|
||||
last_digit = (
|
||||
digit | pynini.cross("mốt", "1") | pynini.cross("tư", "4") | pynini.cross("lăm", "5")
|
||||
)
|
||||
|
||||
graph_number_part = pynini.closure(digit + delete_space, 2) + last_digit
|
||||
number_part = pynutil.insert('number_part: "') + graph_number_part + pynutil.insert('"')
|
||||
|
||||
graph = number_part
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,111 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying time
|
||||
e.g. hai rưỡi -> time { hours: "2" minutes: "30" }
|
||||
e.g. chín giờ kém hai mươi -> time { hours: "8" minutes: "40" }
|
||||
e.g. ba phút hai giây -> time { minutes: "3" seconds: "2" }
|
||||
e.g. mười giờ chín phút bốn mươi lăm giây -> time { hours: "10" minutes: "9" seconds: "45" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="time", kind="classify")
|
||||
# hours, minutes, seconds, suffix, zone, style, speak_period
|
||||
|
||||
graph_hours_to = pynini.string_file(get_abs_path("data/time/hours_to.tsv"))
|
||||
graph_minutes_to = pynini.string_file(get_abs_path("data/time/minutes_to.tsv"))
|
||||
graph_hours = pynini.string_file(get_abs_path("data/time/hours.tsv"))
|
||||
graph_minutes = pynini.string_file(get_abs_path("data/time/minutes.tsv"))
|
||||
time_zone_graph = pynini.invert(pynini.string_file(get_abs_path("data/time/time_zone.tsv")))
|
||||
|
||||
graph_half = pynini.cross("rưỡi", "30")
|
||||
oclock = pynini.cross("giờ", "")
|
||||
minute = pynini.cross("phút", "")
|
||||
optional_minute = pynini.closure(delete_space + minute, 0, 1)
|
||||
second = pynini.cross("giây", "")
|
||||
|
||||
final_graph_hour = (
|
||||
pynutil.insert('hours: "') + graph_hours + pynutil.insert('"') + delete_space + oclock
|
||||
)
|
||||
graph_minute = graph_minutes + optional_minute
|
||||
graph_second = graph_minutes + delete_space + second
|
||||
final_time_zone_optional = pynini.closure(
|
||||
delete_space
|
||||
+ insert_space
|
||||
+ pynutil.insert('zone: "')
|
||||
+ convert_space(time_zone_graph)
|
||||
+ pynutil.insert('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
graph_hm = (
|
||||
final_graph_hour
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('minutes: "')
|
||||
+ (graph_minute | graph_half)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_hms = (
|
||||
final_graph_hour
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('minutes: "')
|
||||
+ graph_minutes
|
||||
+ delete_space
|
||||
+ minute
|
||||
+ pynutil.insert('"')
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('seconds: "')
|
||||
+ graph_second
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_ms = (
|
||||
pynutil.insert('minutes: "')
|
||||
+ graph_minutes
|
||||
+ delete_space
|
||||
+ minute
|
||||
+ pynutil.insert('"')
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('seconds: "')
|
||||
+ (graph_second | graph_half)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_hours_to_component = graph_hours @ graph_hours_to
|
||||
graph_minutes_to_component = graph_minutes @ graph_minutes_to
|
||||
|
||||
graph_time_to = (
|
||||
pynutil.insert('hours: "')
|
||||
+ graph_hours_to_component
|
||||
+ pynutil.insert('"')
|
||||
+ delete_space
|
||||
+ oclock
|
||||
+ delete_space
|
||||
+ pynutil.delete("kém")
|
||||
+ delete_extra_space
|
||||
+ pynutil.insert('minutes: "')
|
||||
+ graph_minutes_to_component
|
||||
+ pynutil.insert('"')
|
||||
+ optional_minute
|
||||
)
|
||||
|
||||
final_graph = (final_graph_hour | graph_hm | graph_hms) + final_time_zone_optional
|
||||
final_graph |= graph_ms
|
||||
final_graph |= graph_time_to
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,107 @@
|
||||
import os
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.date import DateFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.electronic import ElectronicFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.fraction import FractionFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.ordinal import OrdinalFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.punctuation import PunctuationFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.telephone import TelephoneFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.time import TimeFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.whitelist import WhiteListFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.taggers.word import WordFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class ClassifyFst(GraphFst):
|
||||
"""
|
||||
Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
|
||||
super().__init__(name="tokenize_and_classify", kind="classify")
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(cache_dir, "_vi_itn.far")
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
|
||||
logging.info(f"ClassifyFst.fst was restored from {far_file}.")
|
||||
else:
|
||||
logging.info(f"Creating ClassifyFst grammars.")
|
||||
cardinal = CardinalFst()
|
||||
cardinal_graph = cardinal.fst
|
||||
|
||||
fraction = FractionFst(cardinal)
|
||||
fraction_graph = fraction.fst
|
||||
|
||||
ordinal = OrdinalFst()
|
||||
ordinal_graph = ordinal.fst
|
||||
|
||||
decimal = DecimalFst(cardinal)
|
||||
decimal_graph = decimal.fst
|
||||
|
||||
measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst
|
||||
date_graph = DateFst(cardinal=cardinal).fst
|
||||
word_graph = WordFst().fst
|
||||
time_graph = TimeFst().fst
|
||||
money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst
|
||||
whitelist_graph = WhiteListFst().fst
|
||||
punct_graph = PunctuationFst().fst
|
||||
electronic_graph = ElectronicFst().fst
|
||||
telephone_graph = TelephoneFst().fst
|
||||
|
||||
classify = (
|
||||
pynutil.add_weight(whitelist_graph, 1.01)
|
||||
| pynutil.add_weight(time_graph, 1.05)
|
||||
| pynutil.add_weight(date_graph, 1.09)
|
||||
| pynutil.add_weight(decimal_graph, 1.08)
|
||||
| pynutil.add_weight(measure_graph, 1.1)
|
||||
| pynutil.add_weight(cardinal_graph, 1.1)
|
||||
| pynutil.add_weight(ordinal_graph, 1.1)
|
||||
| pynutil.add_weight(fraction_graph, 1.09)
|
||||
| pynutil.add_weight(money_graph, 1.07)
|
||||
| pynutil.add_weight(telephone_graph, 1.1)
|
||||
| pynutil.add_weight(electronic_graph, 1.1)
|
||||
| pynutil.add_weight(word_graph, 100)
|
||||
)
|
||||
|
||||
punct = (
|
||||
pynutil.insert("tokens { ")
|
||||
+ pynutil.add_weight(punct_graph, weight=1.1)
|
||||
+ pynutil.insert(" }")
|
||||
)
|
||||
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
|
||||
token_plus_punct = (
|
||||
pynini.closure(punct + pynutil.insert(" "))
|
||||
+ token
|
||||
+ pynini.closure(pynutil.insert(" ") + punct)
|
||||
)
|
||||
|
||||
graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
|
||||
graph = delete_space + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
|
||||
if far_file:
|
||||
generator_main(far_file, {"tokenize_and_classify": self.fst})
|
||||
logging.info(f"ClassifyFst grammars are saved to {far_file}.")
|
||||
@@ -0,0 +1,19 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst, convert_space
|
||||
from fun_text_processing.inverse_text_normalization.vi.utils import get_abs_path
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying whitelisted tokens
|
||||
e.g. misses -> tokens { name: "mrs." }
|
||||
This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="whitelist", kind="classify")
|
||||
|
||||
whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert()
|
||||
graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,15 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_SPACE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class.
|
||||
e.g. sleep -> tokens { name: "sleep" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="word", kind="classify")
|
||||
word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
|
||||
self.fst = word.optimize()
|
||||
@@ -0,0 +1,13 @@
|
||||
import os
|
||||
|
||||
|
||||
def get_abs_path(rel_path):
|
||||
"""
|
||||
Get absolute path
|
||||
|
||||
Args:
|
||||
rel_path: relative path to this file
|
||||
|
||||
Returns absolute path
|
||||
"""
|
||||
return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinal
|
||||
e.g. cardinal { integer: "23" negative: "-" } -> -23
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="cardinal", kind="verbalize")
|
||||
optional_sign = pynini.closure(
|
||||
pynutil.delete("negative:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ DAMO_NOT_QUOTE
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
graph = (
|
||||
pynutil.delete("integer:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.numbers = graph
|
||||
graph = optional_sign + graph
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,67 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing date, e.g.
|
||||
date { month: "1" year: "2012"} -> tháng 1 năm 2012
|
||||
date { day: "5" month: "10" year: "2021" preserve_order: true } -> 5 tháng 10 năm 2021
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="date", kind="verbalize")
|
||||
day = (
|
||||
pynutil.delete("day:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
month = (
|
||||
pynutil.delete("month:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
year = (
|
||||
pynutil.delete("year:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
# (day) month year
|
||||
# day month
|
||||
graph_dm = day + delete_space + pynutil.insert(" tháng ") + month
|
||||
graph_dmy = graph_dm + delete_space + pynutil.insert(" năm ") + year
|
||||
graph_m = pynutil.insert("tháng ") + month
|
||||
graph_my = pynutil.insert("tháng ") + month + delete_space + pynutil.insert(" năm ") + year
|
||||
graph_y = pynutil.insert("năm ") + year
|
||||
|
||||
optional_preserve_order = pynini.closure(
|
||||
pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
|
||||
| pynutil.delete("field_order:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ DAMO_NOT_QUOTE
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
(graph_y | graph_m | graph_dm | graph_dmy | graph_my)
|
||||
+ delete_space
|
||||
+ optional_preserve_order
|
||||
)
|
||||
|
||||
delete_tokens = self.delete_tokens(final_graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,48 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "12" fractional_part: "5006" quantity: "tỷ" } -> -12.5006 tỷ
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="decimal", kind="verbalize")
|
||||
optionl_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
|
||||
integer = (
|
||||
pynutil.delete("integer_part:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_integer = pynini.closure(integer + delete_space, 0, 1)
|
||||
fractional = (
|
||||
pynutil.insert(".")
|
||||
+ pynutil.delete("fractional_part:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
|
||||
quantity = (
|
||||
pynutil.delete("quantity:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1)
|
||||
graph = optional_integer + optional_fractional + optional_quantity
|
||||
self.numbers = graph
|
||||
graph = optionl_sign + graph
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,45 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic
|
||||
e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="electronic", kind="verbalize")
|
||||
user_name = (
|
||||
pynutil.delete("username:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
domain = (
|
||||
pynutil.delete("domain:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
protocol = (
|
||||
pynutil.delete("protocol:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = user_name + delete_space + pynutil.insert("@") + domain
|
||||
graph |= protocol
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,34 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing fraction,
|
||||
e.g. fraction { numerator: "2" denominator: "3" } } -> 2/3
|
||||
e.g. fraction { numerator: "20" denominator: "3" negative: "true"} } -> 2/3
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="fraction", kind="verbalize")
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
|
||||
numerator = (
|
||||
pynutil.delete('numerator: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
||||
)
|
||||
|
||||
denominator = (
|
||||
pynutil.insert("/")
|
||||
+ pynutil.delete('denominator: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = (numerator + delete_space + denominator).optimize()
|
||||
self.numbers = graph
|
||||
delete_tokens = self.delete_tokens(optional_sign + graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,68 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, cardinal: GraphFst):
|
||||
super().__init__(name="measure", kind="verbalize")
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
|
||||
unit = (
|
||||
pynutil.delete("units:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space
|
||||
)
|
||||
graph_decimal = (
|
||||
pynutil.delete("decimal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ decimal.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph_cardinal = (
|
||||
pynutil.delete("cardinal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ cardinal.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
fractional = (
|
||||
pynutil.insert(".")
|
||||
+ pynutil.delete("fractional_part:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
|
||||
graph = (
|
||||
(graph_cardinal | graph_decimal)
|
||||
+ delete_space
|
||||
+ optional_fractional
|
||||
+ pynutil.insert(" ")
|
||||
+ unit
|
||||
+ delete_space
|
||||
)
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,30 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { integer_part: "12" fractional_part: "05" currency: "$" } -> 12.05$
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst):
|
||||
super().__init__(name="money", kind="verbalize")
|
||||
unit = (
|
||||
pynutil.delete("currency:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = decimal.numbers + delete_space + unit
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,28 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class OrdinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing ordinal, e.g.
|
||||
ordinal { integer: "2" } -> thứ 2
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="ordinal", kind="verbalize")
|
||||
graph = (
|
||||
pynutil.delete("integer:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = pynutil.insert("thứ ") + graph
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,22 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing telephone, e.g.
|
||||
telephone { number_part: "1231235678" }
|
||||
-> 1231235678
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="telephone", kind="verbalize")
|
||||
|
||||
number_part = (
|
||||
pynutil.delete('number_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
delete_tokens = self.delete_tokens(number_part)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,82 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing time, e.g.
|
||||
time { hours: "3" } -> 3h
|
||||
time { hours: "12" minutes: "30" } -> 12:30
|
||||
time { hours: "1" minutes: "12" second: "22"} -> 1:12:22
|
||||
time { minutes: "36" second: "45"} -> 36p45s
|
||||
time { hours: "2" zone: "gmt" } -> 2h gmt
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="time", kind="verbalize")
|
||||
add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
|
||||
pynutil.insert("0") + DAMO_DIGIT
|
||||
)
|
||||
hour = (
|
||||
pynutil.delete("hours:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
minute = (
|
||||
pynutil.delete("minutes:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
second = (
|
||||
pynutil.delete("seconds:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
zone = (
|
||||
delete_space
|
||||
+ insert_space
|
||||
+ pynutil.delete("zone:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_zone = pynini.closure(zone, 0, 1)
|
||||
optional_second = pynini.closure(
|
||||
delete_space + pynutil.insert(":") + (second @ add_leading_zero_to_double_digit),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
graph_h = hour + pynutil.insert("h")
|
||||
graph_hms = (
|
||||
hour
|
||||
+ delete_space
|
||||
+ pynutil.insert(":")
|
||||
+ (minute @ add_leading_zero_to_double_digit)
|
||||
+ optional_second
|
||||
)
|
||||
graph_ms = (
|
||||
minute
|
||||
+ delete_space
|
||||
+ pynutil.insert("p")
|
||||
+ (second @ add_leading_zero_to_double_digit)
|
||||
+ pynutil.insert("s")
|
||||
)
|
||||
|
||||
graph = (graph_h | graph_ms | graph_hms) + optional_zone
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,51 @@
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import GraphFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.date import DateFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.electronic import ElectronicFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.fraction import FractionFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.ordinal import OrdinalFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.telephone import TelephoneFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.time import TimeFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.whitelist import WhiteListFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="verbalize", kind="verbalize")
|
||||
cardinal = CardinalFst()
|
||||
cardinal_graph = cardinal.fst
|
||||
ordinal_graph = OrdinalFst().fst
|
||||
decimal = DecimalFst()
|
||||
decimal_graph = decimal.fst
|
||||
fraction = FractionFst()
|
||||
fraction_graph = fraction.fst
|
||||
measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst
|
||||
money_graph = MoneyFst(decimal=decimal).fst
|
||||
time_graph = TimeFst().fst
|
||||
date_graph = DateFst().fst
|
||||
whitelist_graph = WhiteListFst().fst
|
||||
telephone_graph = TelephoneFst().fst
|
||||
electronic_graph = ElectronicFst().fst
|
||||
graph = (
|
||||
time_graph
|
||||
| date_graph
|
||||
| money_graph
|
||||
| measure_graph
|
||||
| ordinal_graph
|
||||
| fraction_graph
|
||||
| decimal_graph
|
||||
| cardinal_graph
|
||||
| whitelist_graph
|
||||
| telephone_graph
|
||||
| electronic_graph
|
||||
)
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,33 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.vi.verbalizers.word import WordFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence, e.g.
|
||||
tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="verbalize_final", kind="verbalize")
|
||||
verbalize = VerbalizeFst().fst
|
||||
word = WordFst().fst
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,27 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing whitelist
|
||||
e.g. tokens { name: "mrs." } -> mrs.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="whitelist", kind="verbalize")
|
||||
graph = (
|
||||
pynutil.delete("name:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,29 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.vi.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing plain tokens
|
||||
e.g. tokens { name: "sleep" } -> sleep
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="word", kind="verbalize")
|
||||
chars = pynini.closure(DAMO_CHAR - " ", 1)
|
||||
char = (
|
||||
pynutil.delete("name:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ chars
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = char @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
|
||||
self.fst = graph.optimize()
|
||||
Reference in New Issue
Block a user