chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,7 @@
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.tokenize_and_classify import (
|
||||
ClassifyFst,
|
||||
)
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.verbalize_final import (
|
||||
VerbalizeFinalFst,
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
一日 01
|
||||
二日 02
|
||||
三日 03
|
||||
四日 04
|
||||
五日 05
|
||||
六日 06
|
||||
七日 07
|
||||
八日 08
|
||||
九日 09
|
||||
十日 10
|
||||
十一日 11
|
||||
十二日 12
|
||||
十三日 13
|
||||
十四日 14
|
||||
十五日 15
|
||||
十六日 16
|
||||
十七日 17
|
||||
十八日 18
|
||||
十九日 19
|
||||
二十日 20
|
||||
二十一日 21
|
||||
二十二日 22
|
||||
二十三日 23
|
||||
二十四日 24
|
||||
二十五日 25
|
||||
二十六日 26
|
||||
二十七日 27
|
||||
二十八日 28
|
||||
二十九日 29
|
||||
三十日 30
|
||||
三十一日 31
|
||||
一号 01
|
||||
二号 02
|
||||
三号 03
|
||||
四号 04
|
||||
五号 05
|
||||
六号 06
|
||||
七号 07
|
||||
八号 08
|
||||
九号 09
|
||||
十号 10
|
||||
十一号 11
|
||||
十二号 12
|
||||
十三号 13
|
||||
十四号 14
|
||||
十五号 15
|
||||
十六号 16
|
||||
十七号 17
|
||||
十八号 18
|
||||
十九号 19
|
||||
二十号 20
|
||||
二十一号 21
|
||||
二十二号 22
|
||||
二十三号 23
|
||||
二十四号 24
|
||||
二十五号 25
|
||||
二十六号 26
|
||||
二十七号 27
|
||||
二十八号 28
|
||||
二十九号 29
|
||||
三十号 30
|
||||
三十一号 31
|
||||
|
+1
@@ -0,0 +1 @@
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
com
|
||||
uk
|
||||
fr
|
||||
net
|
||||
br
|
||||
in
|
||||
ru
|
||||
de
|
||||
it
|
||||
ai
|
||||
edu
|
||||
cn
|
||||
org
|
||||
|
+17
@@ -0,0 +1,17 @@
|
||||
g mail gmail
|
||||
gmail
|
||||
n vidia nvidia
|
||||
nvidia
|
||||
outlook
|
||||
hotmail
|
||||
yahoo
|
||||
aol
|
||||
gmx
|
||||
msn
|
||||
live
|
||||
yandex
|
||||
orange
|
||||
wanadoo
|
||||
web
|
||||
comcast
|
||||
google
|
||||
|
+24
@@ -0,0 +1,24 @@
|
||||
. 点
|
||||
- 横杠
|
||||
- 横线
|
||||
_ 下划线
|
||||
! 感叹号
|
||||
! 叹号
|
||||
# 井号
|
||||
$ dollar 符
|
||||
% 百分号
|
||||
& ampersand
|
||||
' 引号
|
||||
' 单引号
|
||||
* 星号
|
||||
+ 加号
|
||||
/ 斜杆
|
||||
= 等号
|
||||
? 问号
|
||||
^ circumflex
|
||||
` right single quote
|
||||
{ 左大括号
|
||||
| 竖线
|
||||
} 右大括号
|
||||
~ 代字号
|
||||
, 逗号
|
||||
|
@@ -0,0 +1,4 @@
|
||||
k thousand
|
||||
m million
|
||||
b billion
|
||||
t trillion
|
||||
|
@@ -0,0 +1,82 @@
|
||||
原子质量 amu
|
||||
巴 bar
|
||||
平方厘米 cm²
|
||||
立方厘米 cm³
|
||||
厘米 cm
|
||||
美担 cwt
|
||||
摄氏度 °C
|
||||
分贝 db
|
||||
立方分米 dm³
|
||||
分米 dm
|
||||
英尺 ft
|
||||
华氏度 °F
|
||||
吉字节 gb
|
||||
吉赫兹 ghz
|
||||
吉帕斯卡 gpa
|
||||
吉瓦时 gwh
|
||||
吉瓦 gw
|
||||
戈瑞 gy
|
||||
小时 h
|
||||
公顷 ha
|
||||
赫兹 hz
|
||||
千比特每秒 kbps
|
||||
千比特一秒 kbps
|
||||
千卡 kcal
|
||||
千克力 kgf
|
||||
千克 kg
|
||||
千赫兹 khz
|
||||
平方千米 km²
|
||||
公里 km
|
||||
千米 km
|
||||
千米每小时 km/h
|
||||
千米一小时 km/h
|
||||
千帕 kpa
|
||||
千瓦时 kwh
|
||||
千瓦 kw
|
||||
磅 lbs
|
||||
平方米 m²
|
||||
立方米 m³
|
||||
兆比特每秒 mbps
|
||||
兆比特一秒 mbps
|
||||
克 g
|
||||
毫克 mg
|
||||
兆赫兹 mhz
|
||||
平方英里 mi²
|
||||
分钟 min
|
||||
英里 mi
|
||||
毫升 ml
|
||||
平方毫米 mm²
|
||||
毫米 mm
|
||||
摩尔 mol
|
||||
兆帕 mpa
|
||||
英里每小时 mph
|
||||
英里一小时 mph
|
||||
毫秒 ms
|
||||
毫伏 mv
|
||||
毫瓦 mw
|
||||
千伏 kv
|
||||
米 m
|
||||
纳克 ng
|
||||
纳米 nm
|
||||
纳秒 ns
|
||||
盎司 oz
|
||||
度 º
|
||||
帕斯卡 pa
|
||||
皮克 pg
|
||||
皮秒 ps
|
||||
弧度 rad
|
||||
转每分 rpm
|
||||
平方英尺 sq ft
|
||||
平方英里 sq mi
|
||||
系沃特 sv
|
||||
秒 s
|
||||
太字节 tb
|
||||
万亿焦耳 tj
|
||||
台两 tl
|
||||
伏特 v
|
||||
码 yd
|
||||
微克 μg
|
||||
微米 μm
|
||||
微秒 μs
|
||||
欧米茄 ω
|
||||
度 °
|
||||
|
+192
@@ -0,0 +1,192 @@
|
||||
年来
|
||||
年前
|
||||
年后
|
||||
年内
|
||||
年之前
|
||||
年之后
|
||||
人
|
||||
篇
|
||||
帧
|
||||
把
|
||||
封
|
||||
艘
|
||||
套
|
||||
段
|
||||
号
|
||||
匹
|
||||
张
|
||||
座
|
||||
回
|
||||
场
|
||||
尾
|
||||
条
|
||||
个
|
||||
首
|
||||
阙
|
||||
阵
|
||||
网
|
||||
炮
|
||||
顶
|
||||
丘
|
||||
棵
|
||||
只
|
||||
支
|
||||
袭
|
||||
辆
|
||||
挑
|
||||
担
|
||||
颗
|
||||
壳
|
||||
窠
|
||||
曲
|
||||
墙
|
||||
群
|
||||
腔
|
||||
砣
|
||||
座
|
||||
客
|
||||
贯
|
||||
扎
|
||||
捆
|
||||
刀
|
||||
令
|
||||
打
|
||||
手
|
||||
罗
|
||||
坡
|
||||
山
|
||||
岭
|
||||
江
|
||||
溪
|
||||
钟
|
||||
队
|
||||
单
|
||||
双
|
||||
对
|
||||
口
|
||||
头
|
||||
脚
|
||||
板
|
||||
跳
|
||||
枝
|
||||
件
|
||||
贴
|
||||
针
|
||||
线
|
||||
管
|
||||
名
|
||||
位
|
||||
身
|
||||
堂
|
||||
课
|
||||
本
|
||||
页
|
||||
家
|
||||
户
|
||||
层
|
||||
丝
|
||||
毫
|
||||
厘
|
||||
分
|
||||
钱
|
||||
斤
|
||||
担
|
||||
铢
|
||||
石
|
||||
钧
|
||||
锱
|
||||
忽
|
||||
克
|
||||
毫
|
||||
厘
|
||||
寸
|
||||
尺
|
||||
丈
|
||||
里
|
||||
寻
|
||||
常
|
||||
铺
|
||||
程
|
||||
米
|
||||
撮
|
||||
勺
|
||||
合
|
||||
升
|
||||
斗
|
||||
石
|
||||
盘
|
||||
碗
|
||||
碟
|
||||
叠
|
||||
桶
|
||||
笼
|
||||
盆
|
||||
盒
|
||||
杯
|
||||
钟
|
||||
斛
|
||||
锅
|
||||
簋
|
||||
篮
|
||||
盘
|
||||
桶
|
||||
罐
|
||||
瓶
|
||||
壶
|
||||
卮
|
||||
盏
|
||||
箩
|
||||
箱
|
||||
煲
|
||||
啖
|
||||
袋
|
||||
钵
|
||||
季
|
||||
年
|
||||
月
|
||||
日
|
||||
刻
|
||||
时
|
||||
周
|
||||
天
|
||||
秒
|
||||
旬
|
||||
纪
|
||||
岁
|
||||
世
|
||||
更
|
||||
夜
|
||||
春
|
||||
夏
|
||||
秋
|
||||
冬
|
||||
代
|
||||
伏
|
||||
辈
|
||||
丸
|
||||
泡
|
||||
粒
|
||||
颗
|
||||
幢
|
||||
堆
|
||||
条
|
||||
根
|
||||
支
|
||||
道
|
||||
面
|
||||
片
|
||||
张
|
||||
颗
|
||||
块
|
||||
架
|
||||
角
|
||||
毛
|
||||
字
|
||||
元
|
||||
两
|
||||
两米饭
|
||||
两酒
|
||||
吨
|
||||
顿
|
||||
牛
|
||||
次
|
||||
|
@@ -0,0 +1,117 @@
|
||||
澳元 A$
|
||||
阿联酋迪拉姆 AED
|
||||
阿富汗 阿富汗尼 AFN
|
||||
阿尔巴尼亚列克 ALL
|
||||
荷属安的列斯盾 ANG
|
||||
阿根廷比索 ARS
|
||||
澳元 AUD
|
||||
阿鲁巴盾 AWG
|
||||
阿塞拜疆马纳特 AZN
|
||||
波斯尼亚和黑塞哥维那可兑换马克 BAM
|
||||
巴巴多斯元 BBD
|
||||
保加利亚列弗 BGN
|
||||
百慕大元 BMD
|
||||
文莱达鲁萨兰国元 BND
|
||||
玻利维亚玻利维亚诺 BOB
|
||||
巴西雷亚尔 BRL
|
||||
巴哈马元 BSD
|
||||
博茨瓦纳普拉 BWP
|
||||
白俄罗斯卢布 BYN
|
||||
伯利兹元 BZD
|
||||
加元 CAD$
|
||||
加元 CAD
|
||||
瑞士法郎 CHF
|
||||
智利比索 CLP
|
||||
人民币 CNY
|
||||
哥伦比亚比索 COP
|
||||
哥斯达黎加科隆 CRC
|
||||
古巴比索 CUP
|
||||
捷克克朗 CZK
|
||||
丹麦克朗 DKK
|
||||
多米尼加共和国比索 DOP
|
||||
埃及镑 EGP
|
||||
欧元成员国 EUR
|
||||
斐济元 FJD
|
||||
福克兰群岛（马尔维纳斯）镑 FKP
|
||||
英镑 GBP
|
||||
根西岛镑 GGP
|
||||
加纳塞地 GHS
|
||||
直布罗陀镑 GIP
|
||||
危地马拉格查尔 GTQ
|
||||
圭亚那元 GYD
|
||||
港元 HK＄
|
||||
港元 HKD
|
||||
洪都拉斯伦皮拉 HNL
|
||||
克罗地亚库纳 HRK
|
||||
匈牙利福林 HUF
|
||||
印尼盾 IDR
|
||||
以色列谢克尔 ILS
|
||||
马恩岛英镑 IMP
|
||||
印度卢比 INR
|
||||
伊朗里亚尔 IRR
|
||||
冰岛克朗 ISK
|
||||
日元 J￥
|
||||
泽西镑 JEP
|
||||
牙买加元 JMD
|
||||
日元 JPY￥
|
||||
日元 JPY
|
||||
吉尔吉斯斯坦索姆 KGS
|
||||
柬埔寨瑞尔 KHR
|
||||
朝鲜元 KPW
|
||||
韩元 KRW
|
||||
韩元 KRW
|
||||
开曼群岛元 KYD
|
||||
哈萨克斯坦腾格 KZT
|
||||
老挝基普 LAK
|
||||
黎巴嫩镑 LBP
|
||||
斯里兰卡卢比 LKR
|
||||
利比里亚元 LRD
|
||||
马其顿代纳尔 MKD
|
||||
摩洛哥迪拉姆 MNT
|
||||
蒙古图格里克 MNT
|
||||
毛里求斯卢比 MUR
|
||||
墨西哥比索 MXN
|
||||
马来西亚令吉 MYR
|
||||
莫桑比克梅蒂卡尔 MZN
|
||||
纳米比亚元 NAD
|
||||
尼日利亚奈拉 NGN
|
||||
尼加拉瓜科尔多瓦 NIO
|
||||
挪威克朗 NOK
|
||||
尼泊尔卢比 NPR
|
||||
新西兰元 NZD
|
||||
阿曼里亚尔 OMR
|
||||
巴拿马巴尔博亚 PAB
|
||||
秘鲁索尔 PEN
|
||||
菲律宾比索 PHP
|
||||
巴基斯坦卢比 PKR
|
||||
波兰兹罗提 PLN
|
||||
巴拉圭瓜拉尼 PYG
|
||||
卡塔尔里亚尔 QAR
|
||||
罗马尼亚列伊 RON
|
||||
塞尔维亚第纳尔 RSD
|
||||
俄罗斯卢布 RUB
|
||||
沙特阿拉伯里亚尔 SAR
|
||||
所罗门群岛元 SBD
|
||||
塞舌尔卢比 SCR
|
||||
瑞典克朗 SEK
|
||||
新加坡元 SGD
|
||||
圣赫勒拿镑 SHP
|
||||
索马里先令 SOS
|
||||
苏里南元 SRD
|
||||
萨尔瓦多科隆 SVC
|
||||
叙利亚镑 SYP
|
||||
泰铢 THB
|
||||
土耳其里拉 TRY
|
||||
特立尼达和多巴哥元 TTD
|
||||
图瓦卢元 TVD
|
||||
新台币 TWD
|
||||
乌克兰格里夫纳 UAH
|
||||
美元 USD
|
||||
乌拉圭比索 UYU
|
||||
乌兹别克斯坦索姆 UZS
|
||||
委内瑞拉玻利瓦尔 VEF
|
||||
越南东 VND
|
||||
东加勒比元 XCD
|
||||
也门里亚尔 YER
|
||||
南非兰特 ZAR
|
||||
津巴布韦元 ZWD
|
||||
|
@@ -0,0 +1,63 @@
|
||||
加纳塞地 ¢
|
||||
玻利维亚玻利维亚诺 $b
|
||||
乌拉圭比索 $U
|
||||
美元 $
|
||||
英镑 £
|
||||
英镑 ￡
|
||||
元 ¥
|
||||
元 ￥
|
||||
泰铢 ฿
|
||||
柬埔寨瑞尔 ៛
|
||||
哥斯达黎加科隆 ₡
|
||||
尼日利亚奈拉 ₦
|
||||
朝鲜园 ₩
|
||||
以色列谢克尔 ₪
|
||||
越南东 ₫
|
||||
欧元 €
|
||||
老挝基普 ₭
|
||||
蒙古图格里克 ₮
|
||||
古巴比索 ₱
|
||||
菲律宾比索 ₱
|
||||
乌克兰格里夫纳 ₴
|
||||
印度卢比 ₹
|
||||
土耳其里拉 ₺
|
||||
卢布 ₽
|
||||
白俄罗斯卢布 Br
|
||||
委内瑞拉玻利瓦尔 Bs
|
||||
伯利兹元 BZ$
|
||||
巴拿马巴尔博亚 B/.
|
||||
尼加拉瓜科尔多瓦 C$
|
||||
瑞士法郎 CHF
|
||||
匈牙利福林 Ft
|
||||
阿鲁巴盾 ƒ
|
||||
巴拉圭瓜拉尼 Gs
|
||||
牙买加元 J$
|
||||
捷克克朗 Kč
|
||||
波斯尼亚和黑塞哥维那可兑换马克 KM
|
||||
克罗地亚库纳 kn
|
||||
丹麦克朗 kr
|
||||
罗马尼亚列伊 lei
|
||||
阿尔巴尼亚列克 Lek
|
||||
洪都拉斯伦皮拉 L
|
||||
莫桑比克梅蒂卡尔 MT
|
||||
新台币 NT$
|
||||
博茨瓦纳普拉 P
|
||||
危地马拉格查尔 Q
|
||||
巴西雷亚尔 R$
|
||||
多米尼加共和国比索 RD$
|
||||
马来西亚令吉 RM
|
||||
印尼盾 Rp
|
||||
巴基斯坦卢比 ₨
|
||||
毛里求斯卢比 ₨
|
||||
南非兰特 R
|
||||
秘鲁索尔 S/.
|
||||
索马里先令 S
|
||||
特立尼达和多巴哥元 TT$
|
||||
津巴布韦元 Z$
|
||||
波兰兹罗提 zł
|
||||
马其顿代纳尔 ден
|
||||
塞尔维亚第纳尔 Дин.
|
||||
乌兹别克斯坦索姆 лв
|
||||
保加利亚列弗 лв
|
||||
吉尔吉斯斯坦索姆 лв
|
||||
哈萨克斯坦腾格 лв
|
||||
|
@@ -0,0 +1,12 @@
|
||||
一月 01
|
||||
二月 02
|
||||
三月 03
|
||||
四月 04
|
||||
五月 05
|
||||
六月 06
|
||||
七月 07
|
||||
八月 08
|
||||
九月 09
|
||||
十月 10
|
||||
十一月 11
|
||||
十二月 12
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
一 1
|
||||
幺 1
|
||||
二 2
|
||||
两 2
|
||||
三 3
|
||||
四 4
|
||||
五 5
|
||||
六 6
|
||||
七 7
|
||||
八 8
|
||||
九 9
|
||||
|
@@ -0,0 +1 @@
|
||||
点 .
|
||||
|
@@ -0,0 +1 @@
|
||||
hundred
|
||||
|
@@ -0,0 +1,4 @@
|
||||
正 +
|
||||
正负 ±
|
||||
负 -
|
||||
负的 -
|
||||
|
@@ -0,0 +1,10 @@
|
||||
ten 10
|
||||
eleven 11
|
||||
twelve 12
|
||||
thirteen 13
|
||||
fourteen 14
|
||||
fifteen 15
|
||||
sixteen 16
|
||||
seventeen 17
|
||||
eighteen 18
|
||||
nineteen 19
|
||||
|
+22
@@ -0,0 +1,22 @@
|
||||
thousand
|
||||
million
|
||||
billion
|
||||
trillion
|
||||
quadrillion
|
||||
quintillion
|
||||
sextillion
|
||||
septillion
|
||||
octillion
|
||||
nonillion
|
||||
decillion
|
||||
undecillion
|
||||
duodecillion
|
||||
tredecillion
|
||||
quattuordecillion
|
||||
quindecillion
|
||||
sexdecillion
|
||||
septendecillion
|
||||
octodecillion
|
||||
novemdecillion
|
||||
vigintillion
|
||||
centillion
|
||||
|
@@ -0,0 +1,9 @@
|
||||
twenty 2
|
||||
thirty 3
|
||||
forty 4
|
||||
fourty 4
|
||||
fifty 5
|
||||
sixty 6
|
||||
seventy 7
|
||||
eighty 8
|
||||
ninety 9
|
||||
|
@@ -0,0 +1 @@
|
||||
零 0
|
||||
|
@@ -0,0 +1,83 @@
|
||||
deer
|
||||
fish
|
||||
sheep
|
||||
foot feet
|
||||
goose geese
|
||||
man men
|
||||
mouse mice
|
||||
tooth teeth
|
||||
woman women
|
||||
won
|
||||
child children
|
||||
ox oxen
|
||||
wife wives
|
||||
wolf wolves
|
||||
analysis analyses
|
||||
criterion criteria
|
||||
lbs
|
||||
focus foci
|
||||
percent
|
||||
hertz
|
||||
kroner krone
|
||||
inch inches
|
||||
calory calories
|
||||
yen
|
||||
megahertz
|
||||
gigahertz
|
||||
kilohertz
|
||||
hertz
|
||||
CC
|
||||
c c
|
||||
horsepower
|
||||
hundredweight
|
||||
kilogram force kilograms force
|
||||
mega siemens
|
||||
revolution per minute revolutions per minute
|
||||
mile per hour miles per hour
|
||||
megabit per second megabits per second
|
||||
square foot square feet
|
||||
kilobit per second kilobits per second
|
||||
degree Celsius degrees Celsius
|
||||
degree Fahrenheit degrees Fahrenheit
|
||||
ATM
|
||||
AU
|
||||
BQ
|
||||
CC
|
||||
CD
|
||||
DA
|
||||
EB
|
||||
EV
|
||||
F
|
||||
GB
|
||||
G
|
||||
GL
|
||||
GPA
|
||||
GY
|
||||
HA
|
||||
H
|
||||
HL
|
||||
GP
|
||||
HS
|
||||
KB
|
||||
KL
|
||||
KN
|
||||
KT
|
||||
KV
|
||||
LM
|
||||
MA
|
||||
MA
|
||||
MB
|
||||
MC
|
||||
MF
|
||||
M
|
||||
MM
|
||||
MS
|
||||
MV
|
||||
MW
|
||||
PB
|
||||
PG
|
||||
PS
|
||||
S
|
||||
TB
|
||||
YB
|
||||
ZB
|
||||
|
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
一点 1
|
||||
两点 2
|
||||
三点 3
|
||||
四点 4
|
||||
五点 5
|
||||
六点 6
|
||||
七点 7
|
||||
八点 8
|
||||
九点 9
|
||||
零点 00
|
||||
一点 01
|
||||
两点 02
|
||||
三点 03
|
||||
四点 04
|
||||
五点 05
|
||||
六点 06
|
||||
七点 07
|
||||
八点 08
|
||||
九点 09
|
||||
十点 10
|
||||
十一点 11
|
||||
十二点 12
|
||||
十三点 13
|
||||
十四点 14
|
||||
十五点 15
|
||||
十六点 16
|
||||
十七点 17
|
||||
十八点 18
|
||||
十九点 19
|
||||
二十点 20
|
||||
二十一点 21
|
||||
二十二点 22
|
||||
二十三点 23
|
||||
二十四点 24
|
||||
|
@@ -0,0 +1,60 @@
|
||||
半 30
|
||||
零一 01
|
||||
零二 02
|
||||
零三 03
|
||||
零四 04
|
||||
零五 05
|
||||
零六 06
|
||||
零七 07
|
||||
零八 08
|
||||
零九 09
|
||||
十 10
|
||||
十一 11
|
||||
十二 12
|
||||
十三 13
|
||||
十四 14
|
||||
十五 15
|
||||
十六 16
|
||||
十七 17
|
||||
十八 18
|
||||
十九 19
|
||||
二十 20
|
||||
二十一 21
|
||||
二十二 22
|
||||
二十三 23
|
||||
二十四 24
|
||||
二十五 25
|
||||
二十六 26
|
||||
二十七 27
|
||||
二十八 28
|
||||
二十九 29
|
||||
三十 30
|
||||
三十一 31
|
||||
三十二 32
|
||||
三十三 33
|
||||
三十四 34
|
||||
三十五 35
|
||||
三十六 36
|
||||
三十七 37
|
||||
三十八 38
|
||||
三十九 39
|
||||
四十 40
|
||||
四十一 41
|
||||
四十二 42
|
||||
四十三 43
|
||||
四十四 44
|
||||
四十五 45
|
||||
四十六 46
|
||||
四十七 47
|
||||
四十八 48
|
||||
四十九 49
|
||||
五十 50
|
||||
五十一 51
|
||||
五十二 52
|
||||
五十三 53
|
||||
五十四 54
|
||||
五十五 55
|
||||
五十六 56
|
||||
五十七 57
|
||||
五十八 58
|
||||
五十九 59
|
||||
|
@@ -0,0 +1,6 @@
|
||||
上午 a.m.
|
||||
早上 a.m.
|
||||
早晨 a.m.
|
||||
下午 p.m.
|
||||
晚上 p.m.
|
||||
傍晚 p.m.
|
||||
|
@@ -0,0 +1,60 @@
|
||||
00
|
||||
一秒 01
|
||||
二秒 02
|
||||
三秒 03
|
||||
四秒 04
|
||||
五秒 05
|
||||
六秒 06
|
||||
七秒 07
|
||||
八秒 08
|
||||
九秒 09
|
||||
十秒 10
|
||||
十一秒 11
|
||||
十二秒 12
|
||||
十三秒 13
|
||||
十四秒 14
|
||||
十五秒 15
|
||||
十六秒 16
|
||||
十七秒 17
|
||||
十八秒 18
|
||||
十九秒 19
|
||||
二十秒 20
|
||||
二十一秒 21
|
||||
二十二秒 22
|
||||
二十三秒 23
|
||||
二十四秒 24
|
||||
二十五秒 25
|
||||
二十六秒 26
|
||||
二十七秒 27
|
||||
二十八秒 28
|
||||
二十九秒 29
|
||||
三十秒 30
|
||||
三十一秒 31
|
||||
三十二秒 32
|
||||
三十三秒 33
|
||||
三十四秒 34
|
||||
三十五秒 35
|
||||
三十六秒 36
|
||||
三十七秒 37
|
||||
三十八秒 38
|
||||
三十九秒 39
|
||||
四十秒 40
|
||||
四十一秒 41
|
||||
四十二秒 42
|
||||
四十三秒 43
|
||||
四十四秒 44
|
||||
四十五秒 45
|
||||
四十六秒 46
|
||||
四十七秒 47
|
||||
四十八秒 48
|
||||
四十九秒 49
|
||||
五十秒 50
|
||||
五十一秒 51
|
||||
五十二秒 52
|
||||
五十三秒 53
|
||||
五十四秒 54
|
||||
五十五秒 55
|
||||
五十六秒 56
|
||||
五十七秒 57
|
||||
五十八秒 58
|
||||
五十九秒 59
|
||||
|
@@ -0,0 +1,12 @@
|
||||
e.g. for example
|
||||
dr. doctor
|
||||
mr. mister
|
||||
mrs. misses
|
||||
st. saint
|
||||
7-eleven seven eleven
|
||||
es3 e s three
|
||||
s&p s and p
|
||||
ASAP a s a p
|
||||
AT&T a t and t
|
||||
LLP l l p
|
||||
ATM a t m
|
||||
|
@@ -0,0 +1,206 @@
|
||||
import os
|
||||
import string
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from pynini import Far
|
||||
from pynini.examples import plurals
|
||||
from pynini.export import export
|
||||
from pynini.lib import byte, pynutil, utf8
|
||||
|
||||
DAMO_CHAR = utf8.VALID_UTF8_CHAR
|
||||
|
||||
DAMO_DIGIT = byte.DIGIT
|
||||
DAMO_LOWER = pynini.union(*string.ascii_lowercase).optimize()
|
||||
DAMO_UPPER = pynini.union(*string.ascii_uppercase).optimize()
|
||||
DAMO_ALPHA = pynini.union(DAMO_LOWER, DAMO_UPPER).optimize()
|
||||
DAMO_ALNUM = pynini.union(DAMO_DIGIT, DAMO_ALPHA).optimize()
|
||||
|
||||
DAMO_HEX = pynini.union(*string.hexdigits).optimize()
|
||||
DAMO_NON_BREAKING_SPACE = "\u00A0"
|
||||
|
||||
DAMO_SPACE = " "
|
||||
DAMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize()
|
||||
DAMO_NOT_SPACE = pynini.difference(DAMO_CHAR, DAMO_WHITE_SPACE).optimize()
|
||||
DAMO_NOT_QUOTE = pynini.difference(DAMO_CHAR, r'"').optimize()
|
||||
|
||||
DAMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize()
|
||||
DAMO_GRAPH = pynini.union(DAMO_ALNUM, DAMO_PUNCT).optimize()
|
||||
|
||||
DAMO_SIGMA = pynini.closure(DAMO_CHAR)
|
||||
|
||||
delete_space = pynutil.delete(pynini.closure(DAMO_WHITE_SPACE))
|
||||
delete_zero_or_one_space = pynutil.delete(pynini.closure(DAMO_WHITE_SPACE, 0, 1))
|
||||
insert_space = pynutil.insert(" ")
|
||||
delete_extra_space = pynini.cross(pynini.closure(DAMO_WHITE_SPACE, 1), " ")
|
||||
|
||||
delete_preserve_order = pynini.closure(
|
||||
pynutil.delete(" preserve_order: true")
|
||||
| (pynutil.delete(' field_order: "') + DAMO_NOT_QUOTE + pynutil.delete('"'))
|
||||
)
|
||||
|
||||
suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv"))
|
||||
# _v = pynini.union("a", "e", "i", "o", "u")
|
||||
_c = pynini.union(
|
||||
"b",
|
||||
"c",
|
||||
"d",
|
||||
"f",
|
||||
"g",
|
||||
"h",
|
||||
"j",
|
||||
"k",
|
||||
"l",
|
||||
"m",
|
||||
"n",
|
||||
"p",
|
||||
"q",
|
||||
"r",
|
||||
"s",
|
||||
"t",
|
||||
"v",
|
||||
"w",
|
||||
"x",
|
||||
"y",
|
||||
"z",
|
||||
)
|
||||
_ies = DAMO_SIGMA + _c + pynini.cross("y", "ies")
|
||||
_es = DAMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es")
|
||||
_s = DAMO_SIGMA + pynutil.insert("s")
|
||||
|
||||
graph_plural = plurals._priority_union(
|
||||
suppletive,
|
||||
plurals._priority_union(_ies, plurals._priority_union(_es, _s, DAMO_SIGMA), DAMO_SIGMA),
|
||||
DAMO_SIGMA,
|
||||
).optimize()
|
||||
|
||||
SINGULAR_TO_PLURAL = graph_plural
|
||||
PLURAL_TO_SINGULAR = pynini.invert(graph_plural)
|
||||
TO_LOWER = pynini.union(
|
||||
*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]
|
||||
)
|
||||
TO_UPPER = pynini.invert(TO_LOWER)
|
||||
MIN_NEG_WEIGHT = -0.0001
|
||||
MIN_POS_WEIGHT = 0.0001
|
||||
|
||||
|
||||
def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]):
|
||||
"""
|
||||
Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name.
|
||||
|
||||
Args:
|
||||
file_name: exported file name
|
||||
graphs: Mapping of a rule name and Pynini WFST graph to be exported
|
||||
"""
|
||||
exporter = export.Exporter(file_name)
|
||||
for rule, graph in graphs.items():
|
||||
exporter[rule] = graph.optimize()
|
||||
exporter.close()
|
||||
print(f"Created {file_name}")
|
||||
|
||||
|
||||
def get_plurals(fst):
|
||||
"""
|
||||
Given singular returns plurals
|
||||
Args:
|
||||
fst: Fst
|
||||
Returns plurals to given singular forms
|
||||
"""
|
||||
return SINGULAR_TO_PLURAL @ fst
|
||||
|
||||
|
||||
def get_singulars(fst):
|
||||
"""
|
||||
Given plural returns singulars
|
||||
Args:
|
||||
fst: Fst
|
||||
Returns singulars to given plural forms
|
||||
"""
|
||||
return PLURAL_TO_SINGULAR @ fst
|
||||
|
||||
|
||||
def convert_space(fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Converts space to nonbreaking space.
|
||||
Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty"
|
||||
This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it.
|
||||
Args:
|
||||
fst: input fst
|
||||
Returns output fst where breaking spaces are converted to non breaking spaces
|
||||
"""
|
||||
return fst @ pynini.cdrewrite(
|
||||
pynini.cross(DAMO_SPACE, DAMO_NON_BREAKING_SPACE), "", "", DAMO_SIGMA
|
||||
)
|
||||
|
||||
|
||||
class GraphFst:
|
||||
"""
|
||||
Base class for all grammar fsts.
|
||||
|
||||
Args:
|
||||
name: name of grammar class
|
||||
kind: either 'classify' or 'verbalize'
|
||||
deterministic: if True will provide a single transduction option,
|
||||
for False multiple transduction are generated (used for audio-based normalization)
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, kind: str, deterministic: bool = True):
|
||||
self.name = name
|
||||
self.kind = kind
|
||||
self._fst = None
|
||||
self.deterministic = deterministic
|
||||
|
||||
self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far")
|
||||
if self.far_exist():
|
||||
self._fst = Far(
|
||||
self.far_path, mode="r", arc_type="standard", far_type="default"
|
||||
).get_fst()
|
||||
|
||||
def far_exist(self) -> bool:
|
||||
"""
|
||||
Returns true if FAR can be loaded
|
||||
"""
|
||||
return self.far_path.exists()
|
||||
|
||||
@property
|
||||
def fst(self) -> "pynini.FstLike":
|
||||
return self._fst
|
||||
|
||||
@fst.setter
|
||||
def fst(self, fst):
|
||||
self._fst = fst
|
||||
|
||||
def add_tokens(self, fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Wraps class name around to given fst
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns:
|
||||
Fst: fst
|
||||
"""
|
||||
return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }")
|
||||
|
||||
def delete_tokens(self, fst) -> "pynini.FstLike":
|
||||
"""
|
||||
Deletes class name wrap around output of given fst
|
||||
|
||||
Args:
|
||||
fst: input fst
|
||||
|
||||
Returns:
|
||||
Fst: fst
|
||||
"""
|
||||
res = (
|
||||
pynutil.delete(f"{self.name}")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ fst
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
DAMO_SIGMA,
|
||||
DAMO_SPACE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying cardinals
|
||||
e.g. minus twenty three -> cardinal { integer: "23" negative: "-" } }
|
||||
Numbers below thirteen are not converted.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="cardinal", kind="classify")
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_sign = pynini.string_file(get_abs_path("data/numbers/sign.tsv"))
|
||||
graph_dot = pynini.string_file(get_abs_path("data/numbers/dot.tsv"))
|
||||
|
||||
addzero = pynutil.insert("0")
|
||||
graph_digits = graph_digit | graph_zero
|
||||
|
||||
# 单位， 十
|
||||
# 十一
|
||||
graph_teen = pynini.cross("十", "1") + (graph_digit | addzero)
|
||||
|
||||
##二十一
|
||||
graph_tens = graph_digit + pynutil.delete("十") + (graph_digit | addzero)
|
||||
|
||||
# 单位， 百
|
||||
# 一百一十 一百零六 一百二十 两百
|
||||
graph_hundred = (
|
||||
graph_digit
|
||||
+ pynutil.delete("百")
|
||||
+ (
|
||||
graph_tens
|
||||
| graph_teen
|
||||
| pynutil.add_weight(graph_zero + graph_digit, 0.1)
|
||||
| pynutil.add_weight(graph_digit + addzero, 0.5)
|
||||
| pynutil.add_weight(addzero**2, 0.1)
|
||||
)
|
||||
)
|
||||
|
||||
graph_hundred_component_at_least_one_none_zero_digit = graph_hundred @ (
|
||||
pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT)
|
||||
)
|
||||
|
||||
self.graph_hundred_component_at_least_one_none_zero_digit = (
|
||||
graph_hundred_component_at_least_one_none_zero_digit
|
||||
)
|
||||
|
||||
# 单位， 千
|
||||
# 一千两百三十四 一千零三十四， 一千零四 一千二百
|
||||
# 五千
|
||||
graph_thousand = (
|
||||
(graph_hundred | graph_teen | graph_tens | graph_digits)
|
||||
+ pynutil.delete("千")
|
||||
+ (
|
||||
graph_hundred
|
||||
| pynutil.add_weight(graph_zero + graph_tens, 0.1)
|
||||
| pynutil.add_weight(addzero + graph_zero + graph_digit, 0.5)
|
||||
| pynutil.add_weight(graph_digit + addzero**2, 0.8)
|
||||
| pynutil.add_weight(addzero**3, 1.0)
|
||||
)
|
||||
)
|
||||
|
||||
# 单位，万
|
||||
graph_ten_thousand = (
|
||||
(graph_thousand | graph_hundred | graph_teen | graph_tens | graph_digit)
|
||||
+ pynutil.delete("万")
|
||||
+ (
|
||||
graph_thousand
|
||||
| pynutil.add_weight(graph_zero + graph_hundred, 0.1)
|
||||
| pynutil.add_weight(addzero + graph_zero + graph_tens, 0.5)
|
||||
| pynutil.add_weight(addzero + addzero + graph_zero + graph_digit, 0.5)
|
||||
| pynutil.add_weight(graph_digit + addzero**3, 0.8)
|
||||
| pynutil.add_weight(addzero**4, 1.0)
|
||||
)
|
||||
)
|
||||
|
||||
# 单位， 亿/兆
|
||||
graph_number = (
|
||||
graph_digits
|
||||
| graph_teen
|
||||
| graph_tens
|
||||
| graph_hundred
|
||||
| graph_thousand
|
||||
| graph_ten_thousand
|
||||
)
|
||||
|
||||
graph_number = (
|
||||
(graph_number + pynini.accep("兆") + pynutil.delete("零").ques).ques
|
||||
+ (graph_number + pynini.accep("亿") + pynutil.delete("零").ques).ques
|
||||
+ graph_number
|
||||
)
|
||||
|
||||
##符号
|
||||
graph = graph_number
|
||||
# IP
|
||||
graph |= graph_digit.plus + (graph_dot + graph_digit.plus).plus
|
||||
|
||||
# just fill
|
||||
labels_exception = ["ttttt"]
|
||||
graph_exception = pynini.union(*labels_exception)
|
||||
|
||||
self.graph_no_exception = graph
|
||||
|
||||
self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph
|
||||
|
||||
optional_minus_graph = pynini.closure(
|
||||
pynutil.insert("negative: ") + pynini.cross("负", '"-"'), 0, 1
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,77 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize()
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize()
|
||||
|
||||
|
||||
def _get_month_graph():
|
||||
"""
|
||||
Transducer for month, e.g. march -> march
|
||||
"""
|
||||
month_graph = pynini.string_file(get_abs_path("data/months.tsv"))
|
||||
return month_graph
|
||||
|
||||
|
||||
def _get_day_graph():
|
||||
"""
|
||||
Tranducer for day,
|
||||
"""
|
||||
day_graph = pynini.string_file(get_abs_path("data/days.tsv"))
|
||||
return day_graph
|
||||
|
||||
|
||||
def _get_year_graph():
|
||||
"""
|
||||
Transducer for year,
|
||||
"""
|
||||
year_graph = graph_digit + (graph_digit | graph_zero) ** 3
|
||||
year_graph |= graph_digit**2
|
||||
|
||||
return year_graph
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying date,
|
||||
e.g. january fifth twenty twelve -> date { month: "january" day: "5" year: "2012" preserve_order: true }
|
||||
e.g. the fifth of january twenty twelve -> date { day: "5" month: "january" year: "2012" preserve_order: true }
|
||||
e.g. twenty twenty -> date { year: "2012" preserve_order: true }
|
||||
|
||||
Args:
|
||||
ordinal: OrdinalFst
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="date", kind="classify")
|
||||
|
||||
year_graph = _get_year_graph()
|
||||
|
||||
year_graph = (
|
||||
pynutil.insert('year: "') + year_graph + pynutil.delete("年") + pynutil.insert('"')
|
||||
)
|
||||
|
||||
month_graph = _get_month_graph()
|
||||
month_graph = pynutil.insert('month: "') + month_graph + pynutil.insert('"')
|
||||
|
||||
day_graph = _get_day_graph()
|
||||
day_graph = pynutil.insert('day: "') + day_graph + pynutil.insert('"')
|
||||
|
||||
graph_md = month_graph + pynutil.insert(" ") + day_graph
|
||||
graph_ym = year_graph + pynutil.insert(" ") + month_graph
|
||||
graph_ymd = year_graph + pynutil.insert(" ") + month_graph + pynutil.insert(" ") + day_graph
|
||||
|
||||
graph_year = year_graph
|
||||
|
||||
final_graph = graph_ymd | graph_ym | graph_md | graph_year
|
||||
final_graph += pynutil.insert(" preserve_order: true")
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,59 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying decimal
|
||||
e.g. minus twelve point five o o six billion -> decimal { negative: "true" integer_part: "12" fractional_part: "5006" quantity: "billion" }
|
||||
e.g. one billion -> decimal { integer_part: "1" quantity: "billion" }
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="decimal", kind="classify")
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
graph_dot = pynini.string_file(get_abs_path("data/numbers/dot.tsv"))
|
||||
|
||||
graph_digits = graph_digit | graph_zero
|
||||
|
||||
graph_decimal = graph_digits.plus
|
||||
|
||||
self.graph = graph_decimal
|
||||
|
||||
point = pynini.cross("点", "")
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ")
|
||||
+ (pynini.cross("负", '"true"') | pynini.cross("负的", '"true"'))
|
||||
+ delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
graph_fractional = (
|
||||
pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"')
|
||||
)
|
||||
graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
|
||||
final_graph_wo_sign = (
|
||||
pynini.closure(graph_integer, 0, 1) + point + pynutil.insert(" ") + graph_fractional
|
||||
)
|
||||
|
||||
final_graph = optional_graph_negative + final_graph_wo_sign
|
||||
|
||||
self.final_graph_wo_negative = final_graph_wo_sign
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,102 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_ALPHA,
|
||||
GraphFst,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying electronic: as URLs, email addresses, etc.
|
||||
e.g. c d f one at a b c dot e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="electronic", kind="classify")
|
||||
|
||||
delete_extra_space = pynutil.delete(" ")
|
||||
alpha_num = (
|
||||
DAMO_ALPHA
|
||||
| pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
| pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
)
|
||||
|
||||
symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert()
|
||||
|
||||
accepted_username = alpha_num | symbols
|
||||
process_dot = pynini.cross("点", ".")
|
||||
username = (
|
||||
alpha_num + pynini.closure(delete_extra_space + accepted_username)
|
||||
) | pynutil.add_weight(pynini.closure(DAMO_ALPHA, 1), weight=0.0001)
|
||||
username = pynutil.insert('username: "') + username + pynutil.insert('"')
|
||||
single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num
|
||||
server = single_alphanum | pynini.string_file(
|
||||
get_abs_path("data/electronic/server_name.tsv")
|
||||
)
|
||||
domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
||||
domain_graph = (
|
||||
pynutil.insert('domain: "')
|
||||
+ server
|
||||
+ delete_extra_space
|
||||
+ process_dot
|
||||
+ delete_extra_space
|
||||
+ domain
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
graph = (
|
||||
username
|
||||
+ delete_extra_space
|
||||
+ (pynutil.delete("at") | pynutil.delete("艾特"))
|
||||
+ insert_space
|
||||
+ delete_extra_space
|
||||
+ domain_graph
|
||||
)
|
||||
|
||||
############# url ###
|
||||
protocol_end = pynini.cross(pynini.union("w w w", "www"), "www")
|
||||
protocol_start = (
|
||||
pynini.cross("h t t p", "http") | pynini.cross("h t t p s", "https")
|
||||
) + pynini.cross("冒号斜杆斜杠", "://")
|
||||
# .com,
|
||||
ending = (
|
||||
delete_extra_space
|
||||
+ symbols
|
||||
+ delete_extra_space
|
||||
+ (
|
||||
domain
|
||||
| pynini.closure(
|
||||
accepted_username + delete_extra_space,
|
||||
)
|
||||
+ accepted_username
|
||||
)
|
||||
)
|
||||
|
||||
protocol_default = (
|
||||
(
|
||||
(pynini.closure(delete_extra_space + accepted_username, 1) | server)
|
||||
| pynutil.add_weight(pynini.closure(DAMO_ALPHA, 1), weight=0.0001)
|
||||
)
|
||||
+ pynini.closure(ending, 1)
|
||||
).optimize()
|
||||
protocol = (
|
||||
pynini.closure(protocol_start, 0, 1)
|
||||
+ protocol_end
|
||||
+ pynini.closure(pynini.cross(" ", ""), 0, 1)
|
||||
+ process_dot
|
||||
+ pynini.closure(pynini.cross(" ", ""), 0, 1)
|
||||
+ protocol_default
|
||||
).optimize()
|
||||
|
||||
protocol |= (
|
||||
pynini.closure(protocol_end + delete_extra_space + process_dot, 0, 1) + protocol_default
|
||||
)
|
||||
|
||||
protocol = pynutil.insert('protocol: "') + protocol.optimize() + pynutil.insert('"')
|
||||
graph |= protocol
|
||||
########
|
||||
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,57 @@
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_space,
|
||||
delete_extra_space,
|
||||
DAMO_SIGMA,
|
||||
DAMO_CHAR,
|
||||
DAMO_SPACE,
|
||||
)
|
||||
|
||||
import pynini
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying fraction
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="fraction", kind="classify")
|
||||
# integer_part # numerator # denominator
|
||||
graph_cardinal = cardinal.graph_no_exception
|
||||
|
||||
# without the integerate part
|
||||
# 分子
|
||||
numerator = pynutil.insert('numerator: "') + graph_cardinal + pynutil.insert('"')
|
||||
# 分母
|
||||
denominator = (
|
||||
pynutil.insert('denominator: "')
|
||||
+ graph_cardinal
|
||||
+ pynutil.delete("分之")
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
##
|
||||
graph_fraction_component = denominator + pynutil.insert(" ") + numerator
|
||||
|
||||
self.graph_fraction_component = graph_fraction_component
|
||||
|
||||
graph = graph_fraction_component
|
||||
graph = graph.optimize()
|
||||
self.final_graph_wo_negative = graph
|
||||
|
||||
##负
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ")
|
||||
+ (pynini.cross("负", '"true"') | pynini.cross("负的", '"true"'))
|
||||
+ DAMO_SPACE,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
graph = optional_graph_negative + graph
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,75 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
get_singulars,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying measure
|
||||
e.g. minus twelve kilograms -> measure { negative: "true" cardinal { integer: "12" } units: "kg" }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst):
|
||||
super().__init__(name="measure", kind="classify")
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
|
||||
graph_unit_en = pynini.string_file(get_abs_path("data/measurements_en.tsv"))
|
||||
graph_unit_zh = pynini.string_file(get_abs_path("data/measurements_zh.tsv"))
|
||||
graph_sign = pynini.string_file(get_abs_path("data/numbers/sign.tsv"))
|
||||
|
||||
graph_units = graph_unit_en | (
|
||||
(pynini.accep("亿") | pynini.accep("兆") | pynini.accep("万")).ques + graph_unit_zh
|
||||
)
|
||||
|
||||
optional_graph_negative = pynini.closure(
|
||||
pynutil.insert("negative: ")
|
||||
+ (pynini.cross("负", '"true"') | pynini.cross("负的", '"true"'))
|
||||
+ delete_extra_space,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
|
||||
percent = (
|
||||
pynutil.delete("百分")
|
||||
+ pynutil.delete("之").ques
|
||||
+ (cardinal_graph | pynini.cross("百", "100"))
|
||||
+ pynutil.insert("%")
|
||||
)
|
||||
|
||||
unit_all = pynutil.insert('units: "') + graph_units + pynutil.insert('"')
|
||||
|
||||
subgraph_decimal = (
|
||||
pynutil.insert("decimal { ")
|
||||
+ optional_graph_negative
|
||||
+ decimal.final_graph_wo_negative
|
||||
+ pynutil.insert(" }")
|
||||
+ pynutil.insert(" ")
|
||||
+ unit_all
|
||||
)
|
||||
subgraph_cardinal = (
|
||||
pynutil.insert("cardinal { ")
|
||||
+ optional_graph_negative
|
||||
+ pynutil.insert('integer: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(" }")
|
||||
+ pynutil.insert(" ")
|
||||
+ unit_all
|
||||
)
|
||||
|
||||
final_graph = subgraph_decimal | subgraph_cardinal
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,53 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_DIGIT,
|
||||
DAMO_NOT_SPACE,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
get_singulars,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying money
|
||||
e.g. twelve dollars and five cents -> money { integer_part: "12" fractional_part: 05 currency: "$" }
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst, decimal: GraphFst):
|
||||
super().__init__(name="money", kind="classify")
|
||||
# quantity, integer_part, fractional_part, currency
|
||||
|
||||
cardinal_graph = cardinal.graph_no_exception
|
||||
decimal_graph = decimal.final_graph_wo_negative
|
||||
graph_code = pynini.string_file(get_abs_path("data/money/code.tsv"))
|
||||
graph_symbol = pynini.string_file(get_abs_path("data/money/symbol.tsv"))
|
||||
|
||||
graph_unit = (
|
||||
pynutil.insert('currency: "') + (graph_code | graph_symbol) + pynutil.insert('"')
|
||||
)
|
||||
|
||||
graph_integer = (
|
||||
pynutil.insert('integer_part: "')
|
||||
+ cardinal_graph
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(" ")
|
||||
+ graph_unit
|
||||
)
|
||||
|
||||
graph_decimal = decimal_graph + pynutil.insert(" ") + graph_unit
|
||||
|
||||
final_graph = graph_integer | graph_decimal
|
||||
|
||||
final_graph = self.add_tokens(final_graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,20 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class PunctuationFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying punctuation
|
||||
e.g. a, -> tokens { name: "a" } tokens { name: "," }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="punctuation", kind="classify")
|
||||
|
||||
s = "!#$%&'()*+,-./:;<=>?@^_`{|}~"
|
||||
punct = pynini.union(*s)
|
||||
|
||||
graph = pynutil.insert('name: "') + punct + pynutil.insert('"')
|
||||
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,102 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_ALNUM,
|
||||
DAMO_ALPHA,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
def get_serial_number(cardinal):
|
||||
"""
|
||||
any alphanumerical character sequence with at least one number with length greater equal to 3
|
||||
"""
|
||||
digit = pynini.compose(cardinal.graph_no_exception, DAMO_DIGIT)
|
||||
character = digit | DAMO_ALPHA
|
||||
sequence = character + pynini.closure(pynutil.delete(" ") + character, 2)
|
||||
sequence = sequence @ (pynini.closure(DAMO_ALNUM) + DAMO_DIGIT + pynini.closure(DAMO_ALNUM))
|
||||
return sequence.optimize()
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying telephone numbers, e.g.
|
||||
one two three one two three five six seven eight -> { number_part: "123-123-5678" }
|
||||
|
||||
This class also support card number and IP format.
|
||||
"one two three dot one double three dot o dot four o" -> { number_part: "123.133.0.40"}
|
||||
|
||||
"three two double seven three two one four three two one four three double zero five" ->
|
||||
{ number_part: 3277 3214 3214 3005}
|
||||
|
||||
Args:
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, cardinal: GraphFst):
|
||||
super().__init__(name="telephone", kind="classify")
|
||||
# country code, number_part, extension
|
||||
|
||||
graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
|
||||
graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
|
||||
graph_dot = pynini.string_file(get_abs_path("data/numbers/dot.tsv"))
|
||||
|
||||
graph_digits = graph_digit | graph_zero
|
||||
|
||||
# to handle cases like "one twenty three"
|
||||
phone_number_graph_w_country_code = graph_digits**11
|
||||
phone_number_graph_wo_country_code = (
|
||||
graph_digits**3
|
||||
| graph_digits**5
|
||||
| graph_digits**11
|
||||
| graph_digits**8
|
||||
| graph_digits**12
|
||||
)
|
||||
|
||||
country_code = (
|
||||
pynutil.insert('country_code: "')
|
||||
+ pynini.closure(pynini.cross("加", "+"), 0, 1)
|
||||
+ ((pynini.closure(graph_digits, 0, 2) + graph_digits))
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
optional_country_code = pynini.closure(country_code + insert_space, 0, 1).optimize()
|
||||
|
||||
grpah_phone_number_with_country_code = (
|
||||
pynutil.insert('number_part: "')
|
||||
+ pynutil.add_weight(phone_number_graph_w_country_code.optimize(), 0.1)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
# grpah_phone_number_with_country_code = (
|
||||
# pynutil.insert("number_part: \"")
|
||||
# + phone_number_graph_w_country_code.optimize()
|
||||
# + pynutil.insert("\"")
|
||||
# )
|
||||
|
||||
graph_phone_with_country_code = optional_country_code + grpah_phone_number_with_country_code
|
||||
|
||||
grpah_phone_number_wo_country_code = (
|
||||
pynutil.insert('number_part: "')
|
||||
+ phone_number_graph_wo_country_code
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
# phone number
|
||||
graph = graph_phone_with_country_code | grpah_phone_number_wo_country_code
|
||||
|
||||
# ip
|
||||
ip_graph = graph_digit.plus + (graph_dot + graph_digits.plus).plus
|
||||
|
||||
graph |= pynutil.insert('number_part: "') + ip_graph.optimize() + pynutil.insert('"')
|
||||
graph |= (
|
||||
pynutil.insert('number_part: "')
|
||||
+ pynutil.add_weight(get_serial_number(cardinal=cardinal), weight=0.0001)
|
||||
+ pynutil.insert('"')
|
||||
)
|
||||
|
||||
final_graph = self.add_tokens(graph)
|
||||
self.fst = final_graph.optimize()
|
||||
@@ -0,0 +1,48 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path, num_to_word
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
GraphFst,
|
||||
convert_space,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying time
|
||||
e.g. twelve thirty -> time { hours: "12" minutes: "30" }
|
||||
e.g. twelve past one -> time { minutes: "12" hours: "1" }
|
||||
e.g. two o clock a m -> time { hours: "2" suffix: "a.m." }
|
||||
e.g. quarter to two -> time { hours: "1" minutes: "45" }
|
||||
e.g. quarter past two -> time { hours: "2" minutes: "15" }
|
||||
e.g. half past two -> time { hours: "2" minutes: "30" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="time", kind="classify")
|
||||
# hours, minutes, seconds, suffix, zone, style, speak_period
|
||||
|
||||
noon_graph = pynini.string_file(get_abs_path("data/time/noon.tsv"))
|
||||
hour_graph = pynini.string_file(get_abs_path("data/time/hour.tsv"))
|
||||
minute_graph = pynini.string_file(get_abs_path("data/time/minute.tsv"))
|
||||
second_graph = pynini.string_file(get_abs_path("data/time/second.tsv"))
|
||||
|
||||
final_graph = (
|
||||
(pynutil.insert('noon: "') + noon_graph + pynutil.insert('" ')).ques
|
||||
+ pynutil.insert('hour: "')
|
||||
+ hour_graph
|
||||
+ pynutil.insert('"')
|
||||
+ pynutil.insert(' minute: "')
|
||||
+ minute_graph
|
||||
+ pynutil.delete("分").ques
|
||||
+ pynutil.insert('"')
|
||||
+ (pynutil.insert(' second: "') + second_graph + pynutil.insert('"')).ques
|
||||
)
|
||||
|
||||
final_graph = self.add_tokens(final_graph.optimize())
|
||||
|
||||
self.fst = final_graph.optimize()
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
import os
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.date import DateFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.fraction import FractionFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.electronic import ElectronicFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.punctuation import PunctuationFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.telephone import TelephoneFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.time import TimeFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.whitelist import WhiteListFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.taggers.word import WordFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
generator_main,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
import logging
|
||||
|
||||
|
||||
class ClassifyFst(GraphFst):
|
||||
"""
|
||||
Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
|
||||
Args:
|
||||
cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
|
||||
overwrite_cache: set to True to overwrite .far files
|
||||
"""
|
||||
|
||||
def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
|
||||
super().__init__(name="tokenize_and_classify", kind="classify")
|
||||
|
||||
far_file = None
|
||||
if cache_dir is not None and cache_dir != "None":
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
far_file = os.path.join(cache_dir, "_zh_itn.far")
|
||||
if not overwrite_cache and far_file and os.path.exists(far_file):
|
||||
self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
|
||||
logging.info(f"ClassifyFst.fst was restored from {far_file}.")
|
||||
else:
|
||||
logging.info(f"Creating ClassifyFst grammars.")
|
||||
cardinal = CardinalFst()
|
||||
cardinal_graph = cardinal.fst
|
||||
|
||||
decimal = DecimalFst(cardinal)
|
||||
decimal_graph = decimal.fst
|
||||
|
||||
fraction = FractionFst(cardinal)
|
||||
fraction_graph = fraction.fst
|
||||
|
||||
measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst
|
||||
date_graph = DateFst().fst
|
||||
word_graph = WordFst().fst
|
||||
time_graph = TimeFst().fst
|
||||
money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst
|
||||
whitelist_graph = WhiteListFst().fst
|
||||
punct_graph = PunctuationFst().fst
|
||||
electronic_graph = ElectronicFst().fst
|
||||
telephone_graph = TelephoneFst(cardinal).fst
|
||||
|
||||
classify = (
|
||||
pynutil.add_weight(whitelist_graph, 1.01)
|
||||
| pynutil.add_weight(time_graph, 1.1)
|
||||
| pynutil.add_weight(date_graph, 1.09)
|
||||
| pynutil.add_weight(decimal_graph, 1.1)
|
||||
| pynutil.add_weight(fraction_graph, 1.1)
|
||||
| pynutil.add_weight(measure_graph, 1.1)
|
||||
| pynutil.add_weight(cardinal_graph, 1.1)
|
||||
| pynutil.add_weight(money_graph, 1.1)
|
||||
| pynutil.add_weight(telephone_graph, 1.1)
|
||||
| pynutil.add_weight(electronic_graph, 1.1)
|
||||
| pynutil.add_weight(word_graph, 100)
|
||||
)
|
||||
|
||||
punct = (
|
||||
pynutil.insert("tokens { ")
|
||||
+ pynutil.add_weight(punct_graph, weight=1.1)
|
||||
+ pynutil.insert(" }")
|
||||
)
|
||||
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
|
||||
token_plus_punct = (
|
||||
pynini.closure(punct + pynutil.insert(" "))
|
||||
+ token
|
||||
+ pynini.closure(pynutil.insert(" ") + punct)
|
||||
)
|
||||
|
||||
graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
|
||||
graph = delete_space + graph + delete_space
|
||||
|
||||
self.fst = graph.optimize()
|
||||
|
||||
if far_file:
|
||||
generator_main(far_file, {"tokenize_and_classify": self.fst})
|
||||
logging.info(f"ClassifyFst grammars are saved to {far_file}.")
|
||||
@@ -0,0 +1,19 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.utils import get_abs_path
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst, convert_space
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying whitelisted tokens
|
||||
e.g. misses -> tokens { name: "mrs." }
|
||||
This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="whitelist", kind="classify")
|
||||
|
||||
whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert()
|
||||
graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"')
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,15 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_SPACE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class.
|
||||
e.g. sleep -> tokens { name: "sleep" }
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="word", kind="classify")
|
||||
word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
|
||||
self.fst = word.optimize()
|
||||
@@ -0,0 +1,68 @@
|
||||
import csv
|
||||
import os
|
||||
from typing import Union
|
||||
import inflect
|
||||
|
||||
UNIT_1e01 = "十"
|
||||
UNIT_1e02 = "百"
|
||||
UNIT_1e03 = "千"
|
||||
UNIT_1e04 = "万"
|
||||
|
||||
_inflect = inflect.engine()
|
||||
|
||||
|
||||
def num_to_word(x: Union[str, int]):
|
||||
"""
|
||||
converts integer to spoken representation
|
||||
Args
|
||||
x: integer
|
||||
Returns: spoken representation
|
||||
"""
|
||||
if isinstance(x, int):
|
||||
x = str(x)
|
||||
x = _inflect.number_to_words(str(x)).replace("-", " ").replace(",", "")
|
||||
return x
|
||||
|
||||
|
||||
def get_abs_path(rel_path):
|
||||
"""
|
||||
Get absolute path
|
||||
|
||||
Args:
|
||||
rel_path: relative path to this file
|
||||
|
||||
Returns absolute path
|
||||
"""
|
||||
return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path
|
||||
|
||||
|
||||
def load_labels(abs_path):
|
||||
"""
|
||||
loads relative path file as dictionary
|
||||
|
||||
Args:
|
||||
abs_path: absolute path
|
||||
|
||||
Returns dictionary of mappings
|
||||
"""
|
||||
label_tsv = open(abs_path, encoding="utf-8")
|
||||
labels = list(csv.reader(label_tsv, delimiter="\t"))
|
||||
return labels
|
||||
|
||||
|
||||
def augment_labels_with_punct_at_end(labels):
|
||||
"""
|
||||
augments labels: if key ends on a punctuation that value does not have, add a new label
|
||||
where the value maintains the punctuation
|
||||
|
||||
Args:
|
||||
labels : input labels
|
||||
Returns:
|
||||
additional labels
|
||||
"""
|
||||
res = []
|
||||
for label in labels:
|
||||
if len(label) > 1:
|
||||
if label[0][-1] == "." and label[1][-1] != ".":
|
||||
res.append([label[0], label[1] + "."] + label[2:])
|
||||
return res
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_SPACE,
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class CardinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing cardinal
|
||||
e.g. cardinal { integer: "23" negative: "-" } -> -23
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="cardinal", kind="verbalize")
|
||||
optional_sign = pynini.closure(
|
||||
pynutil.delete("negative:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ DAMO_NOT_QUOTE
|
||||
+ pynutil.delete('"'),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
graph = (
|
||||
pynutil.delete("integer:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
self.numbers = graph
|
||||
graph = optional_sign + graph
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,72 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DateFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing date, e.g.
|
||||
date { month: "january" day: "5" year: "2012" preserve_order: true } -> february 5 2012
|
||||
date { day: "5" month: "january" year: "2012" preserve_order: true } -> 5 february 2012
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="date", kind="verbalize")
|
||||
month = (
|
||||
pynutil.delete("month:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
day = (
|
||||
pynutil.delete("day:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
year = (
|
||||
pynutil.delete("year:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph_ymd = (
|
||||
year
|
||||
+ pynini.cross(" ", "年")
|
||||
+ month
|
||||
+ pynini.cross(" ", "月")
|
||||
+ day
|
||||
+ pynutil.insert("日")
|
||||
)
|
||||
|
||||
graph_md = month + pynini.cross(" ", "月") + day + pynutil.insert("日")
|
||||
|
||||
graph_ym = year + pynini.cross(" ", "年") + month + pynutil.insert("月")
|
||||
|
||||
graph_year = year + pynutil.insert("年")
|
||||
optional_preserve_order = pynini.closure(
|
||||
pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
|
||||
| pynutil.delete("field_order:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ DAMO_NOT_QUOTE
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space
|
||||
)
|
||||
|
||||
final_graph = (
|
||||
(graph_ymd | graph_ym | graph_md | graph_year) + delete_space + optional_preserve_order
|
||||
)
|
||||
|
||||
delete_tokens = self.delete_tokens(final_graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,48 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class DecimalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing decimal, e.g.
|
||||
decimal { negative: "true" integer_part: "12" fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="decimal", kind="verbalize")
|
||||
optionl_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
|
||||
integer = (
|
||||
pynutil.delete("integer_part:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_integer = pynini.closure(integer + delete_space, 0, 1)
|
||||
fractional = (
|
||||
pynutil.insert(".")
|
||||
+ pynutil.delete("fractional_part:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
|
||||
quantity = (
|
||||
pynutil.delete("quantity:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1)
|
||||
graph = optional_integer + optional_fractional + optional_quantity
|
||||
self.numbers = graph
|
||||
graph = optionl_sign + graph
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class ElectronicFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing electronic
|
||||
e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="electronic", kind="verbalize")
|
||||
user_name = (
|
||||
pynutil.delete("username:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
domain = (
|
||||
pynutil.delete("domain:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
protocol = (
|
||||
pynutil.delete("protocol:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = user_name + delete_space + pynutil.insert("@") + domain
|
||||
graph |= protocol
|
||||
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,41 @@
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_NOT_QUOTE,
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
|
||||
import pynini
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class FractionFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing fraction,
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="fraction", kind="verbalize")
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
|
||||
numerator = (
|
||||
pynutil.delete("numerator:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
denominator = (
|
||||
pynutil.delete("denominator:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
graph = (
|
||||
optional_sign + numerator + delete_space + pynutil.insert("/") + denominator
|
||||
).optimize()
|
||||
self.numbers = graph
|
||||
delete_tokens = self.delete_tokens(optional_sign + graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,51 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MeasureFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing measure, e.g.
|
||||
measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
cardinal: CardinalFst
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst, cardinal: GraphFst):
|
||||
super().__init__(name="measure", kind="verbalize")
|
||||
optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
|
||||
unit = (
|
||||
pynutil.delete("units:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
+ delete_space
|
||||
)
|
||||
graph_decimal = (
|
||||
pynutil.delete("decimal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ decimal.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph_cardinal = (
|
||||
pynutil.delete("cardinal {")
|
||||
+ delete_space
|
||||
+ optional_sign
|
||||
+ delete_space
|
||||
+ cardinal.numbers
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = (graph_cardinal | graph_decimal) + delete_space + unit
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,30 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class MoneyFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing money, e.g.
|
||||
money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05
|
||||
|
||||
Args:
|
||||
decimal: DecimalFst
|
||||
"""
|
||||
|
||||
def __init__(self, decimal: GraphFst):
|
||||
super().__init__(name="money", kind="verbalize")
|
||||
unit = (
|
||||
pynutil.delete("currency:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = unit + delete_space + decimal.numbers
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,30 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_QUOTE, GraphFst
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TelephoneFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing telephone, e.g.
|
||||
telephone { number_part: "123-123-5678" }
|
||||
-> 123-123-5678
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="telephone", kind="verbalize")
|
||||
|
||||
number_part = (
|
||||
pynutil.delete('number_part: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_country_code = pynini.closure(
|
||||
pynutil.delete('country_code: "')
|
||||
+ pynini.closure(DAMO_NOT_QUOTE, 1)
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.accep(" "),
|
||||
0,
|
||||
1,
|
||||
)
|
||||
delete_tokens = self.delete_tokens(optional_country_code + number_part)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,79 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_DIGIT,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
insert_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class TimeFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing time, e.g.
|
||||
time { hours: "12" minutes: "30" } -> 12:30
|
||||
time { hours: "1" minutes: "12" } -> 01:12
|
||||
time { hours: "2" suffix: "a.m." } -> 02:00 a.m.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="time", kind="verbalize")
|
||||
add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
|
||||
pynutil.insert("0") + DAMO_DIGIT
|
||||
)
|
||||
|
||||
hour = (
|
||||
pynutil.delete("hour:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
minute = (
|
||||
pynutil.delete("minute:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
second = (
|
||||
pynutil.delete("second:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_DIGIT, 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
|
||||
suffix = (
|
||||
delete_space
|
||||
+ pynutil.insert(" ")
|
||||
+ pynutil.delete("noon:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_suffix = pynini.closure(suffix, 0, 1)
|
||||
zone = (
|
||||
delete_space
|
||||
+ pynutil.insert(" ")
|
||||
+ pynutil.delete("zone:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
optional_zone = pynini.closure(zone, 0, 1)
|
||||
graph = (
|
||||
hour @ add_leading_zero_to_double_digit
|
||||
+ delete_space
|
||||
+ pynutil.insert(":")
|
||||
+ (minute @ add_leading_zero_to_double_digit)
|
||||
+ (pynutil.insert(":") + second @ add_leading_zero_to_double_digit).ques
|
||||
+ optional_suffix
|
||||
+ optional_zone
|
||||
)
|
||||
delete_tokens = self.delete_tokens(graph)
|
||||
self.fst = delete_tokens.optimize()
|
||||
@@ -0,0 +1,48 @@
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.cardinal import CardinalFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.date import DateFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.decimal import DecimalFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.fraction import FractionFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.electronic import ElectronicFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.measure import MeasureFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.money import MoneyFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.telephone import TelephoneFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.time import TimeFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.whitelist import WhiteListFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import GraphFst
|
||||
|
||||
|
||||
class VerbalizeFst(GraphFst):
|
||||
"""
|
||||
Composes other verbalizer grammars.
|
||||
For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
|
||||
More details to deployment at NeMo/tools/text_processing_deployment.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="verbalize", kind="verbalize")
|
||||
cardinal = CardinalFst()
|
||||
cardinal_graph = cardinal.fst
|
||||
decimal = DecimalFst()
|
||||
decimal_graph = decimal.fst
|
||||
fraction = FractionFst()
|
||||
fraction_graph = fraction.fst
|
||||
measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst
|
||||
money_graph = MoneyFst(decimal=decimal).fst
|
||||
time_graph = TimeFst().fst
|
||||
date_graph = DateFst().fst
|
||||
whitelist_graph = WhiteListFst().fst
|
||||
telephone_graph = TelephoneFst().fst
|
||||
electronic_graph = ElectronicFst().fst
|
||||
graph = (
|
||||
time_graph
|
||||
| date_graph
|
||||
| money_graph
|
||||
| measure_graph
|
||||
| decimal_graph
|
||||
| fraction_graph
|
||||
| cardinal_graph
|
||||
| whitelist_graph
|
||||
| telephone_graph
|
||||
| electronic_graph
|
||||
)
|
||||
self.fst = graph
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.verbalize import VerbalizeFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.verbalizers.word import WordFst
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
GraphFst,
|
||||
delete_extra_space,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class VerbalizeFinalFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer that verbalizes an entire sentence, e.g.
|
||||
tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="verbalize_final", kind="verbalize")
|
||||
verbalize = VerbalizeFst().fst
|
||||
word = WordFst().fst
|
||||
types = verbalize | word
|
||||
graph = (
|
||||
pynutil.delete("tokens")
|
||||
+ delete_space
|
||||
+ pynutil.delete("{")
|
||||
+ delete_space
|
||||
+ types
|
||||
+ delete_space
|
||||
+ pynutil.delete("}")
|
||||
)
|
||||
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
|
||||
self.fst = graph
|
||||
@@ -0,0 +1,27 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WhiteListFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing whitelist
|
||||
e.g. tokens { name: "mrs." } -> mrs.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="whitelist", kind="verbalize")
|
||||
graph = (
|
||||
pynutil.delete("name:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ pynini.closure(DAMO_CHAR - " ", 1)
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
self.fst = graph.optimize()
|
||||
@@ -0,0 +1,29 @@
|
||||
import pynini
|
||||
from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
|
||||
DAMO_CHAR,
|
||||
DAMO_SIGMA,
|
||||
GraphFst,
|
||||
delete_space,
|
||||
)
|
||||
from pynini.lib import pynutil
|
||||
|
||||
|
||||
class WordFst(GraphFst):
|
||||
"""
|
||||
Finite state transducer for verbalizing plain tokens
|
||||
e.g. tokens { name: "sleep" } -> sleep
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="word", kind="verbalize")
|
||||
chars = pynini.closure(DAMO_CHAR - " ", 1)
|
||||
char = (
|
||||
pynutil.delete("name:")
|
||||
+ delete_space
|
||||
+ pynutil.delete('"')
|
||||
+ chars
|
||||
+ pynutil.delete('"')
|
||||
)
|
||||
graph = char @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
|
||||
|
||||
self.fst = graph.optimize()
|
||||
Reference in New Issue
Block a user