chore: import upstream snapshot with attribution

fun_text_processing/inverse_text_normalization/ja/__init__.py

@@ -0,0 +1,7 @@
+from fun_text_processing.inverse_text_normalization.en.taggers.tokenize_and_classify import (
+    ClassifyFst,
+)
+from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.en.verbalizers.verbalize_final import (
+    VerbalizeFinalFst,
+)

fun_text_processing/inverse_text_normalization/ja/clean_eval_data.py

@@ -0,0 +1,384 @@
+from argparse import ArgumentParser
+from typing import List
+
+import regex as re
+from fun_text_processing.text_normalization.data_loader_utils import (
+    EOS_TYPE,
+    Instance,
+    load_files,
+    training_data_to_sentences,
+)
+
+
+"""
+This file is for evaluation purposes.
+filter_loaded_data() cleans data (list of instances) for inverse text normalization. Filters and cleaners can be specified for each semiotic class individually.
+For example, normalized text should only include characters and whitespace characters but no punctuation. 
+            Cardinal unnormalized instances should contain at least one integer and all other characters are removed.
+"""
+
+
+class Filter:
+    """
+    Filter class
+
+    Args:
+        class_type: semiotic class used in dataset
+        process_func: function to transform text
+        filter_func:  function to filter text
+
+    """
+
+    def __init__(self, class_type: str, process_func: object, filter_func: object):
+        self.class_type = class_type
+        self.process_func = process_func
+        self.filter_func = filter_func
+
+    def filter(self, instance: Instance) -> bool:
+        """
+        filter function
+
+        Args:
+            filters given instance with filter function
+
+        Returns: True if given instance fulfills criteria or does not belong to class type
+        """
+        if instance.token_type != self.class_type:
+            return True
+        return self.filter_func(instance)
+
+    def process(self, instance: Instance) -> Instance:
+        """
+        process function
+
+        Args:
+            processes given instance with process function
+
+        Returns: processed instance if instance belongs to expected class type or original instance
+        """
+        if instance.token_type != self.class_type:
+            return instance
+        return self.process_func(instance)
+
+
+def filter_cardinal_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_cardinal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r"[^0-9]", "", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_ordinal_1(instance: Instance) -> bool:
+    ok = re.search(r"(st|nd|rd|th)\s*$", instance.un_normalized)
+    return ok
+
+
+def process_ordinal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r"[,\s]", "", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_decimal_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_decimal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_measure_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_measure_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    un_normalized = re.sub(r"m2", "m²", un_normalized)
+    un_normalized = re.sub(r"(\d)([^\d.\s])", r"\1 \2", un_normalized)
+    normalized = re.sub(r"[^a-z\s]", "", normalized)
+    normalized = re.sub(r"per ([a-z\s]*)s$", r"per \1", normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_money_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_money_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    un_normalized = re.sub(r"a\$", r"$", un_normalized)
+    un_normalized = re.sub(r"us\$", r"$", un_normalized)
+    un_normalized = re.sub(r"(\d)m\s*$", r"\1 million", un_normalized)
+    un_normalized = re.sub(r"(\d)bn?\s*$", r"\1 billion", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_time_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_time_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r": ", ":", un_normalized)
+    un_normalized = re.sub(r"(\d)\s?a\s?m\s?", r"\1 a.m.", un_normalized)
+    un_normalized = re.sub(r"(\d)\s?p\s?m\s?", r"\1 p.m.", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_plain_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_plain_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_punct_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_punct_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_date_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_date_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_letters_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_letters_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_verbatim_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_verbatim_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_digit_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_digit_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_telephone_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_telephone_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_electronic_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_electronic_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_fraction_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_fraction_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_address_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_address_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+filters = []
+filters.append(
+    Filter(class_type="CARDINAL", process_func=process_cardinal_1, filter_func=filter_cardinal_1)
+)
+filters.append(
+    Filter(class_type="ORDINAL", process_func=process_ordinal_1, filter_func=filter_ordinal_1)
+)
+filters.append(
+    Filter(class_type="DECIMAL", process_func=process_decimal_1, filter_func=filter_decimal_1)
+)
+filters.append(
+    Filter(class_type="MEASURE", process_func=process_measure_1, filter_func=filter_measure_1)
+)
+filters.append(Filter(class_type="MONEY", process_func=process_money_1, filter_func=filter_money_1))
+filters.append(Filter(class_type="TIME", process_func=process_time_1, filter_func=filter_time_1))
+
+filters.append(Filter(class_type="DATE", process_func=process_date_1, filter_func=filter_date_1))
+filters.append(Filter(class_type="PLAIN", process_func=process_plain_1, filter_func=filter_plain_1))
+filters.append(Filter(class_type="PUNCT", process_func=process_punct_1, filter_func=filter_punct_1))
+filters.append(
+    Filter(class_type="LETTERS", process_func=process_letters_1, filter_func=filter_letters_1)
+)
+filters.append(
+    Filter(class_type="VERBATIM", process_func=process_verbatim_1, filter_func=filter_verbatim_1)
+)
+filters.append(Filter(class_type="DIGIT", process_func=process_digit_1, filter_func=filter_digit_1))
+filters.append(
+    Filter(class_type="TELEPHONE", process_func=process_telephone_1, filter_func=filter_telephone_1)
+)
+filters.append(
+    Filter(
+        class_type="ELECTRONIC", process_func=process_electronic_1, filter_func=filter_electronic_1
+    )
+)
+filters.append(
+    Filter(class_type="FRACTION", process_func=process_fraction_1, filter_func=filter_fraction_1)
+)
+filters.append(
+    Filter(class_type="ADDRESS", process_func=process_address_1, filter_func=filter_address_1)
+)
+filters.append(Filter(class_type=EOS_TYPE, process_func=lambda x: x, filter_func=lambda x: True))
+
+
+def filter_loaded_data(data: List[Instance], verbose: bool = False) -> List[Instance]:
+    """
+    Filters list of instances
+
+    Args:
+        data: list of instances
+
+    Returns: filtered and transformed list of instances
+    """
+    updates_instances = []
+    for instance in data:
+        updated_instance = False
+        for fil in filters:
+            if fil.class_type == instance.token_type and fil.filter(instance):
+                instance = fil.process(instance)
+                updated_instance = True
+        if updated_instance:
+            if verbose:
+                print(instance)
+            updates_instances.append(instance)
+    return updates_instances
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--input", help="input file path", type=str, default="./en_with_types/output-00001-of-00100"
+    )
+    parser.add_argument("--verbose", help="print filtered instances", action="store_true")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    file_path = args.input
+
+    print("Loading training data: " + file_path)
+    instance_list = load_files([file_path])  # List of instances
+    filtered_instance_list = filter_loaded_data(instance_list, args.verbose)
+    training_data_to_sentences(filtered_instance_list)

fun_text_processing/inverse_text_normalization/ja/data/char/fullwidth_to_halfwidth.tsv

@@ -0,0 +1,92 @@
+！	!
+＂	"
+＃	#
+＄	$
+％	%
+＆	&
+＇	'
+（	(
+）	)
+＊	*
+＋	+
+，	,
+－	-
+．	.
+／	/
+０	0
+１	1
+２	2
+３	3
+４	4
+５	5
+６	6
+７	7
+８	8
+９	9
+；	;
+＜	<
+＝	=
+＞	>
+？	?
+＠	@
+Ａ	A
+Ｂ	B
+Ｃ	C
+Ｄ	D
+Ｅ	E
+Ｆ	F
+Ｇ	G
+Ｈ	H
+Ｉ	I
+Ｊ	J
+Ｋ	K
+Ｌ	L
+Ｍ	M
+Ｎ	N
+Ｏ	O
+Ｐ	P
+Ｑ	Q
+Ｒ	R
+Ｓ	S
+Ｔ	T
+Ｕ	U
+Ｖ	V
+Ｗ	W
+Ｘ	X
+Ｙ	Y
+Ｚ	Z
+＼	\
+＾	^
+＿	_
+｀	`
+ａ	a
+ｂ	b
+ｃ	c
+ｄ	d
+ｅ	e
+ｆ	f
+ｇ	g
+ｈ	h
+ｉ	i
+ｊ	j
+ｋ	k
+ｌ	l
+ｍ	m
+ｎ	n
+ｏ	o
+ｐ	p
+ｑ	q
+ｒ	r
+ｓ	s
+ｔ	t
+ｕ	u
+ｖ	v
+ｗ	w
+ｘ	x
+ｙ	y
+ｚ	z
+｛	{
+｜	| 
+：	:
+｝	}
+～	~

fun_text_processing/inverse_text_normalization/ja/data/char/halfwidth_to_fullwidth.tsv

@@ -0,0 +1,92 @@
+!	！
+"	＂
+#	＃
+$	＄
+%	％
+&	＆
+'	＇
+(	（
+)	）
+*	＊
++	＋
+,	，
+-	－
+.	．
+/	／
+0	０
+1	１
+2	２
+3	３
+4	４
+5	５
+6	６
+7	７
+8	８
+9	９
+;	；
+<	＜
+=	＝
+>	＞
+?	？
+@	＠
+A	Ａ
+B	Ｂ
+C	Ｃ
+D	Ｄ
+E	Ｅ
+F	Ｆ
+G	Ｇ
+H	Ｈ
+I	Ｉ
+J	Ｊ
+K	Ｋ
+L	Ｌ
+M	Ｍ
+N	Ｎ
+O	Ｏ
+P	Ｐ
+Q	Ｑ
+R	Ｒ
+S	Ｓ
+T	Ｔ
+U	Ｕ
+V	Ｖ
+W	Ｗ
+X	Ｘ
+Y	Ｙ
+Z	Ｚ
+\	＼
+^	＾
+_	＿
+`	｀
+a	ａ
+b	ｂ
+c	ｃ
+d	ｄ
+e	ｅ
+f	ｆ
+g	ｇ
+h	ｈ
+i	ｉ
+j	ｊ
+k	ｋ
+l	ｌ
+m	ｍ
+n	ｎ
+o	ｏ
+p	ｐ
+q	ｑ
+r	ｒ
+s	ｓ
+t	ｔ
+u	ｕ
+v	ｖ
+w	ｗ
+x	ｘ
+y	ｙ
+z	ｚ
+{	｛
+| 	｜
+:	：
+}	｝
+~	～

fun_text_processing/inverse_text_normalization/ja/data/char/oov_tags.tsv

@@ -0,0 +1 @@
+<oov>	</oov>

fun_text_processing/inverse_text_normalization/ja/data/char/punctuations_zh.tsv

@@ -0,0 +1,72 @@
+！
+？
+｡
+。
+＂
+＃
+＄
+％
+＆
+＇
+（
+）
+＊
+＋
+，
+－
+／
+：
+；
+＜
+＝
+＞
+＠
+［
+＼
+］
+＾
+＿
+｀
+｛
+｜
+｝
+～
+｟
+｠
+｢
+｣
+､
+、
+〃
+》
+「
+」
+『
+』
+【
+】
+〔
+〕
+〖
+〗
+〘
+〙
+〚
+〛
+〜
+〝
+〞
+〟
+〰
+–
+—
+‘
+’
+‛
+“
+”
+„
+‟
+…
+‧
+﹏

fun_text_processing/inverse_text_normalization/ja/data/currency.tsv

@@ -0,0 +1,35 @@
+$	ドル
+$	米ドル
+$	米ドル
+£	英国ポンド
+€	ユーロ
+₩	勝った
+nzd	ニュージーランドドル
+rs	ルピー
+chf	スイスフラン
+dkk	デンマーククローネ
+fim	フィンランドのマルッカ
+aed	アラブ首長国連邦ディルハム
+¥	円
+czk	チェココルナ
+mro	モーリタニアのウギア
+pkr	パキスタン・ルピー
+crc	コスタリカのコロン
+hk$	香港ドル
+npr	ネパールルピー
+awg	アルバフロリン
+nok	ノルウェークローネ
+tzs	タンザニアシリング
+sek	スウェーデンクローナ
+cyp	キプロスポンド
+r	本物
+sar	サウジアラビアリヤル
+cve	カーボベルデ エスクード
+rsd	セルビアディナール
+dm	ドイツマーク
+shp	セントヘレナ・ポンド
+php	フィリピンペソ
+cad	カナダドル
+ssp	南スーダンポンド
+scr	セーシェル・ルピー
+mvr	モルディブ・ルフィア

fun_text_processing/inverse_text_normalization/ja/data/electronic/domain.tsv

@@ -0,0 +1,11 @@
+com
+uk
+fr
+net
+br
+in
+ru
+de
+it
+ai
+ja

fun_text_processing/inverse_text_normalization/ja/data/electronic/server_name.tsv

@@ -0,0 +1,17 @@
+g mail	gmail
+gmail
+n vidia	nvidia
+nvidia
+outlook
+hotmail
+yahoo
+aol
+gmx
+msn
+live
+yandex
+orange
+wanadoo
+web
+comcast
+google

fun_text_processing/inverse_text_normalization/ja/data/electronic/symbols.tsv

@@ -0,0 +1,22 @@
+.	点
+-	ダッシュ
+-	ハイフン
+_	下線
+!	エクスクラメーション・マーク
+#	番号記号
+$	ドル記号
+%	パーセント記号
+&	アンパサンド
+'	見積もり
+*	アスタリスク
++	プラス
+/	スラッシュ
+=	等号
+?	疑問符
+^	サーカムフレックス
+`	右一重引用符
+{	左ブレース
+|	縦棒
+}	右ブレース
+~	チルダ
+,	コンマ

fun_text_processing/inverse_text_normalization/ja/data/magnitudes.tsv

@@ -0,0 +1,4 @@
+千	k
+百万	m
+十億	b
+兆	t

fun_text_processing/inverse_text_normalization/ja/data/measurements.tsv

@@ -0,0 +1,109 @@
+華氏	f
+摂氏	c
+キロメートル	km
+メートル	m
+センチメートル	cm
+ミリメートル	mm
+ヘクタール	ha
+マイル	mi
+平方メートル	m²
+平方キロメートル	km²
+足	ft
+パーセント	%
+ヘルツ	hz
+キロワット	kw
+馬力	hp
+ミリグラム	mg
+キログラム	kg
+キロ	kg
+ギガヘルツ	ghz
+キロヘルツ	khz
+メガヘルツ	mhz
+ボルト	v
+時	h
+メガクーロン	mc
+秒	s
+ナノメートル	nm
+毎分回転数	rpm
+分	min
+ミリアンペア	mA
+パーセント	%
+キロワット時	kwh
+立方メートル	m³
+時速マイル	mph
+テラワット	tw
+ミリボルト	mv
+メガワット	mw
+マイクロメータ	μm
+インチ	"
+テラバイト	tb
+c c	cc
+グラム	g
+ダルトン	da
+雰囲気	atm
+オーム	ω
+デシベル	db
+ペタ秒	ps
+オンス	oz
+ヘクトリットル	hl
+マイクログラム	μg
+ペタグラム	pg
+ギガバイト	gb
+キロビット	kb
+電子ボルト	ev
+メガバイト	mb
+キロバイト	kb
+キロビット/秒	kbps
+毎秒メガビット	mbps
+結石	st
+キロリットル	kl
+テラジュール	tj
+キロボルト	kv
+メガボルト	mv
+キロニュートン	kn
+メガメーター	mm
+天文単位	au
+ヤード	yd
+ラジアン	rad
+ルーメン	lm
+ヘクト秒	hs
+モル	mol
+ギガパスカル	gpa
+ミリリットル	ml
+ギガワット	gw
+メガアンペア	ma
+結び目	kt
+キログラム力	kgf
+ナノグラム	ng
+ナノ秒	ns
+メガシーメンス	ms
+バー	bar
+ギガリットル	gl
+マイクロ秒	μs
+デシアンペア	da
+パスカル	pa
+デシ秒	ds
+ミリ秒	ms
+デシメートル	dm
+立方デシメートル	dm³
+原子質量単位	amu
+メガビット	mb
+メガファラッド	mf
+ベクレル	bq
+ペタビット	pb
+平方ミリメートル	mm²
+平方センチメートル	cm²
+平方マイル	sq mi
+平方フィート	sq ft
+キロパスカル	kpa
+カンデラ	cd
+テラリットル	tl
+メガ秒	ms
+メガパスカル	mpa
+ペタメーター	pm
+ペタバイト	pb
+ギガワットアワー	gwh
+キロカロリー	kcal
+グレー	gy
+シーベルト	sv
+ハンドレッド	cwt

fun_text_processing/inverse_text_normalization/ja/data/money/currency_code.tsv

@@ -0,0 +1,117 @@
+ALL	阿尔巴尼亚列克
+AFN	阿富汗 阿富汗尼
+ARS	阿根廷比索
+AWG	阿鲁巴盾
+AUD	澳元
+AZN	阿塞拜疆马纳特
+BSD	巴哈马元
+BBD	巴巴多斯元
+BYN	白俄罗斯卢布
+BZD	伯利兹元
+BMD	百慕大元
+BOB	玻利维亚玻利维亚诺
+BAM	波斯尼亚和黑塞哥维那可兑换马克
+BWP	博茨瓦纳普拉
+BGN	保加利亚列弗
+BRL	巴西雷亚尔
+BND	文莱达鲁萨兰国元
+KHR	柬埔寨瑞尔
+CAD	加元
+KYD	开曼群岛元
+CLP	智利比索
+CNY	人民币
+COP	哥伦比亚比索
+CRC	哥斯达黎加科隆
+HRK	克罗地亚库纳
+CUP	古巴比索
+CZK	捷克克朗
+DKK	丹麦克朗
+DOP	多米尼加共和国比索
+XCD	东加勒比元
+EGP	埃及镑
+SVC	萨尔瓦多科隆
+EUR	欧元成员国
+FKP	福克兰群岛（马尔维纳斯）镑
+FJD	斐济元
+GHS	加纳塞地
+GIP	直布罗陀镑
+GTQ	危地马拉格查尔
+GGP	根西岛镑
+GYD	圭亚那元
+HNL	洪都拉斯伦皮拉
+HKD	港元
+HUF	匈牙利福林
+ISK	冰岛克朗
+INR	印度卢比
+IDR	印尼盾
+IRR	伊朗里亚尔
+IMP	马恩岛英镑
+ILS	以色列谢克尔
+JMD	牙买加元
+JPY	日元
+JEP	泽西镑
+KZT	哈萨克斯坦腾格
+KPW	朝鲜园
+KRW	韩元
+KGS	吉尔吉斯斯坦索姆
+LAK	老挝基普
+LBP	黎巴嫩镑
+LRD	利比里亚元
+MKD	马其顿代纳尔
+MYR	马来西亚令吉
+MUR	毛里求斯卢比
+MXN	墨西哥比索
+MNT	蒙古图格里克
+MNT	摩洛哥迪拉姆
+MZN	莫桑比克梅蒂卡尔
+NAD	纳米比亚元
+NPR	尼泊尔卢比
+ANG	荷属安的列斯盾
+NZD	新西兰元
+NIO	尼加拉瓜科尔多瓦
+NGN	尼日利亚奈拉
+NOK	挪威克朗
+OMR	阿曼里亚尔
+PKR	巴基斯坦卢比
+PAB	巴拿马巴尔博亚
+PYG	巴拉圭瓜拉尼
+PEN	秘鲁索尔
+PHP	菲律宾比索
+PLN	波兰兹罗提
+QAR	卡塔尔里亚尔
+RON	罗马尼亚列伊
+RUB	俄罗斯卢布
+SHP	圣赫勒拿镑
+SAR	沙特阿拉伯里亚尔
+RSD	塞尔维亚第纳尔
+SCR	塞舌尔卢比
+SGD	新加坡元
+SBD	所罗门群岛元
+SOS	索马里先令
+KRW	韩元
+ZAR	南非兰特
+LKR	斯里兰卡卢比
+SEK	瑞典克朗
+CHF	瑞士法郎
+SRD	苏里南元
+SYP	叙利亚镑
+TWD	新台币
+THB	泰铢
+TTD	特立尼达和多巴哥元
+TRY	土耳其里拉
+TVD	图瓦卢元
+UAH	乌克兰格里夫纳
+AED	阿联酋迪拉姆
+GBP	英镑
+USD	美元
+UYU	乌拉圭比索
+UZS	乌兹别克斯坦索姆
+VEF	委内瑞拉玻利瓦尔
+VND	越南东
+YER	也门里亚尔
+ZWD	津巴布韦元
+J￥	日元
+JPY￥	日元
+HK＄	港元
+A$	澳元
+CAD$	加元

fun_text_processing/inverse_text_normalization/ja/data/money/currency_symbol.tsv

@@ -0,0 +1,63 @@
+Lek	阿尔巴尼亚列克
+ƒ	阿鲁巴盾
+Br	白俄罗斯卢布
+BZ$	伯利兹元
+$b	玻利维亚玻利维亚诺
+KM	波斯尼亚和黑塞哥维那可兑换马克
+P	博茨瓦纳普拉
+лв	保加利亚列弗
+R$	巴西雷亚尔
+៛	柬埔寨瑞尔
+¥	人民币
+₡	哥斯达黎加科隆
+kn	克罗地亚库纳
+₱	古巴比索
+Kč	捷克克朗
+kr	丹麦克朗
+RD$	多米尼加共和国比索
+€	欧元
+¢	加纳塞地
+Q	危地马拉格查尔
+L	洪都拉斯伦皮拉
+Ft	匈牙利福林
+₹	印度卢比
+Rp	印尼盾
+£	英镑
+￡	英镑
+₪	以色列谢克尔
+J$	牙买加元
+лв	哈萨克斯坦腾格
+₩	朝鲜园
+лв	吉尔吉斯斯坦索姆
+₭	老挝基普
+ден	马其顿代纳尔
+RM	马来西亚令吉
+₨	毛里求斯卢比
+₮	蒙古图格里克
+MT	莫桑比克梅蒂卡尔
+C$	尼加拉瓜科尔多瓦
+₦	尼日利亚奈拉
+₨	巴基斯坦卢比
+B/.	巴拿马巴尔博亚
+Gs	巴拉圭瓜拉尼
+S/.	秘鲁索尔
+₱	菲律宾比索
+zł	波兰兹罗提
+lei	罗马尼亚列伊
+₽	卢布
+Дин.	塞尔维亚第纳尔
+S	索马里先令
+R	南非兰特
+CHF	瑞士法郎
+NT$	新台币
+฿	泰铢
+TT$	特立尼达和多巴哥元
+₺	土耳其里拉
+₴	乌克兰格里夫纳
+$	美元
+$U	乌拉圭比索
+лв	乌兹别克斯坦索姆
+Bs	委内瑞拉玻利瓦尔
+₫	越南东
+Z$	津巴布韦元
+￥	人民币

fun_text_processing/inverse_text_normalization/ja/data/months.tsv

@@ -0,0 +1,12 @@
+一月
+二月
+三月
+四月
+五月
+六月
+七月
+八月
+九月
+十月
+十一月
+十二月

fun_text_processing/inverse_text_normalization/ja/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+一	1
+二	2
+三	3
+四	4
+五	5
+六	6
+七	7
+八	8
+九	9

fun_text_processing/inverse_text_normalization/ja/data/numbers/dot.tsv

@@ -0,0 +1 @@
+点	.

fun_text_processing/inverse_text_normalization/ja/data/numbers/hundred.tsv

@@ -0,0 +1 @@
+百

fun_text_processing/inverse_text_normalization/ja/data/numbers/hundred_digit.tsv

@@ -0,0 +1,9 @@
+百一	101
+百二	102
+百三	103
+百四	104
+百五	105
+百六	106
+百七	107
+百八	108
+百九	109

fun_text_processing/inverse_text_normalization/ja/data/numbers/sign.tsv

@@ -0,0 +1,4 @@
+正	+
+プラス	+
+プラスマイナス	±
+マイナス	-

fun_text_processing/inverse_text_normalization/ja/data/numbers/teen.tsv

@@ -0,0 +1,10 @@
+十	10
+十一	11
+十二	12
+十三	13
+十四	14
+十五	15
+十六	16
+十七	17
+十八	18
+十九	19

fun_text_processing/inverse_text_normalization/ja/data/numbers/thousands.tsv

@@ -0,0 +1,9 @@
+千
+万
+亿
+兆
+京
+垓
+秭
+穰
+沟

fun_text_processing/inverse_text_normalization/ja/data/numbers/ties.tsv

@@ -0,0 +1,8 @@
+二十	2
+三十	3
+四十	4
+五十	5
+六十	6
+七十	7
+八十	8
+九十	9

fun_text_processing/inverse_text_normalization/ja/data/numbers/zero.tsv

@@ -0,0 +1,2 @@
+〇	0
+零	0

fun_text_processing/inverse_text_normalization/ja/data/ordinals/digit.tsv

@@ -0,0 +1,9 @@
+第一	1
+第二	2
+第三	3
+第四	4
+第五	5
+第六	6
+第七	7
+第八	8
+第九	9

fun_text_processing/inverse_text_normalization/ja/data/ordinals/teen.tsv

@@ -0,0 +1,10 @@
+第十	10
+第十一	11
+第十二	12
+第十三	13
+第十四	14
+第十五	15
+第十六	16
+第十七	17
+第十八	18
+第十九	19

fun_text_processing/inverse_text_normalization/ja/data/ordinals/ties.tsv

@@ -0,0 +1,9 @@
+第二十	2
+第三十	3
+第四十	4
+第五十	5
+第六十	6
+第七十	7
+第八十	8
+第九十	9
+第百	10

fun_text_processing/inverse_text_normalization/ja/data/time/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/inverse_text_normalization/ja/data/time/minute_to.tsv

@@ -0,0 +1,59 @@
+1	59
+2	58
+3	57
+4	56
+5	55
+6	54
+7	53
+8	52
+9	51
+10	50
+11	49
+12	48
+13	47
+14	46
+15	45
+16	44
+17	43
+18	42
+19	41
+20	40
+21	39
+22	38
+23	37
+24	36
+25	35
+26	34
+27	33
+28	32
+29	31
+30	30
+31	29
+32	28
+33	27
+34	26
+35	25
+36	24
+37	23
+38	22
+39	21
+40	20
+41	19
+42	18
+43	17
+44	16
+45	15
+46	14
+47	13
+48	12
+49	11
+50	10
+51	9
+52	8
+53	7
+54	6
+55	5
+56	4
+57	3
+58	2
+59	1

fun_text_processing/inverse_text_normalization/ja/data/time/time_suffix.tsv

@@ -0,0 +1,8 @@
+p m	p.m.
+pm	p.m.
+p.m.
+p.m	p.m.
+am	a.m.
+a.m.
+a.m	a.m.
+a m	a.m.

fun_text_processing/inverse_text_normalization/ja/data/time/time_zone.tsv

@@ -0,0 +1,7 @@
+cst	c s t
+cet	c e t
+pst	p s t
+est	e s t
+pt	p t
+et	e t
+gmt	g m t

fun_text_processing/inverse_text_normalization/ja/data/time/to_hour.tsv

@@ -0,0 +1,12 @@
+一	12
+二	1
+三	2
+四	3
+五	4
+六	5
+七	6
+八	7
+九	8
+十	9
+十一	10
+十二	11

fun_text_processing/inverse_text_normalization/ja/graph_utils.py

@@ -0,0 +1,143 @@
+import os
+import string
+from pathlib import Path
+from typing import Dict
+
+import pynini
+from pynini import Far
+from pynini.export import export
+from pynini.lib import byte, pynutil, utf8
+
+DAMO_CHAR = utf8.VALID_UTF8_CHAR
+
+DAMO_DIGIT = byte.DIGIT
+DAMO_LOWER = pynini.union(*string.ascii_lowercase).optimize()
+DAMO_UPPER = pynini.union(*string.ascii_uppercase).optimize()
+DAMO_ALPHA = pynini.union(DAMO_LOWER, DAMO_UPPER).optimize()
+DAMO_ALNUM = pynini.union(DAMO_DIGIT, DAMO_ALPHA).optimize()
+DAMO_HEX = pynini.union(*string.hexdigits).optimize()
+DAMO_NON_BREAKING_SPACE = "\u00A0"
+DAMO_SPACE = " "
+DAMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize()
+DAMO_NOT_SPACE = pynini.difference(DAMO_CHAR, DAMO_WHITE_SPACE).optimize()
+DAMO_NOT_QUOTE = pynini.difference(DAMO_CHAR, r'"').optimize()
+
+DAMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize()
+DAMO_GRAPH = pynini.union(DAMO_ALNUM, DAMO_PUNCT).optimize()
+
+DAMO_SIGMA = pynini.closure(DAMO_CHAR)
+
+delete_space = pynutil.delete(pynini.closure(DAMO_WHITE_SPACE))
+insert_space = pynutil.insert(" ")
+delete_extra_space = pynini.cross(pynini.closure(DAMO_WHITE_SPACE, 1), " ")
+
+# French frequently compounds numbers with hyphen.
+delete_hyphen = pynutil.delete(pynini.closure("-", 0, 1))
+insert_hyphen = pynutil.insert("-")
+
+TO_LOWER = pynini.union(
+    *[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]
+)
+TO_UPPER = pynini.invert(TO_LOWER)
+
+
+def generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]):
+    """
+    Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name.
+
+    Args:
+        file_name: exported file name
+        graphs: Mapping of a rule name and Pynini WFST graph to be exported
+    """
+    exporter = export.Exporter(file_name)
+    for rule, graph in graphs.items():
+        exporter[rule] = graph.optimize()
+    exporter.close()
+    print(f"Created {file_name}")
+
+
+def convert_space(fst) -> "pynini.FstLike":
+    """
+    Converts space to nonbreaking space.
+    Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty"
+    This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it.
+
+    Args:
+        fst: input fst
+
+    Returns output fst where breaking spaces are converted to non breaking spaces
+    """
+    return fst @ pynini.cdrewrite(
+        pynini.cross(DAMO_SPACE, DAMO_NON_BREAKING_SPACE), "", "", DAMO_SIGMA
+    )
+
+
+class GraphFst:
+    """
+    Base class for all grammar fsts.
+
+    Args:
+        name: name of grammar class
+        kind: either 'classify' or 'verbalize'
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, name: str, kind: str, deterministic: bool = True):
+        self.name = name
+        self.kind = kind
+        self._fst = None
+        self.deterministic = deterministic
+
+        self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far")
+        if self.far_exist():
+            self._fst = Far(
+                self.far_path, mode="r", arc_type="standard", far_type="default"
+            ).get_fst()
+
+    def far_exist(self) -> bool:
+        """
+        Returns true if FAR can be loaded
+        """
+        return self.far_path.exists()
+
+    @property
+    def fst(self) -> "pynini.FstLike":
+        return self._fst
+
+    @fst.setter
+    def fst(self, fst):
+        self._fst = fst
+
+    def add_tokens(self, fst) -> "pynini.FstLike":
+        """
+        Wraps class name around to given fst
+
+        Args:
+            fst: input fst
+
+        Returns:
+            Fst: fst
+        """
+        return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }")
+
+    def delete_tokens(self, fst) -> "pynini.FstLike":
+        """
+        Deletes class name wrap around output of given fst
+
+        Args:
+            fst: input fst
+
+        Returns:
+            Fst: fst
+        """
+        res = (
+            pynutil.delete(f"{self.name}")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + fst
+            + delete_space
+            + pynutil.delete("}")
+        )
+        return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)

fun_text_processing/inverse_text_normalization/ja/ja_itn_test_expected.txt

@@ -0,0 +1,20 @@
+ps3，ps4，ATM，十数か二十，幸四郎，白雪姫と七人のこびと，七夕，千夜一夜，三百六十行，十五の月，冷凍三足，三十年河東，三十年河西，千年後，五月天，安倍晋三，小泉純一郎，山本五十六
+第1、第2、第3、第4、第5、第6、第7、第8、第9、第10、第11、第12、第13、第14、第15、第16、第17、第18、第19、第20、20、第24、24、第56、第100、第101
+で、わたしに入った原稿料からの1/5をあなたに渡すということなのだが。堅あげ愛は深く「体の1/3、1/4、1/2は堅あげになっています。
+06年4月～19年12月に日本で承認された医療機器は529件、そのうち小児用はわずか12件。
+１９９３年に誕生した同商品にちなみ、約３０年前、２０歳の頃の幸四郎の写真を公開。
+奮闘する自身の姿を収めたドキュメンタリー映画「１０億円稼ぐ」（１１月２０日東京で公開）をＰＲするため東京都内で学生向けセミナーを開催。ゲストの「ＴＨＥ　虎舞竜」の高橋ジョージが、作詞作曲した「ロード」で印税１６億円を稼いだと明かし、会場はどよめいた。
+発売の曲は２２０万枚を売り、高橋は印税をなんと２年で使い切ったそうだが「皆さんがカラオケで１回歌うと７円入りますし、今も年間１２００万円ぐらい、黙ってても入ってきます」。何でもないようなことが幸せだったと思うと歌った曲は、とんでもない印税を生み出していて、テリーも降参。TikTokerゆりにゃ体重39kg“15kg減量”林みなほアナ。
+15日に第64回日本レコード大賞（主催日本作曲家協会）の各賞が発表されたが、これまで12年連続で優秀作品賞を受賞していたAK47はリストに入らず、記録が途絶えていた。
+開票が続くアメリカの中間選挙で複数のアメリカ主要メディアは11月16日、野党・共和党が定数435の連邦議会の下院で218議席を獲得し4年ぶりに多数派を奪還したと報じた。
+これで2021月1月に起こった連邦議会議事堂襲撃事件に関する下院の特別調査委員会は解散させられることになりそうだ。
+「昨年は新谷が本調子じゃない中で、それでもあれだけ走ってくれて助かりました」と現在の好調ぶりが伝えられていた新谷選手は11月13日に行われた東日本女子駅伝でアンカーを務め、10キロを31分08秒の区間賞で東京の逆転優勝の立役者となりました。
+フルマラソンのベストタイムは2時間40分34秒。
+久留米市では今朝一時間に92点5mmの猛烈な雨を観測一時間当たりの雨量としては1977年の統計開始以来最大です
+治療を必要とする動脈管開存症のある赤ちゃんは、1500グラム未満では約30%、1000グラム未満では約50%とされる。薬で血管が閉じることも多いが、彩葉ちゃんは薬では血管が閉じなかった。668km。
+100|150|123|111|123|0|0|1|2|3|10|11|12|13|15|19|20|50|99|1200|1234|1011|1997|01234567890|123456789|102|324|100|200|1000|5000|1万|50万|1000000|4000万|6億|10億|9兆
+1679
+10086
+08613794568
+ソーシャルディス0，0，1，2，3，120，324，100，200，1000，1500，1679，5000，10000，1000000，10000000
+タンスにも遊び心326が隠されていましたが

fun_text_processing/inverse_text_normalization/ja/ja_itn_test_input.txt

@@ -0,0 +1,20 @@
+ps三，ps四，ATM，十数か二十，幸四郎，白雪姫と七人のこびと，七夕，千夜一夜，三百六十行，十五の月，冷凍三足，三十年河東，三十年河西，千年後，五月天，安倍晋三，小泉純一郎，山本五十六
+第一、第二、第三、第四、第五、第六、第七、第八、第九、第十、第十一、第十二、第十三、第十四、第十五、第十六、第十七、第十八、第十九、第二十、二十、第二十四、二十四、第五十六、第百、第百一
+で、わたしに入った原稿料からの五分の一、五分の三をあなたに渡すということなのだが。堅あげ愛は深く「体の三分の一、四分の一、二分の一は堅あげになっています。
+零六年四月～一九年十二月に日本で承認された医療機器は五百二十九件、そのうち小児用はわずか十二件。
+一九九三年に誕生した同商品にちなみ、約三十年前、二十歳の頃の幸四郎の写真を公開。
+奮闘する自身の姿を収めたドキュメンタリー映画「十億円稼ぐ」（十一月二十日東京で公開）をＰＲするため東京都内で学生向けセミナーを開催。ゲストの「ＴＨＥ　虎舞竜」の高橋ジョージが、作詞作曲した「ロード」で印税十六億円を稼いだと明かし、会場はどよめいた。
+発売の曲は二百二十万枚を売り、高橋は印税をなんと二年で使い切ったそうだが「皆さんがカラオケで一回歌うと七円入りますし、今も年間一千二百万円ぐらい、黙ってても入ってきます」。何でもないようなことが幸せだったと思うと歌った曲は、とんでもない印税を生み出していて、テリーも降参。TikTokerゆりにゃ体重三十九キロ“十五キロ減量”林みなほアナ。
+十五日に第六十四回日本レコード大賞（主催日本作曲家協会）の各賞が発表されたが、これまで十二年連続で優秀作品賞を受賞していたAK四十七はリストに入らず、記録が途絶えていた。
+開票が続くアメリカの中間選挙で複数のアメリカ主要メディアは十一月十六日、野党・共和党が定数四百三十五の連邦議会の下院で二百十八議席を獲得し四年ぶりに多数派を奪還したと報じた。
+これで二〇二一月一月に起こった連邦議会議事堂襲撃事件に関する下院の特別調査委員会は解散させられることになりそうだ。
+「昨年は新谷が本調子じゃない中で、それでもあれだけ走ってくれて助かりました」と現在の好調ぶりが伝えられていた新谷選手は十一月十三日に行われた東日本女子駅伝でアンカーを務め、十キロを三十一分〇八秒の区間賞で東京の逆転優勝の立役者となりました。
+フルマラソンのベストタイムは二時間四十分三十四秒。
+久留米市では今朝一時間に九十二点五ミリの猛烈な雨を観測一時間当たりの雨量としては千九百七十七年の統計開始以来最大です
+治療を必要とする動脈管開存症のある赤ちゃんは、一千五グラム未満では約三十パーセント、一千グラム未満では約五十パーセントとされる。薬で血管が閉じることも多いが、彩葉ちゃんは薬では血管が閉じなかった。六百六十八キロメートル。
+百|百五十|百二十三|百十一|一百二十三|〇|零|一|二|三|十|十一|十二|十三|十五|十九|二十|五十|九十九|一千二|一千二百三十四|千十一|千九百九十七|〇一二三四五六七八九零|一二三四五六七八九|百二|三百二十四|一百|二百|一千|五千|一万|五十万|一百万|四千万|六億|十億|九兆
+一千六百七十九
+一〇〇八六
+〇八六一三七九四五六八
+ソーシャルディス〇，零，一，二，三，百二，三百二十四，一百，二百，一千，一千五百，一千六百七十九，五千，一万，一百万，一千万
+タンスにも遊び心三百二十六が隠されていましたが

fun_text_processing/inverse_text_normalization/ja/ja_unit_test.tsv

@@ -0,0 +1,20 @@
+ps三，ps四，ATM，十数か二十，幸四郎，白雪姫と七人のこびと，七夕，千夜一夜，三百六十行，十五の月，冷凍三足，三十年河東，三十年河西，千年後，五月天，安倍晋三，小泉純一郎，山本五十六	ps3，ps4，ATM，十数か二十，幸四郎，白雪姫と七人のこびと，七夕，千夜一夜，三百六十行，十五の月，冷凍三足，三十年河東，三十年河西，千年後，五月天，安倍晋三，小泉純一郎，山本五十六
+第一、第二、第三、第四、第五、第六、第七、第八、第九、第十、第十一、第十二、第十三、第十四、第十五、第十六、第十七、第十八、第十九、第二十、二十、第二十四、二十四、第五十六、第百、第百一	第1、第2、第3、第4、第5、第6、第7、第8、第9、第10、第11、第12、第13、第14、第15、第16、第17、第18、第19、第20、20、第24、24、第56、第100、第101
+で、わたしに入った原稿料からの五分の一、五分の三をあなたに渡すということなのだが。堅あげ愛は深く「体の三分の一、四分の一、二分の一は堅あげになっています。	で、わたしに入った原稿料からの1/5をあなたに渡すということなのだが。堅あげ愛は深く「体の1/3、1/4、1/2は堅あげになっています。
+零六年四月～一九年十二月に日本で承認された医療機器は五百二十九件、そのうち小児用はわずか十二件。	06年4月～19年12月に日本で承認された医療機器は529件、そのうち小児用はわずか12件。
+一九九三年に誕生した同商品にちなみ、約三十年前、二十歳の頃の幸四郎の写真を公開。	１９９３年に誕生した同商品にちなみ、約３０年前、２０歳の頃の幸四郎の写真を公開。
+奮闘する自身の姿を収めたドキュメンタリー映画「十億円稼ぐ」（十一月二十日東京で公開）をＰＲするため東京都内で学生向けセミナーを開催。ゲストの「ＴＨＥ　虎舞竜」の高橋ジョージが、作詞作曲した「ロード」で印税十六億円を稼いだと明かし、会場はどよめいた。	奮闘する自身の姿を収めたドキュメンタリー映画「１０億円稼ぐ」（１１月２０日東京で公開）をＰＲするため東京都内で学生向けセミナーを開催。ゲストの「ＴＨＥ　虎舞竜」の高橋ジョージが、作詞作曲した「ロード」で印税１６億円を稼いだと明かし、会場はどよめいた。
+発売の曲は二百二十万枚を売り、高橋は印税をなんと二年で使い切ったそうだが「皆さんがカラオケで一回歌うと七円入りますし、今も年間一千二百万円ぐらい、黙ってても入ってきます」。何でもないようなことが幸せだったと思うと歌った曲は、とんでもない印税を生み出していて、テリーも降参。TikTokerゆりにゃ体重三十九キロ“十五キロ減量”林みなほアナ。	発売の曲は２２０万枚を売り、高橋は印税をなんと２年で使い切ったそうだが「皆さんがカラオケで１回歌うと７円入りますし、今も年間１２００万円ぐらい、黙ってても入ってきます」。何でもないようなことが幸せだったと思うと歌った曲は、とんでもない印税を生み出していて、テリーも降参。TikTokerゆりにゃ体重39kg“15kg減量”林みなほアナ。
+十五日に第六十四回日本レコード大賞（主催日本作曲家協会）の各賞が発表されたが、これまで十二年連続で優秀作品賞を受賞していたAK四十七はリストに入らず、記録が途絶えていた。	15日に第64回日本レコード大賞（主催日本作曲家協会）の各賞が発表されたが、これまで12年連続で優秀作品賞を受賞していたAK47はリストに入らず、記録が途絶えていた。
+開票が続くアメリカの中間選挙で複数のアメリカ主要メディアは十一月十六日、野党・共和党が定数四百三十五の連邦議会の下院で二百十八議席を獲得し四年ぶりに多数派を奪還したと報じた。	開票が続くアメリカの中間選挙で複数のアメリカ主要メディアは11月16日、野党・共和党が定数435の連邦議会の下院で218議席を獲得し4年ぶりに多数派を奪還したと報じた。
+これで二〇二一月一月に起こった連邦議会議事堂襲撃事件に関する下院の特別調査委員会は解散させられることになりそうだ。	これで2021月1月に起こった連邦議会議事堂襲撃事件に関する下院の特別調査委員会は解散させられることになりそうだ。
+「昨年は新谷が本調子じゃない中で、それでもあれだけ走ってくれて助かりました」と現在の好調ぶりが伝えられていた新谷選手は十一月十三日に行われた東日本女子駅伝でアンカーを務め、十キロを三十一分〇八秒の区間賞で東京の逆転優勝の立役者となりました。	「昨年は新谷が本調子じゃない中で、それでもあれだけ走ってくれて助かりました」と現在の好調ぶりが伝えられていた新谷選手は11月13日に行われた東日本女子駅伝でアンカーを務め、10キロを31分08秒の区間賞で東京の逆転優勝の立役者となりました。
+フルマラソンのベストタイムは二時間四十分三十四秒。	フルマラソンのベストタイムは2時間40分34秒。
+久留米市では今朝一時間に九十二点五ミリの猛烈な雨を観測一時間当たりの雨量としては千九百七十七年の統計開始以来最大です	久留米市では今朝一時間に92点5mmの猛烈な雨を観測一時間当たりの雨量としては1977年の統計開始以来最大です
+治療を必要とする動脈管開存症のある赤ちゃんは、一千五グラム未満では約三十パーセント、一千グラム未満では約五十パーセントとされる。薬で血管が閉じることも多いが、彩葉ちゃんは薬では血管が閉じなかった。六百六十八キロメートル。	治療を必要とする動脈管開存症のある赤ちゃんは、1500グラム未満では約30%、1000グラム未満では約50%とされる。薬で血管が閉じることも多いが、彩葉ちゃんは薬では血管が閉じなかった。668km。
+百|百五十|百二十三|百十一|一百二十三|〇|零|一|二|三|十|十一|十二|十三|十五|十九|二十|五十|九十九|一千二|一千二百三十四|千十一|千九百九十七|〇一二三四五六七八九零|一二三四五六七八九|百二|三百二十四|一百|二百|一千|五千|一万|五十万|一百万|四千万|六億|十億|九兆	100|150|123|111|123|0|0|1|2|3|10|11|12|13|15|19|20|50|99|1200|1234|1011|1997|01234567890|123456789|102|324|100|200|1000|5000|1万|50万|1000000|4000万|6億|10億|9兆
+一千六百七十九	1679
+一〇〇八六	10086
+〇八六一三七九四五六八	08613794568
+ソーシャルディス〇，零，一，二，三，百二，三百二十四，一百，二百，一千，一千五百，一千六百七十九，五千，一万，一百万，一千万	ソーシャルディス0，0，1，2，3，120，324，100，200，1000，1500，1679，5000，10000，1000000，10000000
+タンスにも遊び心三百二十六が隠されていましたが	タンスにも遊び心326が隠されていましたが

fun_text_processing/inverse_text_normalization/ja/taggers/cardinal.py

@@ -0,0 +1,242 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import pynini
+from pynini import accep, cross, string_file, union
+from pynini.lib.pynutil import delete, insert, add_weight
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    DAMO_SPACE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+import unicodedata
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for classifying cardinals
+        e.g. minus twenty three -> cardinal { integer: "23" negative: "-" } }
+    Numbers below thirteen are not converted.
+    """
+
+    def __init__(self, enable_standalone_number: bool = True, enable_0_to_9: bool = True):
+        super().__init__(name="cardinal", kind="classify")
+        self.enable_standalone_number = enable_standalone_number
+        self.enable_0_to_9 = enable_0_to_9
+        zero = string_file(get_abs_path("data/numbers/zero.tsv"))
+        digit = string_file(get_abs_path("data/numbers/digit.tsv"))
+        hundred_digit = string_file(get_abs_path("data/numbers/hundred_digit.tsv"))
+        sign = string_file(get_abs_path("data/numbers/sign.tsv"))
+        dot = string_file(get_abs_path("data/numbers/dot.tsv"))
+        ties = string_file(get_abs_path("data/numbers/ties.tsv"))
+        graph_teen = string_file(get_abs_path("data/numbers/teen.tsv"))
+
+        addzero = insert("0")
+        digits = zero | digit  # 0 ~ 9
+        teen = graph_teen
+        teen |= cross("十", "1") + (digit | addzero)
+        tens = ties + addzero | (ties + (digit | addzero))
+
+        hundred = (
+            digit
+            + delete("百")
+            + (
+                tens
+                | teen
+                | add_weight(zero + digit, 0.1)
+                | add_weight(digit + addzero, 0.5)
+                | add_weight(addzero**2, 1.0)
+            )
+        )
+        hundred |= cross("百", "1") + (
+            tens
+            | teen
+            | add_weight(zero + digit, 0.1)
+            | add_weight(digit + addzero, 0.5)
+            | add_weight(addzero**2, 1.0)
+        )
+        hundred |= hundred_digit
+
+        thousand = (
+            (hundred | teen | tens | digits)
+            + delete("千")
+            + (
+                hundred
+                | add_weight(zero + tens, 0.1)
+                | add_weight(addzero + zero + digit, 0.5)
+                | add_weight(digit + addzero**2, 0.8)
+                | add_weight(addzero**3, 1.0)
+            )
+        )
+        ten_thousand = (
+            (thousand | hundred | teen | tens | digits)
+            + delete("万")
+            + (
+                thousand
+                | add_weight(zero + hundred, 0.1)
+                | add_weight(addzero + zero + tens, 0.5)
+                | add_weight(addzero + addzero + zero + digit, 0.5)
+                | add_weight(digit + addzero**3, 0.8)
+                | add_weight(addzero**4, 1.0)
+            )
+        )
+
+        hundred_thousand = (
+            (ten_thousand | thousand | hundred | teen | tens | digits)
+            + delete("十万")
+            + (
+                ten_thousand
+                | add_weight(zero + thousand, 0.1)
+                | add_weight(addzero + zero + hundred, 0.5)
+                | add_weight(addzero + addzero + zero + tens, 0.5)
+                | add_weight(addzero**3 + zero + digit, 0.5)
+                | add_weight(digit + addzero**4, 0.8)
+                | add_weight(addzero**5, 1.0)
+            )
+        )
+
+        million = (
+            (hundred_thousand | ten_thousand | thousand | hundred | teen | tens | digits)
+            + delete("百万")
+            + (
+                hundred_thousand
+                | add_weight(zero + ten_thousand, 0.1)
+                | add_weight(addzero + zero + thousand, 0.5)
+                | add_weight(addzero + addzero + zero + hundred, 0.5)
+                | add_weight(addzero**3 + zero + tens, 0.5)
+                | add_weight(addzero**4 + zero + digit, 0.5)
+                | add_weight(digit + addzero**5, 0.8)
+                | add_weight(addzero**6, 1.0)
+            )
+        )
+        # 1亿
+        hundred_million = (
+            (million | hundred_thousand | ten_thousand | thousand | hundred | teen | tens | digits)
+            + delete("億")
+            + (
+                add_weight(zero + million, 0.1)
+                | add_weight(addzero + zero + hundred_thousand, 0.5)
+                | add_weight(addzero**2 + zero + ten_thousand, 0.5)
+                | add_weight(addzero**3 + zero + thousand, 0.5)
+                | add_weight(addzero**4 + hundred, 0.5)
+                | add_weight(addzero**5 + tens, 0.5)
+                | add_weight(addzero**6 + digit, 0.5)
+                | add_weight(digit + addzero**7, 0.8)
+                | add_weight(addzero**8, 1.0)
+            )
+        )
+        # 1兆
+        hundred_billion = (
+            (
+                hundred_million
+                | million
+                | hundred_thousand
+                | ten_thousand
+                | thousand
+                | hundred
+                | teen
+                | tens
+                | digits
+            )
+            + delete("兆")
+            + (
+                add_weight(addzero**3 + zero + hundred_million, 0.1)
+                | add_weight(addzero**4 + zero + million, 0.5)
+                | add_weight(addzero**5 + zero + hundred_thousand, 0.5)
+                | add_weight(addzero**6 + zero + ten_thousand, 0.5)
+                | add_weight(addzero**7 + zero + thousand, 0.5)
+                | add_weight(addzero**8 + hundred, 0.5)
+                | add_weight(addzero**9 + tens, 0.5)
+                | add_weight(addzero**10 + digit, 0.5)
+                | add_weight(digit + addzero**11, 0.8)
+                | add_weight(addzero**12, 1.0)
+            )
+        )
+        # 1.11, 1.01
+        number = (
+            digits | teen | tens | hundred | thousand | ten_thousand | hundred_thousand | million
+        )
+        # number = digits | teen | tens | hundred | thousand | ten_thousand | hundred_thousand | million | hundred_million | hundred_billion
+        # 兆/亿
+        number = (number + accep("兆") + delete("零").ques).ques + (
+            number + accep("億") + delete("零").ques
+        ).ques + number | (number + accep("兆") + delete("〇").ques).ques + (
+            number + accep("億") + delete("〇").ques
+        ).ques + number
+
+        number = sign.ques + number + (dot + digits.plus).ques
+        self.number = number.optimize()
+        self.digits = digits.optimize()
+
+        # cardinal string like 127.0.0.1, used in ID, IP, etc.
+        cardinal = digit.plus + (dot + digits.plus).plus
+        # float number like 1.11
+        cardinal |= number + dot + digits.plus
+        # cardinal string like 110 or 12306 or 13125617878, used in phone
+        cardinal |= digits**3 | digits**5 | digits**10 | digits**11 | digits**12
+        # cardinal string like 23
+        if self.enable_standalone_number:
+            if self.enable_0_to_9:
+                cardinal |= number
+            else:
+                number_two_plus = (
+                    (digits + digits.plus)
+                    | teen
+                    | tens
+                    | hundred
+                    | thousand
+                    | ten_thousand
+                    | hundred_thousand
+                    | million
+                    | hundred_million
+                    | hundred_billion
+                )
+                cardinal |= number_two_plus
+        labels_exception = [""]
+        graph_exception = pynini.union(*labels_exception)
+
+        self.graph_no_exception = cardinal
+        self.graph = (pynini.project(cardinal, "input") - graph_exception.arcsort()) @ cardinal
+
+        optional_minus_graph = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("マイナス", '"-"') + DAMO_SPACE, 0, 1
+        )
+
+        final_graph = (
+            optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
+        )
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()
+
+        # ########
+        graph_hundred = pynini.cross("百", "")
+        graph_a_hundred_digit_component = pynini.union(pynini.cross("百", "10") + digit)
+        graph_one_hundred_component = pynini.union(pynini.cross("百", "100"))
+        graph_hundred_ties_component = pynini.cross("百", "1") + pynini.union(
+            graph_teen | pynutil.insert("00"),
+            (ties | pynutil.insert("0")) + (digit | pynutil.insert("0")),
+        )
+        graph_hundred_component = pynini.union(digit + graph_hundred, pynutil.insert("0"))
+        graph_hundred_component += pynini.union(
+            graph_teen | pynutil.insert("00"),
+            (ties | pynutil.insert("0")) + (digit | pynutil.insert("0")),
+        )
+        graph_hundred_component = (
+            graph_hundred_component
+            | graph_a_hundred_digit_component
+            | graph_one_hundred_component
+            | graph_hundred_ties_component
+        )
+        #
+        graph_hundred_component_at_least_one_none_zero_digit = graph_hundred_component @ (
+            pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT)
+        )
+        self.graph_hundred_component_at_least_one_none_zero_digit = (
+            graph_hundred_component_at_least_one_none_zero_digit
+        )

fun_text_processing/inverse_text_normalization/ja/taggers/date.py

@@ -0,0 +1,159 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")).optimize()
+graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize()
+ties_graph = pynini.string_file(get_abs_path("data/numbers/ties.tsv")).optimize()
+
+
+def _get_month_graph():
+    """
+    Transducer for month, e.g. march -> march
+    """
+    month_graph = pynini.string_file(get_abs_path("data/months.tsv"))
+    return month_graph
+
+
+def _get_ties_graph():
+    """
+    Transducer for 20-99 e.g
+    twenty three -> 23
+    """
+    graph = ties_graph + (delete_space + graph_digit | pynutil.insert("0"))
+    return graph
+
+
+def _get_range_graph():
+    """
+    Transducer for decades (1**0s, 2**0s), centuries (2*00s, 1*00s), millennia (2000s)
+    """
+    graph_ties = _get_ties_graph()
+    graph = (graph_ties | graph_teen) + delete_space + pynini.cross("百", "00s")
+    graph |= pynini.cross("二", "2") + delete_space + pynini.cross("千", "000s")
+    graph |= (
+        (graph_ties | graph_teen)
+        + delete_space
+        + (pynini.closure(DAMO_ALPHA, 1) + (pynini.cross("ies", "y") | pynutil.delete("s")))
+        @ (graph_ties | pynini.cross("十", "10"))
+        + pynutil.insert("s")
+    )
+    graph @= pynini.union("1", "2") + DAMO_DIGIT + DAMO_DIGIT + DAMO_DIGIT + "s"
+    return graph
+
+
+def _get_year_graph():
+    """
+    Transducer for year, e.g. twenty twenty -> 2020
+    """
+
+    def _get_digits_graph():
+        zero = pynini.cross((pynini.accep("〇") | pynini.accep("零")), "0")
+        graph = zero + delete_space + graph_digit
+        graph.optimize()
+        return graph
+
+    def _get_thousands_graph():
+        graph_ties = _get_ties_graph()
+        graph_hundred_component = (
+            graph_digit + delete_space + pynutil.delete("百")
+        ) | pynutil.insert("0")
+        graph_hundred_component |= (pynini.cross("百", "1")) | pynutil.insert("0")
+        graph = (
+            graph_digit
+            + delete_space
+            + pynutil.delete("千")
+            + delete_space
+            + graph_hundred_component
+            + delete_space
+            + (graph_teen | graph_ties)
+        )
+        graph |= (
+            pynini.cross("千", "1")
+            + delete_space
+            + graph_hundred_component
+            + delete_space
+            + (graph_teen | graph_ties)
+        )
+
+        return graph
+
+    graph_ties = _get_ties_graph()
+    graph_digits = _get_digits_graph()
+    graph_thousands = _get_thousands_graph()
+    year_graph = (
+        # 20 19, 40 12, 2012 - assuming no limit on the year
+        (graph_teen + delete_space + (graph_ties | graph_digits | graph_teen))
+        | (graph_ties + delete_space + (graph_ties | graph_digits | graph_teen))
+        | graph_thousands
+    )
+    year_graph.optimize()
+    return year_graph
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for classifying date,
+        e.g. january fifth twenty twelve -> date { month: "january" day: "5" year: "2012" preserve_order: true }
+        e.g. the fifth of january twenty twelve -> date { day: "5" month: "january" year: "2012" preserve_order: true }
+        e.g. twenty twenty -> date { year: "2012" preserve_order: true }
+
+    Args:
+        ordinal: OrdinalFst
+    """
+
+    def __init__(self, ordinal: GraphFst):
+        super().__init__(name="date", kind="classify")
+
+        ordinal_graph = ordinal.graph
+        year_graph = _get_year_graph()
+        YEAR_WEIGHT = 0.001
+        year_graph = pynutil.add_weight(year_graph, YEAR_WEIGHT)
+        month_graph = _get_month_graph()
+
+        month_graph = pynutil.insert('month: "') + month_graph + pynutil.insert('"')
+
+        day_graph = (
+            pynutil.insert('day: "') + pynutil.add_weight(ordinal_graph, -0.7) + pynutil.insert('"')
+        )
+        graph_year = (
+            delete_extra_space
+            + pynutil.insert('year: "')
+            + pynutil.add_weight(year_graph, -YEAR_WEIGHT)
+            + pynutil.insert('"')
+        )
+        optional_graph_year = pynini.closure(
+            graph_year,
+            0,
+            1,
+        )
+        graph_mdy = month_graph + (
+            (delete_extra_space + day_graph)
+            | graph_year
+            | (delete_extra_space + day_graph + graph_year)
+        )
+        graph_dmy = (
+            pynutil.delete("the")
+            + delete_space
+            + day_graph
+            + delete_space
+            + pynutil.delete("of")
+            + delete_extra_space
+            + month_graph
+            + optional_graph_year
+        )
+        graph_year = (
+            pynutil.insert('year: "') + (year_graph | _get_range_graph()) + pynutil.insert('"')
+        )
+
+        final_graph = graph_mdy | graph_dmy | graph_year
+        final_graph += pynutil.insert(" preserve_order: true")
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/decimal.py

@@ -0,0 +1,102 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_DIGIT,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+def get_quantity(
+    decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike"
+) -> "pynini.FstLike":
+    """
+    Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
+    e.g. one million -> integer_part: "1" quantity: "million"
+    e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million"
+
+    Args:
+        decimal: decimal FST
+        cardinal_up_to_hundred: cardinal FST
+    """
+    numbers = cardinal_up_to_hundred @ (
+        pynutil.delete(pynini.closure("0"))
+        + pynini.difference(DAMO_DIGIT, "0")
+        + pynini.closure(DAMO_DIGIT)
+    )
+    suffix = pynini.union(
+        "万", "百万", "千万", "億" "十億", "trillion", "quadrillion", "quintillion", "sextillion"
+    )
+    res = (
+        pynutil.insert('integer_part: "')
+        + numbers
+        + pynutil.insert('"')
+        + delete_extra_space
+        + pynutil.insert('quantity: "')
+        + suffix
+        + pynutil.insert('"')
+    )
+    res |= (
+        decimal
+        + delete_extra_space
+        + pynutil.insert('quantity: "')
+        + (suffix | "千")
+        + pynutil.insert('"')
+    )
+    return res
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for classifying decimal
+        e.g. minus twelve point five o o six billion -> decimal { negative: "true" integer_part: "12"  fractional_part: "5006" quantity: "billion" }
+        e.g. one billion -> decimal { integer_part: "1" quantity: "billion" }
+    Args:
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, cardinal: GraphFst):
+        super().__init__(name="decimal", kind="classify")
+
+        cardinal_graph = cardinal.graph_no_exception
+
+        graph_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+        graph_decimal |= (
+            pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+            | pynini.cross("零", "０")
+            | pynini.cross("〇", "０")
+        )
+
+        graph_decimal = pynini.closure(graph_decimal + delete_space) + graph_decimal
+        self.graph = graph_decimal
+
+        point = pynutil.delete("点")
+
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("マイナス", '"true"') + delete_extra_space,
+            0,
+            1,
+        )
+
+        graph_fractional = (
+            pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"')
+        )
+        graph_integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
+        final_graph_wo_sign = (
+            pynini.closure(graph_integer + delete_extra_space, 0, 1)
+            + point
+            + delete_extra_space
+            + graph_fractional
+        )
+        final_graph = optional_graph_negative + final_graph_wo_sign
+
+        self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(
+            final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit
+        )
+        final_graph |= optional_graph_negative + get_quantity(
+            final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit
+        )
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/electronic.py

@@ -0,0 +1,103 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_ALPHA,
+    GraphFst,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for classifying electronic: as URLs, email addresses, etc.
+        e.g. c d f one at a b c dot e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } }
+    """
+
+    def __init__(self):
+        super().__init__(name="electronic", kind="classify")
+
+        delete_extra_space = pynutil.delete(" ")
+        alpha_num = (
+            DAMO_ALPHA
+            | pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+            | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        )
+
+        symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert()
+
+        accepted_username = alpha_num | symbols
+        process_dot = pynini.cross("ドット", ".")
+        username = (
+            alpha_num + pynini.closure(delete_extra_space + accepted_username)
+        ) | pynutil.add_weight(pynini.closure(DAMO_ALPHA, 1), weight=0.0001)
+        username = pynutil.insert('username: "') + username + pynutil.insert('"')
+        single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num
+        server = single_alphanum | pynini.string_file(
+            get_abs_path("data/electronic/server_name.tsv")
+        )
+        domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
+        domain_graph = (
+            pynutil.insert('domain: "')
+            + server
+            + delete_extra_space
+            + process_dot
+            + delete_extra_space
+            + domain
+            + pynutil.insert('"')
+        )
+        graph = (
+            username
+            + delete_extra_space
+            + pynutil.delete("at")
+            + insert_space
+            + delete_extra_space
+            + domain_graph
+        )
+
+        ############# url ###
+        protocol_end = pynini.cross(pynini.union("w w w", "www"), "www")
+        protocol_start = (
+            pynini.cross("h t t p", "http") | pynini.cross("h t t p s", "https")
+        ) + pynini.cross(
+            "コロンスラッシュスラッシュ", "://"  # colon slash slash
+        )
+        # .com,
+        ending = (
+            delete_extra_space
+            + symbols
+            + delete_extra_space
+            + (
+                domain
+                | pynini.closure(
+                    accepted_username + delete_extra_space,
+                )
+                + accepted_username
+            )
+        )
+
+        protocol_default = (
+            (
+                (pynini.closure(delete_extra_space + accepted_username, 1) | server)
+                | pynutil.add_weight(pynini.closure(DAMO_ALPHA, 1), weight=0.0001)
+            )
+            + pynini.closure(ending, 1)
+        ).optimize()
+        protocol = (
+            pynini.closure(protocol_start, 0, 1)
+            + protocol_end
+            + delete_extra_space
+            + process_dot
+            + protocol_default
+        ).optimize()
+
+        protocol |= (
+            pynini.closure(protocol_end + delete_extra_space + process_dot, 0, 1) + protocol_default
+        )
+
+        protocol = pynutil.insert('protocol: "') + protocol.optimize() + pynutil.insert('"')
+        graph |= protocol
+        ########
+
+        final_graph = self.add_tokens(graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/fraction.py

@@ -0,0 +1,51 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+    insert_space,
+    DAMO_CHAR,
+)
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for classifying fraction
+        e.g. 2 phần 3 -> tokens { fraction { numerator: "2" denominator: "3" } }
+        e.g. 2 trên 3 -> tokens { fraction { numerator: "2" denominator: "3" } }
+        e.g. 2 chia 3 -> tokens { fraction { numerator: "2" denominator: "3" } }
+
+    Args:
+        cardinal: OrdinalFst
+    """
+
+    def __init__(self, cardinal: GraphFst):
+        super().__init__(name="fraction", kind="classify")
+
+        graph_cardinal = cardinal.graph_no_exception
+        graph_four = pynini.cross("クォーター", "4")  # quarter
+
+        denominator = (
+            pynutil.insert('denominator: "') + (graph_cardinal | graph_four) + pynutil.insert('"')
+        )
+        fraction_component = pynutil.delete(pynini.union("分の", "割る"))
+        numerator = pynutil.insert('numerator: "') + graph_cardinal + pynutil.insert('"')
+
+        graph_fraction_component = denominator + insert_space + fraction_component + numerator
+
+        self.graph_fraction_component = graph_fraction_component
+
+        graph = graph_fraction_component
+        graph = graph.optimize()
+        self.final_graph_wo_negative = graph
+
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("マイナス", '"true"') + delete_extra_space,
+            0,
+            1,
+        )
+
+        graph = optional_graph_negative + graph
+        final_graph = self.add_tokens(graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/measure.py

@@ -0,0 +1,110 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_SIGMA,
+    GraphFst,
+    convert_space,
+    delete_extra_space,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for classifying measure
+        e.g. minus twelve kilograms -> measure { negative: "true" cardinal { integer: "12" } units: "kg" }
+
+    Args:
+        cardinal: CardinalFst
+        decimal: DecimalFst
+    """
+
+    def __init__(self, cardinal: GraphFst, decimal: GraphFst):
+        super().__init__(name="measure", kind="classify")
+
+        cardinal_graph = cardinal.graph_no_exception
+
+        graph_unit = pynini.string_file(get_abs_path("data/measurements.tsv"))
+        graph_unit_singular = graph_unit  # singular -> abbr
+        graph_unit_plural = graph_unit_singular  # plural -> abbr
+
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("マイナス", '"true"') + delete_extra_space,
+            0,
+            1,
+        )
+
+        unit_singular = graph_unit_singular
+        unit_plural = graph_unit_plural
+        unit_misc = pynutil.insert("/") + pynutil.delete("每") + delete_space + graph_unit_singular
+
+        unit_singular = (
+            pynutil.insert('units: "')
+            + (
+                unit_singular
+                | unit_misc
+                | pynutil.add_weight(unit_singular + delete_space + unit_misc, 0.01)
+            )
+            + pynutil.insert('"')
+        )
+
+        unit_plural = (
+            pynutil.insert('units: "')
+            + (
+                unit_plural
+                | unit_misc
+                | pynutil.add_weight(unit_plural + delete_space + unit_misc, 0.01)
+            )
+            + pynutil.insert('"')
+        )
+
+        subgraph_decimal = (
+            pynutil.insert("decimal { ")
+            + optional_graph_negative
+            + decimal.final_graph_wo_negative
+            + pynutil.insert(" }")
+            + delete_extra_space
+            + unit_plural
+        )
+        subgraph_decimal |= (
+            pynutil.insert("decimal { ")
+            + optional_graph_negative
+            + decimal.final_graph_wo_negative
+            + pynutil.insert(" }")
+            # + delete_extra_space
+            + unit_plural
+        )
+        subgraph_cardinal = (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert('integer: "')
+            + ((DAMO_SIGMA - "一") @ cardinal_graph)
+            + pynutil.insert('"')
+            + pynutil.insert(" }")
+            + delete_extra_space
+            + unit_plural
+        )
+        subgraph_cardinal |= (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert('integer: "')
+            + pynini.cross("一", "1")
+            + pynutil.insert('"')
+            + pynutil.insert(" }")
+            + delete_extra_space
+            + unit_singular
+        )
+        subgraph_cardinal |= (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert('integer: "')
+            + ((DAMO_SIGMA - "一") @ cardinal_graph)
+            + pynutil.insert('"')
+            + pynutil.insert(" }")
+            + unit_singular
+        )
+        final_graph = subgraph_decimal | subgraph_cardinal
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/money.py

@@ -0,0 +1,108 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_DIGIT,
+    DAMO_NOT_SPACE,
+    DAMO_SIGMA,
+    GraphFst,
+    convert_space,
+    delete_extra_space,
+    delete_space,
+    # get_singulars,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for classifying money
+        e.g. twelve dollars and five cents -> money { integer_part: "12" fractional_part: 05 currency: "$" }
+
+    Args:
+        cardinal: CardinalFst
+        decimal: DecimalFst
+    """
+
+    def __init__(self, cardinal: GraphFst, decimal: GraphFst):
+        super().__init__(name="money", kind="classify")
+        # quantity, integer_part, fractional_part, currency
+
+        cardinal_graph = cardinal.graph_no_exception
+        # add support for missing hundred (only for 3 digit numbers)
+        # "one fifty" -> "one hundred fifty"
+        with_hundred = pynini.compose(
+            pynini.closure(DAMO_NOT_SPACE) + pynini.accep(" ") + pynutil.insert("百") + DAMO_SIGMA,
+            pynini.compose(cardinal_graph, DAMO_DIGIT**3),
+        )
+        cardinal_graph |= with_hundred
+        graph_decimal_final = decimal.final_graph_wo_negative
+
+        unit = pynini.string_file(get_abs_path("data/currency.tsv"))
+        unit_singular = pynini.invert(unit)
+        unit_plural = unit_singular
+        # unit_plural = get_singulars(unit_singular)
+
+        graph_unit_singular = (
+            pynutil.insert('currency: "') + convert_space(unit_singular) + pynutil.insert('"')
+        )
+        graph_unit_plural = (
+            pynutil.insert('currency: "') + convert_space(unit_plural) + pynutil.insert('"')
+        )
+
+        add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
+            pynutil.insert("0") + DAMO_DIGIT
+        )
+        # twelve dollars (and) fifty cents, zero cents
+        cents_standalone = (
+            pynutil.insert('fractional_part: "')
+            + pynini.union(
+                pynutil.add_weight(((DAMO_SIGMA - "一") @ cardinal_graph), -0.7)
+                @ add_leading_zero_to_double_digit
+                + delete_space
+                + pynutil.delete("セント"),  # cent
+                pynini.cross("一", "01") + delete_space + pynutil.delete("セント"),  # cent
+            )
+            + pynutil.insert('"')
+        )
+
+        optional_cents_standalone = pynini.closure(
+            delete_space
+            + pynini.closure(pynutil.delete("と") + delete_space, 0, 1)  # and
+            + insert_space
+            + cents_standalone,
+            0,
+            1,
+        )
+        # twelve dollars fifty, only after integer
+        optional_cents_suffix = pynini.closure(
+            delete_extra_space
+            + pynutil.insert('fractional_part: "')
+            + pynutil.add_weight(cardinal_graph @ add_leading_zero_to_double_digit, -0.7)
+            + pynutil.insert('"'),
+            0,
+            1,
+        )
+
+        graph_integer = (
+            pynutil.insert('integer_part: "')
+            + ((DAMO_SIGMA - "一") @ cardinal_graph)
+            + pynutil.insert('"')
+            + delete_extra_space
+            + graph_unit_plural
+            + (optional_cents_standalone | optional_cents_suffix)
+        )
+        graph_integer |= (
+            pynutil.insert('integer_part: "')
+            + pynini.cross("一", "1")
+            + pynutil.insert('"')
+            + delete_extra_space
+            + graph_unit_singular
+            + (optional_cents_standalone | optional_cents_suffix)
+        )
+        graph_decimal = graph_decimal_final + delete_extra_space + graph_unit_plural
+        graph_decimal |= pynutil.insert('currency: "$" integer_part: "0" ') + cents_standalone
+        final_graph = graph_integer | graph_decimal
+
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/ordinal.py

@@ -0,0 +1,55 @@
+import pynini
+from pynini import cross
+from pynini.lib.pynutil import delete, insert, add_weight
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_CHAR, GraphFst
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for classifying ordinal
+        e.g. thirteenth -> ordinal { integer: "13" }
+
+    Args:
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, cardinal: GraphFst):
+        super().__init__(name="ordinal", kind="classify")
+
+        cardinal_graph = cardinal.graph_no_exception
+        digit = pynini.string_file(get_abs_path("data/ordinals/digit.tsv"))
+        ties = pynini.string_file(get_abs_path("data/ordinals/ties.tsv"))
+        teen = pynini.string_file(get_abs_path("data/ordinals/teen.tsv"))
+        zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        hundred_digit = pynini.string_file(get_abs_path("data/numbers/hundred_digit.tsv"))
+        addzero = insert("0")
+        tens = ties + addzero | (digit + delete("十") + (digit | addzero))
+        hundred = (
+            digit
+            + delete("百")
+            + (
+                tens
+                | teen
+                | add_weight(zero + digit, 0.1)
+                | add_weight(digit + addzero, 0.5)
+                | add_weight(addzero**2, 1.0)
+            )
+        )
+        hundred |= cross("百", "1") + (
+            tens
+            | teen
+            | add_weight(zero + digit, 0.1)
+            | add_weight(digit + addzero, 0.5)
+            | add_weight(addzero**2, 1.0)
+        )
+        hundred |= hundred_digit
+
+        ordinal = digit | teen | tens | hundred
+        graph = pynini.closure(DAMO_CHAR, 1) + ordinal
+
+        self.graph = graph @ cardinal_graph
+        final_graph = pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/preprocessor.py

@@ -0,0 +1,22 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_SIGMA, GraphFst
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from pynini.lib import pynutil
+
+
+class PreProcessor(GraphFst):
+    def __init__(
+        self,
+        halfwidth_to_fullwidth: bool = True,
+    ):
+        super().__init__(name="PreProcessor", kind="processor")
+
+        graph = pynini.cdrewrite("", "", "", DAMO_SIGMA)
+
+        if halfwidth_to_fullwidth:
+            halfwidth_to_fullwidth_graph = pynini.string_file(
+                get_abs_path("data/char/halfwidth_to_fullwidth.tsv")
+            )
+            graph @= pynini.cdrewrite(halfwidth_to_fullwidth_graph, "", "", DAMO_SIGMA)
+
+        self.fst = graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/punctuation.py

@@ -0,0 +1,20 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst
+from pynini.lib import pynutil
+
+
+class PunctuationFst(GraphFst):
+    """
+    Finite state transducer for classifying punctuation
+        e.g. a, -> tokens { name: "a" } tokens { name: "," }
+    """
+
+    def __init__(self):
+        super().__init__(name="punctuation", kind="classify")
+
+        s = "!#$%&'()*+,-./:;<=>?@^_`{|}~、。，！【】「」《》￥（）——・"
+        punct = pynini.union(*s)
+
+        graph = pynutil.insert('name: "') + punct + pynutil.insert('"')
+
+        self.fst = graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/telephone.py

@@ -0,0 +1,150 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_ALNUM,
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    GraphFst,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+def get_serial_number(cardinal):
+    """
+    any alphanumerical character sequence with at least one number with length greater equal to 3
+    """
+    digit = pynini.compose(cardinal.graph_no_exception, DAMO_DIGIT)
+    character = digit
+    sequence = character + pynini.closure(character, 2)
+    sequence = sequence @ (pynini.closure(DAMO_ALNUM) + DAMO_DIGIT + pynini.closure(DAMO_ALNUM))
+    return sequence.optimize()
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for classifying telephone numbers, e.g.
+        one two three one two three five six seven eight -> { number_part: "123-123-5678" }
+
+    This class also support card number and IP format.
+        "one two three dot one double three dot o dot four o" -> { number_part: "123.133.0.40"}
+
+        "three two double seven three two one four three two one four three double zero five" ->
+            { number_part: 3277 3214 3214 3005}
+
+    Args:
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, cardinal: GraphFst):
+        super().__init__(name="telephone", kind="classify")
+        # country code, number_part, extension
+        digit_to_str = (
+            pynini.invert(pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize())
+            # | pynini.cross("０", pynini.union("o")).optimize()
+            | pynini.cross("０", pynini.union("〇", "零")).optimize()
+        )
+
+        str_to_digit = pynini.invert(digit_to_str)
+
+        double_digit = pynini.union(
+            *[
+                pynini.cross(
+                    pynini.project(str(i) @ digit_to_str, "output")
+                    + pynini.accep(" ")
+                    + pynini.project(str(i) @ digit_to_str, "output"),
+                    pynutil.insert("double ") + pynini.project(str(i) @ digit_to_str, "output"),
+                )
+                for i in range(10)
+            ]
+        )
+        double_digit.invert()
+
+        # to handle cases like "one twenty three"
+        two_digit_cardinal = pynini.compose(cardinal.graph_no_exception, DAMO_DIGIT**2)
+        double_digit_to_digit = (
+            pynini.compose(double_digit, str_to_digit + pynutil.delete(" ") + str_to_digit)
+            | two_digit_cardinal
+        )
+
+        single_or_double_digit = (
+            pynutil.add_weight(double_digit_to_digit, -0.0001) | str_to_digit
+        ).optimize()
+        single_or_double_digit |= (
+            single_or_double_digit
+            + pynini.closure(
+                pynutil.add_weight(pynutil.delete(" ") + single_or_double_digit, 0.0001)
+            )
+        ).optimize()
+
+        number_part = pynini.compose(
+            single_or_double_digit,
+            DAMO_DIGIT**3
+            + pynutil.insert("-")
+            + DAMO_DIGIT**3
+            + pynutil.insert("-")
+            + DAMO_DIGIT**4,
+        ).optimize()
+        number_part = (
+            pynutil.insert('number_part: "') + number_part.optimize() + pynutil.insert('"')
+        )
+
+        cardinal_option = pynini.compose(single_or_double_digit, DAMO_DIGIT ** (2, 3))
+
+        country_code = (
+            pynutil.insert('country_code: "')
+            + pynini.closure(pynini.cross("プラス ", "+"), 0, 1)  # plus
+            + (
+                (pynini.closure(str_to_digit + pynutil.delete(" "), 0, 2) + str_to_digit)
+                | cardinal_option
+            )
+            + pynutil.insert('"')
+        )
+
+        optional_country_code = pynini.closure(
+            country_code + pynutil.delete(" ") + insert_space, 0, 1
+        ).optimize()
+        graph = optional_country_code + number_part
+
+        # credit card number
+        space_four_digits = insert_space + DAMO_DIGIT**4
+        credit_card_graph = pynini.compose(
+            single_or_double_digit, DAMO_DIGIT**4 + space_four_digits**3
+        ).optimize()
+        graph |= (
+            pynutil.insert('number_part: "') + credit_card_graph.optimize() + pynutil.insert('"')
+        )
+
+        # SSN
+        ssn_graph = pynini.compose(
+            single_or_double_digit,
+            DAMO_DIGIT**3
+            + pynutil.insert("-")
+            + DAMO_DIGIT**2
+            + pynutil.insert("-")
+            + DAMO_DIGIT**4,
+        ).optimize()
+        graph |= pynutil.insert('number_part: "') + ssn_graph.optimize() + pynutil.insert('"')
+
+        # ip
+        digit_or_double = (
+            pynini.closure(str_to_digit + pynutil.delete(" "), 0, 1) + double_digit_to_digit
+        )
+        digit_or_double |= double_digit_to_digit + pynini.closure(
+            pynutil.delete(" ") + str_to_digit, 0, 1
+        )
+        digit_or_double |= str_to_digit + (pynutil.delete(" ") + str_to_digit) ** (0, 2)
+        digit_or_double |= cardinal_option
+        digit_or_double = digit_or_double.optimize()
+
+        ip_graph = digit_or_double + (pynini.cross(" 点 ", ".") + digit_or_double) ** 3  # dot
+
+        graph |= pynutil.insert('number_part: "') + ip_graph.optimize() + pynutil.insert('"')
+        graph |= (
+            pynutil.insert('number_part: "')
+            + pynutil.add_weight(get_serial_number(cardinal=cardinal), weight=0.0001)
+            + pynutil.insert('"')
+        )
+
+        final_graph = self.add_tokens(graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/time.py

@@ -0,0 +1,140 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.taggers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path, num_to_word
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    GraphFst,
+    convert_space,
+    delete_extra_space,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for classifying time
+        e.g. twelve thirty -> time { hours: "12" minutes: "30" }
+        e.g. twelve past one -> time { minutes: "12" hours: "1" }
+        e.g. two o clock a m -> time { hours: "2" suffix: "a.m." }
+        e.g. quarter to two -> time { hours: "1" minutes: "45" }
+        e.g. quarter past two -> time { hours: "2" minutes: "15" }
+        e.g. half past two -> time { hours: "2" minutes: "30" }
+    """
+
+    def __init__(self):
+        super().__init__(name="time", kind="classify")
+        # hours, minutes, seconds, suffix, zone, style, speak_period
+
+        suffix_graph = pynini.string_file(get_abs_path("data/time/time_suffix.tsv"))
+        time_zone_graph = pynini.invert(pynini.string_file(get_abs_path("data/time/time_zone.tsv")))
+        to_hour_graph = pynini.string_file(get_abs_path("data/time/to_hour.tsv"))
+        minute_to_graph = pynini.string_file(get_abs_path("data/time/minute_to.tsv"))
+
+        # only used for < 1000 thousand -> 0 weight
+        cardinal = pynutil.add_weight(CardinalFst().graph_no_exception, weight=-0.7)
+
+        labels_hour = [num_to_word(x) for x in range(0, 24)]
+        labels_minute_single = [num_to_word(x) for x in range(1, 10)]
+        labels_minute_double = [num_to_word(x) for x in range(10, 60)]
+
+        graph_hour = pynini.union(*labels_hour) @ cardinal
+
+        graph_minute_single = pynini.union(*labels_minute_single) @ cardinal
+        graph_minute_double = pynini.union(*labels_minute_double) @ cardinal
+        graph_minute_verbose = pynini.cross("半", "30") | pynini.cross(
+            "クォーター", "15"
+        )  # half, quarter
+        oclock = pynini.cross(pynini.union("時", "o' clock", "o clock", "o'clock", "oclock"), "")
+
+        final_graph_hour = pynutil.insert('hours: "') + graph_hour + pynutil.insert('"')
+        graph_minute = (
+            oclock + pynutil.insert("00")
+            | pynutil.delete(pynini.union("〇", "零")) + delete_space + graph_minute_single
+            | graph_minute_double
+        )
+        final_suffix = (
+            pynutil.insert('suffix: "') + convert_space(suffix_graph) + pynutil.insert('"')
+        )
+        final_suffix = delete_space + insert_space + final_suffix
+        final_suffix_optional = pynini.closure(final_suffix, 0, 1)
+        final_time_zone_optional = pynini.closure(
+            delete_space
+            + insert_space
+            + pynutil.insert('zone: "')
+            + convert_space(time_zone_graph)
+            + pynutil.insert('"'),
+            0,
+            1,
+        )
+
+        # five o' clock
+        # two o eight, two thirty five (am/pm)
+        # two pm/am
+        graph_hm = (
+            final_graph_hour
+            + delete_extra_space
+            + pynutil.insert('minutes: "')
+            + graph_minute
+            + pynutil.insert('"')
+        )
+        # 10 past four, quarter past four, half past four
+        graph_m_past_h = (
+            pynutil.insert('minutes: "')
+            + pynini.union(graph_minute_single, graph_minute_double, graph_minute_verbose)
+            + pynutil.insert('"')
+            + delete_extra_space
+            + final_graph_hour
+        )
+
+        graph_quarter_time = (
+            pynutil.insert('minutes: "')
+            + pynini.cross("クォーター", "45")  # quarter
+            + pynutil.insert('"')
+            + delete_space
+            + pynutil.delete(pynini.union("から", "to", "till"))  # to, till
+            + delete_extra_space
+            + pynutil.insert('hours: "')
+            + to_hour_graph
+            + pynutil.insert('"')
+        )
+
+        graph_m_to_h_suffix_time = (
+            pynutil.insert('minutes: "')
+            + ((graph_minute_single | graph_minute_double).optimize() @ minute_to_graph)
+            + pynutil.insert('"')
+            + pynini.closure(
+                delete_space
+                + pynutil.delete(pynini.union("分", "min", "mins", "minute", "minutes")),
+                0,
+                1,
+            )
+            + delete_space
+            + pynutil.delete(pynini.union("から", "to", "till"))  # to, till
+            + delete_extra_space
+            + pynutil.insert('hours: "')
+            + to_hour_graph
+            + pynutil.insert('"')
+            + final_suffix
+        )
+
+        graph_h = (
+            final_graph_hour
+            + delete_extra_space
+            + pynutil.insert('minutes: "')
+            + (pynutil.insert("00") | graph_minute)
+            + pynutil.insert('"')
+            + final_suffix
+            + final_time_zone_optional
+        )
+        final_graph = (
+            (graph_hm | graph_m_past_h | graph_quarter_time)
+            + final_suffix_optional
+            + final_time_zone_optional
+        )
+        final_graph |= graph_h
+        final_graph |= graph_m_to_h_suffix_time
+
+        final_graph = self.add_tokens(final_graph.optimize())
+
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/tokenize_and_classify.py

@@ -0,0 +1,123 @@
+import os
+
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.taggers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.date import DateFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.fraction import FractionFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.ordinal import OrdinalFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.punctuation import PunctuationFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.time import TimeFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.whitelist import WhiteListFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.word import WordFst
+from fun_text_processing.inverse_text_normalization.ja.taggers.preprocessor import PreProcessor
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    GraphFst,
+    DAMO_SIGMA,
+    delete_extra_space,
+    delete_space,
+    generator_main,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+import logging
+
+
+class ClassifyFst(GraphFst):
+    """
+    Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+
+    Args:
+        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
+        overwrite_cache: set to True to overwrite .far files
+    """
+
+    def __init__(
+        self,
+        cache_dir: str = None,
+        overwrite_cache: bool = False,
+        enable_standalone_number: bool = True,
+        enable_0_to_9: bool = True,
+    ):
+        super().__init__(name="tokenize_and_classify", kind="classify")
+        self.convert_number = enable_standalone_number
+        self.enable_0_to_9 = enable_0_to_9
+
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            far_file = os.path.join(cache_dir, "_ja_itn.far")
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
+            logging.info(f"ClassifyFst.fst was restored from {far_file}.")
+        else:
+            logging.info(f"Creating ClassifyFst grammars.")
+            cardinal = CardinalFst(self.convert_number, self.enable_0_to_9)
+            cardinal_graph = cardinal.fst
+
+            fraction = FractionFst(cardinal)
+            fraction_graph = fraction.fst
+
+            ordinal = OrdinalFst(cardinal)
+            ordinal_graph = ordinal.fst
+
+            decimal = DecimalFst(cardinal)
+            decimal_graph = decimal.fst
+
+            measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst
+            date_graph = DateFst(ordinal=ordinal).fst
+            word_graph = WordFst().fst
+            time_graph = TimeFst().fst
+            money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst
+            whitelist_graph = WhiteListFst().fst
+            punct_graph = PunctuationFst().fst
+            preprocessor = PreProcessor(halfwidth_to_fullwidth=True).fst
+            electronic_graph = ElectronicFst().fst
+            telephone_graph = TelephoneFst(cardinal).fst
+
+            classify = (
+                pynutil.add_weight(whitelist_graph, 1.01)
+                | pynutil.add_weight(time_graph, 1.1)
+                | pynutil.add_weight(date_graph, 1.09)
+                | pynutil.add_weight(decimal_graph, 1.1)
+                | pynutil.add_weight(measure_graph, 1.1)
+                | pynutil.add_weight(cardinal_graph, 1.1)
+                | pynutil.add_weight(ordinal_graph, 1.1)
+                | pynutil.add_weight(fraction_graph, 1.09)
+                | pynutil.add_weight(money_graph, 1.1)
+                | pynutil.add_weight(telephone_graph, 1.1)
+                | pynutil.add_weight(electronic_graph, 1.1)
+                | pynutil.add_weight(word_graph, 100)
+            )
+
+            punct = (
+                pynutil.insert("tokens { ")
+                + pynutil.add_weight(punct_graph, weight=1.1)
+                + pynutil.insert(" }")
+            )
+            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
+            token_plus_punct = (
+                pynini.closure(punct + pynutil.insert(" "))
+                + token
+                + pynini.closure(pynutil.insert(" ") + punct)
+            )
+            graph = token_plus_punct + pynini.closure(
+                pynini.union(insert_space, delete_extra_space) + token_plus_punct
+            )
+
+            graph = delete_space + graph + delete_space
+
+            self.fst = graph.optimize()
+
+            if far_file:
+                generator_main(far_file, {"tokenize_and_classify": self.fst})
+                logging.info(f"ClassifyFst grammars are saved to {far_file}.")
+
+            self.token_plus_punct = token_plus_punct.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/whitelist.py

@@ -0,0 +1,19 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst, convert_space
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for classifying whitelisted tokens
+        e.g. misses -> tokens { name: "mrs." }
+    This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
+    """
+
+    def __init__(self):
+        super().__init__(name="whitelist", kind="classify")
+
+        whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert()
+        graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"')
+        self.fst = graph.optimize()

fun_text_processing/inverse_text_normalization/ja/taggers/word.py

@@ -0,0 +1,20 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_NOT_SPACE,
+    GraphFst,
+    DAMO_CHAR,
+)
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class.
+        e.g. sleep -> tokens { name: "sleep" }
+    """
+
+    def __init__(self):
+        super().__init__(name="word", kind="classify")
+        word = pynutil.insert('name: "') + DAMO_NOT_SPACE + pynutil.insert('"')
+
+        self.fst = word.optimize()

fun_text_processing/inverse_text_normalization/ja/utils.py

@@ -0,0 +1,33 @@
+import os
+from typing import Union
+
+import inflect
+
+_inflect = inflect.engine()
+
+
+def num_to_word(x: Union[str, int]):
+    """
+    converts integer to spoken representation
+
+    Args
+        x: integer
+
+    Returns: spoken representation
+    """
+    if isinstance(x, int):
+        x = str(x)
+        x = _inflect.number_to_words(str(x)).replace("-", " ").replace(",", "")
+    return x
+
+
+def get_abs_path(rel_path):
+    """
+    Get absolute path
+
+    Args:
+        rel_path: relative path to this file
+
+    Returns absolute path
+    """
+    return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path

fun_text_processing/inverse_text_normalization/ja/verbalizers/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/inverse_text_normalization/ja/verbalizers/cardinal.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+    DAMO_CHAR,
+)
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing cardinal
+        e.g. cardinal { integer: "23" negative: "-" } -> -23
+    """
+
+    def __init__(self):
+        # enable_standalone_number: bool = True,
+        # enable_0_to_9: bool = True):
+        super().__init__(name="cardinal", kind="verbalize")
+        # self.enable_standalone_number = enable_standalone_number
+        # self.enable_0_to_9 = enable_0_to_9
+        optional_sign = pynini.closure(
+            pynutil.delete("negative:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space,
+            0,
+            1,
+        )
+        graph = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        # if self.enable_standalone_number:
+        #     if self.enable_0_to_9:
+        #     else:
+        self.numbers = graph
+        graph = optional_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/date.py

@@ -0,0 +1,70 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for verbalizing date, e.g.
+        date { month: "january" day: "5" year: "2012" preserve_order: true } -> february 5 2012
+        date { day: "5" month: "january" year: "2012" preserve_order: true } -> 5 february 2012
+    """
+
+    def __init__(self):
+        super().__init__(name="date", kind="verbalize")
+        month = (
+            pynutil.delete("month:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        day = (
+            pynutil.delete("day:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        year = (
+            pynutil.delete("year:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + delete_space
+            + pynutil.delete('"')
+        )
+
+        # month (day) year
+        graph_mdy = (
+            month
+            + pynini.closure(delete_extra_space + day, 0, 1)
+            + pynini.closure(delete_extra_space + year, 0, 1)
+        )
+
+        # (day) month year
+        graph_dmy = (
+            pynini.closure(day + delete_extra_space, 0, 1)
+            + month
+            + pynini.closure(delete_extra_space + year, 0, 1)
+        )
+
+        optional_preserve_order = pynini.closure(
+            pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
+            | pynutil.delete("field_order:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space
+        )
+
+        final_graph = (graph_mdy | year | graph_dmy) + delete_space + optional_preserve_order
+
+        delete_tokens = self.delete_tokens(final_graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/decimal.py

@@ -0,0 +1,48 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing decimal, e.g.
+        decimal { negative: "true" integer_part: "12"  fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
+    """
+
+    def __init__(self):
+        super().__init__(name="decimal", kind="verbalize")
+        optionl_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        integer = (
+            pynutil.delete("integer_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_integer = pynini.closure(integer + delete_space, 0, 1)
+        fractional = (
+            pynutil.insert(".")
+            + pynutil.delete("fractional_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
+        quantity = (
+            pynutil.delete("quantity:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1)
+        graph = optional_integer + optional_fractional + optional_quantity
+        self.numbers = graph
+        graph = optionl_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/electronic.py

@@ -0,0 +1,45 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu
+    """
+
+    def __init__(self):
+        super().__init__(name="electronic", kind="verbalize")
+        user_name = (
+            pynutil.delete("username:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        domain = (
+            pynutil.delete("domain:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        protocol = (
+            pynutil.delete("protocol:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = user_name + delete_space + pynutil.insert("@") + domain
+        graph |= protocol
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/fraction.py

@@ -0,0 +1,47 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+    DAMO_SIGMA,
+    delete_extra_space,
+)
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for verbalizing fraction,
+        e.g. fraction { numerator: "2" denominator: "3" } } -> 2/3
+        e.g. fraction { numerator: "20" denominator: "3" negative: "true"} } -> 2/3
+    """
+
+    def __init__(self):
+        super().__init__(name="fraction", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        numerator = (
+            pynutil.delete('numerator: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
+        )
+
+        denominator = (
+            pynutil.insert("/")
+            + pynutil.delete('denominator: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = (numerator + delete_space + denominator).optimize()
+
+        # numerator = pynutil.delete('numerator: "') + DAMO_NOT_QUOTE + pynutil.delete('"')
+        #
+        # denominator = (
+        #     pynutil.delete('denominator: "')
+        #     + DAMO_NOT_QUOTE
+        #     + pynutil.delete('"')
+        # )
+        #
+        # graph = (numerator + pynutil.insert("/") + denominator).optimize()
+
+        self.numbers = graph
+        delete_tokens = self.delete_tokens(optional_sign + graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/measure.py

@@ -0,0 +1,51 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure, e.g.
+        measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg
+
+    Args:
+        decimal: DecimalFst
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, decimal: GraphFst, cardinal: GraphFst):
+        super().__init__(name="measure", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
+        unit = (
+            pynutil.delete("units:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+            + delete_space
+        )
+        graph_decimal = (
+            pynutil.delete("decimal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + decimal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph_cardinal = (
+            pynutil.delete("cardinal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + cardinal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = (graph_cardinal | graph_decimal) + delete_space + unit
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/money.py

@@ -0,0 +1,30 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing money, e.g.
+        money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05
+
+    Args:
+        decimal: DecimalFst
+    """
+
+    def __init__(self, decimal: GraphFst):
+        super().__init__(name="money", kind="verbalize")
+        unit = (
+            pynutil.delete("currency:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = unit + delete_space + decimal.numbers
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/ordinal.py

@@ -0,0 +1,51 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing ordinal, e.g.
+       ordinal { integer: "13" } -> 13th
+    """
+
+    def __init__(self):
+        super().__init__(name="ordinal", kind="verbalize")
+        # convert_rest = pynutil.insert("第", weight=0.01)
+        graph = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynutil.insert("第", weight=0.01)
+            # + DAMO_NOT_QUOTE
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        # convert_hundred = pynini.cross("第百", "第100")
+        # convert_eleven = pynini.cross("11", "十一")
+        # convert_twelve = pynini.cross("12", "十二")
+        # convert_thirteen = pynini.cross("13", "第十三")
+        # convert_one = pynini.cross("1", "第一")
+        # convert_two = pynini.cross("2", "第二")
+        # convert_three = pynini.cross("3", "第三")
+
+        # suffix = pynini.cdrewrite(
+        #     convert_hundred
+        # #     convert_eleven
+        # #     | convert_twelve
+        # #     | convert_thirteen
+        # #     | convert_one
+        # #     | convert_two
+        # #     | convert_three,
+        #     "",
+        #     "[EOS]",
+        #     DAMO_SIGMA,
+        # )
+        # graph = graph @ suffix
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/telephone.py

@@ -0,0 +1,30 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for verbalizing telephone, e.g.
+        telephone { number_part: "123-123-5678" }
+        -> 123-123-5678
+    """
+
+    def __init__(self):
+        super().__init__(name="telephone", kind="verbalize")
+
+        number_part = (
+            pynutil.delete('number_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_country_code = pynini.closure(
+            pynutil.delete('country_code: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynini.accep(" "),
+            0,
+            1,
+        )
+        delete_tokens = self.delete_tokens(optional_country_code + number_part)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/time.py

@@ -0,0 +1,68 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_CHAR,
+    DAMO_DIGIT,
+    GraphFst,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing time, e.g.
+        time { hours: "12" minutes: "30" } -> 12:30
+        time { hours: "1" minutes: "12" } -> 01:12
+        time { hours: "2" suffix: "a.m." } -> 02:00 a.m.
+    """
+
+    def __init__(self):
+        super().__init__(name="time", kind="verbalize")
+        add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
+            pynutil.insert("0") + DAMO_DIGIT
+        )
+        hour = (
+            pynutil.delete("hours:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+        minute = (
+            pynutil.delete("minutes:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+        suffix = (
+            delete_space
+            + insert_space
+            + pynutil.delete("suffix:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        optional_suffix = pynini.closure(suffix, 0, 1)
+        zone = (
+            delete_space
+            + insert_space
+            + pynutil.delete("zone:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        optional_zone = pynini.closure(zone, 0, 1)
+        graph = (
+            hour @ add_leading_zero_to_double_digit
+            + delete_space
+            + pynutil.insert(":")
+            + (minute @ add_leading_zero_to_double_digit)
+            + optional_suffix
+            + optional_zone
+        )
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize.py

@@ -0,0 +1,51 @@
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.date import DateFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.ordinal import OrdinalFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.time import TimeFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.whitelist import WhiteListFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.fraction import FractionFst
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import GraphFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize", kind="verbalize")
+        cardinal = CardinalFst()
+        cardinal_graph = cardinal.fst
+        ordinal_graph = OrdinalFst().fst
+        decimal = DecimalFst()
+        decimal_graph = decimal.fst
+        fraction = FractionFst()
+        fraction_graph = fraction.fst
+        measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst
+        money_graph = MoneyFst(decimal=decimal).fst
+        time_graph = TimeFst().fst
+        date_graph = DateFst().fst
+        whitelist_graph = WhiteListFst().fst
+        telephone_graph = TelephoneFst().fst
+        electronic_graph = ElectronicFst().fst
+        graph = (
+            time_graph
+            | date_graph
+            | money_graph
+            | fraction_graph
+            | measure_graph
+            | ordinal_graph
+            | decimal_graph
+            | cardinal_graph
+            | whitelist_graph
+            | telephone_graph
+            | electronic_graph
+        )
+        self.fst = graph

fun_text_processing/inverse_text_normalization/ja/verbalizers/verbalize_final.py

@@ -0,0 +1,33 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.ja.verbalizers.word import WordFst
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence, e.g.
+    tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize_final", kind="verbalize")
+        verbalize = VerbalizeFst().fst
+        word = WordFst().fst
+        types = verbalize | word
+        graph = (
+            pynutil.delete("tokens")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + types
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = delete_space + pynini.closure(graph + delete_space) + graph + delete_space
+        self.fst = graph

fun_text_processing/inverse_text_normalization/ja/verbalizers/whitelist.py

@@ -0,0 +1,27 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for verbalizing whitelist
+        e.g. tokens { name: "mrs." } -> mrs.
+    """
+
+    def __init__(self):
+        super().__init__(name="whitelist", kind="verbalize")
+        graph = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+        self.fst = graph.optimize()

fun_text_processing/inverse_text_normalization/ja/verbalizers/word.py

@@ -0,0 +1,29 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ja.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for verbalizing plain tokens
+        e.g. tokens { name: "sleep" } -> sleep
+    """
+
+    def __init__(self):
+        super().__init__(name="word", kind="verbalize")
+        chars = pynini.closure(DAMO_CHAR - " ", 1)
+        char = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + chars
+            + pynutil.delete('"')
+        )
+        graph = char @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+
+        self.fst = graph.optimize()