chore: import upstream snapshot with attribution

fun_text_processing/inverse_text_normalization/ko/__init__.py

@@ -0,0 +1,7 @@
+from fun_text_processing.inverse_text_normalization.ko.taggers.tokenize_and_classify import (
+    ClassifyFst,
+)
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.verbalize_final import (
+    VerbalizeFinalFst,
+)

fun_text_processing/inverse_text_normalization/ko/clean_eval_data.py

@@ -0,0 +1,384 @@
+from argparse import ArgumentParser
+from typing import List
+
+import regex as re
+from fun_text_processing.text_normalization.data_loader_utils import (
+    EOS_TYPE,
+    Instance,
+    load_files,
+    training_data_to_sentences,
+)
+
+
+"""
+This file is for evaluation purposes.
+filter_loaded_data() cleans data (list of instances) for inverse text normalization. Filters and cleaners can be specified for each semiotic class individually.
+For example, normalized text should only include characters and whitespace characters but no punctuation. 
+            Cardinal unnormalized instances should contain at least one integer and all other characters are removed.
+"""
+
+
+class Filter:
+    """
+    Filter class
+
+    Args:
+        class_type: semiotic class used in dataset
+        process_func: function to transform text
+        filter_func:  function to filter text
+
+    """
+
+    def __init__(self, class_type: str, process_func: object, filter_func: object):
+        self.class_type = class_type
+        self.process_func = process_func
+        self.filter_func = filter_func
+
+    def filter(self, instance: Instance) -> bool:
+        """
+        filter function
+
+        Args:
+            filters given instance with filter function
+
+        Returns: True if given instance fulfills criteria or does not belong to class type
+        """
+        if instance.token_type != self.class_type:
+            return True
+        return self.filter_func(instance)
+
+    def process(self, instance: Instance) -> Instance:
+        """
+        process function
+
+        Args:
+            processes given instance with process function
+
+        Returns: processed instance if instance belongs to expected class type or original instance
+        """
+        if instance.token_type != self.class_type:
+            return instance
+        return self.process_func(instance)
+
+
+def filter_cardinal_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_cardinal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r"[^0-9]", "", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_ordinal_1(instance: Instance) -> bool:
+    ok = re.search(r"(st|nd|rd|th)\s*$", instance.un_normalized)
+    return ok
+
+
+def process_ordinal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r"[,\s]", "", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_decimal_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_decimal_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_measure_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_measure_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    un_normalized = re.sub(r"m2", "m²", un_normalized)
+    un_normalized = re.sub(r"(\d)([^\d.\s])", r"\1 \2", un_normalized)
+    normalized = re.sub(r"[^a-z\s]", "", normalized)
+    normalized = re.sub(r"per ([a-z\s]*)s$", r"per \1", normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_money_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_money_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    un_normalized = re.sub(r"a\$", r"$", un_normalized)
+    un_normalized = re.sub(r"us\$", r"$", un_normalized)
+    un_normalized = re.sub(r"(\d)m\s*$", r"\1 million", un_normalized)
+    un_normalized = re.sub(r"(\d)bn?\s*$", r"\1 billion", un_normalized)
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_time_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_time_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r": ", ":", un_normalized)
+    un_normalized = re.sub(r"(\d)\s?a\s?m\s?", r"\1 a.m.", un_normalized)
+    un_normalized = re.sub(r"(\d)\s?p\s?m\s?", r"\1 p.m.", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_plain_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_plain_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_punct_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_punct_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_date_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_date_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    un_normalized = re.sub(r",", "", un_normalized)
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_letters_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_letters_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_verbatim_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_verbatim_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_digit_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_digit_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_telephone_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_telephone_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_electronic_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_electronic_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_fraction_1(instance: Instance) -> bool:
+    ok = re.search(r"[0-9]", instance.un_normalized)
+    return ok
+
+
+def process_fraction_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+def filter_address_1(instance: Instance) -> bool:
+    ok = True
+    return ok
+
+
+def process_address_1(instance: Instance) -> Instance:
+    un_normalized = instance.un_normalized
+    normalized = instance.normalized
+    normalized = re.sub(r"[^a-z ]", "", normalized)
+    return Instance(
+        token_type=instance.token_type, un_normalized=un_normalized, normalized=normalized
+    )
+
+
+filters = []
+filters.append(
+    Filter(class_type="CARDINAL", process_func=process_cardinal_1, filter_func=filter_cardinal_1)
+)
+filters.append(
+    Filter(class_type="ORDINAL", process_func=process_ordinal_1, filter_func=filter_ordinal_1)
+)
+filters.append(
+    Filter(class_type="DECIMAL", process_func=process_decimal_1, filter_func=filter_decimal_1)
+)
+filters.append(
+    Filter(class_type="MEASURE", process_func=process_measure_1, filter_func=filter_measure_1)
+)
+filters.append(Filter(class_type="MONEY", process_func=process_money_1, filter_func=filter_money_1))
+filters.append(Filter(class_type="TIME", process_func=process_time_1, filter_func=filter_time_1))
+
+filters.append(Filter(class_type="DATE", process_func=process_date_1, filter_func=filter_date_1))
+filters.append(Filter(class_type="PLAIN", process_func=process_plain_1, filter_func=filter_plain_1))
+filters.append(Filter(class_type="PUNCT", process_func=process_punct_1, filter_func=filter_punct_1))
+filters.append(
+    Filter(class_type="LETTERS", process_func=process_letters_1, filter_func=filter_letters_1)
+)
+filters.append(
+    Filter(class_type="VERBATIM", process_func=process_verbatim_1, filter_func=filter_verbatim_1)
+)
+filters.append(Filter(class_type="DIGIT", process_func=process_digit_1, filter_func=filter_digit_1))
+filters.append(
+    Filter(class_type="TELEPHONE", process_func=process_telephone_1, filter_func=filter_telephone_1)
+)
+filters.append(
+    Filter(
+        class_type="ELECTRONIC", process_func=process_electronic_1, filter_func=filter_electronic_1
+    )
+)
+filters.append(
+    Filter(class_type="FRACTION", process_func=process_fraction_1, filter_func=filter_fraction_1)
+)
+filters.append(
+    Filter(class_type="ADDRESS", process_func=process_address_1, filter_func=filter_address_1)
+)
+filters.append(Filter(class_type=EOS_TYPE, process_func=lambda x: x, filter_func=lambda x: True))
+
+
+def filter_loaded_data(data: List[Instance], verbose: bool = False) -> List[Instance]:
+    """
+    Filters list of instances
+
+    Args:
+        data: list of instances
+
+    Returns: filtered and transformed list of instances
+    """
+    updates_instances = []
+    for instance in data:
+        updated_instance = False
+        for fil in filters:
+            if fil.class_type == instance.token_type and fil.filter(instance):
+                instance = fil.process(instance)
+                updated_instance = True
+        if updated_instance:
+            if verbose:
+                print(instance)
+            updates_instances.append(instance)
+    return updates_instances
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--input", help="input file path", type=str, default="./en_with_types/output-00001-of-00100"
+    )
+    parser.add_argument("--verbose", help="print filtered instances", action="store_true")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    file_path = args.input
+
+    print("Loading training data: " + file_path)
+    instance_list = load_files([file_path])  # List of instances
+    filtered_instance_list = filter_loaded_data(instance_list, args.verbose)
+    training_data_to_sentences(filtered_instance_list)

fun_text_processing/inverse_text_normalization/ko/data/charactor.tsv

@@ -0,0 +1,57 @@
+%	퍼센트
+#	파운드
+=	등호
+@	골뱅이
+≥	보다 크거나 같음
+≤	보다 작거나 같음
+≠	부등호
+≈	근삿값
+±	플러스마이너스
+×	곱셈기호
+Α	알파
+Β	베타
+Γ	감마
+Δ	델타
+Ε	엡실론
+Ζ	제타
+Θ	세타
+Ι	이오타
+Κ	카파
+∧	람다
+Μ	뮤
+Ν	뉴
+Ξ	크시
+Ο	오미크론
+∏	파이
+Ρ	로
+∑	싱마
+Τ	타우
+Υ	입실론
+Φ	피
+Χ	키
+Ψ	프시
+Ω	오메가
+α	알파
+β	베타
+γ	감마
+δ	델타
+ε	엡실론
+ζ	제타
+η	에타
+θ	세타
+ι	이오타
+κ	카파
+λ	람다
+μ	뮤
+ν	뉴
+ξ	크시
+ο	오미크론
+π	파이
+ρ	로
+σ	싱마
+τ	타우
+υ	입실론
+φ	피
+χ	키
+ψ	프시
+ω	오메가

fun_text_processing/inverse_text_normalization/ko/data/currency.tsv

@@ -0,0 +1,34 @@
+$	달러
+$	미국 달러
+$	미국 달러
+£	영국 파운드
+€	유로
+₩	원
+nzd	뉴질랜드 달러
+rs	루피
+chf	스위스 프랑
+dkk	덴마크 크로네
+fim	핀란드 마르카
+aed	아랍 에미리트 디르함
+¥	엔
+czk	체코 코루나
+mro	모리타니 우기야
+pkr	파키스탄 루피
+crc	코스타리카 콜론
+hk$	홍콩 달러
+npr	네팔 루피
+awg	아루반 플로린
+nok	노르웨이 크로네
+tzs	탄자니아 실링
+sek	스웨덴 크로나
+cyp	키프로스 파운드
+sar	사우디 리얄
+cve	케이프 베르데 에스쿠도
+rsd	세르비아 디나르
+dm	독일 마크
+shp	세인트 헬레나 파운드
+php	필리핀 페소
+cad	캐나다 달러
+ssp	남수단 파운드
+scr	세이셸 루피
+mvr	몰디브 루피야

fun_text_processing/inverse_text_normalization/ko/data/day.tsv

@@ -0,0 +1,31 @@
+일일	01
+이일	02
+삼일	03
+사일	04
+오일	05
+육일	06
+칠일	07
+팔일	08
+구일	09
+십일	10
+십일일	11
+십이일	12
+십삼일	13
+십사일	14
+십오일	15
+십육일	16
+십칠일	17
+십팔일	18
+십구일	19
+이십일	20
+이십일일	21
+이십이일	22
+이십삼일	23
+이십사일	24
+이십오일	25
+이십육일	26
+이십칠일	27
+이십팔일	28
+이십구일	29
+삼십일	30
+삼십일일	31

fun_text_processing/inverse_text_normalization/ko/data/day_inherent.tsv

@@ -0,0 +1,17 @@
+하루	01
+이틀	02
+사흘	03
+나흘	04
+닷새	05
+엿새	06
+이레	07
+여드래	08
+아흐레	09
+열흘	10
+열하루	11
+열이틀	12
+열사흘	13
+스무날	20
+스무하루	21
+스무아흐레	29
+그믐	30

fun_text_processing/inverse_text_normalization/ko/data/electronic/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/inverse_text_normalization/ko/data/electronic/domain.tsv

@@ -0,0 +1,10 @@
+com
+uk
+fr
+net
+br
+in
+ru
+de
+it
+ai

fun_text_processing/inverse_text_normalization/ko/data/electronic/server_name.tsv

@@ -0,0 +1,17 @@
+g mail	gmail
+gmail
+n vidia	nvidia
+nvidia
+outlook
+hotmail
+yahoo
+aol
+gmx
+msn
+live
+yandex
+orange
+wanadoo
+web
+comcast
+google

fun_text_processing/inverse_text_normalization/ko/data/electronic/symbols.tsv

@@ -0,0 +1,22 @@
+.	점
+-	대시
+-	하이픈
+_	밑줄
+!	느낌표
+#	숫자 기호
+$	달러 기호
+%	퍼센트 기호
+&	앰퍼샌드
+'	인용하다
+*	별표
++	플러스
+/	슬래시
+=	등호
+?	물음표
+^	곡절 악센트
+`	오른쪽 작은따옴표
+{	왼쪽 중괄호
+|	세로 막대
+}	오른쪽 중괄호
+~	물결표
+,	반점

fun_text_processing/inverse_text_normalization/ko/data/measurements.tsv

@@ -0,0 +1,90 @@
+μm	마이크로미터
+mm	밀리미터
+cm	센치미터
+km	킬로미터
+mm²	평방밀리미터
+cm²	평방센치미터
+dm²	평방데시미터
+m²	평방미터
+km²	평방킬로미터
+mm³	세제곱 밀리미터
+cm³	세제곱 센치미터
+dm³	세제곱 데시미터
+m³	세제곱 미터
+km³	세제곱 킬로미터
+μg	마이크로그램
+mg	밀리그램
+kg	킬로그램
+msec	밀리초
+sec	초
+hr	시간
+m/s	미터매초
+km/h	킬로미터매시
+mph	마일시간
+bit/s	비트매초
+byte/s	바이트매초
+%	퍼센트
+°	도
+℃	섭씨
+℉	화씨
+kcal	킬로칼로리
+fl.oz	액량 온스
+F/m	파라미터
+g/l	그램매리터
+g/mL	그램매밀리미터
+hz	헤르츠
+khz	킬로헤르츠
+mhz	메가헤르츠
+ghz	기가헤르츠
+km/h	길로미터매시
+kw·h	킬로와트시
+ml	밀리리터
+mg/ml	밀리그램매밀리리터
+mg/l	밀리그램매리터
+mA	밀리 암페어
+mA⋅h	밀리 암페어시
+mol	몰
+Ω·m	옴표
+S/m	일미터당 일센스
+tsp	티스푼
+μA	마이크로 암페어
+Ω	오메가
+kPa	킬로파스칼
+mmHg	밀리미터 수은주
+Vol	볼륨
+cc	입방 센티미터
+rpm	분당 횟 수
+bpm	분당 박자 수
+px	픽셀
+V	볼트
+kV	킬로볼트
+ha	헥타르
+ac	에이커
+ct	캐럿
+L	리터
+gal	갤런
+mol	몰
+pa	파스칼
+mpa	메가파스칼
+mA	밀리암페어
+mAh	밀리암페어시
+in	인치
+ft	피트
+yd	야드
+nm	나노미터
+m	미터
+dm	데시미터
+g	그램
+KB	킬로바이트
+MB	메가바이트
+GB	기가바이트
+TB	테라바이트
+hp	마력
+db	데시벨
+J	줄
+kJ	킬로줄
+oz	온스
+kw	킬로와트
+min	분
+sec	초
+cal	칼로리

fun_text_processing/inverse_text_normalization/ko/data/months.tsv

@@ -0,0 +1,14 @@
+일월	01
+이월	02
+삼월	03
+사월	04
+오월	05
+육월	06
+유월	06
+칠월	07
+팔월	08
+구월	09
+십월	10
+시월	10
+십일월	11
+십이월	12

fun_text_processing/inverse_text_normalization/ko/data/numbers/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/inverse_text_normalization/ko/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+일	1
+이	2
+삼	3
+사	4
+오	5
+육	6
+칠	7
+팔	8
+구	9

fun_text_processing/inverse_text_normalization/ko/data/numbers/digit_inherent_digit.tsv

@@ -0,0 +1,9 @@
+하나	1
+둘	2
+셋	3
+넷	4
+다섯	5
+여섯	6
+일곱	7
+여덟	8
+아홉	9

fun_text_processing/inverse_text_normalization/ko/data/numbers/digit_inherent_others.tsv

@@ -0,0 +1,7 @@
+여든아홉	89
+예순셋	63
+쉰여덟	58
+서른다섯	35
+스물일곱	27
+열둘	27
+열하나	11

fun_text_processing/inverse_text_normalization/ko/data/numbers/digit_inherent_teens.tsv

@@ -0,0 +1,9 @@
+열	10
+스물	20
+서른	30
+마흔	40
+쉰	50
+예순	60
+일흔	70
+여든	80
+아흔	90

fun_text_processing/inverse_text_normalization/ko/data/numbers/digit_inherent_teens_without_zero.tsv

@@ -0,0 +1,9 @@
+열	1
+스물	2
+서른	3
+마흔	4
+쉰	5
+예순	6
+일흔	7
+여든	8
+아흔	9

fun_text_processing/inverse_text_normalization/ko/data/numbers/digit_teens.tsv

@@ -0,0 +1,9 @@
+십	10
+이십	20
+삼십	30
+사십	40
+오십	50
+육십	60
+칠십	70
+팔십	80
+구십	90

fun_text_processing/inverse_text_normalization/ko/data/numbers/digit_teens_without_zero.tsv

@@ -0,0 +1,9 @@
+십	1
+이십	2
+삼십	3
+사십	4
+오십	5
+육십	6
+칠십	7
+팔십	8
+구십	9

fun_text_processing/inverse_text_normalization/ko/data/numbers/dot.tsv

@@ -0,0 +1 @@
+점	.

fun_text_processing/inverse_text_normalization/ko/data/numbers/zero.tsv

@@ -0,0 +1 @@
+영	0

fun_text_processing/inverse_text_normalization/ko/data/ordinals/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/inverse_text_normalization/ko/data/ordinals/digit.tsv

@@ -0,0 +1,9 @@
+first	one
+second	two
+third	three
+fourth	four
+fifth	five
+sixth	sixth
+seventh	seven
+eighth	eight
+ninth	nine

fun_text_processing/inverse_text_normalization/ko/data/ordinals/teen.tsv

@@ -0,0 +1 @@
+twelfth	twelve

fun_text_processing/inverse_text_normalization/ko/data/suppletive.tsv

@@ -0,0 +1,83 @@
+deer
+fish
+sheep
+foot	feet
+goose	geese
+man	men
+mouse	mice
+tooth	teeth
+woman	women
+won
+child	children
+ox	oxen
+wife	wives
+wolf	wolves
+analysis	analyses
+criterion	criteria
+lbs
+focus	foci
+percent
+hertz
+kroner	krone
+inch	inches
+calory	calories
+yen
+megahertz
+gigahertz
+kilohertz
+hertz
+CC
+c c
+horsepower
+hundredweight
+kilogram force	kilograms force
+mega siemens
+revolution per minute	revolutions per minute
+mile per hour	miles per hour
+megabit per second	megabits per second
+square foot	square feet
+kilobit per second	kilobits per second
+degree Celsius	degrees Celsius
+degree Fahrenheit	degrees Fahrenheit
+ATM
+AU
+BQ
+CC
+CD
+DA
+EB
+EV
+F
+GB
+G
+GL
+GPA
+GY
+HA
+H
+HL
+GP
+HS
+KB
+KL
+KN
+KT
+KV
+LM
+MA
+MA
+MB
+MC
+MF
+M
+MM
+MS
+MV
+MW
+PB
+PG
+PS
+S
+TB
+YB
+ZB

fun_text_processing/inverse_text_normalization/ko/data/time/__init__.py

@@ -0,0 +1 @@
+

fun_text_processing/inverse_text_normalization/ko/data/time/hours.tsv

@@ -0,0 +1,12 @@
+한시	01
+두시	02
+세시	03
+네시	04
+다섯시	05
+여섯시	06
+일곱시	07
+여덟시	08
+아홉시	09
+열시	10
+열한시	11
+열두시	12

fun_text_processing/inverse_text_normalization/ko/data/time/minute_to.tsv

@@ -0,0 +1,59 @@
+1	59
+2	58
+3	57
+4	56
+5	55
+6	54
+7	53
+8	52
+9	51
+10	50
+11	49
+12	48
+13	47
+14	46
+15	45
+16	44
+17	43
+18	42
+19	41
+20	40
+21	39
+22	38
+23	37
+24	36
+25	35
+26	34
+27	33
+28	32
+29	31
+30	30
+31	29
+32	28
+33	27
+34	26
+35	25
+36	24
+37	23
+38	22
+39	21
+40	20
+41	19
+42	18
+43	17
+44	16
+45	15
+46	14
+47	13
+48	12
+49	11
+50	10
+51	9
+52	8
+53	7
+54	6
+55	5
+56	4
+57	3
+58	2
+59	1

fun_text_processing/inverse_text_normalization/ko/data/time/minutes.tsv

@@ -0,0 +1,59 @@
+일분	01
+이분	02
+삼분	03
+사분	04
+오분	05
+육분	06
+칠분	07
+팔분	08
+구분	09
+십분	10
+십일분	11
+십이분	12
+십삼분	13
+십사분	14
+십오분	15
+십육분	16
+십칠분	17
+십팔분	18
+십구분	19
+이십분	20
+이십일분	21
+이십이분	22
+이십삼분	23
+이십사분	24
+이십오분	25
+이십육분	26
+이십칠분	27
+이십팔분	28
+이십구분	29
+삼십분	30
+삼십일분	31
+삼십이분	32
+삼십삼분	33
+삼십사분	34
+삼십오분	35
+삼십육분	36
+삼십칠분	37
+삼십팔분	38
+삼십구분	39
+사십분	40
+사십일분	41
+사십이분	42
+사십삼분	43
+사십사분	44
+사십오분	45
+사십육분	46
+사십칠분	47
+사십팔분	48
+사십구분	49
+오십분	50
+오십일분	51
+오십이분	52
+오십삼분	53
+오십사분	54
+오십오분	55
+오십육분	56
+오십칠분	57
+오십팔분	58
+오십구분	59

fun_text_processing/inverse_text_normalization/ko/data/time/seconds.tsv

@@ -0,0 +1,59 @@
+일초	01
+이초	02
+삼초	03
+사초	04
+오초	05
+육초	06
+칠초	07
+팔초	08
+구초	09
+십초	10
+십일초	11
+십이초	12
+십삼초	13
+십사초	14
+십오초	15
+십육초	16
+십칠초	17
+십팔초	18
+십구초	19
+이십초	20
+이십일초	21
+이십이초	22
+이십삼초	23
+이십사초	24
+이십오초	25
+이십육초	26
+이십칠초	27
+이십팔초	28
+이십구초	29
+삼십초	30
+삼십일초	31
+삼십이초	32
+삼십삼초	33
+삼십사초	34
+삼십오초	35
+삼십육초	36
+삼십칠초	37
+삼십팔초	38
+삼십구초	39
+사십초	40
+사십일초	41
+사십이초	42
+사십삼초	43
+사십사초	44
+사십오초	45
+사십육초	46
+사십칠초	47
+사십팔초	48
+사십구초	49
+오십초	50
+오십일초	51
+오십이초	52
+오십삼초	53
+오십사초	54
+오십오초	55
+오십육초	56
+오십칠초	57
+오십팔초	58
+오십구초	59

fun_text_processing/inverse_text_normalization/ko/data/time/time_suffix.tsv

@@ -0,0 +1,8 @@
+p m	p.m.
+pm	p.m.
+p.m.
+p.m	p.m.
+am	a.m.
+a.m.
+a.m	a.m.
+a m	a.m.

fun_text_processing/inverse_text_normalization/ko/data/time/time_zone.tsv

@@ -0,0 +1,7 @@
+cst	c s t
+cet	c e t
+pst	p s t
+est	e s t
+pt	p t
+et	e t
+gmt	g m t

fun_text_processing/inverse_text_normalization/ko/data/time/to_hour.tsv

@@ -0,0 +1,12 @@
+one	12
+two	1
+three	2
+four	3
+five	4
+six	5
+seven	6
+eigh	7
+nine	8
+ten	9
+eleven	10
+twelve	11

fun_text_processing/inverse_text_normalization/ko/data/whitelist.tsv

@@ -0,0 +1,12 @@
+e.g.	for example
+dr.	doctor
+mr.	mister
+mrs.	misses
+st.	saint
+7-eleven	seven eleven
+es3	e s three
+s&p	s and p
+ASAP	a s a p
+AT&T	a t and t
+LLP	l l p
+ATM	a t m

fun_text_processing/inverse_text_normalization/ko/graph_utils.py

@@ -0,0 +1,209 @@
+import os
+import string
+from pathlib import Path
+from typing import Dict
+
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from pynini import Far
+from pynini.examples import plurals
+from pynini.export import export
+from pynini.lib import byte, pynutil, utf8
+
+DAMO_CHAR = utf8.VALID_UTF8_CHAR
+
+DAMO_DIGIT = byte.DIGIT
+DAMO_LOWER = pynini.union(*string.ascii_lowercase).optimize()
+DAMO_UPPER = pynini.union(*string.ascii_uppercase).optimize()
+DAMO_ALPHA = pynini.union(DAMO_LOWER, DAMO_UPPER).optimize()
+DAMO_ALNUM = pynini.union(DAMO_DIGIT, DAMO_ALPHA).optimize()
+DAMO_HEX = pynini.union(*string.hexdigits).optimize()
+DAMO_NON_BREAKING_SPACE = "\u00A0"
+DAMO_SPACE = " "
+DAMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize()
+DAMO_NOT_SPACE = pynini.difference(DAMO_CHAR, DAMO_WHITE_SPACE).optimize()
+DAMO_NOT_QUOTE = pynini.difference(DAMO_CHAR, r'"').optimize()
+
+DAMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize()
+DAMO_GRAPH = pynini.union(DAMO_ALNUM, DAMO_PUNCT).optimize()
+
+DAMO_SIGMA = pynini.closure(DAMO_CHAR)
+
+delete_space = pynutil.delete(pynini.closure(DAMO_WHITE_SPACE))
+delete_zero_or_one_space = pynutil.delete(pynini.closure(DAMO_WHITE_SPACE, 0, 1))
+insert_space = pynutil.insert(" ")
+delete_extra_space = pynini.cross(pynini.closure(DAMO_WHITE_SPACE, 1), " ")
+delete_preserve_order = pynini.closure(
+    pynutil.delete(" preserve_order: true")
+    | (pynutil.delete(' field_order: "') + DAMO_NOT_QUOTE + pynutil.delete('"'))
+)
+
+suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv"))
+# _v = pynini.union("a", "e", "i", "o", "u")
+_c = pynini.union(
+    "b",
+    "c",
+    "d",
+    "f",
+    "g",
+    "h",
+    "j",
+    "k",
+    "l",
+    "m",
+    "n",
+    "p",
+    "q",
+    "r",
+    "s",
+    "t",
+    "v",
+    "w",
+    "x",
+    "y",
+    "z",
+)
+_ies = DAMO_SIGMA + _c + pynini.cross("y", "ies")
+_es = DAMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es")
+_s = DAMO_SIGMA + pynutil.insert("s")
+
+graph_plural = plurals._priority_union(
+    suppletive,
+    plurals._priority_union(_ies, plurals._priority_union(_es, _s, DAMO_SIGMA), DAMO_SIGMA),
+    DAMO_SIGMA,
+).optimize()
+
+SINGULAR_TO_PLURAL = graph_plural
+PLURAL_TO_SINGULAR = pynini.invert(graph_plural)
+TO_LOWER = pynini.union(
+    *[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]
+)
+TO_UPPER = pynini.invert(TO_LOWER)
+MIN_NEG_WEIGHT = -0.0001
+MIN_POS_WEIGHT = 0.0001
+
+
+def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]):
+    """
+    Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name.
+
+    Args:
+        file_name: exported file name
+        graphs: Mapping of a rule name and Pynini WFST graph to be exported
+    """
+    exporter = export.Exporter(file_name)
+    for rule, graph in graphs.items():
+        exporter[rule] = graph.optimize()
+    exporter.close()
+    print(f"Created {file_name}")
+
+
+def get_plurals(fst):
+    """
+    Given singular returns plurals
+
+    Args:
+        fst: Fst
+
+    Returns plurals to given singular forms
+    """
+    return SINGULAR_TO_PLURAL @ fst
+
+
+def get_singulars(fst):
+    """
+    Given plural returns singulars
+
+    Args:
+        fst: Fst
+
+    Returns singulars to given plural forms
+    """
+    return PLURAL_TO_SINGULAR @ fst
+
+
+def convert_space(fst) -> "pynini.FstLike":
+    """
+    Converts space to nonbreaking space.
+    Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty"
+    This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it.
+
+    Args:
+        fst: input fst
+
+    Returns output fst where breaking spaces are converted to non breaking spaces
+    """
+    return fst @ pynini.cdrewrite(
+        pynini.cross(DAMO_SPACE, DAMO_NON_BREAKING_SPACE), "", "", DAMO_SIGMA
+    )
+
+
+class GraphFst:
+    """
+    Base class for all grammar fsts.
+
+    Args:
+        name: name of grammar class
+        kind: either 'classify' or 'verbalize'
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, name: str, kind: str, deterministic: bool = True):
+        self.name = name
+        self.kind = kind
+        self._fst = None
+        self.deterministic = deterministic
+
+        self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far")
+        if self.far_exist():
+            self._fst = Far(
+                self.far_path, mode="r", arc_type="standard", far_type="default"
+            ).get_fst()
+
+    def far_exist(self) -> bool:
+        """
+        Returns true if FAR can be loaded
+        """
+        return self.far_path.exists()
+
+    @property
+    def fst(self) -> "pynini.FstLike":
+        return self._fst
+
+    @fst.setter
+    def fst(self, fst):
+        self._fst = fst
+
+    def add_tokens(self, fst) -> "pynini.FstLike":
+        """
+        Wraps class name around to given fst
+
+        Args:
+            fst: input fst
+
+        Returns:
+            Fst: fst
+        """
+        return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }")
+
+    def delete_tokens(self, fst) -> "pynini.FstLike":
+        """
+        Deletes class name wrap around output of given fst
+
+        Args:
+            fst: input fst
+
+        Returns:
+            Fst: fst
+        """
+        res = (
+            pynutil.delete(f"{self.name}")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + fst
+            + delete_space
+            + pynutil.delete("}")
+        )
+        return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)

fun_text_processing/inverse_text_normalization/ko/taggers/cardinal.py

@@ -0,0 +1,142 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_SIGMA,
+    DAMO_DIGIT,
+    DAMO_SPACE,
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for classifying cardinals
+        e.g. minus twenty three -> cardinal { integer: "23" negative: "-" } }
+    Numbers below thirteen are not converted.
+    """
+
+    def __init__(self):
+        super().__init__(name="cardinal", kind="classify")
+        graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+
+        graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+        graph_teens_without_zero = pynini.string_file(
+            get_abs_path("data/numbers/digit_teens_without_zero.tsv")
+        )
+        graph_teens = pynini.string_file(get_abs_path("data/numbers/digit_teens.tsv"))
+
+        graph_inh_digit = pynini.string_file(get_abs_path("data/numbers/digit_inherent_digit.tsv"))
+        graph_inh_teen_without_zero = pynini.string_file(
+            get_abs_path("data/numbers/digit_inherent_teens_without_zero.tsv")
+        )
+        graph_inh_teen = pynini.string_file(get_abs_path("data/numbers/digit_inherent_teens.tsv"))
+        graph_inh_teen_others = pynini.string_file(
+            get_abs_path("data/numbers/digit_inherent_others.tsv")
+        )
+
+        graph_less_hundred_num_inh_p1 = graph_inh_teen_without_zero + graph_inh_digit
+        graph_less_hundred_num_inh = pynini.union(
+            graph_inh_teen, graph_less_hundred_num_inh_p1, graph_inh_teen_others
+        )
+
+        graph_less_hundred_num_p1 = graph_teens_without_zero + graph_digit
+        graph_less_hundred_num = pynini.union(graph_less_hundred_num_p1, graph_teens)
+
+        # digits
+        addzero = pynutil.insert("0")
+        zero = graph_zero
+        digits_combine = graph_digit | graph_inh_digit | zero
+        digits = graph_digit | zero
+        digit = graph_digit
+
+        # teens
+        teens_combine = graph_less_hundred_num | graph_less_hundred_num_inh
+        # teens = graph_less_hundred_num
+        teens = teens_combine
+
+        # hundred, #백 单位 百
+        hundred = (
+            digit
+            + pynutil.delete("백")
+            + (
+                teens
+                | pynutil.add_weight(zero + digit, 0.1)
+                | pynutil.add_weight(digit + addzero, 0.5)
+                | pynutil.add_weight(addzero**2, 1.0)
+            )
+        )
+
+        graph_hundred_component_at_least_one_none_zero_digit = hundred @ (
+            pynini.closure(DAMO_DIGIT) + (DAMO_DIGIT - "0") + pynini.closure(DAMO_DIGIT)
+        )
+
+        self.graph_hundred_component_at_least_one_none_zero_digit = (
+            graph_hundred_component_at_least_one_none_zero_digit
+        )
+
+        ##thousand 천 千单位
+        thousand = (
+            (hundred | teens | digits)
+            + pynutil.delete("천")
+            + (
+                hundred
+                | pynutil.add_weight(zero + teens, 0.1)
+                | pynutil.add_weight(addzero + zero + digit, 0.5)
+                | pynutil.add_weight(digit + addzero**2, 0.8)
+                | pynutil.add_weight(addzero**3, 1.0)
+            )
+        )
+
+        ##만 单位万
+        ten_thousand = (
+            (thousand | hundred | teens | digits)
+            + pynutil.delete("만")
+            + pynini.cross(" ", "").ques
+            + (
+                thousand
+                | pynutil.add_weight(zero + hundred, 0.1)
+                | pynutil.add_weight(addzero + zero + teens, 0.5)
+                | pynutil.add_weight(addzero + addzero + zero + digit, 0.5)
+                | pynutil.add_weight(digit + addzero**3, 0.8)
+                | pynutil.add_weight(addzero**4, 1.0)
+            )
+        )
+
+        ##조, 单位兆，  억, 单位亿
+        number = digits | teens | hundred | thousand | ten_thousand
+
+        ## ques is equal to pynini.closure(, 0, 1)
+        number = (
+            (number + pynini.accep("조").ques + pynini.cross(" ", "").ques).ques
+            + (number + pynini.accep("억").ques + pynini.cross(" ", "").ques).ques
+            + number
+        )
+
+        graph = (
+            number
+            | graph_less_hundred_num_inh
+            | graph_inh_digit
+            | graph_inh_teen
+            | graph_inh_teen_others
+        )
+        # labels_exception = [num_to_word(x) for x in range(0, 13)]
+        labels_exception = ["zzzzzzzzz"]
+        graph_exception = pynini.union(*labels_exception)
+
+        self.graph_no_exception = graph
+
+        self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph
+
+        optional_minus_graph = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("마이너스", '"-"') + DAMO_SPACE, 0, 1
+        )
+
+        final_graph = (
+            optional_minus_graph + pynutil.insert('integer: "') + self.graph + pynutil.insert('"')
+        )
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/date.py

@@ -0,0 +1,95 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).optimize()
+graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).optimize()
+graph_digit_inh = pynini.string_file(
+    get_abs_path("data/numbers/digit_inherent_digit.tsv")
+).optimize()
+
+
+def _get_month_graph():
+    """
+    Transducer for month, e.g. march -> march
+    """
+    month_graph = pynini.string_file(get_abs_path("data/months.tsv"))
+    # print(month_graph)
+    return month_graph
+
+
+def _get_day_graph():
+    """
+    Transducer for month, e.g. march -> march
+    """
+    day_graph_num = pynini.string_file(get_abs_path("data/day.tsv"))
+    day_graph_inh = pynini.string_file(get_abs_path("data/day_inherent.tsv"))
+    day_graph = pynini.union(day_graph_num, day_graph_inh)
+    # print(day_graph)
+    return day_graph
+
+
+def _get_year_graph():
+    """
+    Transducer for year, e.g. twenty twenty -> 2020
+    """
+    digit = graph_digit | graph_digit_inh
+    zero = graph_zero
+    year_graph_4num = digit + (digit | zero) ** 3
+    year_graph_2num = digit**2
+
+    year_graph = pynini.union(year_graph_4num, year_graph_2num)
+    return year_graph
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for classifying date,
+        e.g. january fifth twenty twelve -> date { month: "january" day: "5" year: "2012" preserve_order: true }
+        e.g. the fifth of january twenty twelve -> date { day: "5" month: "january" year: "2012" preserve_order: true }
+        e.g. twenty twenty -> date { year: "2012" preserve_order: true }
+    Args:
+        ordinal: OrdinalFst
+    """
+
+    def __init__(self):
+        super().__init__(name="date", kind="classify")
+
+        year_graph = _get_year_graph() + pynini.accep("년")
+        YEAR_WEIGHT = 0.001
+        year_graph = (
+            pynutil.insert('year: "')
+            + pynutil.add_weight(year_graph, YEAR_WEIGHT)
+            + pynutil.insert('"')
+        )
+        # year_graph_space = pynutil.insert("year: \"") + pynutil.add_weight(year_graph, YEAR_WEIGHT) + pynutil.insert("\"") + pynutil.insert(" ")
+        # year_graph = pynutil.insert("year: \"") + year_graph + pynutil.insert("\"")
+
+        MONTH_WEIGHT = -0.001
+        month_graph = _get_month_graph() + pynini.cross("", "월")
+        # month_graph = pynutil.insert("month: \"") + pynutil.add_weight(month_graph, MONTH_WEIGHT) + pynutil.insert("\"")
+        month_graph = pynutil.insert('month: "') + month_graph + pynutil.insert('"')
+        # month_graph_space = pynutil.insert("month: \"") + month_graph + pynutil.insert("\"") + pynutil.insert(" ")
+
+        day_graph = _get_day_graph() + pynini.cross("", "일")
+        DAY_WEIGHT = -0.7
+        # day_graph = pynutil.insert("day: \"") + pynutil.add_weight(day_graph, DAY_WEIGHT) + pynutil.insert("\"")
+        day_graph = pynutil.insert('day: "') + day_graph + pynutil.insert('"')
+        # day_graph_space = pynutil.insert("day: \"") + day_graph + pynutil.insert("\"") + pynutil.insert(" ")
+
+        graph_ymd = year_graph + delete_space + month_graph + delete_space + day_graph
+        graph_md = month_graph + delete_space + day_graph
+        graph_ym = year_graph + delete_space + month_graph
+
+        final_graph = graph_ymd | graph_md | graph_ym | year_graph | month_graph | day_graph
+
+        final_graph += pynutil.insert(" preserve_order: true")
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/decimal.py

@@ -0,0 +1,101 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_DIGIT,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+def get_quantity(
+    decimal: "pynini.FstLike", cardinal_up_to_hundred: "pynini.FstLike"
+) -> "pynini.FstLike":
+    """
+    Returns FST that transforms either a cardinal or decimal followed by a quantity into a numeral,
+    e.g. one million -> integer_part: "1" quantity: "million"
+    e.g. one point five million -> integer_part: "1" fractional_part: "5" quantity: "million"
+
+    Args:
+        decimal: decimal FST
+        cardinal_up_to_hundred: cardinal FST
+    """
+    numbers = cardinal_up_to_hundred @ (
+        pynutil.delete(pynini.closure("0"))
+        + pynini.difference(DAMO_DIGIT, "0")
+        + pynini.closure(DAMO_DIGIT)
+    )
+    # "만", "백만", "천만", "억", "조", 万、百万、千万、亿、兆
+    # 천 千
+    suffix = pynini.union("만", "백만", "천만", "억", "조")
+    res = (
+        pynutil.insert('integer_part: "')
+        + numbers
+        + pynutil.insert('"')
+        + delete_extra_space
+        + pynutil.insert('quantity: "')
+        + suffix
+        + pynutil.insert('"')
+    )
+    res |= (
+        decimal
+        + delete_extra_space
+        + pynutil.insert('quantity: "')
+        + (suffix | "천")
+        + pynutil.insert('"')
+    )
+    return res
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for classifying decimal
+        e.g. minus twelve point five o o six billion -> decimal { negative: "true" integer_part: "12"  fractional_part: "5006" quantity: "billion" }
+        e.g. one billion -> decimal { integer_part: "1" quantity: "billion" }
+    Args:
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, cardinal: GraphFst):
+        super().__init__(name="decimal", kind="classify")
+
+        cardinal_graph = cardinal.graph_no_exception
+
+        graph_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+        graph_decimal |= pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+
+        graph_decimal = pynini.closure(graph_decimal)
+        self.graph = graph_decimal
+
+        ##마이너스 负
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("마이너스", '"true"') + delete_extra_space,
+            0,
+            1,
+        )
+
+        graph_fractional = (
+            pynutil.insert('fractional_part: "') + graph_decimal + pynutil.insert('"')
+        )
+
+        # 점 点
+        graph_integer = (
+            pynutil.insert('integer_part: "')
+            + cardinal_graph
+            + pynutil.delete("점")
+            + pynutil.insert('"')
+        )
+
+        final_graph_wo_sign = graph_integer + pynini.cross(" ", " ") + graph_fractional
+
+        final_graph = optional_graph_negative + delete_space + final_graph_wo_sign
+
+        self.final_graph_wo_negative = final_graph_wo_sign | get_quantity(
+            final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit
+        )
+        final_graph |= optional_graph_negative + get_quantity(
+            final_graph_wo_sign, cardinal.graph_hundred_component_at_least_one_none_zero_digit
+        )
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/electronic.py

@@ -0,0 +1,100 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_ALPHA,
+    GraphFst,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for classifying electronic: as URLs, email addresses, etc.
+        e.g. c d f one at a b c dot e d u -> tokens { electronic { username: "cdf1" domain: "abc.edu" } }
+    """
+
+    def __init__(self):
+        super().__init__(name="electronic", kind="classify")
+
+        delete_extra_space = pynutil.delete(" ")
+        alpha_num = (
+            DAMO_ALPHA
+            | pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+            | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        )
+
+        symbols = pynini.string_file(get_abs_path("data/electronic/symbols.tsv")).invert()
+
+        accepted_username = alpha_num | symbols
+        process_dot = pynini.cross("점", ".")
+        username = (
+            alpha_num + pynini.closure(delete_extra_space + accepted_username)
+        ) | pynutil.add_weight(pynini.closure(DAMO_ALPHA, 1), weight=0.0001)
+        username = pynutil.insert('username: "') + username + pynutil.insert('"')
+        single_alphanum = pynini.closure(alpha_num + delete_extra_space) + alpha_num
+        server = single_alphanum | pynini.string_file(
+            get_abs_path("data/electronic/server_name.tsv")
+        )
+        domain = single_alphanum | pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
+        domain_graph = (
+            pynutil.insert('domain: "')
+            + server
+            + delete_extra_space
+            + process_dot
+            + delete_extra_space
+            + domain
+            + pynutil.insert('"')
+        )
+        graph = (
+            username
+            + delete_extra_space
+            + pynutil.delete("에서")
+            + insert_space
+            + delete_extra_space
+            + domain_graph
+        )
+
+        ############# url ###
+        protocol_end = pynini.cross(pynini.union("w w w", "www"), "www")
+        protocol_start = (
+            pynini.cross("h t t p", "http") | pynini.cross("h t t p s", "https")
+        ) + pynini.cross(" 콜론 슬래시 슬래시 ", "://")
+        # .com,
+        ending = (
+            delete_extra_space
+            + symbols
+            + delete_extra_space
+            + (
+                domain
+                | pynini.closure(
+                    accepted_username + delete_extra_space,
+                )
+                + accepted_username
+            )
+        )
+
+        protocol_default = (
+            (
+                (pynini.closure(delete_extra_space + accepted_username, 1) | server)
+                | pynutil.add_weight(pynini.closure(DAMO_ALPHA, 1), weight=0.0001)
+            )
+            + pynini.closure(ending, 1)
+        ).optimize()
+        protocol = (
+            pynini.closure(protocol_start, 0, 1)
+            + protocol_end
+            + delete_extra_space
+            + process_dot
+            + protocol_default
+        ).optimize()
+
+        protocol |= (
+            pynini.closure(protocol_end + delete_extra_space + process_dot, 0, 1) + protocol_default
+        )
+
+        protocol = pynutil.insert('protocol: "') + protocol.optimize() + pynutil.insert('"')
+        graph |= protocol
+
+        final_graph = self.add_tokens(graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/fraction.py

@@ -0,0 +1,53 @@
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    convert_space,
+    delete_space,
+    delete_extra_space,
+    DAMO_SIGMA,
+    DAMO_CHAR,
+    DAMO_SPACE,
+)
+import pynini
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for classifying fraction
+    """
+
+    def __init__(self, cardinal: GraphFst):
+        super().__init__(name="fraction", kind="classify")
+        # integer_part # numerator # denominator
+
+        graph_cardinal = cardinal.graph_no_exception
+
+        # without the integerate part
+        # 分子
+        numerator = pynutil.insert('numerator: "') + graph_cardinal + pynutil.insert('"')
+        # 分母
+        denominator = (
+            pynutil.insert('denominator: "')
+            + graph_cardinal
+            + pynutil.delete("분의")
+            + pynutil.insert('"')
+        )
+
+        ##
+        graph_fraction_component = denominator + pynini.cross(" ", " ") + numerator
+
+        self.graph_fraction_component = graph_fraction_component
+
+        graph = graph_fraction_component
+        graph = graph.optimize()
+        self.final_graph_wo_negative = graph
+
+        ##负
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("마이너스", '"true"') + DAMO_SPACE, 0, 1
+        )
+
+        graph = optional_graph_negative + graph
+        final_graph = self.add_tokens(graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/measure.py

@@ -0,0 +1,64 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_SIGMA,
+    GraphFst,
+    convert_space,
+    delete_extra_space,
+    delete_space,
+    get_singulars,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for classifying measure
+        e.g. minus twelve kilograms -> measure { negative: "true" cardinal { integer: "12" } units: "kg" }
+
+    Args:
+        cardinal: CardinalFst
+        decimal: DecimalFst
+    """
+
+    def __init__(self, cardinal: GraphFst, decimal: GraphFst):
+        super().__init__(name="measure", kind="classify")
+
+        cardinal_graph = cardinal.graph_no_exception
+        decimal_graph = decimal.final_graph_wo_negative
+
+        unit_graph = pynini.string_file(get_abs_path("data/measurements.tsv"))
+
+        graph_unit = pynini.invert(unit_graph)  # singular -> abbr
+
+        ## 마이너 负
+        optional_graph_negative = pynini.closure(
+            pynutil.insert("negative: ") + pynini.cross("마이너", '"true"') + delete_extra_space,
+            0,
+            1,
+        )
+
+        graph_units = pynutil.insert('units: "') + graph_unit + pynutil.insert('"')
+
+        subgraph_decimal = (
+            pynutil.insert("decimal { ")
+            + optional_graph_negative
+            + decimal_graph
+            + pynutil.insert(" }")
+            + delete_extra_space
+            + graph_units
+        )
+        subgraph_cardinal = (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert('integer: "')
+            + cardinal_graph
+            + pynutil.insert('"')
+            + pynutil.insert(" }")
+            + delete_extra_space
+            + graph_units
+        )
+
+        final_graph = subgraph_decimal | subgraph_cardinal
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/money.py

@@ -0,0 +1,51 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_DIGIT,
+    DAMO_NOT_SPACE,
+    DAMO_SIGMA,
+    GraphFst,
+    convert_space,
+    delete_extra_space,
+    delete_space,
+    get_singulars,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for classifying money
+        e.g. twelve dollars and five cents -> money { integer_part: "12" fractional_part: 05 currency: "$" }
+
+    Args:
+        cardinal: CardinalFst
+        decimal: DecimalFst
+    """
+
+    def __init__(self, cardinal: GraphFst, decimal: GraphFst):
+        super().__init__(name="money", kind="classify")
+        # quantity, integer_part, fractional_part, currency
+
+        cardinal_graph = cardinal.graph_no_exception
+        decimal_graph = decimal.final_graph_wo_negative
+
+        unit = pynini.string_file(get_abs_path("data/currency.tsv")).invert()
+
+        graph_unit = pynutil.insert('currency: "') + unit + pynutil.insert('"')
+
+        graph_integer = (
+            pynutil.insert('integer_part: "')
+            + cardinal_graph
+            + pynutil.insert('"')
+            + delete_extra_space
+            + graph_unit
+        )
+
+        graph_decimal = decimal_graph + pynutil.insert(" ") + graph_unit
+
+        final_graph = graph_integer | graph_decimal
+
+        final_graph = self.add_tokens(final_graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/punctuation.py

@@ -0,0 +1,20 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
+from pynini.lib import pynutil
+
+
+class PunctuationFst(GraphFst):
+    """
+    Finite state transducer for classifying punctuation
+        e.g. a, -> tokens { name: "a" } tokens { name: "," }
+    """
+
+    def __init__(self):
+        super().__init__(name="punctuation", kind="classify")
+
+        s = ",.?"  # here, we only support three type of punctuation
+        punct = pynini.union(*s)
+
+        graph = pynutil.insert('name: "') + punct + pynutil.insert('"')
+
+        self.fst = graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/telephone.py

@@ -0,0 +1,79 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_ALNUM,
+    DAMO_ALPHA,
+    DAMO_DIGIT,
+    GraphFst,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+def get_serial_number(cardinal):
+    """
+    any alphanumerical character sequence with at least one number with length greater equal to 3
+    """
+    digit = pynini.compose(cardinal.graph_no_exception, DAMO_DIGIT)
+    character = digit | DAMO_ALPHA
+    sequence = character + pynini.closure(pynutil.delete(" ") + character, 2)
+    sequence = sequence @ (pynini.closure(DAMO_ALNUM) + DAMO_DIGIT + pynini.closure(DAMO_ALNUM))
+    return sequence.optimize()
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for classifying telephone numbers, e.g.
+        one two three one two three five six seven eight -> { number_part: "123-123-5678" }
+
+    This class also support card number and IP format.
+        "one two three dot one double three dot o dot four o" -> { number_part: "123.133.0.40"}
+
+        "three two double seven three two one four three two one four three double zero five" ->
+            { number_part: 3277 3214 3214 3005}
+
+    Args:
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, cardinal: GraphFst):
+        super().__init__(name="telephone", kind="classify")
+        # country code, number_part, extension
+
+        graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
+        graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
+        graph_dot = pynini.string_file(get_abs_path("data/numbers/dot.tsv"))
+
+        graph_digits = graph_digit | graph_zero
+
+        phone_number_graph = graph_digits**9 | graph_digits**10 | graph_digits**11
+
+        country_code = (
+            pynutil.insert('country_code: "')
+            + pynini.closure(pynini.cross("더한", "+"), 0, 1)
+            + (pynini.closure(graph_digits, 0, 2) + graph_digits)
+            + pynutil.insert('"')
+        )
+
+        optional_country_code = pynini.closure(
+            country_code + pynutil.delete(" ") + insert_space, 0, 1
+        ).optimize()
+
+        grpah_phone_number = (
+            pynutil.insert('number_part: "') + phone_number_graph + pynutil.insert('"')
+        )
+
+        graph = optional_country_code + grpah_phone_number
+
+        # ip
+        ip_graph = graph_digit.plus + (graph_dot + graph_digits.plus).plus
+
+        graph |= pynutil.insert('number_part: "') + ip_graph.optimize() + pynutil.insert('"')
+        graph |= (
+            pynutil.insert('number_part: "')
+            + pynutil.add_weight(get_serial_number(cardinal=cardinal), weight=0.0001)
+            + pynutil.insert('"')
+        )
+
+        final_graph = self.add_tokens(graph)
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/time.py

@@ -0,0 +1,96 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path, num_to_word
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    GraphFst,
+    convert_space,
+    delete_extra_space,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for classifying time
+        e.g. twelve thirty -> time { hours: "12" minutes: "30" }
+        e.g. twelve past one -> time { minutes: "12" hours: "1" }
+        e.g. two o clock a m -> time { hours: "2" suffix: "a.m." }
+        e.g. quarter to two -> time { hours: "1" minutes: "45" }
+        e.g. quarter past two -> time { hours: "2" minutes: "15" }
+        e.g. half past two -> time { hours: "2" minutes: "30" }
+    """
+
+    def __init__(self):
+        super().__init__(name="time", kind="classify")
+        # hours, minutes, seconds, suffix, zone, style, speak_period
+
+        suffix_graph = pynini.string_file(get_abs_path("data/time/time_suffix.tsv"))
+        time_zone_graph = pynini.invert(pynini.string_file(get_abs_path("data/time/time_zone.tsv")))
+
+        hour_graph = pynini.string_file(get_abs_path("data/time/hours.tsv"))
+        minute_graph = pynini.string_file(get_abs_path("data/time/minutes.tsv"))
+        second_graph = pynini.string_file(get_abs_path("data/time/seconds.tsv"))
+
+        # only used for < 1000 thousand -> 0 weight
+        # cardinal = pynutil.add_weight(CardinalFst().graph_no_exception, weight=-0.7)
+
+        graph_hour = hour_graph
+        graph_minute = minute_graph
+        graph_second = second_graph
+
+        final_graph_hour = pynutil.insert('hours: "') + graph_hour + pynutil.insert('"')
+
+        final_suffix = (
+            pynutil.insert('suffix: "') + convert_space(suffix_graph) + pynutil.insert('"')
+        )
+        final_suffix = delete_space + insert_space + final_suffix
+        final_suffix_optional = pynini.closure(final_suffix, 0, 1)
+        final_time_zone_optional = pynini.closure(
+            delete_space
+            + insert_space
+            + pynutil.insert('zone: "')
+            + convert_space(time_zone_graph)
+            + pynutil.insert('"'),
+            0,
+            1,
+        )
+
+        graph_hm = (
+            final_graph_hour
+            + delete_extra_space
+            + pynutil.insert('minutes: "')
+            + graph_minute
+            + pynutil.insert('"')
+        )
+
+        graph_hms = (
+            final_graph_hour
+            + delete_extra_space
+            + pynutil.insert('minutes: "')
+            + graph_minute
+            + pynutil.insert('"')
+            + delete_extra_space
+            + pynutil.insert('seconds: "')
+            + graph_second
+            + pynutil.insert('"')
+        )
+
+        graph_h = (
+            final_graph_hour
+            + delete_extra_space
+            + pynutil.insert('minutes: "')
+            + (pynutil.insert("00") | graph_minute)
+            + pynutil.insert('"')
+            + final_suffix
+            + final_time_zone_optional
+        )
+
+        final_graph = (graph_hm | graph_hms) + final_suffix_optional + final_time_zone_optional
+
+        final_graph |= graph_h
+
+        final_graph = self.add_tokens(final_graph.optimize())
+
+        self.fst = final_graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/tokenize_and_classify.py

@@ -0,0 +1,102 @@
+import os
+
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.taggers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.date import DateFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.fraction import FractionFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.punctuation import PunctuationFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.time import TimeFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.whitelist import WhiteListFst
+from fun_text_processing.inverse_text_normalization.ko.taggers.word import WordFst
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+    generator_main,
+)
+from pynini.lib import pynutil
+
+import logging
+
+
+class ClassifyFst(GraphFst):
+    """
+    Final class that composes all other classification grammars. This class can process an entire sentence, that is lower cased.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+
+    Args:
+        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
+        overwrite_cache: set to True to overwrite .far files
+    """
+
+    def __init__(self, cache_dir: str = None, overwrite_cache: bool = False):
+        super().__init__(name="tokenize_and_classify", kind="classify")
+
+        far_file = None
+        if cache_dir is not None and cache_dir != "None":
+            os.makedirs(cache_dir, exist_ok=True)
+            far_file = os.path.join(cache_dir, "_ko_itn.far")
+        if not overwrite_cache and far_file and os.path.exists(far_file):
+            self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
+            logging.info(f"ClassifyFst.fst was restored from {far_file}.")
+        else:
+            logging.info(f"Creating ClassifyFst grammars.")
+            cardinal = CardinalFst()
+            cardinal_graph = cardinal.fst
+
+            decimal = DecimalFst(cardinal)
+            decimal_graph = decimal.fst
+
+            fraction = FractionFst(cardinal)
+            fraction_graph = fraction.fst
+
+            measure_graph = MeasureFst(cardinal=cardinal, decimal=decimal).fst
+            date_graph = DateFst().fst
+            word_graph = WordFst().fst
+            time_graph = TimeFst().fst
+            money_graph = MoneyFst(cardinal=cardinal, decimal=decimal).fst
+            whitelist_graph = WhiteListFst().fst
+            punct_graph = PunctuationFst().fst
+            electronic_graph = ElectronicFst().fst
+            telephone_graph = TelephoneFst(cardinal).fst
+
+            classify = (
+                pynutil.add_weight(whitelist_graph, 1.01)
+                | pynutil.add_weight(time_graph, 1.1)
+                | pynutil.add_weight(date_graph, 1.09)
+                | pynutil.add_weight(decimal_graph, 1.1)
+                | pynutil.add_weight(fraction_graph, 1.1)
+                | pynutil.add_weight(measure_graph, 1.1)
+                | pynutil.add_weight(cardinal_graph, 1.1)
+                | pynutil.add_weight(money_graph, 1.1)
+                | pynutil.add_weight(telephone_graph, 1.1)
+                | pynutil.add_weight(electronic_graph, 1.1)
+                | pynutil.add_weight(word_graph, 100)
+            )
+
+            punct = (
+                pynutil.insert("tokens { ")
+                + pynutil.add_weight(punct_graph, weight=1.1)
+                + pynutil.insert(" }")
+            )
+            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
+            token_plus_punct = (
+                pynini.closure(punct + pynutil.insert(" "))
+                + token
+                + pynini.closure(pynutil.insert(" ") + punct)
+            )
+
+            graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct)
+            graph = delete_space + graph + delete_space
+
+            self.fst = graph.optimize()
+
+            if far_file:
+                generator_main(far_file, {"tokenize_and_classify": self.fst})
+                logging.info(f"ClassifyFst grammars are saved to {far_file}.")

fun_text_processing/inverse_text_normalization/ko/taggers/whitelist.py

@@ -0,0 +1,19 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.utils import get_abs_path
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst, convert_space
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for classifying whitelisted tokens
+        e.g. misses -> tokens { name: "mrs." }
+    This class has highest priority among all classifier grammars. Whitelisted tokens are defined and loaded from "data/whitelist.tsv".
+    """
+
+    def __init__(self):
+        super().__init__(name="whitelist", kind="classify")
+
+        whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")).invert()
+        graph = pynutil.insert('name: "') + convert_space(whitelist) + pynutil.insert('"')
+        self.fst = graph.optimize()

fun_text_processing/inverse_text_normalization/ko/taggers/word.py

@@ -0,0 +1,15 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_SPACE, GraphFst
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class.
+        e.g. sleep -> tokens { name: "sleep" }
+    """
+
+    def __init__(self):
+        super().__init__(name="word", kind="classify")
+        word = pynutil.insert('name: "') + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert('"')
+        self.fst = word.optimize()

fun_text_processing/inverse_text_normalization/ko/utils.py

@@ -0,0 +1,64 @@
+import csv
+import os
+from typing import Union
+
+import inflect
+
+_inflect = inflect.engine()
+
+
+def num_to_word(x: Union[str, int]):
+    """
+    converts integer to spoken representation
+    Args
+        x: integer
+    Returns: spoken representation
+    """
+    if isinstance(x, int):
+        x = str(x)
+        x = _inflect.number_to_words(str(x)).replace("-", " ").replace(",", "")
+    return x
+
+
+def get_abs_path(rel_path):
+    """
+    Get absolute path
+
+    Args:
+        rel_path: relative path to this file
+
+    Returns absolute path
+    """
+    return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path
+
+
+def load_labels(abs_path):
+    """
+    loads relative path file as dictionary
+
+    Args:
+        abs_path: absolute path
+
+    Returns dictionary of mappings
+    """
+    label_tsv = open(abs_path, encoding="utf-8")
+    labels = list(csv.reader(label_tsv, delimiter="\t"))
+    return labels
+
+
+def augment_labels_with_punct_at_end(labels):
+    """
+    augments labels: if key ends on a punctuation that value does not have, add a new label
+    where the value maintains the punctuation
+
+    Args:
+        labels : input labels
+    Returns:
+        additional labels
+    """
+    res = []
+    for label in labels:
+        if len(label) > 1:
+            if label[0][-1] == "." and label[1][-1] != ".":
+                res.append([label[0], label[1] + "."] + label[2:])
+    return res

fun_text_processing/inverse_text_normalization/ko/verbalizers/cardinal.py

@@ -0,0 +1,38 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class CardinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing cardinal
+        e.g. cardinal { integer: "23" negative: "-" } -> -23
+    """
+
+    def __init__(self):
+        super().__init__(name="cardinal", kind="verbalize")
+        optional_sign = pynini.closure(
+            pynutil.delete("negative:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space,
+            0,
+            1,
+        )
+        graph = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        self.numbers = graph
+        graph = optional_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/date.py

@@ -0,0 +1,86 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for verbalizing date, e.g.
+        date { month: "january" day: "5" year: "2012" preserve_order: true } -> february 5 2012
+        date { day: "5" month: "january" year: "2012" preserve_order: true } -> 5 february 2012
+    """
+
+    def __init__(self):
+        super().__init__(name="date", kind="verbalize")
+        month = (
+            pynutil.delete("month:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynutil.insert(" ")
+        )
+        day = (
+            pynutil.delete("day:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynutil.insert(" ")
+        )
+        year = (
+            pynutil.delete("year:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynutil.insert(" ")
+        )
+
+        # month (day) year
+        graph_mdy = (
+            month
+            + pynini.closure(delete_extra_space + day, 0, 1)
+            + pynini.closure(delete_extra_space + year, 0, 1)
+        )
+
+        # (day) month year
+        graph_dmy = (
+            pynini.closure(day + delete_extra_space, 0, 1)
+            + month
+            + pynini.closure(delete_extra_space + year, 0, 1)
+        )
+
+        optional_preserve_order = pynini.closure(
+            pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space
+            | pynutil.delete("field_order:")
+            + delete_space
+            + pynutil.delete('"')
+            + DAMO_NOT_QUOTE
+            + pynutil.delete('"')
+            + delete_space
+        )
+
+        # year month day
+        graph_ymd = year + month + day
+
+        # month day
+        graph_md = month + day
+
+        # year month
+        graph_ym = year + month
+
+        # add some grammars
+        final_graph = (
+            (graph_mdy | year | graph_dmy | graph_ymd | graph_md | graph_ym | month | day)
+            + delete_space
+            + optional_preserve_order
+        )
+
+        delete_tokens = self.delete_tokens(final_graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/decimal.py

@@ -0,0 +1,48 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class DecimalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing decimal, e.g.
+        decimal { negative: "true" integer_part: "12"  fractional_part: "5006" quantity: "billion" } -> -12.5006 billion
+    """
+
+    def __init__(self):
+        super().__init__(name="decimal", kind="verbalize")
+        optionl_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        integer = (
+            pynutil.delete("integer_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_integer = pynini.closure(integer + delete_space, 0, 1)
+        fractional = (
+            pynutil.insert(".")
+            + pynutil.delete("fractional_part:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_fractional = pynini.closure(fractional + delete_space, 0, 1)
+        quantity = (
+            pynutil.delete("quantity:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_quantity = pynini.closure(pynutil.insert(" ") + quantity + delete_space, 0, 1)
+        graph = optional_integer + optional_fractional + optional_quantity
+        self.numbers = graph
+        graph = optionl_sign + graph
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/electronic.py

@@ -0,0 +1,45 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class ElectronicFst(GraphFst):
+    """
+    Finite state transducer for verbalizing electronic
+        e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> cdf1@abc.edu
+    """
+
+    def __init__(self):
+        super().__init__(name="electronic", kind="verbalize")
+        user_name = (
+            pynutil.delete("username:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        domain = (
+            pynutil.delete("domain:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        protocol = (
+            pynutil.delete("protocol:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = user_name + delete_space + pynutil.insert("@") + domain
+        graph |= protocol
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/fraction.py

@@ -0,0 +1,41 @@
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+
+import pynini
+from pynini.lib import pynutil
+
+
+class FractionFst(GraphFst):
+    """
+    Finite state transducer for verbalizing fraction,
+    """
+
+    def __init__(self):
+        super().__init__(name="fraction", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-") + delete_space, 0, 1)
+        numerator = (
+            pynutil.delete("numerator:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        denominator = (
+            pynutil.delete("denominator:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+
+        graph = (
+            optional_sign + numerator + delete_space + pynutil.insert("/") + denominator
+        ).optimize()
+        self.numbers = graph
+        delete_tokens = self.delete_tokens(optional_sign + graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/measure.py

@@ -0,0 +1,51 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MeasureFst(GraphFst):
+    """
+    Finite state transducer for verbalizing measure, e.g.
+        measure { negative: "true" cardinal { integer: "12" } units: "kg" } -> -12 kg
+
+    Args:
+        decimal: DecimalFst
+        cardinal: CardinalFst
+    """
+
+    def __init__(self, decimal: GraphFst, cardinal: GraphFst):
+        super().__init__(name="measure", kind="verbalize")
+        optional_sign = pynini.closure(pynini.cross('negative: "true"', "-"), 0, 1)
+        unit = (
+            pynutil.delete("units:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+            + delete_space
+        )
+        graph_decimal = (
+            pynutil.delete("decimal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + decimal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph_cardinal = (
+            pynutil.delete("cardinal {")
+            + delete_space
+            + optional_sign
+            + delete_space
+            + cardinal.numbers
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = (graph_cardinal | graph_decimal) + delete_space + pynutil.insert(" ") + unit
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/money.py

@@ -0,0 +1,30 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class MoneyFst(GraphFst):
+    """
+    Finite state transducer for verbalizing money, e.g.
+        money { integer_part: "12" fractional_part: "05" currency: "$" } -> $12.05
+
+    Args:
+        decimal: DecimalFst
+    """
+
+    def __init__(self, decimal: GraphFst):
+        super().__init__(name="money", kind="verbalize")
+        unit = (
+            pynutil.delete("currency:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = unit + delete_space + decimal.numbers
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/ordinal.py

@@ -0,0 +1,48 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_NOT_QUOTE,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class OrdinalFst(GraphFst):
+    """
+    Finite state transducer for verbalizing ordinal, e.g.
+       ordinal { integer: "13" } -> 13th
+    """
+
+    def __init__(self):
+        super().__init__(name="ordinal", kind="verbalize")
+        graph = (
+            pynutil.delete("integer:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        convert_eleven = pynini.cross("11", "11th")
+        convert_twelve = pynini.cross("12", "12th")
+        convert_thirteen = pynini.cross("13", "13th")
+        convert_one = pynini.cross("1", "1st")
+        convert_two = pynini.cross("2", "2nd")
+        convert_three = pynini.cross("3", "3rd")
+        convert_rest = pynutil.insert("th", weight=0.01)
+
+        suffix = pynini.cdrewrite(
+            convert_eleven
+            | convert_twelve
+            | convert_thirteen
+            | convert_one
+            | convert_two
+            | convert_three
+            | convert_rest,
+            "",
+            "[EOS]",
+            DAMO_SIGMA,
+        )
+        graph = graph @ suffix
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/telephone.py

@@ -0,0 +1,30 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import DAMO_NOT_QUOTE, GraphFst
+from pynini.lib import pynutil
+
+
+class TelephoneFst(GraphFst):
+    """
+    Finite state transducer for verbalizing telephone, e.g.
+        telephone { number_part: "123-123-5678" }
+        -> 123-123-5678
+    """
+
+    def __init__(self):
+        super().__init__(name="telephone", kind="verbalize")
+
+        number_part = (
+            pynutil.delete('number_part: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+        )
+        optional_country_code = pynini.closure(
+            pynutil.delete('country_code: "')
+            + pynini.closure(DAMO_NOT_QUOTE, 1)
+            + pynutil.delete('"')
+            + pynini.accep(" "),
+            0,
+            1,
+        )
+        delete_tokens = self.delete_tokens(optional_country_code + number_part)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/time.py

@@ -0,0 +1,92 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    DAMO_DIGIT,
+    GraphFst,
+    delete_space,
+    insert_space,
+)
+from pynini.lib import pynutil
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing time, e.g.
+        time { hours: "12" minutes: "30" } -> 12:30
+        time { hours: "1" minutes: "12" } -> 01:12
+        time { hours: "2" suffix: "a.m." } -> 02:00 a.m.
+    """
+
+    def __init__(self):
+        super().__init__(name="time", kind="verbalize")
+        add_leading_zero_to_double_digit = (DAMO_DIGIT + DAMO_DIGIT) | (
+            pynutil.insert("0") + DAMO_DIGIT
+        )
+        # hour
+        hour = (
+            pynutil.delete("hours:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+        # minute
+        minute = (
+            pynutil.delete("minutes:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+        # seconds
+        second = (
+            pynutil.delete("seconds:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_DIGIT, 1)
+            + pynutil.delete('"')
+        )
+
+        suffix = (
+            delete_space
+            + insert_space
+            + pynutil.delete("suffix:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        optional_suffix = pynini.closure(suffix, 0, 1)
+        zone = (
+            delete_space
+            + insert_space
+            + pynutil.delete("zone:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        optional_zone = pynini.closure(zone, 0, 1)
+
+        # hms
+        graph_hms = (
+            (hour @ add_leading_zero_to_double_digit)
+            + delete_space
+            + pynutil.insert(":")
+            + (minute @ add_leading_zero_to_double_digit)
+            + delete_space
+            + pynutil.insert(":")
+            + second
+        )
+        # hm
+        graph_hm = (
+            (hour @ add_leading_zero_to_double_digit)
+            + delete_space
+            + pynutil.insert(":")
+            + (minute @ add_leading_zero_to_double_digit)
+        )
+
+        graph = (graph_hms | graph_hm) + optional_suffix + optional_zone
+
+        delete_tokens = self.delete_tokens(graph)
+        self.fst = delete_tokens.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize.py

@@ -0,0 +1,51 @@
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.cardinal import CardinalFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.date import DateFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.decimal import DecimalFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.fraction import FractionFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.electronic import ElectronicFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.measure import MeasureFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.money import MoneyFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.ordinal import OrdinalFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.telephone import TelephoneFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.time import TimeFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.whitelist import WhiteListFst
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import GraphFst
+
+
+class VerbalizeFst(GraphFst):
+    """
+    Composes other verbalizer grammars.
+    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
+    More details to deployment at NeMo/tools/text_processing_deployment.
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize", kind="verbalize")
+        cardinal = CardinalFst()
+        cardinal_graph = cardinal.fst
+        ordinal_graph = OrdinalFst().fst
+        decimal = DecimalFst()
+        decimal_graph = decimal.fst
+        fraction = FractionFst()
+        fraction_graph = fraction.fst
+        measure_graph = MeasureFst(decimal=decimal, cardinal=cardinal).fst
+        money_graph = MoneyFst(decimal=decimal).fst
+        time_graph = TimeFst().fst
+        date_graph = DateFst().fst
+        whitelist_graph = WhiteListFst().fst
+        telephone_graph = TelephoneFst().fst
+        electronic_graph = ElectronicFst().fst
+        graph = (
+            time_graph
+            | date_graph
+            | money_graph
+            | measure_graph
+            | ordinal_graph
+            | decimal_graph
+            | fraction_graph
+            | cardinal_graph
+            | whitelist_graph
+            | telephone_graph
+            | electronic_graph
+        )
+        self.fst = graph

fun_text_processing/inverse_text_normalization/ko/verbalizers/verbalize_final.py

@@ -0,0 +1,33 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.verbalize import VerbalizeFst
+from fun_text_processing.inverse_text_normalization.ko.verbalizers.word import WordFst
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    GraphFst,
+    delete_extra_space,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class VerbalizeFinalFst(GraphFst):
+    """
+    Finite state transducer that verbalizes an entire sentence, e.g.
+    tokens { name: "its" } tokens { time { hours: "12" minutes: "30" } } tokens { name: "now" } -> its 12:30 now
+    """
+
+    def __init__(self):
+        super().__init__(name="verbalize_final", kind="verbalize")
+        verbalize = VerbalizeFst().fst
+        word = WordFst().fst
+        types = verbalize | word
+        graph = (
+            pynutil.delete("tokens")
+            + delete_space
+            + pynutil.delete("{")
+            + delete_space
+            + types
+            + delete_space
+            + pynutil.delete("}")
+        )
+        graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space
+        self.fst = graph

fun_text_processing/inverse_text_normalization/ko/verbalizers/whitelist.py

@@ -0,0 +1,27 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WhiteListFst(GraphFst):
+    """
+    Finite state transducer for verbalizing whitelist
+        e.g. tokens { name: "mrs." } -> mrs.
+    """
+
+    def __init__(self):
+        super().__init__(name="whitelist", kind="verbalize")
+        graph = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + pynini.closure(DAMO_CHAR - " ", 1)
+            + pynutil.delete('"')
+        )
+        graph = graph @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+        self.fst = graph.optimize()

fun_text_processing/inverse_text_normalization/ko/verbalizers/word.py

@@ -0,0 +1,29 @@
+import pynini
+from fun_text_processing.inverse_text_normalization.ko.graph_utils import (
+    DAMO_CHAR,
+    DAMO_SIGMA,
+    GraphFst,
+    delete_space,
+)
+from pynini.lib import pynutil
+
+
+class WordFst(GraphFst):
+    """
+    Finite state transducer for verbalizing plain tokens
+        e.g. tokens { name: "sleep" } -> sleep
+    """
+
+    def __init__(self):
+        super().__init__(name="word", kind="verbalize")
+        chars = pynini.closure(DAMO_CHAR - " ", 1)
+        char = (
+            pynutil.delete("name:")
+            + delete_space
+            + pynutil.delete('"')
+            + chars
+            + pynutil.delete('"')
+        )
+        graph = char @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", DAMO_SIGMA)
+
+        self.fst = graph.optimize()