chore: import upstream snapshot with attribution

scripts/tokenizers/add_special_tokens_to_sentencepiece.py

@@ -0,0 +1,110 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+from argparse import ArgumentParser
+
+import sentencepiece as spm
+
+try:
+    import sentencepiece_model_pb2 as spt
+except (ImportError, ModuleNotFoundError):
+    raise Exception("Ensure that sentencepiece_model_pb2.py has been generated from the protoc compiler")
+
+
+"""Utility to add special tokens to existing sentencepiece models.
+
+Generate sentencepiece_model_pb2.py in the directory of this script before running
+To generate run `protoc --python_out=<path_to_NeMo>/scripts/tokenizers/ sentencepiece_model.proto`
+inside the src folder in sentencepiece repo
+Refer: https://github.com/google/sentencepiece/issues/121
+
+Usage:
+python edit_spt_model.py \
+    --input_file <input_model_dir> \
+    --output_file <output_model_dir> \
+    --tokens <space separated special tokens> 
+
+Example:
+python edit_spt_model.py \
+    --input_file test.model \
+    --output_file test.model \
+    --tokens [CLS] [SEP]
+"""
+
+
+def edit_spt_model():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--input_file",
+        type=str,
+        required=True,
+        help="Path to sentencepiece model file",
+    )
+    parser.add_argument(
+        "--output_file",
+        type=str,
+        required=True,
+        help="Path to sentencepiece model file",
+    )
+    parser.add_argument(
+        "--tokens",
+        type=str,
+        nargs='+',
+        required=True,
+        help="Special tokens to add to tokenizer",
+    )
+    parser.add_argument(
+        "--is_userdefined",
+        action="store_true",
+        help="When set, the new tokens are set as user_defined tokens",
+    )
+    args = parser.parse_args()
+
+    token_type = 3
+    if args.is_userdefined:
+        token_type = 4
+
+    model = spt.ModelProto()
+    model.ParseFromString(open(args.input_file, 'rb').read())
+
+    for token in args.tokens:
+        piece = model.SentencePiece(piece=token, score=0.0, type=token_type)
+        if piece in model.pieces:
+            logging.error(f"Special Token '{token}' already exists in the input model!")
+            sys.exit(1)
+        model.pieces.append(piece)
+
+    sp = spm.SentencePieceProcessor()
+    try:
+        sp.LoadFromSerializedProto(model.SerializeToString())
+        for token in args.tokens:
+            id = sp.piece_to_id(token)
+            logging.info(f"Created token '{token}' at ID {id}")
+        logging.info(f"New tokenizer vocab size: {sp.get_piece_size()}")
+    except:
+        logging.error("Could not appropriately configure new tokenizer. Verify if the special tokens already exist.")
+        sys.exit(1)
+
+    with open(args.output_file, 'wb') as outf:
+        outf.write(model.SerializeToString())
+
+    logging.info(f"Created new tokenizer at: {args.output_file}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+    edit_spt_model()

scripts/tokenizers/conf/huggingface_data_tokenizer.yaml

@@ -0,0 +1,34 @@
+
+# num workers to use for extracting text from datasets.
+num_workers: 8
+
+# simple text cleaning, by default converts all chars to lower-case and only keeps alpha-numeric chars.
+normalize_text: false
+symbols_to_keep: ["'"]  # a list of symbols to keep during text cleaning.
+
+# the key for groundtruth transcription, e.g., MCV usually uses "sentence" while some others use "text"
+text_key:  "text"  # the key for groundtruth transcription, e.g., MCV usually uses "sentence" while some others use "text"
+num_proc: 4  # num processes to use for downloading HF datasets
+
+data_path: "librispeech_asr"
+data_name: null
+streaming: true
+
+hf_data_cfg:  # hf_data_cfg can be a ListConfig or DictConfig. Params for each data are passed into huggingface load_dataset(). Add more params if needed
+  - path: ${data_path}
+    name: ${data_name}
+    split: 'train.clean.360'
+    streaming: ${streaming}
+    num_proc: ${num_proc}
+  - path: ${data_path}
+    name: ${data_name}
+    split: 'train.clean.100'
+    streaming: ${streaming}
+    num_proc: ${num_proc}
+  - path: ${data_path}
+    name: ${data_name}
+    split: 'train.other.500'
+    streaming: ${streaming}
+    num_proc: ${num_proc}
+
+output_file: "librispeech_asr_train960.txt"

scripts/tokenizers/conf/tabular_data_tokenizer.yaml

@@ -0,0 +1,33 @@
+table_structure:
+  - name: col_a
+    code_type: float
+    args:
+      code_len: 4  # number of tokens used to code the column
+      base: 16   # the positional base number. ie. it uses 16 tokens for one digit
+      fillall: False # whether to use full base number for each token or derive it from the data.
+      hasnan: False # can it handles nan or not
+      transform: yeo-johnson # can be ['yeo-johnson', 'quantile', 'robust'], check https://scikit-learn.org/stable/modules/classes.html#module-sklearn.preprocessing 
+  - name: col_b
+    code_type: float
+    args:
+      code_len: 4
+      base: 32
+      fillall: True
+      hasnan: True
+      transform: quantile
+  - name: col_c
+    code_type: int
+    args:
+      code_len: 3
+      base: 12
+      fillall: True
+      hasnan: True
+  - name: col_d
+    code_type: category
+    args:
+      code_len: 3
+      base: 12
+      fillall: True
+      hasnan: True
+tokenizer_file: ???  # tabular tokneizer output file path
+table_csv_file: ???  # input table csv file

scripts/tokenizers/get_hf_text_data.py

@@ -0,0 +1,103 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This script is used to download text corpus from HuggingFace datasets,
+where the saved corpus can be further used to train a tokenizer using `process_asr_text_tokenizer.py`.
+
+Usage:
+```
+python get_hf_text_data.py --config-path="conf" --config-name="huggingface_data_tokenizer"
+```
+
+Please refer to "conf/huggingface_data_tokenizer.yaml" for more details.
+"""
+
+
+import os
+from itertools import repeat
+from multiprocessing import Pool
+from pathlib import Path
+
+import datasets as hf_datasets
+from omegaconf import OmegaConf, open_dict
+
+from nemo.core.config import hydra_runner
+from nemo.utils import logging
+
+
+def clean_text(text: str, symbols_to_keep=None):
+    symbols_to_keep = [x for x in symbols_to_keep] if symbols_to_keep is not None else []
+    text = text.lower()
+    # only keep alphanumeric characters, spaces and symbols defined in self.symbols_to_keep
+    text = ''.join([c for c in text if c.isalnum() or c.isspace() or c in symbols_to_keep])
+    return text
+
+
+def get_nested_dict_value(dictionary: dict, key: str):
+    """
+    the key should be a string of nested keys separated by `.`, e.g. `key1.key2.key3`,
+    then the returned value will be `dictionary[key1][key2][key3]`
+    """
+    nested_keys = key.split(".")
+    result = dictionary
+    for k in nested_keys:
+        if k not in result:
+            raise KeyError(
+                f"Key `{key}` not found in [{result.keys()}], target is {nested_keys}, input is {dictionary}"
+            )
+        result = result[k]
+    return result
+
+
+def worker(x):
+    sample, cfg = x
+    text = get_nested_dict_value(sample, cfg.text_key)
+    if cfg.normalize_text:
+        text = clean_text(text, cfg.symbols_to_keep)
+    return text
+
+
+@hydra_runner(config_path="conf", config_name="huggingface_data_tokenizer")
+def main(cfg) -> None:
+    logging.info("\n\n************** Experiment configuration ***********")
+    logging.info(OmegaConf.to_yaml(cfg, resolve=True))
+
+    if cfg.output_file is None:
+        cfg.output_file = 'huggingface_text_corpus.txt'
+
+    if Path(cfg.output_file).exists():
+        logging.info(f"Output file {cfg.output_file} already exists, removing it...")
+        os.system(f"rm {cfg.output_file}")
+
+    for data_cfg in cfg.hf_data_cfg:
+        if 'num_proc' in data_cfg and data_cfg.get('streaming', False):
+            logging.warning("num_proc is not supported for streaming datasets, removing it from config")
+            with open_dict(data_cfg):
+                data_cfg.pop('num_proc')
+        logging.info(
+            f"Loading from HuggingFace datasets library with config: {OmegaConf.to_container(data_cfg, resolve=True)}"
+        )
+        dataset = hf_datasets.load_dataset(**data_cfg)
+        logging.info("Start extracting text from dataset...")
+        with Pool(cfg.num_workers) as p:
+            text_corpus = p.map(worker, zip(dataset, repeat(cfg)))
+            with Path(cfg.output_file).open('a') as f:
+                for line in text_corpus:
+                    f.write(f"{line}\n")
+            logging.info(f"Finished processing {len(text_corpus)} samples from {data_cfg}")
+    logging.info("All Done!")
+
+
+if __name__ == '__main__':
+    main()

scripts/tokenizers/process_asr_text_tokenizer.py

@@ -0,0 +1,386 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python process_asr_text_tokenizer.py --manifest=<path to train manifest files, seperated by commas> \
+#         --data_root="<output directory>" \
+#         --vocab_size=<number of tokens in vocabulary> \
+#         --tokenizer=<"spe" or "wpe"> \
+#         --log
+# where <manifest> can be: train_clean_100, train_clean_360, train_other_500
+# You can also put more than one data_set comma-separated:
+# --manifest="train_clean_100,train_clean_360,train_other_500"
+# or
+#       python process_asr_text_tokenizer.py --data_file=<path to train text file> \
+#         --data_root="<output directory>" \
+#         --vocab_size=<number of tokens in vocabulary> \
+#         --tokenizer=<"bpe" or "wpe"> \
+#         --log
+# where <manifest> can be: train_clean_100, train_clean_360, train_other_500
+# You can also put more than one data_set comma-separated:
+# --manifest="train_clean_100,train_clean_360,train_other_500"
+#
+# Args:
+#   --manifest or --data_file: If your text data lies inside of an ASR manifest file,
+#       then use the --manifest path. If instead the text data is inside a file with separate lines
+#       corresponding to different text lines, then use --data_file.
+#       In either case, you can add commas to concatenate different manifests or different data files.
+#
+#   --data_root: The output directory (whose subdirectories will be created if not present) where
+#       the tokenizers will be placed.
+#
+#   --vocab_size: The size of the tokenizer vocabulary. Larger vocabularies can accommodate almost entire,
+#       words but the decoder size of any model will grow proportionally.
+#
+#   --tokenizer: Can be either spe or wpe . spe refers to the Google sentencepiece library tokenizer.
+#       wpe refers to the HuggingFace BERT Word Piece tokenizer.
+#
+#   --no_lower_case: When this flag is passed, it will force the tokenizer to create seperate tokens for
+#       upper and lower case characters. By default, the script will turn all the text to lower case
+#       before tokenization (and if upper case characters are passed during training/inference, the
+#       tokenizer will emit a token equivalent to Out-Of-Vocabulary). Used primarily for the
+#       English language.
+#
+#    --spe_type: The sentencepiece library has a few implementations of the tokenization technique, and
+#       spe_type refers to these implementations. Currently supported types are unigram, bpe, char, word.
+#       Defaults to bpe.
+#
+#   --spe_character_coverage: The sentencepiece library considers how much of the original vocabulary it
+#       should cover in its "base set" of tokens (akin to the lower and upper case characters of the
+#       English language). For almost all languages with small base token sets (<1000 tokens), this
+#       should be kept at its default of 1.0. For languages with larger vocabularies (say Japanese,
+#       Mandarin, Korean etc), the suggested value is 0.9995.
+#
+#   --spe_user_defined_symbols: The sentencepiece library allows you to add your own tokens to the base set.
+#      This flag allows you to pass a space separated list of tokens that you want to add to the base set.
+#      These tokens remain in the decoded text and are encoded automatically when present in the input text.
+#
+#   --spe_control_symbols: The sentencepiece library allows you to add your own tokens to the base set.
+#      This flag allows you to pass a space separated list of tokens that you want to add to the base set.
+#      These tokens get removed at decode time and are not encoded from the text - can only be added to the
+#      input programatically.
+#
+#   --spe_byte_fallback: If <unk>, fallback to a byte sequence of the characters.
+#
+#   --spe_split_digits: If true, digits are split into individual tokens.
+#
+#   --spe_sample_size: If the dataset is too large, consider using a sampled dataset indicated by a
+#       positive integer. By default, any negative value (default = -1) will use the entire dataset.
+#
+#   --spe_train_extremely_large_corpus: When training a sentencepiece tokenizer on very large amounts of text,
+#       sometimes the tokenizer will run out of memory or wont be able to process so much data on RAM.
+#       At some point you might receive the following error - "Input corpus too large, try with
+#       train_extremely_large_corpus=true". If your machine has large amounts of RAM, it might still be possible
+#       to build the tokenizer using the above flag. Will silently fail if it runs out of RAM.
+#
+#   --spe_max_sentencepiece_length: Limits the maximum length that any any SentencePiece subword can be.
+#       Using this will change the subword tokens generated.
+#
+#   --spe_pad: Adds <pad> as special token.
+#
+#   --spe_bos: Adds <s> as Begining-of-Sentence special token.
+#
+#   --spe_eos: Adds </s> as End-of-Sentence special token.
+#
+#   --log: Whether the script should display log messages
+
+
+import argparse
+import json
+import logging
+import os
+from typing import List, Optional
+
+import tokenizers
+
+from nemo.collections.common.tokenizers.sentencepiece_tokenizer import create_spt_model
+from nemo.utils.data_utils import DataStoreObject
+
+parser = argparse.ArgumentParser(description='Create tokenizer')
+group = parser.add_mutually_exclusive_group(required=True)
+group.add_argument("--manifest", default=None, type=str, help='Comma separated list of manifest files')
+group.add_argument("--data_file", default=None, help='data file from which to create tokenizer model')
+parser.add_argument("--data_root", required=True, default=None, type=str, help='Output directory')
+parser.add_argument("--vocab_size", default=1024, type=int, help='Vocabulary size')
+parser.add_argument("--tokenizer", default="wpe", choices=["spe", "wpe"], help='Type of tokenization to perform')
+parser.add_argument(
+    "--spe_type",
+    default="bpe",
+    choices=['bpe', 'unigram', 'char', 'word'],
+    help='Type of the SentencePiece model. Can be `bpe`, `unigram`, `char` or `word`.'
+    'Used only if --tokenizer == `spe`',
+)
+parser.add_argument(
+    '--spe_character_coverage',
+    type=float,
+    default=1.0,
+    help="Character coverage percentage for SentencePiece tokenization. For languages "
+    "with large vocabulary, should be close to 0.9995, otherwise kept as 1.0",
+)
+parser.add_argument('--spe_bos', action='store_true', help='Add <s> token to SentencePiece Tokenizer.')
+parser.add_argument('--spe_eos', action='store_true', help='Add </s> token to SentencePiece Tokenizer.')
+parser.add_argument('--spe_pad', action='store_true', help='Add <pad> token to SentencePiece Tokenizer.')
+parser.add_argument(
+    '--spe_user_defined_symbols', default=None, type=str, nargs='+', help='User defined symbols for SentencePiece'
+)
+parser.add_argument(
+    '--spe_control_symbols', default=None, type=str, nargs='+', help='Control symbols for SentencePiece'
+)
+parser.add_argument('--spe_split_digits', action='store_true', help='Split digits into separate tokens.')
+parser.add_argument(
+    '--spe_remove_extra_whitespaces',
+    action='store_true',
+    help='Remove leading, trailing, and duplicate internal whitespace.',
+)
+
+parser.add_argument(
+    '--spe_sample_size',
+    type=int,
+    default=-1,
+    help="Samples the dataset by `sample_size` if positive integer, otherwise uses whole dataset",
+)
+parser.add_argument('--spe_train_extremely_large_corpus', action='store_true', help='')
+parser.add_argument(
+    '--spe_max_sentencepiece_length',
+    type=int,
+    default=-1,
+    help='Limit the maximum number of tokens in each SentencePiece subword. '
+    'Must be a positive integer > 0. By default places no limit on subword length.',
+)
+parser.add_argument(
+    '--spe_no_split_by_unicode_script',
+    dest='spe_split_by_unicode_script',
+    action='store_false',
+    help="Don't use Unicode script to split sentence pieces.",
+)
+parser.add_argument(
+    '--spe_byte_fallback',
+    dest='spe_byte_fallback',
+    action='store_true',
+    help="If <unk>, fallback to a byte sequence of the characters.",
+)
+parser.add_argument('--no_lower_case', dest='lower_case', action='store_false')
+parser.add_argument("--log", action='store_true')
+parser.set_defaults(log=False, lower_case=True, spe_train_extremely_large_corpus=False)
+args = parser.parse_args()
+
+
+def __build_document_from_manifests(
+    data_root: str,
+    manifests: str,
+):
+    if ',' in manifests:
+        manifests = manifests.split(',')
+    else:
+        manifests = [manifests]
+
+    document_dir = os.path.join(data_root, 'text_corpus')
+    if not os.path.exists(document_dir):
+        os.makedirs(document_dir)
+
+    document_path = os.path.join(document_dir, 'document.txt')
+
+    if os.path.exists(document_path):
+        logging.info('Corpus already exists at path : %s', document_path)
+        return document_path
+
+    num_lines = 0
+    with open(document_path, 'w') as out_writer:
+        for manifest in manifests:
+            with open(DataStoreObject(manifest).get(), 'r') as in_reader:
+                for line in in_reader:
+                    item = json.loads(line)
+                    text = item['text']
+
+                    out_writer.write(text + '\n')
+                    out_writer.flush()
+
+                    num_lines += 1
+
+            logging.info(f"Finished extracting manifest : {manifest}")
+
+        logging.info("Finished extracting all manifests ! Number of sentences : {}".format(num_lines))
+    return document_path
+
+
+def __process_data(
+    text_path: str,
+    dst_folder: str,
+    vocab_size: int,
+    tokenizer_type: str,
+    spe_type: str,
+    spe_character_coverage: float,
+    spe_train_extremely_large_corpus: bool,
+    spe_sample_size: int,
+    spe_max_sentencepiece_length: int,
+    spe_split_by_unicode_script: bool,
+    spe_bos: bool,
+    spe_eos: bool,
+    spe_pad: bool,
+    spe_control_symbols: Optional[List[str]],
+    spe_user_defined_symbols: Optional[List[str]],
+    spe_byte_fallback: bool,
+    spe_split_digits: bool,
+    spe_remove_extra_whitespaces: bool,
+    lower_case: bool,
+):
+    """
+    Converts flac to wav and build manifests's json
+    Args:
+        text_path: source with text lines
+        dst_folder: where wav files will be stored
+        vocab_size: vocabular size used in encoding the text
+        tokenizer_type: type of tokenization to perform - wpe or spe
+        spe_type: type of tokenization model used for spe.
+        spe_character_coverage: float value between 0 and 1 (as a percentage). For languages with a vast charset,
+            can be < 1.0, but for all other languages, it should be set as 1.0
+        spe_sample_size: int, default of -1. If positive integer is used, samples the dataset
+            by given sample size.
+        spe_train_extremely_large_corpus: bool. If dataset is too large, and user has sufficient RAM,
+            this flag can be set to try to trained the tokenizer. Will silently fail if it runs out of RAM.
+        spe_max_sentencepiece_length: Limits the maximum length of the SentencePiece subword that can be constructed.
+            By default, no limit is placed.
+        spe_bos: Bool flag, whether to add <s> to SentencePiece tokenizer vocabulary.
+        spe_eos: Bool flag, whether to add </s> to SentencePiece tokenizer vocabulary.
+        spe_pad: Bool flag, whether to add <pad> to SentencePiece tokenizer vocabulary.
+        spe_control_symbols: control symbols to add to tokenizer, as defined by sentencepiece.
+            These tokens get removed at decode time and are not encoded from the text - can only be added to the input programatically.
+        spe_user_defined_symbols: user symbols to add to tokenizer, as defined by sentencepiece.
+            These tokens remain in the decoded text and are encoded automatically when present in the input text.
+        spe_byte_fallback: If <unk>, fallback to a byte sequence of the character.
+        spe_split_digits: If true, digits are split into individual tokens.
+        spe_remove_extra_whitespaces: If true, removes leading, trailing, and duplicate internal whitespace.
+        lower_case: whether to tokenize with lower case character set only (for english)
+
+    Returns:
+    """
+    if tokenizer_type == 'spe':
+
+        # Prepare directory of tokenizer
+        if spe_max_sentencepiece_length > 0:
+            tokenizer_dir = os.path.join(dst_folder, 'tokenizer_{}_{}_v{}_max_{}').format(
+                tokenizer_type, spe_type, vocab_size, spe_max_sentencepiece_length
+            )
+        else:
+            tokenizer_dir = os.path.join(dst_folder, 'tokenizer_{}_{}_v{}').format(
+                tokenizer_type, spe_type, vocab_size
+            )
+
+        if spe_pad:
+            tokenizer_dir = f'{tokenizer_dir}_pad'
+        if spe_bos:
+            tokenizer_dir = f'{tokenizer_dir}_bos'
+        if spe_eos:
+            tokenizer_dir = f'{tokenizer_dir}_eos'
+
+        if not os.path.exists(tokenizer_dir):
+            os.makedirs(tokenizer_dir)
+
+        if os.path.exists(os.path.join(tokenizer_dir, 'tokenizer.model')):
+            logging.warning("Model file already exists, overriding old model file !")
+            os.remove(os.path.join(tokenizer_dir, 'tokenizer.model'))
+
+        # Build tokenizer
+        tokenizer_path, vocab_path = create_spt_model(
+            data_file=text_path,
+            vocab_size=vocab_size,
+            sample_size=spe_sample_size,
+            do_lower_case=lower_case,
+            output_dir=tokenizer_dir,
+            tokenizer_type=spe_type,
+            character_coverage=spe_character_coverage,
+            train_extremely_large_corpus=spe_train_extremely_large_corpus,
+            max_sentencepiece_length=spe_max_sentencepiece_length,
+            split_by_unicode_script=spe_split_by_unicode_script,
+            bos=spe_bos,
+            eos=spe_eos,
+            pad=spe_pad,
+            control_symbols=spe_control_symbols,
+            user_defined_symbols=spe_user_defined_symbols,
+            byte_fallback=spe_byte_fallback,
+            split_digits=spe_split_digits,
+            remove_extra_whitespaces=spe_remove_extra_whitespaces,
+        )
+
+    else:
+        tokenizer_dir = os.path.join(dst_folder, 'tokenizer_{}_v{}').format(tokenizer_type, vocab_size)
+
+        if not os.path.exists(tokenizer_dir):
+            os.makedirs(tokenizer_dir)
+
+        tokenizer = tokenizers.BertWordPieceTokenizer(lowercase=lower_case)
+
+        tokenizer.train(text_path, vocab_size=vocab_size)
+        tokenizer.save_model(tokenizer_dir)
+
+    return tokenizer_dir
+
+
+def main():
+    data_root = args.data_root
+    manifests = args.manifest
+    data_file = args.data_file
+    vocab_size = args.vocab_size
+    tokenizer = args.tokenizer
+    spe_type = args.spe_type
+    spe_character_coverage = args.spe_character_coverage
+    spe_sample_size = args.spe_sample_size
+    spe_train_extremely_large_corpus = args.spe_train_extremely_large_corpus
+    spe_max_sentencepiece_length = args.spe_max_sentencepiece_length
+    spe_split_by_unicode_script = args.spe_split_by_unicode_script
+    spe_bos, spe_eos, spe_pad = args.spe_bos, args.spe_eos, args.spe_pad
+    spe_control_symbols = args.spe_control_symbols
+    spe_user_defined_symbols = args.spe_user_defined_symbols
+    spe_byte_fallback = args.spe_byte_fallback
+    spe_split_digits = args.spe_split_digits
+    spe_remove_extra_whitespaces = args.spe_remove_extra_whitespaces
+    lower_case = args.lower_case
+
+    if not os.path.exists(data_root):
+        os.makedirs(data_root)
+
+    if args.log:
+        logging.basicConfig(level=logging.INFO)
+
+    if manifests:
+        text_corpus_path = __build_document_from_manifests(data_root, manifests)
+    else:
+        text_corpus_path = data_file
+    tokenizer_path = __process_data(
+        text_corpus_path,
+        data_root,
+        vocab_size,
+        tokenizer,
+        spe_type,
+        lower_case=lower_case,
+        spe_character_coverage=spe_character_coverage,
+        spe_sample_size=spe_sample_size,
+        spe_train_extremely_large_corpus=spe_train_extremely_large_corpus,
+        spe_max_sentencepiece_length=spe_max_sentencepiece_length,
+        spe_split_by_unicode_script=spe_split_by_unicode_script,
+        spe_bos=spe_bos,
+        spe_eos=spe_eos,
+        spe_pad=spe_pad,
+        spe_control_symbols=spe_control_symbols,
+        spe_user_defined_symbols=spe_user_defined_symbols,
+        spe_byte_fallback=spe_byte_fallback,
+        spe_split_digits=spe_split_digits,
+        spe_remove_extra_whitespaces=spe_remove_extra_whitespaces,
+    )
+
+    print("Serialized tokenizer at location :", tokenizer_path)
+    logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()