chore: import upstream snapshot with attribution

scripts/tokenizers/conf/huggingface_data_tokenizer.yaml

@@ -0,0 +1,34 @@
+
+# num workers to use for extracting text from datasets.
+num_workers: 8
+
+# simple text cleaning, by default converts all chars to lower-case and only keeps alpha-numeric chars.
+normalize_text: false
+symbols_to_keep: ["'"]  # a list of symbols to keep during text cleaning.
+
+# the key for groundtruth transcription, e.g., MCV usually uses "sentence" while some others use "text"
+text_key:  "text"  # the key for groundtruth transcription, e.g., MCV usually uses "sentence" while some others use "text"
+num_proc: 4  # num processes to use for downloading HF datasets
+
+data_path: "librispeech_asr"
+data_name: null
+streaming: true
+
+hf_data_cfg:  # hf_data_cfg can be a ListConfig or DictConfig. Params for each data are passed into huggingface load_dataset(). Add more params if needed
+  - path: ${data_path}
+    name: ${data_name}
+    split: 'train.clean.360'
+    streaming: ${streaming}
+    num_proc: ${num_proc}
+  - path: ${data_path}
+    name: ${data_name}
+    split: 'train.clean.100'
+    streaming: ${streaming}
+    num_proc: ${num_proc}
+  - path: ${data_path}
+    name: ${data_name}
+    split: 'train.other.500'
+    streaming: ${streaming}
+    num_proc: ${num_proc}
+
+output_file: "librispeech_asr_train960.txt"

scripts/tokenizers/conf/tabular_data_tokenizer.yaml

@@ -0,0 +1,33 @@
+table_structure:
+  - name: col_a
+    code_type: float
+    args:
+      code_len: 4  # number of tokens used to code the column
+      base: 16   # the positional base number. ie. it uses 16 tokens for one digit
+      fillall: False # whether to use full base number for each token or derive it from the data.
+      hasnan: False # can it handles nan or not
+      transform: yeo-johnson # can be ['yeo-johnson', 'quantile', 'robust'], check https://scikit-learn.org/stable/modules/classes.html#module-sklearn.preprocessing 
+  - name: col_b
+    code_type: float
+    args:
+      code_len: 4
+      base: 32
+      fillall: True
+      hasnan: True
+      transform: quantile
+  - name: col_c
+    code_type: int
+    args:
+      code_len: 3
+      base: 12
+      fillall: True
+      hasnan: True
+  - name: col_d
+    code_type: category
+    args:
+      code_len: 3
+      base: 12
+      fillall: True
+      hasnan: True
+tokenizer_file: ???  # tabular tokneizer output file path
+table_csv_file: ???  # input table csv file