chore: import upstream snapshot with attribution

scripts/dataset_processing/add_noise.py

@@ -0,0 +1,171 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python add_noise.py --input_manifest=<manifest file of original "clean" dataset>
+#   --noise_manifest=<manifest file poinitng to noise data>
+#   --out_dir=<destination directory for noisy audio and manifests>
+#   --snrs=<list of snrs at which noise should be added to the audio>
+#   --seed=<seed for random number generator>
+#   --num_workers=<number of parallel workers>
+# To be able to reproduce the same noisy dataset, use a fixed seed and num_workers=1
+
+import argparse
+import copy
+import json
+import multiprocessing
+import os
+import random
+
+import numpy as np
+import soundfile as sf
+
+from nemo.collections.asr.parts.preprocessing.perturb import NoisePerturbation
+from nemo.collections.asr.parts.preprocessing.segment import AudioSegment
+
+rng = None
+att_factor = 0.8
+save_noise = False
+sample_rate = 16000
+
+
+def get_out_dir_name(out_dir, input_name, noise_name, snr):
+    return os.path.join(out_dir, input_name, noise_name + "_" + str(snr) + "db")
+
+
+def create_manifest(input_manifest, noise_manifest, snrs, out_path, save_noise):
+    os.makedirs(os.path.join(out_path, "manifests"), exist_ok=True)
+    for snr in snrs:
+        out_dir = get_out_dir_name(
+            out_path,
+            os.path.splitext(os.path.basename(input_manifest))[0],
+            os.path.splitext(os.path.basename(noise_manifest))[0],
+            snr,
+        )
+        out_mfst = os.path.join(
+            os.path.join(out_path, "manifests"),
+            os.path.splitext(os.path.basename(input_manifest))[0]
+            + "_"
+            + os.path.splitext(os.path.basename(noise_manifest))[0]
+            + "_"
+            + str(snr)
+            + "db"
+            + ".json",
+        )
+        with open(input_manifest, "r") as inf, open(out_mfst, "w") as outf:
+            for line in inf:
+                row = json.loads(line.strip())
+                row['audio_filepath'] = os.path.join(out_dir, os.path.basename(row['audio_filepath']))
+                if save_noise:
+                    file_ext = os.path.splitext(row['audio_filepath'])[1]
+                    noise_filename = os.path.basename(row['audio_filepath']).replace(file_ext, "_noise" + file_ext)
+                    row['noise_filepath'] = os.path.join(out_dir, noise_filename)
+                outf.write(json.dumps(row) + "\n")
+
+
+def process_row(row):
+    audio_file = row['audio_filepath']
+    global sample_rate
+    data_orig = AudioSegment.from_file(audio_file, target_sr=sample_rate, offset=0)
+    for snr in row['snrs']:
+        min_snr_db = snr
+        max_snr_db = snr
+        global att_factor
+        perturber = NoisePerturbation(
+            manifest_path=row['noise_manifest'], min_snr_db=min_snr_db, max_snr_db=max_snr_db, rng=rng
+        )
+        out_dir = get_out_dir_name(
+            row['out_dir'],
+            os.path.splitext(os.path.basename(row['input_manifest']))[0],
+            os.path.splitext(os.path.basename(row['noise_manifest']))[0],
+            snr,
+        )
+        os.makedirs(out_dir, exist_ok=True)
+        out_f = os.path.join(out_dir, os.path.basename(audio_file))
+        if os.path.exists(out_f):
+            continue
+        data = copy.deepcopy(data_orig)
+        perturber.perturb(data)
+
+        max_level = np.max(np.abs(data.samples))
+
+        norm_factor = att_factor / max_level
+        new_samples = norm_factor * data.samples
+        sf.write(out_f, new_samples.transpose(), sample_rate)
+
+        global save_noise
+        if save_noise:
+            noise_samples = new_samples - norm_factor * data_orig.samples
+            out_f_ext = os.path.splitext(out_f)[1]
+            out_f_noise = out_f.replace(out_f_ext, "_noise" + out_f_ext)
+            sf.write(out_f_noise, noise_samples.transpose(), sample_rate)
+
+
+def add_noise(infile, snrs, noise_manifest, out_dir, num_workers=1):
+    allrows = []
+
+    with open(infile, "r") as inf:
+        for line in inf:
+            row = json.loads(line.strip())
+            row['snrs'] = snrs
+            row['out_dir'] = out_dir
+            row['noise_manifest'] = noise_manifest
+            row['input_manifest'] = infile
+            allrows.append(row)
+    pool = multiprocessing.Pool(num_workers)
+    pool.map(process_row, allrows)
+    pool.close()
+    print('Done!')
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input_manifest",
+        type=str,
+        required=True,
+        help="clean test set",
+    )
+    parser.add_argument("--noise_manifest", type=str, required=True, help="path to noise manifest file")
+    parser.add_argument("--out_dir", type=str, required=True, help="destination directory for audio and manifests")
+    parser.add_argument("--snrs", type=int, nargs="+", default=[0, 10, 20, 30])
+    parser.add_argument("--seed", type=int, default=None)
+    parser.add_argument("--num_workers", default=1, type=int)
+    parser.add_argument("--sample_rate", default=16000, type=int)
+    parser.add_argument(
+        "--attenuation_factor",
+        default=0.8,
+        type=float,
+        help="Attenuation factor applied on the normalized noise-added samples before writing to wave",
+    )
+    parser.add_argument(
+        "--save_noise", default=False, action="store_true", help="save the noise added to the input signal"
+    )
+
+    args = parser.parse_args()
+    global sample_rate
+    sample_rate = args.sample_rate
+    global att_factor
+    att_factor = args.attenuation_factor
+    global save_noise
+    save_noise = args.save_noise
+    global rng
+    rng = args.seed
+    num_workers = args.num_workers
+
+    add_noise(args.input_manifest, args.snrs, args.noise_manifest, args.out_dir, num_workers=num_workers)
+    create_manifest(args.input_manifest, args.noise_manifest, args.snrs, args.out_dir, args.save_noise)
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/fisher_audio_to_wav.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE:
+# python fisher_audio_to_wav.py \
+#   --data_root=<FisherEnglishTrainingSpeech root> \
+#   --dest_root=<destination dir root>
+#
+# Converts all .sph audio files in the Fisher dataset to .wav.
+# Requires sph2pipe to be installed.
+import argparse
+import concurrent.futures
+import glob
+import logging
+import os
+import subprocess
+
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description='Convert Fisher .sph to .wav')
+parser.add_argument(
+    "--data_root",
+    default=None,
+    type=str,
+    required=True,
+    help="The path to the root Fisher dataset folder.",
+)
+parser.add_argument(
+    "--dest_root",
+    default=None,
+    type=str,
+    required=True,
+    help="Path to the destination root directory.",
+)
+args = parser.parse_args()
+
+
+def __convert_audio(in_path, out_path):
+    """
+    Helper function that's called per thread, converts sph to wav.
+    Args:
+        in_path: source sph file to convert
+        out_path: destination for wav file
+    """
+    cmd = ["sph2pipe", "-f", "wav", "-p", in_path, out_path]
+    subprocess.run(cmd)
+
+
+def __process_set(data_root, dst_root):
+    """
+    Finds and converts all sph audio files in the given directory to wav.
+    Args:
+        data_folder: source directory with sph files to convert
+        dst_root: where wav files will be stored
+    """
+    sph_list = glob.glob(data_root)
+
+    if not os.path.exists(dst_root):
+        os.makedirs(dst_root)
+
+    # Set up and execute concurrent audio conversion
+    tp = concurrent.futures.ProcessPoolExecutor(max_workers=64)
+    futures = []
+
+    for sph_path in tqdm(sph_list, desc="Submitting sph futures", unit="file"):
+        audio_id, _ = os.path.splitext(os.path.basename(sph_path))
+        out_path = os.path.join(dst_root, "{}.wav".format(audio_id))
+        futures.append(tp.submit(__convert_audio, sph_path, out_path))
+
+    pbar = tqdm(total=len(sph_list), desc="Converting sph files", unit="file")
+    count = 0
+    for f in concurrent.futures.as_completed(futures):
+        count += 1
+        pbar.update()
+    tp.shutdown()
+    pbar.close()
+
+
+def main():
+    data_root = args.data_root
+    dest_root = args.dest_root
+
+    logging.info("\n\nConverting audio for Part 1")
+    __process_set(
+        os.path.join(
+            data_root,
+            "LDC2004S13-Part1",
+            "fisher_eng_tr_sp_d*",
+            "audio",
+            "*",
+            "*.sph",
+        ),
+        os.path.join(dest_root, "LDC2004S13-Part1", "audio_wav"),
+    )
+
+    logging.info("\n\nConverting audio for Part 2")
+    __process_set(
+        os.path.join(
+            data_root,
+            "LDC2005S13-Part2",
+            "fe_03_p2_sph*",
+            "audio",
+            "*",
+            "*.sph",
+        ),
+        os.path.join(dest_root, "LDC2005S13-Part2", "audio_wav"),
+    )
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/g2p/convert_cmu_arpabet_to_ipa.py

@@ -0,0 +1,141 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import os
+from argparse import ArgumentParser
+from typing import Dict
+
+from syllabify import syllabify
+
+
+"""
+Usage:
+    cd NeMo/scripts && python dataset_processing/g2p/convert_cmu_arpabet_to_ipa.py
+"""
+
+
+def parse_args():
+    parser = ArgumentParser("ARPABET to IPA conversion sctipt")
+    parser.add_argument(
+        '--cmu_arpabet',
+        help="Path to CMU ARPABET dictionary file",
+        type=str,
+        default="tts_dataset_files/cmudict-0.7b_nv22.10",
+    )
+    parser.add_argument("--ipa_out", help="Path to save IPA version of the dictionary", type=str, required=True)
+    parser.add_argument(
+        "--mapping",
+        help="ARPABET to IPA phoneme mapping file",
+        type=str,
+        default="tts_dataset_files/cmudict-arpabet_to_ipa_nv22.10.tsv",
+    )
+    return parser.parse_args()
+
+
+def convert_arp_to_ipa(arp_to_ipa_dict: Dict[str, str], arp_input: str, remove_space: bool = False) -> str:
+    """
+    Converts ARPABET phoneme to IPA based on arp_to_ipa_dict mapping
+
+    Args:
+        arp_to_ipa_dict: ARPABET to IPA phonemes mapping
+        arp_input: ARPABET input
+        remove_space: set to TRUE to remove spaces between IPA phonemes
+
+    Returns:
+        input word in IPA form
+    """
+
+    primary_stress = "ˈ"
+    secondary_stress = "ˌ"
+    stress_dict = {"0": "", "1": primary_stress, "2": secondary_stress}
+
+    word_ipa = ""
+    phonemes = arp_input.split()
+
+    # split ARPABET phoneme input into syllables,
+    # e.g. syllabify(["HH", "AH0", "L", "OW1"]) -> [(['HH'], ['AH0'], []), (['L'], ['OW1'], [])]
+    syllables = syllabify(phonemes)
+
+    for syl_idx, syll in enumerate(syllables):
+        syll_stress = ""
+        syll_ipa = ""
+
+        # syll is a tuple of lists of phonemes, here we flatten it and get rid of empty entries,
+        # e.g. (['HH'], ['AH0'], []) -> ['HH', 'AH0']
+        syll = [x for x in itertools.chain.from_iterable(syll)]
+        for phon_idx, phon in enumerate(syll):
+            if phon[-1].isdigit():
+                syll_stress = phon[-1]
+                if syll_stress not in stress_dict:
+                    raise ValueError(f"{syll_stress} unknown")
+                syll_stress = stress_dict[syll_stress]
+
+            # some phonemes are followed by a digit that represents stress, e.g., `AH0`
+            if phon not in arp_to_ipa_dict and phon[-1].isdigit():
+                phon = phon[:-1]
+
+            if phon not in arp_to_ipa_dict:
+                raise ValueError(f"|{phon}| phoneme not found in |{arp_input}|")
+            else:
+                ipa_phone = arp_to_ipa_dict[phon]
+                syll_ipa += ipa_phone + " "
+
+        word_ipa += " " + syll_stress + syll_ipa.strip()
+
+    word_ipa = word_ipa.strip()
+    if remove_space:
+        word_ipa = word_ipa.replace(" ", "")
+    return word_ipa
+
+
+def _get_arpabet_to_ipa_mapping(arp_ipa_map_file: str) -> Dict[str, str]:
+    """
+    arp_ipa_map_file: Arpabet to IPA phonemes mapping
+    """
+    arp_to_ipa = {}
+    with open(arp_ipa_map_file, "r", encoding="utf-8") as f:
+        for line in f:
+            arp, ipa = line.strip().split("\t")
+            arp_to_ipa[arp] = ipa
+    return arp_to_ipa
+
+
+def convert_cmu_arpabet_to_ipa(arp_ipa_map_file: str, arp_dict_file: str, output_ipa_file: str):
+    """
+    Converts CMU ARPABET-based dictionary to IPA.
+
+    Args:
+        arp_ipa_map_file: ARPABET to IPA phoneme mapping file
+        arp_dict_file: path to ARPABET version of CMU dictionary
+        output_ipa_file: path to output IPA version of CMU dictionary
+    """
+    arp_to_ipa_dict = _get_arpabet_to_ipa_mapping(arp_ipa_map_file)
+    with open(arp_dict_file, "r", encoding="utf-8") as f_arp, open(output_ipa_file, "w", encoding="utf-8") as f_ipa:
+        for line in f_arp:
+            if line.startswith(";;;"):
+                f_ipa.write(line)
+            else:
+                # First, split the line at " #" if there are comments in the dictionary file following the mapping entries.
+                # Next, split at default "  " separator.
+                graphemes, phonemes = line.split(" #")[0].strip().split("  ")
+                ipa_form = convert_arp_to_ipa(arp_to_ipa_dict, phonemes, remove_space=True)
+                f_ipa.write(f"{graphemes}  {ipa_form}\n")
+
+    print(f"IPA version of {os.path.abspath(arp_dict_file)} saved in {os.path.abspath(output_ipa_file)}")
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    convert_cmu_arpabet_to_ipa(args.mapping, args.cmu_arpabet, args.ipa_out)

scripts/dataset_processing/g2p/syllabify.py

@@ -0,0 +1,288 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Copyright (c) 2012-2013 Kyle Gorman <gormanky@ohsu.edu>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# syllabify.py: prosodic parsing of ARPABET entries
+# source: https://github.com/kylebgorman/syllabify
+
+from itertools import chain
+
+## constants
+SLAX = {
+    "IH1",
+    "IH2",
+    "EH1",
+    "EH2",
+    "AE1",
+    "AE2",
+    "AH1",
+    "AH2",
+    "UH1",
+    "UH2",
+}
+VOWELS = {
+    "IY1",
+    "IY2",
+    "IY0",
+    "EY1",
+    "EY2",
+    "EY0",
+    "AA1",
+    "AA2",
+    "AA0",
+    "ER1",
+    "ER2",
+    "ER0",
+    "AW1",
+    "AW2",
+    "AW0",
+    "AO1",
+    "AO2",
+    "AO0",
+    "AY1",
+    "AY2",
+    "AY0",
+    "OW1",
+    "OW2",
+    "OW0",
+    "OY1",
+    "OY2",
+    "OY0",
+    "IH0",
+    "EH0",
+    "AE0",
+    "AH0",
+    "UH0",
+    "UW1",
+    "UW2",
+    "UW0",
+    "UW",
+    "IY",
+    "EY",
+    "AA",
+    "ER",
+    "AW",
+    "AO",
+    "AY",
+    "OW",
+    "OY",
+    "UH",
+    "IH",
+    "EH",
+    "AE",
+    "AH",
+    "UH",
+} | SLAX
+
+## licit medial onsets
+
+O2 = {
+    ("P", "R"),
+    ("T", "R"),
+    ("K", "R"),
+    ("B", "R"),
+    ("D", "R"),
+    ("G", "R"),
+    ("F", "R"),
+    ("TH", "R"),
+    ("P", "L"),
+    ("K", "L"),
+    ("B", "L"),
+    ("G", "L"),
+    ("F", "L"),
+    ("S", "L"),
+    ("K", "W"),
+    ("G", "W"),
+    ("S", "W"),
+    ("S", "P"),
+    ("S", "T"),
+    ("S", "K"),
+    ("HH", "Y"),  # "clerihew"
+    ("R", "W"),
+}
+O3 = {("S", "T", "R"), ("S", "K", "L"), ("T", "R", "W")}  # "octroi"
+
+# This does not represent anything like a complete list of onsets, but
+# merely those that need to be maximized in medial position.
+
+
+def syllabify(pron, alaska_rule=True):
+    """
+    Syllabifies a CMU dictionary (ARPABET) word string
+
+    # Alaska rule:
+    >>> pprint(syllabify('AH0 L AE1 S K AH0'.split())) # Alaska
+    '-AH0-.L-AE1-S.K-AH0-'
+    >>> pprint(syllabify('AH0 L AE1 S K AH0'.split(), 0)) # Alaska
+    '-AH0-.L-AE1-.S K-AH0-'
+
+    # huge medial onsets:
+    >>> pprint(syllabify('M IH1 N S T R AH0 L'.split())) # minstrel
+    'M-IH1-N.S T R-AH0-L'
+    >>> pprint(syllabify('AA1  K T R W AA0 R'.split())) # octroi
+    '-AA1-K.T R W-AA0-R'
+
+    # destressing
+    >>> pprint(destress(syllabify('M IH1 L AH0 T EH2 R IY0'.split())))
+    'M-IH-.L-AH-.T-EH-.R-IY-'
+
+    # normal treatment of 'j':
+    >>> pprint(syllabify('M EH1 N Y UW0'.split())) # menu
+    'M-EH1-N.Y-UW0-'
+    >>> pprint(syllabify('S P AE1 N Y AH0 L'.split())) # spaniel
+    'S P-AE1-N.Y-AH0-L'
+    >>> pprint(syllabify('K AE1 N Y AH0 N'.split())) # canyon
+    'K-AE1-N.Y-AH0-N'
+    >>> pprint(syllabify('M IH0 N Y UW2 EH1 T'.split())) # minuet
+    'M-IH0-N.Y-UW2-.-EH1-T'
+    >>> pprint(syllabify('JH UW1 N Y ER0'.split())) # junior
+    'JH-UW1-N.Y-ER0-'
+    >>> pprint(syllabify('K L EH R IH HH Y UW'.split())) # clerihew
+    'K L-EH-.R-IH-.HH Y-UW-'
+
+    # nuclear treatment of 'j'
+    >>> pprint(syllabify('R EH1 S K Y UW0'.split())) # rescue
+    'R-EH1-S.K-Y UW0-'
+    >>> pprint(syllabify('T R IH1 B Y UW0 T'.split())) # tribute
+    'T R-IH1-B.Y-UW0-T'
+    >>> pprint(syllabify('N EH1 B Y AH0 L AH0'.split())) # nebula
+    'N-EH1-B.Y-AH0-.L-AH0-'
+    >>> pprint(syllabify('S P AE1 CH UH0 L AH0'.split())) # spatula
+    'S P-AE1-.CH-UH0-.L-AH0-'
+    >>> pprint(syllabify('AH0 K Y UW1 M AH0 N'.split())) # acumen
+    '-AH0-K.Y-UW1-.M-AH0-N'
+    >>> pprint(syllabify('S AH1 K Y AH0 L IH0 N T'.split())) # succulent
+    'S-AH1-K.Y-AH0-.L-IH0-N T'
+    >>> pprint(syllabify('F AO1 R M Y AH0 L AH0'.split())) # formula
+    'F-AO1 R-M.Y-AH0-.L-AH0-'
+    >>> pprint(syllabify('V AE1 L Y UW0'.split())) # value
+    'V-AE1-L.Y-UW0-'
+
+    # everything else
+    >>> pprint(syllabify('N AO0 S T AE1 L JH IH0 K'.split())) # nostalgic
+    'N-AO0-.S T-AE1-L.JH-IH0-K'
+    >>> pprint(syllabify('CH ER1 CH M AH0 N'.split())) # churchmen
+    'CH-ER1-CH.M-AH0-N'
+    >>> pprint(syllabify('K AA1 M P AH0 N S EY2 T'.split())) # compensate
+    'K-AA1-M.P-AH0-N.S-EY2-T'
+    >>> pprint(syllabify('IH0 N S EH1 N S'.split())) # inCENSE
+    '-IH0-N.S-EH1-N S'
+    >>> pprint(syllabify('IH1 N S EH2 N S'.split())) # INcense
+    '-IH1-N.S-EH2-N S'
+    >>> pprint(syllabify('AH0 S EH1 N D'.split())) # ascend
+    '-AH0-.S-EH1-N D'
+    >>> pprint(syllabify('R OW1 T EY2 T'.split())) # rotate
+    'R-OW1-.T-EY2-T'
+    >>> pprint(syllabify('AA1 R T AH0 S T'.split())) # artist
+    '-AA1 R-.T-AH0-S T'
+    >>> pprint(syllabify('AE1 K T ER0'.split())) # actor
+    '-AE1-K.T-ER0-'
+    >>> pprint(syllabify('P L AE1 S T ER0'.split())) # plaster
+    'P L-AE1-S.T-ER0-'
+    >>> pprint(syllabify('B AH1 T ER0'.split())) # butter
+    'B-AH1-.T-ER0-'
+    >>> pprint(syllabify('K AE1 M AH0 L'.split())) # camel
+    'K-AE1-.M-AH0-L'
+    >>> pprint(syllabify('AH1 P ER0'.split())) # upper
+    '-AH1-.P-ER0-'
+    >>> pprint(syllabify('B AH0 L UW1 N'.split())) # balloon
+    'B-AH0-.L-UW1-N'
+    >>> pprint(syllabify('P R OW0 K L EY1 M'.split())) # proclaim
+    'P R-OW0-.K L-EY1-M'
+    >>> pprint(syllabify('IH0 N S EY1 N'.split())) # insane
+    '-IH0-N.S-EY1-N'
+    >>> pprint(syllabify('IH0 K S K L UW1 D'.split())) # exclude
+    '-IH0-K.S K L-UW1-D'
+    """
+    ## main pass
+    mypron = list(pron)
+    nuclei = []
+    onsets = []
+    i = -1
+    for j, seg in enumerate(mypron):
+        if seg in VOWELS:
+            nuclei.append([seg])
+            onsets.append(mypron[i + 1 : j])  # actually interludes, r.n.
+            i = j
+    codas = [mypron[i + 1 :]]
+    ## resolve disputes and compute coda
+    for i in range(1, len(onsets)):
+        coda = []
+        # boundary cases
+        if len(onsets[i]) > 1 and onsets[i][0] == "R":
+            nuclei[i - 1].append(onsets[i].pop(0))
+        if len(onsets[i]) > 2 and onsets[i][-1] == "Y":
+            nuclei[i].insert(0, onsets[i].pop())
+        if len(onsets[i]) > 1 and alaska_rule and nuclei[i - 1][-1] in SLAX and onsets[i][0] == "S":
+            coda.append(onsets[i].pop(0))
+        # onset maximization
+        depth = 1
+        if len(onsets[i]) > 1:
+            if tuple(onsets[i][-2:]) in O2:
+                depth = 3 if tuple(onsets[i][-3:]) in O3 else 2
+        for j in range(len(onsets[i]) - depth):
+            coda.append(onsets[i].pop(0))
+        # store coda
+        codas.insert(i - 1, coda)
+
+    ## verify that all segments are included in the ouput
+    output = list(zip(onsets, nuclei, codas))  # in Python3 zip is a generator
+    flat_output = list(chain.from_iterable(chain.from_iterable(output)))
+    if flat_output != mypron:
+        raise ValueError(f"could not syllabify {mypron}, got {flat_output}")
+    return output
+
+
+def pprint(syllab):
+    """
+    Pretty-print a syllabification
+    """
+    return ".".join("-".join(" ".join(p) for p in syl) for syl in syllab)
+
+
+def destress(syllab):
+    """
+    Generate a syllabification with nuclear stress information removed
+    """
+    syls = []
+    for onset, nucleus, coda in syllab:
+        nuke = [p[:-1] if p[-1] in {"0", "1", "2"} else p for p in nucleus]
+        syls.append((onset, nuke, coda))
+    return syls
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

scripts/dataset_processing/get_aishell_data.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python get_aishell_data.py --data_root=<where to put data>
+
+import argparse
+import json
+import logging
+import os
+import subprocess
+import tarfile
+import urllib.request
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+parser = argparse.ArgumentParser(description="Aishell Data download")
+parser.add_argument("--data_root", required=True, default=None, type=str)
+args = parser.parse_args()
+
+URL = {"data_aishell": "http://www.openslr.org/resources/33/data_aishell.tgz"}
+
+
+def __retrieve_with_progress(source: str, filename: str):
+    """
+    Downloads source to destination
+    Displays progress bar
+    Args:
+        source: url of resource
+        destination: local filepath
+    Returns:
+    """
+    with open(filename, "wb") as f:
+        response = urllib.request.urlopen(source)
+        total = response.length
+
+        if total is None:
+            f.write(response.content)
+        else:
+            with tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024) as pbar:
+                for data in response:
+                    f.write(data)
+                    pbar.update(len(data))
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+
+    """
+    source = URL[source]
+    if not os.path.exists(destination):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        __retrieve_with_progress(source, filename=destination + ".tmp")
+        os.rename(destination + ".tmp", destination)
+        logging.info("Downloaded {0}.".format(destination))
+    else:
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    return destination
+
+
+def __extract_all_files(filepath: str, data_root: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_root)
+        audio_dir = os.path.join(data_dir, "wav")
+        for subfolder, _, filelist in os.walk(audio_dir):
+            for ftar in filelist:
+                extract_file(os.path.join(subfolder, ftar), subfolder)
+    else:
+        logging.info("Skipping extracting. Data already there %s" % data_dir)
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def __process_data(data_folder: str, dst_folder: str):
+    """
+    To generate manifest
+    Args:
+        data_folder: source with wav files
+        dst_folder: where manifest files will be stored
+    Returns:
+
+    """
+
+    if not os.path.exists(dst_folder):
+        os.makedirs(dst_folder)
+
+    transcript_file = os.path.join(data_folder, "transcript", "aishell_transcript_v0.8.txt")
+    transcript_dict = {}
+    with open(transcript_file, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            audio_id, text = line.split(" ", 1)
+            # remove white space
+            text = text.replace(" ", "")
+            transcript_dict[audio_id] = text
+
+    data_types = ["train", "dev", "test"]
+    vocab_count = {}
+    for dt in data_types:
+        json_lines = []
+        audio_dir = os.path.join(data_folder, "wav", dt)
+        for sub_folder, _, file_list in os.walk(audio_dir):
+            for fname in file_list:
+                audio_path = os.path.join(sub_folder, fname)
+                audio_id = fname.strip(".wav")
+                if audio_id not in transcript_dict:
+                    continue
+                text = transcript_dict[audio_id]
+                for li in text:
+                    vocab_count[li] = vocab_count.get(li, 0) + 1
+                duration = subprocess.check_output(["soxi", "-D", audio_path])
+                duration = float(duration)
+                json_lines.append(
+                    json.dumps(
+                        {
+                            "audio_filepath": os.path.abspath(audio_path),
+                            "duration": duration,
+                            "text": text,
+                        },
+                        ensure_ascii=False,
+                    )
+                )
+
+        manifest_path = os.path.join(dst_folder, dt + ".json")
+        with open(manifest_path, "w", encoding="utf-8") as fout:
+            for line in json_lines:
+                fout.write(line + "\n")
+
+    vocab = sorted(vocab_count.items(), key=lambda k: k[1], reverse=True)
+    vocab_file = os.path.join(dst_folder, "vocab.txt")
+    with open(vocab_file, "w", encoding="utf-8") as f:
+        for v, c in vocab:
+            f.write(v + "\n")
+
+
+def main():
+    data_root = args.data_root
+    data_set = "data_aishell"
+    logging.info("\n\nWorking on: {0}".format(data_set))
+    file_path = os.path.join(data_root, data_set + ".tgz")
+    logging.info("Getting {0}".format(data_set))
+    __maybe_download_file(file_path, data_set)
+    logging.info("Extracting {0}".format(data_set))
+    data_folder = os.path.join(data_root, data_set)
+    __extract_all_files(file_path, data_root, data_folder)
+    logging.info("Processing {0}".format(data_set))
+    __process_data(data_folder, data_folder)
+    logging.info("Done!")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/get_commonvoice_data.py

@@ -0,0 +1,229 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Copyright (c) 2020, SeanNaren.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# To convert mp3 files to wav using sox, you must have installed sox with mp3 support
+# For example sudo apt-get install libsox-fmt-mp3
+import argparse
+import csv
+import json
+import logging
+import multiprocessing
+import os
+import sys
+import tarfile
+import urllib.request
+from multiprocessing.pool import ThreadPool
+from pathlib import Path
+from typing import List
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+parser = argparse.ArgumentParser(description='Downloads and processes Mozilla Common Voice dataset.')
+parser.add_argument("--data_root", default='CommonVoice_dataset/', type=str, help="Directory to store the dataset.")
+parser.add_argument('--manifest_dir', default='./', type=str, help='Output directory for manifests')
+parser.add_argument("--num_workers", default=multiprocessing.cpu_count(), type=int, help="Workers to process dataset.")
+parser.add_argument('--sample_rate', default=16000, type=int, help='Sample rate')
+parser.add_argument('--n_channels', default=1, type=int, help='Number of channels for output wav files')
+parser.add_argument("--log", dest="log", action="store_true", default=False)
+parser.add_argument("--cleanup", dest="cleanup", action="store_true", default=False)
+parser.add_argument(
+    '--files_to_process',
+    nargs='+',
+    default=['test.tsv', 'dev.tsv', 'train.tsv'],
+    type=str,
+    help='list of *.csv file names to process',
+)
+parser.add_argument(
+    '--version',
+    default='cv-corpus-5.1-2020-06-22',
+    type=str,
+    help='Version of the dataset (obtainable via https://commonvoice.mozilla.org/en/datasets',
+)
+parser.add_argument(
+    '--language',
+    default='en',
+    type=str,
+    help='Which language to download.(default english,'
+    'check https://commonvoice.mozilla.org/en/datasets for more language codes',
+)
+args = parser.parse_args()
+COMMON_VOICE_URL = (
+    f"https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/"
+    "{}/{}.tar.gz".format(args.version, args.language)
+)
+COMMON_VOICE_USER_AGENT = (
+    'Mozilla/5.0 (Windows NT 10.0; WOW64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
+)
+
+
+def _load_sox():
+    try:
+        import sox
+        from sox import Transformer
+    except ImportError:
+        raise ImportError(
+            "Optional dependency 'sox' is required by this script. Install it with: pip install sox"
+        ) from None
+
+    return sox, Transformer
+
+
+def download_commonvoice_archive(url: str, output_path: str):
+    request = urllib.request.Request(url, headers={'User-Agent': COMMON_VOICE_USER_AGENT})
+    with urllib.request.urlopen(request) as response, open(output_path, 'wb') as f:
+        while True:
+            chunk = response.read(1024 * 1024)
+            if not chunk:
+                break
+            f.write(chunk)
+
+
+def create_manifest(data: List[tuple], output_name: str, manifest_path: str):
+    output_file = Path(manifest_path) / output_name
+    output_file.parent.mkdir(exist_ok=True, parents=True)
+
+    with output_file.open(mode='w') as f:
+        for wav_path, duration, text in tqdm(data, total=len(data)):
+            if wav_path != '':
+                # skip invalid input files that could not be converted
+                f.write(
+                    json.dumps({'audio_filepath': os.path.abspath(wav_path), "duration": duration, 'text': text})
+                    + '\n'
+                )
+
+
+def process_files(csv_file, data_root, num_workers):
+    """Read *.csv file description, convert mp3 to wav, process text.
+        Save results to data_root.
+
+    Args:
+        csv_file: str, path to *.csv file with data description, usually start from 'cv-'
+        data_root: str, path to dir to save results; wav/ dir will be created
+    """
+    sox, Transformer = _load_sox()
+    wav_dir = os.path.join(data_root, 'wav/')
+    os.makedirs(wav_dir, exist_ok=True)
+    audio_clips_path = os.path.dirname(csv_file) + '/clips/'
+
+    def process(x):
+        file_path, text = x
+        file_name = os.path.splitext(os.path.basename(file_path))[0]
+        text = text.lower().strip()
+        audio_path = os.path.join(audio_clips_path, file_path)
+        if os.path.getsize(audio_path) == 0:
+            logging.warning(f'Skipping empty audio file {audio_path}')
+            return '', '', ''
+
+        output_wav_path = os.path.join(wav_dir, file_name + '.wav')
+
+        if not os.path.exists(output_wav_path):
+            tfm = Transformer()
+            tfm.rate(samplerate=args.sample_rate)
+            tfm.channels(n_channels=args.n_channels)
+            tfm.build(input_filepath=audio_path, output_filepath=output_wav_path)
+
+        duration = sox.file_info.duration(output_wav_path)
+        return output_wav_path, duration, text
+
+    logging.info('Converting mp3 to wav for {}.'.format(csv_file))
+    with open(csv_file) as csvfile:
+        reader = csv.DictReader(csvfile, delimiter='\t')
+        next(reader, None)  # skip the headers
+        data = []
+        for row in reader:
+            file_name = row['path']
+            # add the mp3 extension if the tsv entry does not have it
+            if not file_name.endswith('.mp3'):
+                file_name += '.mp3'
+            data.append((file_name, row['sentence']))
+        with ThreadPool(num_workers) as pool:
+            data = list(tqdm(pool.imap(process, data), total=len(data)))
+    return data
+
+
+def main():
+    if args.log:
+        logging.basicConfig(level=logging.INFO)
+
+    data_root = args.data_root
+    os.makedirs(data_root, exist_ok=True)
+
+    target_unpacked_dir = os.path.join(data_root, "CV_unpacked")
+
+    if os.path.exists(target_unpacked_dir):
+        logging.info('Find existing folder {}'.format(target_unpacked_dir))
+    else:
+        logging.info("Could not find Common Voice, Downloading corpus...")
+
+        # some dataset versions are packaged in different named files, so forcing
+        output_archive_filename = args.language + '.tar.gz'
+        output_archive_filename = os.path.join(data_root, output_archive_filename)
+
+        download_commonvoice_archive(COMMON_VOICE_URL, output_archive_filename)
+        filename = f"{args.language}.tar.gz"
+        target_file = os.path.join(data_root, os.path.basename(filename))
+
+        os.makedirs(target_unpacked_dir, exist_ok=True)
+        logging.info("Unpacking corpus to {} ...".format(target_unpacked_dir))
+        with tarfile.open(target_file) as tar:
+            safe_extract(tar, target_unpacked_dir)
+        if args.cleanup:
+            logging.info("removing tar archive to save space")
+            os.remove(target_file)
+
+    folder_path = os.path.join(target_unpacked_dir, args.version + f'/{args.language}/')
+    if not os.path.isdir(folder_path):
+        # try without language
+        folder_path = os.path.join(target_unpacked_dir, args.version)
+        if not os.path.isdir(folder_path):
+            # try without version
+            folder_path = target_unpacked_dir
+            if not os.path.isdir(folder_path):
+                logging.error(f'unable to locate unpacked files in {folder_path}')
+                sys.exit()
+
+    for csv_file in args.files_to_process:
+        data = process_files(
+            csv_file=os.path.join(folder_path, csv_file),
+            data_root=os.path.join(data_root, os.path.splitext(csv_file)[0]),
+            num_workers=args.num_workers,
+        )
+        logging.info('Creating manifests...')
+        create_manifest(
+            data=data,
+            output_name=f'commonvoice_{os.path.splitext(csv_file)[0]}_manifest.json',
+            manifest_path=args.manifest_dir,
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/get_demand_data.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python get_demand_data.py --data_root=<where to put data>
+#        --data_set=<datasets_to_download>
+# where <datasets_to_download> can be: one or more of the 16 kHz noise profiles
+# listed at https://zenodo.org/record/1227121#.Ygb4avXMKJk ,
+# or ALL
+# You can put more than one data_set comma-separated:
+# --data_sets=DKITCHEN,DLIVING,NRIVER
+
+import argparse
+import glob
+import json
+import logging
+import os
+import shutil
+import subprocess
+import urllib.request
+
+parser = argparse.ArgumentParser(description='LibriSpeech Data download')
+parser.add_argument("--data_root", required=True, default=None, type=str)
+parser.add_argument("--data_sets", default="ALL", type=str)
+
+parser.add_argument('--log', dest='log', action='store_true', default=False)
+args = parser.parse_args()
+
+URLS = {
+    'DKITCHEN': ("https://zenodo.org/record/1227121/files/DKITCHEN_16k.zip"),
+    'DLIVING': ("https://zenodo.org/record/1227121/files/DLIVING_16k.zip"),
+    'DWASHING': ("https://zenodo.org/record/1227121/files/DWASHING_16k.zip"),
+    'NFIELD': ("https://zenodo.org/record/1227121/files/NFIELD_16k.zip"),
+    'NPARK': ("https://zenodo.org/record/1227121/files/NPARK_16k.zip"),
+    'NRIVER': ("https://zenodo.org/record/1227121/files/NRIVER_16k.zip"),
+    'OHALLWAY': ("https://zenodo.org/record/1227121/files/OHALLWAY_16k.zip"),
+    'OMEETING': ("https://zenodo.org/record/1227121/files/OMEETING_16k.zip"),
+    'OOFFICE': ("https://zenodo.org/record/1227121/files/OOFFICE_16k.zip"),
+    'PCAFETER': ("https://zenodo.org/record/1227121/files/PCAFETER_16k.zip"),
+    'PRESTO': ("https://zenodo.org/record/1227121/files/PRESTO_16k.zip"),
+    'PSTATION': ("https://zenodo.org/record/1227121/files/PSTATION_16k.zip"),
+    'SPSQUARE': ("https://zenodo.org/record/1227121/files/SPSQUARE_16k.zip"),
+    'STRAFFIC': ("https://zenodo.org/record/1227121/files/STRAFFIC_16k.zip"),
+    'TBUS': ("https://zenodo.org/record/1227121/files/TBUS_16k.zip"),
+    'TCAR': ("https://zenodo.org/record/1227121/files/TCAR_16k.zip"),
+    'TMETRO': ("https://zenodo.org/record/1227121/files/TMETRO_16k.zip"),
+}
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+    Returns:
+    """
+    source = URLS[source]
+    if not os.path.exists(destination):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info("Downloaded {0}.".format(destination))
+    else:
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    return destination
+
+
+def __extract_file(filepath: str, data_dir: str):
+    shutil.unpack_archive(filepath, data_dir)
+
+
+def __create_manifest(dst_folder: str):
+    """
+    Create manifests for the noise files
+    Args:
+        file_path: path to a source transcript  with flac sources
+        dst_folder: path where manifests will be created
+    Returns:
+
+        a list of metadata entries for processed files.
+    """
+    # Read directory
+    # Get all wav file names
+    # create line per wav file in manifest
+    noise_name = os.path.basename(dst_folder)
+    wav_files = glob.glob(dst_folder + "/*.wav")
+    wav_files.sort()
+    os.makedirs(os.path.join(os.path.dirname(dst_folder), "manifests"), exist_ok=True)
+    with open(os.path.join(os.path.dirname(dst_folder), "manifests", noise_name + ".json"), "w") as mfst_f:
+        for wav_f in wav_files:
+            dur = subprocess.check_output(["soxi", "-D", wav_f])
+            row = {"audio_filepath": wav_f, "text": "", "duration": float(dur)}
+            mfst_f.write(json.dumps(row) + "\n")
+
+
+def main():
+    data_root = args.data_root
+    data_sets = args.data_sets
+
+    if args.log:
+        print("here")
+        logging.basicConfig(level=logging.INFO)
+    if not os.path.exists(data_root):
+        os.makedirs(data_root)
+
+    if data_sets == "ALL":
+        data_sets = URLS.keys()
+    else:
+        data_sets = data_sets.split(',')
+
+    for data_set in data_sets:
+        if data_set not in URLS.keys():
+            raise ValueError(f"{data_sets} is not part of demand noise database")
+        logging.info("\n\nWorking on: {0}".format(data_set))
+        filepath = os.path.join(data_root, data_set + "_16k.zip")
+        logging.info("Getting {0}".format(data_set))
+        __maybe_download_file(filepath, data_set.upper())
+        logging.info("Extracting {0}".format(data_set))
+        __extract_file(filepath, data_root)
+        logging.info("Processing {0}".format(data_set))
+        __create_manifest(os.path.join(data_root, data_set))
+    logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/get_librispeech_data.py

@@ -0,0 +1,221 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# USAGE: python get_librispeech_data.py --data_root=<where to put data>
+#        --data_set=<datasets_to_download> --num_workers=<number of parallel workers>
+# where <datasets_to_download> can be: dev_clean, dev_other, test_clean,
+# test_other, train_clean_100, train_clean_360, train_other_500 or ALL
+# You can also put more than one data_set comma-separated:
+# --data_set=dev_clean,train_clean_100
+import argparse
+import fnmatch
+import functools
+import json
+import logging
+import multiprocessing
+import os
+import subprocess
+import tarfile
+import urllib.request
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+parser = argparse.ArgumentParser(description="LibriSpeech Data download")
+parser.add_argument("--data_root", required=True, default=None, type=str)
+parser.add_argument("--data_sets", default="dev_clean", type=str)
+parser.add_argument("--num_workers", default=4, type=int)
+parser.add_argument("--log", dest="log", action="store_true", default=False)
+args = parser.parse_args()
+
+URLS = {
+    "TRAIN_CLEAN_100": ("http://www.openslr.org/resources/12/train-clean-100.tar.gz"),
+    "TRAIN_CLEAN_360": ("http://www.openslr.org/resources/12/train-clean-360.tar.gz"),
+    "TRAIN_OTHER_500": ("http://www.openslr.org/resources/12/train-other-500.tar.gz"),
+    "DEV_CLEAN": "http://www.openslr.org/resources/12/dev-clean.tar.gz",
+    "DEV_OTHER": "http://www.openslr.org/resources/12/dev-other.tar.gz",
+    "TEST_CLEAN": "http://www.openslr.org/resources/12/test-clean.tar.gz",
+    "TEST_OTHER": "http://www.openslr.org/resources/12/test-other.tar.gz",
+    "DEV_CLEAN_2": "https://www.openslr.org/resources/31/dev-clean-2.tar.gz",
+    "TRAIN_CLEAN_5": "https://www.openslr.org/resources/31/train-clean-5.tar.gz",
+}
+
+
+def _load_sox_transformer():
+    try:
+        from sox import Transformer
+    except ImportError:
+        raise ImportError(
+            "Optional dependency 'sox' is required by this script. Install it with: pip install sox"
+        ) from None
+
+    return Transformer
+
+
+def __retrieve_with_progress(source: str, filename: str):
+    """
+    Downloads source to destination
+    Displays progress bar
+    Args:
+        source: url of resource
+        destination: local filepath
+    Returns:
+    """
+    with open(filename, "wb") as f:
+        response = urllib.request.urlopen(source)
+        total = response.length
+
+        if total is None:
+            f.write(response.content)
+        else:
+            with tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024) as pbar:
+                for data in response:
+                    f.write(data)
+                    pbar.update(len(data))
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+    Returns:
+    """
+    source = URLS[source]
+    if not os.path.exists(destination):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+
+        __retrieve_with_progress(source, filename=destination + ".tmp")
+
+        os.rename(destination + ".tmp", destination)
+        logging.info("Downloaded {0}.".format(destination))
+    else:
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    return destination
+
+
+def __extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def __process_transcript(file_path: str, dst_folder: str):
+    """
+    Converts flac files to wav from a given transcript, capturing the metadata.
+    Args:
+        file_path: path to a source transcript  with flac sources
+        dst_folder: path where wav files will be stored
+    Returns:
+        a list of metadata entries for processed files.
+    """
+    Transformer = _load_sox_transformer()
+    entries = []
+    root = os.path.dirname(file_path)
+    with open(file_path, encoding="utf-8") as fin:
+        for line in fin:
+            id, text = line[: line.index(" ")], line[line.index(" ") + 1 :]
+            transcript_text = text.lower().strip()
+
+            # Convert FLAC file to WAV
+            flac_file = os.path.join(root, id + ".flac")
+            wav_file = os.path.join(dst_folder, id + ".wav")
+            if not os.path.exists(wav_file):
+                Transformer().build(flac_file, wav_file)
+            # check duration
+            duration = subprocess.check_output(["soxi", "-D", wav_file])
+
+            entry = {}
+            entry["audio_filepath"] = os.path.abspath(wav_file)
+            entry["duration"] = float(duration)
+            entry["text"] = transcript_text
+            entries.append(entry)
+    return entries
+
+
+def __process_data(data_folder: str, dst_folder: str, manifest_file: str, num_workers: int):
+    """
+    Converts flac to wav and build manifests's json
+    Args:
+        data_folder: source with flac files
+        dst_folder: where wav files will be stored
+        manifest_file: where to store manifest
+        num_workers: number of parallel workers processing files
+    Returns:
+    """
+
+    if not os.path.exists(dst_folder):
+        os.makedirs(dst_folder)
+
+    files = []
+    entries = []
+
+    for root, dirnames, filenames in os.walk(data_folder):
+        for filename in fnmatch.filter(filenames, "*.trans.txt"):
+            files.append(os.path.join(root, filename))
+
+    with multiprocessing.Pool(num_workers) as p:
+        processing_func = functools.partial(__process_transcript, dst_folder=dst_folder)
+        results = p.imap(processing_func, files)
+        for result in tqdm(results, total=len(files)):
+            entries.extend(result)
+
+    with open(manifest_file, "w") as fout:
+        for m in entries:
+            fout.write(json.dumps(m) + "\n")
+
+
+def main():
+    data_root = args.data_root
+    data_sets = args.data_sets
+    num_workers = args.num_workers
+
+    if args.log:
+        logging.basicConfig(level=logging.INFO)
+
+    if data_sets == "ALL":
+        data_sets = "dev_clean,dev_other,train_clean_100,train_clean_360,train_other_500,test_clean,test_other"
+    if data_sets == "mini":
+        data_sets = "dev_clean_2,train_clean_5"
+    for data_set in data_sets.split(","):
+        logging.info("\n\nWorking on: {0}".format(data_set))
+        filepath = os.path.join(data_root, data_set + ".tar.gz")
+        logging.info("Getting {0}".format(data_set))
+        __maybe_download_file(filepath, data_set.upper())
+        logging.info("Extracting {0}".format(data_set))
+        __extract_file(filepath, data_root)
+        logging.info("Processing {0}".format(data_set))
+        __process_data(
+            os.path.join(
+                os.path.join(data_root, "LibriSpeech"),
+                data_set.replace("_", "-"),
+            ),
+            os.path.join(
+                os.path.join(data_root, "LibriSpeech"),
+                data_set.replace("_", "-"),
+            )
+            + "-processed",
+            os.path.join(data_root, data_set + ".json"),
+            num_workers=num_workers,
+        )
+    logging.info("Done!")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/get_openslr_rir_data.py

@@ -0,0 +1,165 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python get_openslr_rir_data.py --data_root=<where to put data>
+# Data is downloaded from OpenSLR's "Room Impulse Response and Noise Database"
+# RIRs in multichannel files are separated into single channel files and
+# a json file that can be used as in input to NeMo is created
+
+import argparse
+import glob
+import json
+import logging
+import os
+import subprocess
+import urllib.request
+from shutil import copy, move
+from zipfile import ZipFile
+
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="OpenSLR RIR Data download and process")
+parser.add_argument("--data_root", required=True, default=None, type=str)
+args = parser.parse_args()
+
+URLS = {
+    "SLR28": ("http://www.openslr.org/resources/28/rirs_noises.zip"),
+}
+
+
+def __retrieve_with_progress(source: str, filename: str):
+    """
+    Downloads source to destination
+    Displays progress bar
+    Args:
+        source: url of resource
+        destination: local filepath
+    Returns:
+    """
+    with open(filename, "wb") as f:
+        response = urllib.request.urlopen(source)
+        total = response.length
+
+        if total is None:
+            f.write(response.content)
+        else:
+            with tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024) as pbar:
+                for data in response:
+                    f.write(data)
+                    pbar.update(len(data))
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+    Returns:
+    """
+    source = URLS[source]
+    if not os.path.exists(destination):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        __retrieve_with_progress(source, filename=destination + ".tmp")
+        os.rename(destination + ".tmp", destination)
+        logging.info("Downloaded {0}.".format(destination))
+    else:
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    return destination
+
+
+def __extract_file(filepath: str, data_dir: str):
+    try:
+        with ZipFile(filepath, "r") as zipObj:
+            zipObj.extractall(data_dir)
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def __process_data(data_folder: str, dst_folder: str, manifest_file: str):
+    """
+    Converts flac to wav and build manifests's json
+    Args:
+        data_folder: source with flac files
+        dst_folder: where wav files will be stored
+        manifest_file: where to store manifest
+    Returns:
+    """
+    if not os.path.exists(dst_folder):
+        os.makedirs(dst_folder)
+
+    real_rir_list = os.path.join(data_folder, "RIRS_NOISES", "real_rirs_isotropic_noises", "rir_list")
+    rirfiles = []
+    with open(real_rir_list, "r") as rir_f:
+        for line in rir_f:
+            rirfiles.append(os.path.join(data_folder, line.rstrip().split(" ")[4]))
+
+    real_rir_folder = os.path.join(dst_folder, "real_rirs")
+    if not os.path.exists(real_rir_folder):
+        os.makedirs(real_rir_folder)
+    # split multi-channel rir files to single channel
+    for rir_f in rirfiles:
+        n_chans = int(subprocess.check_output(["soxi", "-c", rir_f]))
+        if n_chans == 1:
+            copy(rir_f, real_rir_folder)
+        else:
+            for chan in range(1, n_chans + 1):
+                chan_file_name = os.path.join(
+                    real_rir_folder,
+                    os.path.splitext(os.path.basename(rir_f))[0] + "-" + str(chan) + ".wav",
+                )
+                _ = subprocess.check_output(["sox", rir_f, chan_file_name, "remix", str(chan)])
+
+    # move simulated rirs to processed
+    if not os.path.exists(os.path.join(dst_folder, "simulated_rirs")):
+        move(os.path.join(data_folder, "RIRS_NOISES", "simulated_rirs"), dst_folder)
+
+    os.chdir(dst_folder)
+    all_rirs = glob.glob("**/*.wav", recursive=True)
+    with open(manifest_file, "w") as man_f:
+        entry = {}
+        for rir in all_rirs:
+            rir_file = os.path.join(dst_folder, rir)
+            duration = subprocess.check_output(["soxi", "-D", rir_file])
+            entry["audio_filepath"] = rir_file
+            entry["duration"] = float(duration)
+            entry["offset"] = 0
+            entry["text"] = "_"
+            man_f.write(json.dumps(entry) + "\n")
+
+    print("Done!")
+
+
+def main():
+    data_root = os.path.abspath(args.data_root)
+    data_set = "slr28"
+    logging.getLogger().setLevel(logging.INFO)
+    logging.info("\n\nWorking on: {0}".format(data_set))
+    filepath = os.path.join(data_root, data_set + ".zip")
+    logging.info("Getting {0}".format(data_set))
+    __maybe_download_file(filepath, data_set.upper())
+    logging.info("Extracting {0}".format(data_set))
+    __extract_file(filepath, data_root)
+    logging.info("Processing {0}".format(data_set))
+    __process_data(
+        data_root,
+        os.path.join(os.path.join(data_root, "processed")),
+        os.path.join(os.path.join(data_root, "processed", "rir.json")),
+    )
+    logging.info("Done!")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/kaldi2json.py

@@ -0,0 +1,135 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+
+import pandas as pd
+
+from nemo.utils import logging
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert kaldi data folder to manifest.json")
+    parser.add_argument(
+        "--data_dir",
+        required=True,
+        type=str,
+        help="data in kaldi format",
+    )
+    parser.add_argument(
+        "--manifest",
+        required=True,
+        type=str,
+        help="path to store the manifest file",
+    )
+    parser.add_argument(
+        "--with_aux_data",
+        default=False,
+        action="store_true",
+        help="whether to include auxiliary data in the manifest",
+    )
+    args = parser.parse_args()
+
+    kaldi_folder = args.data_dir
+    required_data = {
+        "audio_filepath": os.path.join(kaldi_folder, "wav.scp"),
+        "duration": os.path.join(kaldi_folder, "segments"),
+        "text": os.path.join(kaldi_folder, "text"),
+    }
+    aux_data = {
+        "speaker": os.path.join(kaldi_folder, "utt2spk"),
+        "gender": os.path.join(kaldi_folder, "utt2gender"),
+    }
+    output_names = list(required_data.keys())
+
+    # check if required files exist
+    for name, file in required_data.items():
+        if not os.path.exists(file):
+            raise ValueError(f"{os.path.basename(file)} is not in {kaldi_folder}.")
+
+    # read wav.scp
+    wavscp = pd.read_csv(required_data["audio_filepath"], sep=" ", header=None)
+    if wavscp.shape[1] > 2:
+        logging.warning(
+            f"""More than two columns in 'wav.scp': {wavscp.shape[1]}.
+            Maybe it contains pipes? Pipe processing can be slow at runtime."""
+        )
+        wavscp = pd.read_csv(
+            required_data["audio_filepath"],
+            sep="^([^ ]+) ",
+            engine="python",
+            header=None,
+            usecols=[1, 2],
+            names=["wav_label", "audio_filepath"],
+        )
+    else:
+        wavscp = wavscp.rename(columns={0: "wav_label", 1: "audio_filepath"})
+
+    # read text
+    text = pd.read_csv(
+        required_data["text"],
+        sep="^([^ ]+) ",
+        engine="python",
+        header=None,
+        usecols=[1, 2],
+        names=["label", "text"],
+    )
+
+    # read segments
+    segments = pd.read_csv(
+        required_data["duration"],
+        sep=" ",
+        header=None,
+        names=["label", "wav_label", "offset", "end"],
+    )
+    # add offset if needed
+    if len(segments.offset) > len(segments.offset[segments.offset == 0.0]):
+        logging.info("Adding offset field.")
+        output_names.insert(2, "offset")
+    segments["duration"] = (segments.end - segments.offset).round(decimals=3)
+
+    # merge data
+    wav_segments_text = pd.merge(
+        pd.merge(segments, wavscp, how="inner", on="wav_label"),
+        text,
+        how="inner",
+        on="label",
+    )
+
+    if args.with_aux_data:
+        # check if auxiliary data is present
+        for name, aux_file in aux_data.items():
+            if os.path.exists(aux_file):
+                logging.info(f"Adding info from '{os.path.basename(aux_file)}'.")
+                wav_segments_text = pd.merge(
+                    wav_segments_text,
+                    pd.read_csv(aux_file, sep=" ", header=None, names=["label", name]),
+                    how="left",
+                    on="label",
+                )
+                output_names.append(name)
+            else:
+                logging.info(f"'{os.path.basename(aux_file)}' does not exist. Skipping ...")
+
+    # write data to .json
+    entries = wav_segments_text[output_names].to_dict(orient="records")
+    with open(args.manifest, "w", encoding="utf-8") as fout:
+        for m in entries:
+            fout.write(json.dumps(m, ensure_ascii=False) + "\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/process_aishell2_data.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python process_aishell2_data.py
+#                   --audio_folder=<source data>
+#                   --dest_folder=<where to store the results>
+import argparse
+import json
+import os
+import subprocess
+
+parser = argparse.ArgumentParser(description="Processing Aishell2 Data")
+parser.add_argument("--audio_folder", default=None, type=str, required=True, help="Audio (wav) data directory.")
+parser.add_argument("--dest_folder", default=None, type=str, required=True, help="Destination directory.")
+args = parser.parse_args()
+
+
+def __process_data(data_folder: str, dst_folder: str):
+    """
+    To generate manifest
+    Args:
+        data_folder: source with wav files
+        dst_folder: where manifest files will be stored
+    Returns:
+    """
+    if not os.path.exists(dst_folder):
+        os.makedirs(dst_folder)
+    data_type = ['dev', 'test', 'train']
+    for data in data_type:
+        dst_file = os.path.join(dst_folder, data + ".json")
+        uttrances = []
+        wav_dir = os.path.join(data_folder, "wav", data)
+        transcript_file = os.path.join(data_folder, "transcript", data, "trans.txt")
+        trans_text = {}
+        with open(transcript_file, "r", encoding='utf-8') as f:
+            for line in f:
+                line = line.strip().split()
+                utterance_id, text = line[0], " ".join(line[1:])
+                trans_text[utterance_id] = text.upper()
+        session_list = os.listdir(wav_dir)
+        for sessions in session_list:
+            cur_dir = os.path.join(wav_dir, sessions)
+            for wavs in os.listdir(cur_dir):
+                audio_id = wavs.strip(".wav")
+                audio_filepath = os.path.abspath(os.path.join(cur_dir, wavs))
+                duration = subprocess.check_output(["soxi", "-D", audio_filepath])
+                duration = float(duration)
+                text = trans_text[audio_id]
+                uttrances.append(
+                    json.dumps(
+                        {"audio_filepath": audio_filepath, "duration": duration, "text": text}, ensure_ascii=False
+                    )
+                )
+        with open(dst_file, "w") as f:
+            for line in uttrances:
+                f.write(line + "\n")
+
+
+def __get_vocab(data_folder: str, des_dir: str):
+    """
+    To generate the vocabulary file
+    Args:
+        data_folder: source with the transcript file
+        dst_folder: where the file will be stored
+    Returns:
+    """
+    if not os.path.exists(des_dir):
+        os.makedirs(des_dir)
+    trans_file = os.path.join(data_folder, "transcript", "train", "trans.txt")
+    vocab_dict = {}
+    with open(trans_file, "r", encoding='utf-8') as f:
+        for line in f:
+            line = line.strip().split()
+            text = " ".join(line[1:])
+            for i in text.upper():
+                if i in vocab_dict:
+                    vocab_dict[i] += 1
+                else:
+                    vocab_dict[i] = 1
+    vocab_dict = sorted(vocab_dict.items(), key=lambda k: k[1], reverse=True)
+    vocab = os.path.join(des_dir, "vocab.txt")
+    vocab = open(vocab, "w", encoding='utf-8')
+    for k in vocab_dict:
+        vocab.write(k[0] + "\n")
+    vocab.close()
+
+
+def main():
+    source_data = args.audio_folder
+    des_dir = args.dest_folder
+    print("begin to process data...")
+    __process_data(source_data, des_dir)
+    __get_vocab(source_data, des_dir)
+    print("finish all!")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/process_an4_data.py

@@ -0,0 +1,89 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import glob
+import json
+import logging
+import os
+import subprocess
+
+import librosa
+
+parser = argparse.ArgumentParser(description="AN4 dataset download and processing")
+parser.add_argument("--data_root", required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def build_manifest(data_root, transcripts_path, manifest_path, wav_path):
+    with open(transcripts_path, 'r') as fin:
+        with open(manifest_path, 'w') as fout:
+            for line in fin:
+                # Lines look like this:
+                # <s> transcript </s> (fileID)
+                transcript = line[: line.find('(') - 1].lower()
+                transcript = transcript.replace('<s>', '').replace('</s>', '')
+                transcript = transcript.strip()
+
+                file_id = line[line.find('(') + 1 : -2]  # e.g. "cen4-fash-b"
+                audio_path = os.path.join(
+                    data_root,
+                    wav_path,
+                    file_id[file_id.find('-') + 1 : file_id.rfind('-')],
+                    file_id + '.wav',
+                )
+
+                duration = librosa.core.get_duration(filename=audio_path)
+
+                # Write the metadata to the manifest
+                metadata = {
+                    "audio_filepath": audio_path,
+                    "duration": duration,
+                    "text": transcript,
+                }
+                json.dump(metadata, fout)
+                fout.write('\n')
+
+
+def main():
+    data_root = os.path.abspath(args.data_root)
+
+    # Convert from .sph to .wav
+    logging.info("Converting audio files to .wav...")
+    sph_list = glob.glob(os.path.join(data_root, 'an4/**/*.sph'), recursive=True)
+    for sph_path in sph_list:
+        wav_path = sph_path[:-4] + '.wav'
+        cmd = ['sox', sph_path, wav_path]
+        subprocess.run(cmd)
+    logging.info("Finished conversion.")
+
+    # Build manifests
+    logging.info("Building training manifest...")
+    train_transcripts = os.path.join(data_root, 'an4/etc/an4_train.transcription')
+    train_manifest = os.path.join(data_root, 'an4/train_manifest.json')
+    train_wavs = os.path.join(data_root, 'an4/wav/an4_clstk')
+    build_manifest(data_root, train_transcripts, train_manifest, train_wavs)
+    logging.info("Training manifests created.")
+
+    logging.info("Building test manifest...")
+    test_transcripts = os.path.join(data_root, 'an4/etc/an4_test.transcription')
+    test_manifest = os.path.join(data_root, 'an4/test_manifest.json')
+    test_wavs = os.path.join(data_root, 'an4/wav/an4test_clstk')
+    build_manifest(data_root, test_transcripts, test_manifest, test_wavs)
+    logging.info("Test manifest created.")
+
+    logging.info("Done with AN4 processing!")
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/process_fisher_data.py

@@ -0,0 +1,434 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE:
+# python process_fisher_data.py \
+#   --audio_root=<audio (.wav) directory>
+#   --transcript_root=<LDC Fisher dataset directory> \
+#   --dest_root=<destination directory> \
+#   --data_sets=LDC2004S13-Part1,LDC2005S13-Part2 \
+#   --remove_noises
+#
+# Matches Fisher dataset transcripts to the corresponding audio file (.wav),
+# and slices them into min_slice_duration segments with one speaker.
+# Also performs some other processing on transcripts.
+#
+# Heavily derived from Patter's Fisher processing script.
+
+import argparse
+import glob
+import json
+import os
+import re
+from math import ceil, floor
+
+import numpy as np
+import scipy.io.wavfile as wavfile
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="Fisher Data Processing")
+parser.add_argument(
+    "--audio_root",
+    default=None,
+    type=str,
+    required=True,
+    help="The path to the root of the audio (wav) data folder.",
+)
+parser.add_argument(
+    "--transcript_root",
+    default=None,
+    type=str,
+    required=True,
+    help="The path to the root of the transcript data folder.",
+)
+parser.add_argument(
+    "--dest_root",
+    default=None,
+    type=str,
+    required=True,
+    help="Path to the destination root directory.",
+)
+
+# Optional arguments
+parser.add_argument(
+    "--min_slice_duration",
+    default=10.0,
+    type=float,
+    help="Minimum audio slice duration after processing.",
+)
+parser.add_argument(
+    "--keep_low_conf",
+    action="store_true",
+    help="Keep all utterances with low confidence transcripts",
+)
+parser.add_argument(
+    "--remove_noises",
+    action="store_true",
+    help="Removes transcripted noises such as [laughter].",
+)
+parser.add_argument(
+    "--noises_to_emoji",
+    action="store_true",
+    help="Converts transcripts for noises to an emoji character.",
+)
+args = parser.parse_args()
+
+# Total number of files before segmenting, and train/val/test splits
+NUM_FILES = 5850 + 5849
+TRAIN_END_IDX = int(NUM_FILES * 0.8)
+VAL_END_IDX = int(NUM_FILES * 0.9)
+
+# Known transcription errors and their fixes (from Mozilla)
+TRANSCRIPT_BUGS = {
+    "fe_03_00265-B-3353-3381": "correct",
+    "fe_03_00991-B-52739-52829": "that's one of those",
+    "fe_03_10282-A-34442-34484.wav": "they don't want",
+    "fe_03_10677-B-10104-10641": "uh my mine yeah the german shepherd "
+    + "pitbull mix he snores almost as loud "
+    + "as i do",
+    "fe_03_00027-B-39380-39405": None,
+    "fe_03_11487-B-3109-23406": None,
+    "fe_03_01326-A-30742-30793": None,
+}
+
+TRANSCRIPT_NUMBERS = {
+    "401k": "four o one k",
+    "f16": "f sixteen",
+    "m16": "m sixteen",
+    "ak47": "a k forty seven",
+    "v8": "v eight",
+    "y2k": "y two k",
+    "mp3": "m p three",
+    "vh1": "v h one",
+    "90210": "nine o two one o",
+    "espn2": "e s p n two",
+    "u2": "u two",
+    "dc3s": "d c threes",
+    "book 2": "book two",
+    "s2b": "s two b",
+    "3d": "three d",
+}
+
+TAG_MAP = {
+    "[laughter]": "🤣",
+    "[laugh]": "🤣",
+    "[noise]": "😕",
+    "[sigh]": "😕",
+    "[cough]": "😕",
+    "[mn]": "😕",
+    "[breath]": "😕",
+    "[lipsmack]": "😕",
+    "[[skip]]": "",
+    "[pause]": "",
+    "[sneeze]": "😕",
+}
+
+
+def __write_sample(dest, file_id, count, file_count, sample_rate, audio, duration, transcript):
+    """
+    Writes one slice to the given target directory.
+    Args:
+        dest: the destination directory
+        file_id: name of the transcript/audio file for this block
+        count: the count of segments in the file so far
+        file_count: the total number of filse processed so far
+        sample rate: sample rate of the audio data
+        audio: audio data of the current sample
+        duration: audio duration of the current sample
+        transcript: transcript of the current sample
+    """
+    partition = __partition_name(file_count)
+    audio_path = os.path.join(dest, partition, f"{file_id}_{count:03}.wav")
+
+    # Write audio
+    wavfile.write(audio_path, sample_rate, audio)
+
+    # Write transcript info
+    transcript = {
+        "audio_filepath": audio_path,
+        "duration": duration,
+        "text": transcript,
+    }
+
+    # Append to manifest
+    manifest_path = os.path.join(dest, f"manifest_{partition}.json")
+    with open(manifest_path, 'a') as f:
+        json.dump(transcript, f)
+        f.write('\n')
+
+
+def __normalize(utt):
+    replace_table = str.maketrans(dict.fromkeys('()*;:"!&{},.-?'))
+    utt = (
+        utt.lower()
+        .replace('[uh]', 'uh')
+        .replace('[um]', 'um')
+        .replace('<noise>', '[noise]')
+        .replace('<spoken_noise>', '[vocalized-noise]')
+        .replace('.period', 'period')
+        .replace('.dot', 'dot')
+        .replace('-hyphen', 'hyphen')
+        .replace('._', ' ')
+        .translate(replace_table)
+    )
+    utt = re.sub(r"'([a-z]+)'", r'\1', utt)  # Unquote quoted words
+    return utt
+
+
+def __process_utterance(file_id, trans_path, line, keep_low_conf, rem_noises, emojify):
+    """
+    Processes one utterance (one line of a transcript).
+    Args:
+        file_id: the ID of the transcript file
+        trans_path: transcript path
+        line: one line in the transcript file
+        keep_low_conf: whether to keep low confidence lines
+        rem_noises: whether to remove noise symbols
+        emojify: whether to convert noise symbols to emoji, lower precedence
+    """
+    # Check for lines to skip (comments, empty, low confidence)
+    if line.startswith('#') or not line.strip() or (not keep_low_conf and '((' in line):
+        return None, None, None, None
+
+    # Data and sanity checks
+    line = line.split()
+
+    t_start, t_end = float(line[0]), float(line[1])
+    if (t_start < 0) or (t_end < t_start):
+        print(f"Invalid time: {t_start} to {t_end} in {trans_path}")
+        return None, None, None, None
+
+    channel = line[2]
+    idx = 0 if line[2] == 'A:' else 1
+
+    if channel not in ('A:', 'B:'):
+        print(f"Could not read channel info ({channel}) in {trans_path}")
+        return None, None, None, None
+
+    # Replacements as necessary
+    line_id = '-'.join([file_id, channel[0], str(t_start * 10), str(t_end * 10)])
+
+    content = TRANSCRIPT_BUGS.get(line_id, ' '.join(line[3:]))
+
+    if content is None:
+        return None, None, None, None
+
+    for tag, newtag in TRANSCRIPT_NUMBERS.items():
+        content = content.replace(tag, newtag)
+
+    content = __normalize(content)
+
+    if rem_noises:
+        for k, _ in TAG_MAP.items():
+            content = content.replace(k, '')
+    elif emojify:
+        for k, v in TAG_MAP.items():
+            content = content.replace(k, v)
+
+    return t_start, t_end, idx, content
+
+
+def __process_one_file(
+    trans_path,
+    sample_rate,
+    audio_data,
+    file_id,
+    dst_root,
+    min_slice_duration,
+    file_count,
+    keep_low_conf,
+    rem_noises,
+    emojify,
+):
+    """
+    Creates one block of audio slices and their corresponding transcripts.
+    Args:
+        trans_path: filepath to transcript
+        sample_rate: sample rate of the audio
+        audio_data: numpy array of shape [samples, channels]
+        file_id: identifying label, e.g. 'fe_03_01102'
+        dst_root: path to destination directory
+        min_slice_duration: min number of seconds for an audio slice
+        file_count: total number of files processed so far
+        keep_low_conf: keep utterances with low-confidence transcripts
+        rem_noises: remove noise symbols
+        emojify: convert noise symbols into emoji characters
+    """
+    count = 0
+
+    with open(trans_path, encoding="utf-8") as fin:
+        fin.readline()  # Comment w/ corresponding sph filename
+        fin.readline()  # Comment about transcriber
+
+        transcript_buffers = ['', '']  # [A buffer, B buffer]
+        audio_buffers = [[], []]
+        buffer_durations = [0.0, 0.0]
+
+        for line in fin:
+            t_start, t_end, idx, content = __process_utterance(
+                file_id, trans_path, line, keep_low_conf, rem_noises, emojify
+            )
+
+            if content is None or not content:
+                continue
+
+            duration = t_end - t_start
+
+            # Append utterance to buffer
+            transcript_buffers[idx] += content
+            audio_buffers[idx].append(
+                audio_data[
+                    floor(t_start * sample_rate) : ceil(t_end * sample_rate),
+                    idx,
+                ]
+            )
+            buffer_durations[idx] += duration
+
+            if buffer_durations[idx] < min_slice_duration:
+                transcript_buffers[idx] += ' '
+            else:
+                # Write out segment and transcript
+                count += 1
+                __write_sample(
+                    dst_root,
+                    file_id,
+                    count,
+                    file_count,
+                    sample_rate,
+                    np.concatenate(audio_buffers[idx], axis=0),
+                    buffer_durations[idx],
+                    transcript_buffers[idx],
+                )
+
+                # Clear buffers
+                transcript_buffers[idx] = ''
+                audio_buffers[idx] = []
+                buffer_durations[idx] = 0.0
+
+            # Note: We drop any shorter "scraps" at the end of the file, if
+            #   they end up shorter than min_slice_duration.
+
+
+def __partition_name(file_count):
+    if file_count >= VAL_END_IDX:
+        return "test"
+    elif file_count >= TRAIN_END_IDX:
+        return "val"
+    else:
+        return "train"
+
+
+def __process_data(
+    audio_root,
+    transcript_root,
+    dst_root,
+    min_slice_duration,
+    file_count,
+    keep_low_conf,
+    rem_noises,
+    emojify,
+):
+    """
+    Converts Fisher wav files to numpy arrays, segments audio and transcripts.
+    Args:
+        audio_root: source directory with the wav files
+        transcript_root: source directory with the transcript files
+            (can be the same as audio_root)
+        dst_root: where the processed and segmented files will be stored
+        min_slice_duration: minimum number of seconds for a slice of output
+        file_count: total number of files processed so far
+        keep_low_conf: whether or not to keep low confidence transcriptions
+        rem_noises: whether to remove noise symbols
+        emojify: whether to convert noise symbols to emoji, lower precedence
+    Assumes:
+        1. There is exactly one transcripts directory in data_folder
+        2. Audio files are all: <audio_root>/audio-wav/fe_03_xxxxx.wav
+    """
+    transcript_list = glob.glob(os.path.join(transcript_root, "fe_03_p*_tran*", "data", "trans", "*", "*.txt"))
+    print("Found {} transcripts.".format(len(transcript_list)))
+
+    count = file_count
+
+    # Grab audio file associated with each transcript, and slice
+    for trans_path in tqdm(transcript_list, desc="Matching and segmenting"):
+        file_id, _ = os.path.splitext(os.path.basename(trans_path))
+        audio_path = os.path.join(audio_root, "audio_wav", file_id + ".wav")
+
+        sample_rate, audio_data = wavfile.read(audio_path)
+
+        # Create a set of segments (a block) for each file
+        __process_one_file(
+            trans_path,
+            sample_rate,
+            audio_data,
+            file_id,
+            dst_root,
+            min_slice_duration,
+            count,
+            keep_low_conf,
+            rem_noises,
+            emojify,
+        )
+        count += 1
+
+    return count
+
+
+def main():
+    # Arguments to the script
+    audio_root = args.audio_root
+    transcript_root = args.transcript_root
+    dest_root = args.dest_root
+
+    min_slice_duration = args.min_slice_duration
+    keep_low_conf = args.keep_low_conf
+    rem_noises = args.remove_noises
+    emojify = args.noises_to_emoji
+
+    print(f"Expected number of files to segment: {NUM_FILES}")
+    print("With a 80/10/10 split:")
+    print(f"Number of training files: {TRAIN_END_IDX}")
+    print(f"Number of validation files: {VAL_END_IDX - TRAIN_END_IDX}")
+    print(f"Number of test files: {NUM_FILES - VAL_END_IDX}")
+
+    if not os.path.exists(os.path.join(dest_root, 'train/')):
+        os.makedirs(os.path.join(dest_root, 'train/'))
+        os.makedirs(os.path.join(dest_root, 'val/'))
+        os.makedirs(os.path.join(dest_root, 'test/'))
+    else:
+        # Wipe manifest contents first
+        open(os.path.join(dest_root, "manifest_train.json"), 'w').close()
+        open(os.path.join(dest_root, "manifest_val.json"), 'w').close()
+        open(os.path.join(dest_root, "manifest_test.json"), 'w').close()
+
+    file_count = 0
+
+    for data_set in ['LDC2004S13-Part1', 'LDC2005S13-Part2']:
+        print(f"\n\nWorking on dataset: {data_set}")
+        file_count = __process_data(
+            os.path.join(audio_root, data_set),
+            os.path.join(transcript_root, data_set),
+            dest_root,
+            min_slice_duration,
+            file_count,
+            keep_low_conf,
+            rem_noises,
+            emojify,
+        )
+
+        print(f"Total file count so far: {file_count}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/process_hub5_data.py

@@ -0,0 +1,285 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is heavily derived from the Patter HUB5 processing script written
+# by Ryan Leary
+
+import argparse
+import glob
+import json
+import os
+import re
+import subprocess
+import sys
+from collections import namedtuple
+from math import ceil, floor
+from operator import attrgetter
+
+import numpy as np
+import scipy.io.wavfile as wavfile
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="Prepare HUB5 data for training/eval")
+parser.add_argument(
+    "--data_root",
+    default=None,
+    type=str,
+    required=True,
+    help="The path to the root LDC HUB5 dataset directory.",
+)
+parser.add_argument(
+    "--dest_root",
+    default=None,
+    type=str,
+    required=True,
+    help="Path to the destination root directory for processed files.",
+)
+
+# Optional arguments
+parser.add_argument(
+    "--min_slice_duration",
+    default=10.0,
+    type=float,
+    help="Minimum audio slice duration after processing.",
+)
+
+args = parser.parse_args()
+
+StmUtterance = namedtuple(
+    'StmUtterance',
+    [
+        'filename',
+        'channel',
+        'speaker_id',
+        'begin',
+        'end',
+        'label',
+        'transcript',
+    ],
+)
+STM_LINE_FMT = re.compile(r"^(\w+)\s+(\w+)\s+(\w+)\s+([0-9.]+)\s+([0-9.]+)\s+(<.*>)?\s+(.+)$")
+
+# Transcription errors and their fixes
+TRANSCRIPT_BUGS = {"en_4622-B-12079-12187": "KIND OF WEIRD BUT"}
+
+
+def get_utt_id(segment):
+    """
+    Gives utterance IDs in a form like: en_4156-a-36558-37113
+    """
+    return "{}-{}-{}-{}".format(
+        segment.filename,
+        segment.channel,
+        int(segment.begin * 100),
+        int(segment.end * 100),
+    )
+
+
+def convert_utterances(sph_path, wav_path):
+    """
+    Converts a sphere audio file to wav.
+    """
+    cmd = ["sph2pipe", "-f", "wav", "-p", sph_path, wav_path]
+    subprocess.run(cmd)
+
+
+def create_wavs(data_root, dest_root):
+    """
+    Converts the English sph files to wav using sph2pipe.
+    """
+    sph_root = os.path.join(data_root, "hub5e_00", "english")
+    sph_list = glob.glob(os.path.join(sph_root, "*.sph"))
+
+    # Iterate over each sphere file and conver to wav
+    for sph_path in tqdm(sph_list, desc="Converting to wav", unit="file"):
+        sph_name, _ = os.path.splitext(os.path.basename(sph_path))
+        wav_path = os.path.join(dest_root, 'full_audio_wav', sph_name + ".wav")
+        cmd = ["sph2pipe", "-f", "wav", "-p", sph_path, wav_path]
+        subprocess.run(cmd)
+
+
+def process_transcripts(dataset_root):
+    """
+    Reads in transcripts for each audio segment and processes them.
+    """
+    stm_path = os.path.join(
+        dataset_root,
+        "2000_hub5_eng_eval_tr",
+        "reference",
+        "hub5e00.english.000405.stm",
+    )
+    results = []
+    chars = set()
+
+    with open(stm_path, "r") as fh:
+        for line in fh:
+            # lines with ';;' are comments
+            if line.startswith(";;"):
+                continue
+
+            if "IGNORE_TIME_SEGMENT_" in line:
+                continue
+            line = line.replace("<B_ASIDE>", "").replace("<E_ASIDE>", "")
+            line = line.replace("(%HESITATION)", "UH")
+            line = line.replace("-", "")
+            line = line.replace("(%UH)", "UH")
+            line = line.replace("(%AH)", "UH")
+            line = line.replace("(", "").replace(")", "")
+
+            line = line.lower()
+
+            m = STM_LINE_FMT.search(line.strip())
+            utt = StmUtterance(*m.groups())
+
+            # Convert begin/end times to float
+            utt = utt._replace(begin=float(utt.begin))
+            utt = utt._replace(end=float(utt.end))
+
+            # Check for utterance in dict of transcript mistakes
+            transcript_update = TRANSCRIPT_BUGS.get(get_utt_id(utt))
+            if transcript_update is not None:
+                utt = utt._replace(transcript=transcript_update)
+
+            results.append(utt)
+            chars.update(list(utt.transcript))
+    return results, chars
+
+
+def write_one_segment(dest_root, speaker_id, count, audio, sr, duration, transcript):
+    """
+    Writes out one segment of audio, and writes its corresponding transcript
+    in the manifest.
+
+    Args:
+        dest_root: the path to the output directory root
+        speaker_id: ID of the speaker, used in file naming
+        count: number of segments from this speaker so far
+        audio: the segment's audio data
+        sr: sample rate of the audio
+        duration: duration of the audio
+        transcript: the corresponding transcript
+    """
+    audio_path = os.path.join(dest_root, "audio", f"{speaker_id}_{count:03}.wav")
+
+    manifest_path = os.path.join(dest_root, "manifest_hub5.json")
+
+    # Write audio
+    wavfile.write(audio_path, sr, audio)
+
+    # Write transcript
+    transcript = {
+        "audio_filepath": audio_path,
+        "duration": duration,
+        "text": transcript,
+    }
+    with open(manifest_path, 'a') as f:
+        json.dump(transcript, f)
+        f.write('\n')
+
+
+def segment_audio(info_list, dest_root, min_slice_duration):
+    """
+    Combines audio into >= min_slice_duration segments of the same speaker,
+    and writes the combined transcripts into a manifest.
+
+    Args:
+        info_list: list of StmUtterance objects with transcript information.
+        dest_root: path to output destination
+        min_slice_duration: min number of seconds per output audio slice
+    """
+    info_list = sorted(info_list, key=attrgetter('speaker_id', 'begin'))
+
+    prev_id = None  # For checking audio concatenation
+    id_count = 0
+
+    sample_rate, audio_data = None, None
+    transcript_buffer = ''
+    audio_buffer = []
+    buffer_duration = 0.0
+
+    # Iterate through utterances to build segments
+    for info in info_list:
+        if info.speaker_id != prev_id:
+            # Scrap the remainder in the buffers and start next segment
+            prev_id = info.speaker_id
+            id_count = 0
+
+            sample_rate, audio_data = wavfile.read(os.path.join(dest_root, 'full_audio_wav', info.filename + '.wav'))
+            transcript_buffer = ''
+            audio_buffer = []
+            buffer_duration = 0.0
+
+        # Append utterance info to buffers
+        transcript_buffer += info.transcript
+        channel = 0 if info.channel.lower() == 'a' else 1
+        audio_buffer.append(
+            audio_data[
+                floor(info.begin * sample_rate) : ceil(info.end * sample_rate),
+                channel,
+            ]
+        )
+        buffer_duration += info.end - info.begin
+
+        if buffer_duration < min_slice_duration:
+            transcript_buffer += ' '
+        else:
+            # Write out segment and transcript
+            id_count += 1
+            write_one_segment(
+                dest_root,
+                info.speaker_id,
+                id_count,
+                np.concatenate(audio_buffer, axis=0),
+                sample_rate,
+                buffer_duration,
+                transcript_buffer,
+            )
+
+            transcript_buffer = ''
+            audio_buffer = []
+            buffer_duration = 0.0
+
+
+def main():
+    data_root = args.data_root
+    dest_root = args.dest_root
+
+    min_slice_duration = args.min_slice_duration
+
+    if not os.path.exists(os.path.join(dest_root, 'full_audio_wav')):
+        os.makedirs(os.path.join(dest_root, 'full_audio_wav'))
+    if not os.path.exists(os.path.join(dest_root, 'audio')):
+        os.makedirs(os.path.join(dest_root, 'audio'))
+
+    # Create/wipe manifest contents
+    open(os.path.join(dest_root, "manifest_hub5.json"), 'w').close()
+
+    # Convert full audio files from .sph to .wav
+    create_wavs(data_root, dest_root)
+
+    # Get each audio transcript from transcript file
+    info_list, chars = process_transcripts(data_root)
+
+    print("Writing out vocab file", file=sys.stderr)
+    with open(os.path.join(dest_root, "vocab.txt"), 'w') as fh:
+        for x in sorted(list(chars)):
+            fh.write(x + "\n")
+
+    # Segment the audio data
+    print("Segmenting audio and writing manifest")
+    segment_audio(info_list, dest_root, min_slice_duration)
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/process_speech_commands_data.py

@@ -0,0 +1,521 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Usage:
+
+python process_speech_commands_data.py \
+    --data_root=<absolute path to where the data should be stored> \
+    --data_version=<either 1 or 2, indicating version of the dataset> \
+    --class_split=<either "all" or "sub", indicates whether all 30/35 classes should be used, or the 10+2 split should be used> \
+    --num_processes=<number of processes to use for data preprocessing> \
+    --rebalance \
+    --log
+"""
+
+import argparse
+import glob
+import json
+import logging
+import os
+import re
+import tarfile
+import urllib.request
+from collections import defaultdict
+from functools import partial
+from multiprocessing import Pool
+from typing import Dict, List, Set, Tuple
+
+import librosa
+import numpy as np
+import soundfile
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+URL_v1 = 'http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz'
+URL_v2 = 'http://download.tensorflow.org/data/speech_commands_v0.02.tar.gz'
+
+
+def __maybe_download_file(destination: str, source: str) -> str:
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+        Local filepath of the downloaded file
+    """
+    if not os.path.exists(destination):
+        logging.info(f'{destination} does not exist. Downloading ...')
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info(f'Downloaded {destination}.')
+    else:
+        logging.info(f'Destination {destination} exists. Skipping.')
+    return destination
+
+
+def __extract_all_files(filepath: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_dir)
+    else:
+        logging.info(f'Skipping extracting. Data already there {data_dir}')
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info('Not extracting. Maybe already there?')
+
+
+def __get_mp_chunksize(dataset_size: int, num_processes: int) -> int:
+    """
+    Returns the number of chunks to split the dataset into for multiprocessing.
+
+    Args:
+        dataset_size: size of the dataset
+        num_processes: number of processes to use for multiprocessing
+
+        Returns:
+            Number of chunks to split the dataset into for multiprocessing
+    """
+    chunksize = dataset_size // num_processes
+    return chunksize if chunksize > 0 else 1
+
+
+def __construct_filepaths(
+    all_files: List[str],
+    valset_uids: Set[str],
+    testset_uids: Set[str],
+    class_split: str,
+    class_subset: List[str],
+    pattern: str,
+) -> Tuple[Dict[str, int], Dict[str, List[tuple]], List[tuple], List[tuple], List[tuple], List[tuple], List[tuple]]:
+    """
+    Prepares the filepaths for the dataset.
+
+    Args:
+        all_files: list of all files in the dataset
+        valset_uids: set of uids of files in the validation set
+        testset_uids: set of uids of files in the test set
+        class_split: whether to use all classes as distinct labels, or to use
+            10 classes subset and rest of the classes as noise or background
+        class_subset: list of classes to consider if `class_split` is set to `sub`
+        pattern: regex pattern to match the file names in the dataset
+    """
+
+    label_count = defaultdict(int)
+    label_filepaths = defaultdict(list)
+    unknown_val_filepaths = []
+    unknown_test_filepaths = []
+
+    train, val, test = [], [], []
+    for entry in all_files:
+        r = re.match(pattern, entry)
+        if r:
+            label, uid = r.group(2), r.group(3)
+
+            if label == '_background_noise_' or label == 'silence':
+                continue
+
+        if class_split == 'sub' and label not in class_subset:
+            label = 'unknown'
+
+            if uid in valset_uids:
+                unknown_val_filepaths.append((label, entry))
+            elif uid in testset_uids:
+                unknown_test_filepaths.append((label, entry))
+
+        if uid not in valset_uids and uid not in testset_uids:
+            label_count[label] += 1
+            label_filepaths[label].append((label, entry))
+
+        if label == 'unknown':
+            continue
+
+        if uid in valset_uids:
+            val.append((label, entry))
+        elif uid in testset_uids:
+            test.append((label, entry))
+        else:
+            train.append((label, entry))
+
+    return {
+        'label_count': label_count,
+        'label_filepaths': label_filepaths,
+        'unknown_val_filepaths': unknown_val_filepaths,
+        'unknown_test_filepaths': unknown_test_filepaths,
+        'train': train,
+        'val': val,
+        'test': test,
+    }
+
+
+def __construct_silence_set(
+    rng: np.random.RandomState, sampling_rate: int, silence_stride: int, data_folder: str, background_noise: str
+) -> List[str]:
+    """
+    Creates silence files given a background noise.
+
+    Args:
+        rng: Random state for random number generator
+        sampling_rate: sampling rate of the audio
+        silence_stride: stride for creating silence files
+        data_folder: folder containing the silence directory
+        background_noise: filepath of the background noise
+
+    Returns:
+        List of filepaths of silence files
+    """
+    silence_files = []
+    if '.wav' in background_noise:
+        y, sr = librosa.load(background_noise, sr=sampling_rate)
+
+        for i in range(0, len(y) - sampling_rate, silence_stride):
+            file_path = f'silence/{os.path.basename(background_noise)[:-4]}_{i}.wav'
+            y_slice = y[i : i + sampling_rate] * rng.uniform(0.0, 1.0)
+            out_file_path = os.path.join(data_folder, file_path)
+            soundfile.write(out_file_path, y_slice, sr)
+
+            silence_files.append(('silence', out_file_path))
+
+    return silence_files
+
+
+def __rebalance_files(max_count: int, label_filepath: str) -> Tuple[str, List[str], int]:
+    """
+    Rebalance the number of samples for a class.
+
+    Args:
+        max_count: maximum number of samples for a class
+        label_filepath: list of filepaths for a class
+
+    Returns:
+        Rebalanced list of filepaths along with the label name and the number of samples
+    """
+    command, samples = label_filepath
+    filepaths = [sample[1] for sample in samples]
+
+    rng = np.random.RandomState(0)
+    filepaths = np.asarray(filepaths)
+    num_samples = len(filepaths)
+
+    if num_samples < max_count:
+        difference = max_count - num_samples
+        duplication_ids = rng.choice(num_samples, difference, replace=True)
+        filepaths = np.append(filepaths, filepaths[duplication_ids], axis=0)
+
+    return command, filepaths, num_samples
+
+
+def __prepare_metadata(skip_duration, sample: Tuple[str, str]) -> dict:
+    """
+    Creates the manifest entry for a file.
+
+    Args:
+        skip_duration: Whether to skip the computation of duration
+        sample: Tuple of label and filepath
+
+    Returns:
+        Manifest entry of the file
+    """
+    label, audio_path = sample
+    return json.dumps(
+        {
+            'audio_filepath': audio_path,
+            'duration': 0.0 if skip_duration else librosa.core.get_duration(filename=audio_path),
+            'command': label,
+        }
+    )
+
+
+def __process_data(
+    data_folder: str,
+    dst_folder: str,
+    num_processes: int = 1,
+    rebalance: bool = False,
+    class_split: str = 'all',
+    skip_duration: bool = False,
+):
+    """
+    Processes the data and generates the manifests.
+
+    Args:
+        data_folder: source with wav files and validation / test lists
+        dst_folder: where manifest files will be stored
+        num_processes: number of processes
+        rebalance: rebalance the classes to have same number of samples
+        class_split: whether to use all classes as distinct labels, or to use
+            10 classes subset and rest of the classes as noise or background
+        skip_duration: Bool whether to skip duration computation. Use this only for
+            colab notebooks where knowing duration is not necessary for demonstration
+    """
+
+    os.makedirs(dst_folder, exist_ok=True)
+
+    # Used for 10 classes + silence + unknown class setup - Only used when class_split is 'sub'
+    class_subset = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']
+
+    pattern = re.compile(r'(.+\/)?(\w+)\/([^_]+)_.+wav')
+    all_files = glob.glob(os.path.join(data_folder, '*/*wav'))
+
+    # Get files in the validation set
+    valset_uids = set()
+    with open(os.path.join(data_folder, 'validation_list.txt')) as fin:
+        for line in fin:
+            r = re.match(pattern, line)
+            if r:
+                valset_uids.add(r.group(3))
+
+    # Get files in the test set
+    testset_uids = set()
+    with open(os.path.join(data_folder, 'testing_list.txt')) as fin:
+        for line in fin:
+            r = re.match(pattern, line)
+            if r:
+                testset_uids.add(r.group(3))
+
+    logging.info('Validation and test set lists extracted')
+
+    filepath_info = __construct_filepaths(all_files, valset_uids, testset_uids, class_split, class_subset, pattern)
+    label_count = filepath_info['label_count']
+    label_filepaths = filepath_info['label_filepaths']
+    unknown_val_filepaths = filepath_info['unknown_val_filepaths']
+    unknown_test_filepaths = filepath_info['unknown_test_filepaths']
+    train = filepath_info['train']
+    val = filepath_info['val']
+    test = filepath_info['test']
+
+    logging.info('Prepared filepaths for dataset')
+
+    pool = Pool(num_processes)
+
+    # Add silence and unknown class label samples
+    if class_split == 'sub':
+        logging.info('Perforiming 10+2 class subsplit')
+
+        silence_path = os.path.join(data_folder, 'silence')
+        os.makedirs(silence_path, exist_ok=True)
+
+        silence_stride = 1000  # 0.0625 second stride
+        sampling_rate = 16000
+        folder = os.path.join(data_folder, '_background_noise_')
+
+        silence_files = []
+        rng = np.random.RandomState(0)
+
+        background_noise_files = [os.path.join(folder, x) for x in os.listdir(folder)]
+        silence_set_fn = partial(__construct_silence_set, rng, sampling_rate, silence_stride, data_folder)
+        for silence_flist in tqdm(
+            pool.imap(
+                silence_set_fn, background_noise_files, __get_mp_chunksize(len(background_noise_files), num_processes)
+            ),
+            total=len(background_noise_files),
+            desc='Constructing silence set',
+        ):
+            silence_files.extend(silence_flist)
+
+        rng = np.random.RandomState(0)
+        rng.shuffle(silence_files)
+        logging.info(f'Constructed silence set of {len(silence_files)}')
+
+        # Create the splits
+        rng = np.random.RandomState(0)
+        silence_split = 0.1
+        unknown_split = 0.1
+
+        # train split
+        num_total_samples = sum([label_count[cls] for cls in class_subset])
+        num_silence_samples = int(np.ceil(silence_split * num_total_samples))
+
+        # initialize sample
+        label_count['silence'] = 0
+        label_filepaths['silence'] = []
+
+        for silence_id in range(num_silence_samples):
+            label_count['silence'] += 1
+            label_filepaths['silence'].append(silence_files[silence_id])
+
+        train.extend(label_filepaths['silence'])
+
+        # Update train unknown set
+        unknown_train_samples = label_filepaths['unknown']
+
+        rng.shuffle(unknown_train_samples)
+        unknown_size = int(np.ceil(unknown_split * num_total_samples))
+
+        label_count['unknown'] = unknown_size
+        label_filepaths['unknown'] = unknown_train_samples[:unknown_size]
+
+        train.extend(label_filepaths['unknown'])
+
+        logging.info('Train set prepared')
+
+        # val set silence
+        num_val_samples = len(val)
+        num_silence_samples = int(np.ceil(silence_split * num_val_samples))
+
+        val_idx = label_count['silence'] + 1
+        for silence_id in range(num_silence_samples):
+            val.append(silence_files[val_idx + silence_id])
+
+        # Update val unknown set
+        rng.shuffle(unknown_val_filepaths)
+        unknown_size = int(np.ceil(unknown_split * num_val_samples))
+
+        val.extend(unknown_val_filepaths[:unknown_size])
+
+        logging.info('Validation set prepared')
+
+        # test set silence
+        num_test_samples = len(test)
+        num_silence_samples = int(np.ceil(silence_split * num_test_samples))
+
+        test_idx = val_idx + num_silence_samples + 1
+        for silence_id in range(num_silence_samples):
+            test.append(silence_files[test_idx + silence_id])
+
+        # Update test unknown set
+        rng.shuffle(unknown_test_filepaths)
+        unknown_size = int(np.ceil(unknown_split * num_test_samples))
+
+        test.extend(unknown_test_filepaths[:unknown_size])
+
+        logging.info('Test set prepared')
+
+    max_command = None
+    max_count = -1
+    for command, count in label_count.items():
+        if command == 'unknown':
+            continue
+
+        if count > max_count:
+            max_count = count
+            max_command = command
+
+    if rebalance:
+        logging.info(f'Command with maximum number of samples = {max_command} with {max_count} samples')
+        logging.info(f'Rebalancing dataset by duplicating classes with less than {max_count} samples...')
+
+        rebalance_fn = partial(__rebalance_files, max_count)
+        for command, filepaths, num_samples in tqdm(
+            pool.imap(rebalance_fn, label_filepaths.items(), __get_mp_chunksize(len(label_filepaths), num_processes)),
+            total=len(label_filepaths),
+            desc='Rebalancing dataset',
+        ):
+            if num_samples < max_count:
+                logging.info(f'Extended class label {command} from {num_samples} samples to {len(filepaths)} samples')
+                label_filepaths[command] = [(command, filepath) for filepath in filepaths]
+
+        del train
+        train = []
+        for label, samples in label_filepaths.items():
+            train.extend(samples)
+
+    manifests = [
+        ('train_manifest.json', train),
+        ('validation_manifest.json', val),
+        ('test_manifest.json', test),
+    ]
+
+    metadata_fn = partial(__prepare_metadata, skip_duration)
+    for manifest_filename, dataset in manifests:
+        num_files = len(dataset)
+
+        logging.info(f'Preparing manifest : {manifest_filename} with #{num_files} files')
+
+        manifest = [
+            metadata
+            for metadata in tqdm(
+                pool.imap(metadata_fn, dataset, __get_mp_chunksize(len(dataset), num_processes)),
+                total=num_files,
+                desc=f'Preparing {manifest_filename}',
+            )
+        ]
+
+        with open(os.path.join(dst_folder, manifest_filename), 'w') as fout:
+            for metadata in manifest:
+                fout.write(metadata + '\n')
+
+        logging.info(f'Finished construction of manifest. Path: {os.path.join(dst_folder, manifest_filename)}')
+
+    pool.close()
+
+    if skip_duration:
+        logging.info(
+            f'\n<<NOTE>> Duration computation was skipped for demonstration purposes on Colaboratory.\n'
+            f'In order to replicate paper results and properly perform data augmentation, \n'
+            f'please recompute the manifest file without the `--skip_duration` flag !\n'
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Google Speech Commands Data download and preprocessing')
+    parser.add_argument('--data_root', required=True, help='Root directory for storing data')
+    parser.add_argument(
+        '--data_version',
+        required=True,
+        default=1,
+        type=int,
+        choices=[1, 2],
+        help='Version of the speech commands dataset to download',
+    )
+    parser.add_argument(
+        '--class_split', default='all', choices=['all', 'sub'], help='Whether to consider all classes or only a subset'
+    )
+    parser.add_argument('--num_processes', default=1, type=int, help='Number of processes')
+    parser.add_argument('--rebalance', action='store_true', help='Rebalance the number of samples in each class')
+    parser.add_argument('--skip_duration', action='store_true', help='Skip computing duration of audio files')
+    parser.add_argument('--log', action='store_true', help='Generate logs')
+    args = parser.parse_args()
+
+    if args.log:
+        logging.basicConfig(level=logging.DEBUG)
+
+    data_root = args.data_root
+    data_set = f'google_speech_recognition_v{args.data_version}'
+    data_folder = os.path.join(data_root, data_set)
+
+    logging.info(f'Working on: {data_set}')
+
+    URL = URL_v1 if args.data_version == 1 else URL_v2
+
+    # Download and extract
+    if not os.path.exists(data_folder):
+        file_path = os.path.join(data_root, data_set + '.tar.bz2')
+        logging.info(f'Getting {data_set}')
+        __maybe_download_file(file_path, URL)
+        logging.info(f'Extracting {data_set}')
+        __extract_all_files(file_path, data_folder)
+
+    logging.info(f'Processing {data_set}')
+    __process_data(
+        data_folder,
+        data_folder,
+        num_processes=args.num_processes,
+        rebalance=args.rebalance,
+        class_split=args.class_split,
+        skip_duration=args.skip_duration,
+    )
+    logging.info('Done!')
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/process_vad_data.py

@@ -0,0 +1,503 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage:
+
+python process_vad_data.py \
+    --out_dir=<output path to where the generated manifest should be stored> \
+    --speech_data_root=<path where the speech data are stored> \
+    --background_data_root=<path where the background data are stored> \
+    --rebalance_method=<'under' or 'over' or 'fixed'> \ 
+    --log
+    (Optional --demo (for demonstration in tutorial). If you want to use your own background noise data, make sure to delete --demo)
+"""
+import argparse
+import glob
+import json
+import logging
+import os
+import tarfile
+import urllib.request
+
+import librosa
+import numpy as np
+import soundfile as sf
+from sklearn.model_selection import train_test_split
+
+from nemo.utils.tar_utils import safe_extract
+
+sr = 16000
+
+# google speech command v2
+URL = "http://download.tensorflow.org/data/speech_commands_v0.02.tar.gz"
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+
+    """
+    if not os.path.exists(destination):
+        logging.info(f"{destination} does not exist. Downloading ...")
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info(f"Downloaded {destination}.")
+    else:
+        logging.info(f"Destination {destination} exists. Skipping.")
+    return destination
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info('Not extracting. Maybe already there?')
+
+
+def __extract_all_files(filepath: str, data_root: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_dir)
+    else:
+        logging.info(f'Skipping extracting. Data already there {data_dir}')
+
+
+def split_train_val_test(data_dir, file_type, test_size=0.1, val_size=0.1, demo=False):
+    X = []
+    if file_type == "speech":
+        for o in os.listdir(data_dir):
+            if os.path.isdir(os.path.join(data_dir, o)) and o.split("/")[-1] != "_background_noise_":
+                X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav'))
+
+        if demo:
+            logging.info(
+                f"For Demonstration, we use {int(len(X)/100)}/{len(X)} speech data. Make sure to remove --demo flag when you actually train your model!"
+            )
+            X = np.random.choice(X, int(len(X) / 100), replace=False)
+
+    else:
+        for o in os.listdir(data_dir):
+            if os.path.isdir(os.path.join(data_dir, o)):
+                X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav'))
+            else:  # for using "_background_noise_" from google speech commands as background data
+                if o.endswith(".wav"):
+                    X.append(os.path.join(data_dir, o))
+
+    X_train, X_test = train_test_split(X, test_size=test_size, random_state=1)
+    val_size_tmp = val_size / (1 - test_size)
+    X_train, X_val = train_test_split(X_train, test_size=val_size_tmp, random_state=1)
+
+    with open(os.path.join(data_dir, file_type + "_training_list.txt"), "w") as outfile:
+        outfile.write("\n".join(X_train))
+    with open(os.path.join(data_dir, file_type + "_testing_list.txt"), "w") as outfile:
+        outfile.write("\n".join(X_test))
+    with open(os.path.join(data_dir, file_type + "_validation_list.txt"), "w") as outfile:
+        outfile.write("\n".join(X_val))
+
+    logging.info(f'Overall: {len(X)}, Train: {len(X_train)}, Validatoin: {len(X_val)}, Test: {len(X_test)}')
+    logging.info(f"Finish spliting train, val and test for {file_type}. Write to files!")
+
+
+def process_google_speech_train(data_dir):
+    X = []
+    for o in os.listdir(data_dir):
+        if os.path.isdir(os.path.join(data_dir, o)) and o.split("/")[-1] != "_background_noise_":
+            X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav'))
+
+    short_files = [i.split(data_dir)[1] for i in files]
+
+    with open(os.path.join(data_dir, 'testing_list.txt'), 'r') as allfile:
+        testing_list = allfile.read().splitlines()
+
+    with open(os.path.join(data_dir, 'validation_list.txt'), 'r') as allfile:
+        validation_list = allfile.read().splitlines()
+
+    exist_set = set(testing_list).copy()
+    exist_set.update(set(validation_list))
+
+    training_list = [i for i in short_files if i not in exist_set]
+
+    with open(os.path.join(data_dir, "training_list.txt"), "w") as outfile:
+        outfile.write("\n".join(training_list))
+
+    logging.info(
+        f'Overall: {len(files)}, Train: {len(training_list)}, Validatoin: {len(validation_list)}, Test: {len(testing_list)}'
+    )
+
+
+def write_manifest(
+    out_dir,
+    files,
+    prefix,
+    manifest_name,
+    start=0.0,
+    end=None,
+    duration_stride=1.0,
+    duration_max=None,
+    duration_limit=100.0,
+    filter_long=False,
+):
+    """
+    Given a list of files, segment each file and write them to manifest with restrictions.
+    Args:
+        out_dir: directory of generated manifest
+        files: list of files to be processed
+        prefix: label of samples
+        manifest_name: name of generated manifest
+        start: beginning of audio of generating segment
+        end: end of audio of generating segment
+        duration_stride: stride for segmenting audio samples
+        duration_max: duration for each segment
+        duration_limit: duration threshold for filtering out long audio samples
+        filter_long: boolean to determine whether to filter out long audio samples
+    Returns:
+    """
+    seg_num = 0
+    skip_num = 0
+    if duration_max is None:
+        duration_max = 1e9
+
+    if not os.path.exists(out_dir):
+        logging.info(f'Outdir {out_dir} does not exist. Creat directory.')
+        os.mkdir(out_dir)
+
+    output_path = os.path.join(out_dir, manifest_name + '.json')
+    with open(output_path, 'w') as fout:
+        for file in files:
+            label = prefix
+
+            try:
+                x, _sr = librosa.load(file, sr=sr)
+                duration = librosa.get_duration(y=x, sr=sr)
+
+            except Exception:
+                continue
+
+            if filter_long and duration > duration_limit:
+                skip_num += 1
+                continue
+
+            offsets = []
+            durations = []
+
+            if duration > duration_max:
+                current_offset = start
+
+                while current_offset < duration:
+                    if end is not None and current_offset > end:
+                        break
+
+                    difference = duration - current_offset
+
+                    if difference < duration_max:
+                        break
+
+                    offsets.append(current_offset)
+                    durations.append(duration_max)
+
+                    current_offset += duration_stride
+
+            else:
+                # Duration is not long enough! Skip
+                skip_num += 1
+
+            for duration, offset in zip(durations, offsets):
+                metadata = {
+                    'audio_filepath': file,
+                    'duration': duration,
+                    'label': label,
+                    'text': '_',  # for compatibility with ASRAudioText
+                    'offset': offset,
+                }
+                json.dump(metadata, fout)
+                fout.write('\n')
+                fout.flush()
+                seg_num += 1
+    return skip_num, seg_num, output_path
+
+
+def load_list_write_manifest(
+    data_dir,
+    out_dir,
+    filename,
+    prefix,
+    start,
+    end,
+    duration_stride=1.0,
+    duration_max=1.0,
+    duration_limit=100.0,
+    filter_long=True,
+):
+
+    filename = prefix + '_' + filename
+    file_path = os.path.join(data_dir, filename)
+
+    with open(file_path, 'r') as allfile:
+        files = allfile.read().splitlines()
+
+    manifest_name = filename.split('_list.txt')[0] + '_manifest'
+    skip_num, seg_num, output_path = write_manifest(
+        out_dir,
+        files,
+        prefix,
+        manifest_name,
+        start,
+        end,
+        duration_stride,
+        duration_max,
+        duration_limit,
+        filter_long=True,
+    )
+    return skip_num, seg_num, output_path
+
+
+def rebalance_json(data_dir, data_json, num, prefix):
+    data = []
+    seg = 0
+    with open(data_json, 'r') as f:
+        for line in f:
+            data.append(json.loads(line))
+
+    filename = data_json.split('/')[-1]
+    fout_path = os.path.join(data_dir, prefix + "_" + filename)
+
+    if len(data) >= num:
+        selected_sample = np.random.choice(data, num, replace=False)
+    else:
+        selected_sample = np.random.choice(data, num, replace=True)
+
+    with open(fout_path, 'a') as fout:
+        for i in selected_sample:
+            seg += 1
+            json.dump(i, fout)
+            fout.write('\n')
+            fout.flush()
+
+    logging.info(f'Get {seg}/{num} to  {fout_path} from {data_json}')
+    return fout_path
+
+
+def generate_variety_noise(data_dir, filename, prefix):
+
+    curr_dir = data_dir.split("_background_noise_")[0]
+    silence_path = os.path.join(curr_dir, "_background_noise_more")
+
+    if not os.path.exists(silence_path):
+        os.mkdir(silence_path)
+
+    silence_stride = 1000  # stride = 1/16 seconds
+    sampling_rate = 16000
+
+    silence_files = []
+    rng = np.random.RandomState(0)
+
+    filename = prefix + '_' + filename
+    file_path = os.path.join(data_dir, filename)
+
+    with open(file_path, 'r') as allfile:
+        files = allfile.read().splitlines()
+
+    for file in files:
+        y, sr = librosa.load(path=file, sr=sampling_rate)
+
+        for i in range(
+            0, len(y) - sampling_rate, silence_stride * 100
+        ):  # stride * 100 to generate less samples for demo
+            file_name = "{}_{}.wav".format(file.split("/")[-1], i)
+            y_slice = y[i : i + sampling_rate]
+            magnitude = rng.uniform(0.0, 1.0)
+            y_slice *= magnitude
+            out_file_path = os.path.join(silence_path, file_name)
+            sf.write(out_file_path, y_slice, sr)
+
+            silence_files.append(out_file_path)
+
+    new_list_file = os.path.join(silence_path, filename)
+    with open(new_list_file, "w") as outfile:
+        outfile.write("\n".join(silence_files))
+
+    logging.info(f"Generate {len(out_file_path)} background files for {file_path}. => {new_list_file} !")
+    return len(silence_files)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Speech and backgound data download and preprocess')
+    parser.add_argument("--out_dir", required=False, default='./manifest/', type=str)
+    parser.add_argument("--speech_data_root", required=True, default=None, type=str)
+    parser.add_argument("--background_data_root", required=True, default=None, type=str)
+    parser.add_argument('--test_size', required=False, default=0.1, type=float)
+    parser.add_argument('--val_size', required=False, default=0.1, type=float)
+    parser.add_argument('--window_length_in_sec', required=False, default=0.63, type=float)
+    parser.add_argument('--log', required=False, action='store_true')
+    parser.add_argument('--rebalance_method', required=False, default=None, type=str)
+    parser.add_argument('--demo', required=False, action='store_true')
+    parser.set_defaults(log=False, generate=False)
+    args = parser.parse_args()
+
+    if not args.rebalance_method:
+        rebalance = False
+    else:
+        if args.rebalance_method != 'over' and args.rebalance_method != 'under' and args.rebalance_method != 'fixed':
+            raise NameError("Please select a valid sampling method: over/under/fixed.")
+        else:
+            rebalance = True
+
+    if args.log:
+        logging.basicConfig(level=logging.DEBUG)
+
+    # Download speech data
+    speech_data_root = args.speech_data_root
+    data_set = "google_speech_recognition_v2"
+    speech_data_folder = os.path.join(speech_data_root, data_set)
+
+    background_data_folder = args.background_data_root
+    logging.info(f"Working on: {data_set}")
+
+    # Download and extract speech data
+    if not os.path.exists(speech_data_folder):
+        file_path = os.path.join(speech_data_root, data_set + ".tar.bz2")
+        logging.info(f"Getting {data_set}")
+        __maybe_download_file(file_path, URL)
+        logging.info(f"Extracting {data_set}")
+        __extract_all_files(file_path, speech_data_root, speech_data_folder)
+
+    logging.info(f"Split speech data!")
+    # dataset provide testing.txt and validation.txt feel free to split data using that with process_google_speech_train
+    split_train_val_test(speech_data_folder, "speech", args.test_size, args.val_size, args.demo)
+
+    logging.info(f"Split background data!")
+    split_train_val_test(background_data_folder, "background", args.test_size, args.val_size)
+
+    out_dir = args.out_dir
+
+    # Process Speech manifest
+    logging.info(f"=== Write speech data to manifest!")
+    skip_num_val, speech_seg_num_val, speech_val = load_list_write_manifest(
+        speech_data_folder,
+        out_dir,
+        'validation_list.txt',
+        'speech',
+        0.2,
+        0.8,
+        args.window_length_in_sec,
+        args.window_length_in_sec,
+    )
+    skip_num_test, speech_seg_num_test, speech_test = load_list_write_manifest(
+        speech_data_folder, out_dir, 'testing_list.txt', 'speech', 0.2, 0.8, 0.01, args.window_length_in_sec
+    )
+    skip_num_train, speech_seg_num_train, speech_train = load_list_write_manifest(
+        speech_data_folder,
+        out_dir,
+        'training_list.txt',
+        'speech',
+        0.2,
+        0.8,
+        args.window_length_in_sec,
+        args.window_length_in_sec,
+    )
+
+    logging.info(f'Val: Skip {skip_num_val} samples. Get {speech_seg_num_val} segments! => {speech_val} ')
+    logging.info(f'Test: Skip {skip_num_test} samples. Get {speech_seg_num_test} segments! => {speech_test}')
+    logging.info(f'Train: Skip {skip_num_train} samples. Get {speech_seg_num_train} segments!=> {speech_train}')
+
+    # Process background manifest
+    # if we select to generate more background noise data
+    if args.demo:
+        logging.info("Start generating more background noise data")
+        generate_variety_noise(background_data_folder, 'validation_list.txt', 'background')
+        generate_variety_noise(background_data_folder, 'training_list.txt', 'background')
+        generate_variety_noise(background_data_folder, 'testing_list.txt', 'background')
+        background_data_folder = os.path.join(
+            background_data_folder.split("_background_noise_")[0], "_background_noise_more"
+        )
+
+    logging.info(f"=== Write background data to manifest!")
+    skip_num_val, background_seg_num_val, background_val = load_list_write_manifest(
+        background_data_folder, out_dir, 'validation_list.txt', 'background', 0, None, 0.15, args.window_length_in_sec
+    )
+    skip_num_test, background_seg_num_test, background_test = load_list_write_manifest(
+        background_data_folder, out_dir, 'testing_list.txt', 'background', 0, None, 0.01, args.window_length_in_sec
+    )
+    skip_num_train, background_seg_num_train, background_train = load_list_write_manifest(
+        background_data_folder, out_dir, 'training_list.txt', 'background', 0, None, 0.15, args.window_length_in_sec
+    )
+
+    logging.info(f'Val: Skip {skip_num_val} samples. Get {background_seg_num_val} segments! => {background_val}')
+    logging.info(f'Test: Skip {skip_num_test} samples. Get {background_seg_num_test} segments! => {background_test}')
+    logging.info(
+        f'Train: Skip {skip_num_train} samples. Get {background_seg_num_train} segments! => {background_train}'
+    )
+    min_val, max_val = min(speech_seg_num_val, background_seg_num_val), max(speech_seg_num_val, background_seg_num_val)
+    min_test, max_test = (
+        min(speech_seg_num_test, background_seg_num_test),
+        max(speech_seg_num_test, background_seg_num_test),
+    )
+    min_train, max_train = (
+        min(speech_seg_num_train, background_seg_num_train),
+        max(speech_seg_num_train, background_seg_num_train),
+    )
+
+    logging.info('Finish generating manifest!')
+
+    if rebalance:
+        # Random Oversampling: Randomly duplicate examples in the minority class.
+        # Random Undersampling: Randomly delete examples in the majority class.
+        if args.rebalance_method == 'under':
+            logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.")
+            logging.info(f'Val: {min_val} Test: {min_test} Train: {min_train}!')
+
+            rebalance_json(out_dir, background_val, min_val, 'balanced')
+            rebalance_json(out_dir, background_test, min_test, 'balanced')
+            rebalance_json(out_dir, background_train, min_train, 'balanced')
+
+            rebalance_json(out_dir, speech_val, min_val, 'balanced')
+            rebalance_json(out_dir, speech_test, min_test, 'balanced')
+            rebalance_json(out_dir, speech_train, min_train, 'balanced')
+
+        if args.rebalance_method == 'over':
+            logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.")
+            logging.info(f'Val: {max_val} Test: {max_test} Train: {max_train}!')
+
+            rebalance_json(out_dir, background_val, max_val, 'balanced')
+            rebalance_json(out_dir, background_test, max_test, 'balanced')
+            rebalance_json(out_dir, background_train, max_train, 'balanced')
+
+            rebalance_json(out_dir, speech_val, max_val, 'balanced')
+            rebalance_json(out_dir, speech_test, max_test, 'balanced')
+            rebalance_json(out_dir, speech_train, max_train, 'balanced')
+
+        if args.rebalance_method == 'fixed':
+            fixed_test, fixed_val, fixed_train = 200, 100, 500
+            logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.")
+            logging.info(f'Val: {fixed_val} Test: {fixed_test} Train: {fixed_train}!')
+
+            rebalance_json(out_dir, background_val, fixed_val, 'balanced')
+            rebalance_json(out_dir, background_test, fixed_test, 'balanced')
+            rebalance_json(out_dir, background_train, fixed_train, 'balanced')
+
+            rebalance_json(out_dir, speech_val, fixed_val, 'balanced')
+            rebalance_json(out_dir, speech_test, fixed_test, 'balanced')
+            rebalance_json(out_dir, speech_train, fixed_train, 'balanced')
+    else:
+        logging.info("Don't rebalance number of samples in classes.")
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/speaker_tasks/README.md

@@ -0,0 +1,5 @@
+# Speaker Tasks Dataset Scripts
+
+In this folder are scripts to download speaker tasks (mainly for diarization) datasets. These scripts will return NeMo format manifest files to use with Diarization.
+
+We also have scripts for CallHome and DIHARD3, however the data has to be downloaded separately. If you require the scripts please leave an issue.

scripts/dataset_processing/speaker_tasks/get_aishell_diarization_data.py

@@ -0,0 +1,118 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# downloads the training/eval set for AISHELL Diarization.
+# the training dataset is around 170GiB, to skip pass the --skip_train flag.
+
+import argparse
+import glob
+import logging
+import os
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from nemo.collections.asr.parts.utils.manifest_utils import create_manifest
+from nemo.utils.tar_utils import safe_extract
+
+train_url = "https://www.openslr.org/resources/111/train_{}.tar.gz"
+train_datasets = ["S", "M", "L"]
+
+eval_url = "https://www.openslr.org/resources/111/test.tar.gz"
+
+
+def _load_sox_transformer():
+    try:
+        from sox import Transformer
+    except ImportError:
+        raise ImportError(
+            "Optional dependency 'sox' is required by this script. Install it with: pip install sox"
+        ) from None
+
+    return Transformer
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def __process_data(dataset_url: str, dataset_path: Path, manifest_output_path: Path):
+    os.makedirs(dataset_path, exist_ok=True)
+    tar_file_path = os.path.join(dataset_path, os.path.basename(dataset_url))
+    if not os.path.exists(tar_file_path):
+        urllib.request.urlretrieve(dataset_url, filename=tar_file_path)
+    extract_file(tar_file_path, str(dataset_path))
+    wav_path = dataset_path / 'converted_wav/'
+    extracted_dir = Path(tar_file_path).stem.replace('.tar', '')
+    flac_path = dataset_path / (extracted_dir + '/wav/')
+    __process_flac_audio(flac_path, wav_path)
+
+    audio_files = [os.path.join(os.path.abspath(wav_path), file) for file in os.listdir(str(wav_path))]
+    rttm_files = glob.glob(str(dataset_path / (extracted_dir + '/TextGrid/*.rttm')))
+    rttm_files = [os.path.abspath(file) for file in rttm_files]
+
+    audio_list = dataset_path / 'audio_files.txt'
+    rttm_list = dataset_path / 'rttm_files.txt'
+    with open(audio_list, 'w') as f:
+        f.write('\n'.join(audio_files))
+    with open(rttm_list, 'w') as f:
+        f.write('\n'.join(rttm_files))
+    create_manifest(
+        str(audio_list),
+        manifest_output_path,
+        rttm_path=str(rttm_list),
+    )
+
+
+def __process_flac_audio(flac_path, wav_path):
+    Transformer = _load_sox_transformer()
+    os.makedirs(wav_path, exist_ok=True)
+    flac_files = os.listdir(flac_path)
+    for flac_file in flac_files:
+        # Convert FLAC file to WAV
+        id = Path(flac_file).stem
+        wav_file = os.path.join(wav_path, id + ".wav")
+        if not os.path.exists(wav_file):
+            Transformer().build(os.path.join(flac_path, flac_file), wav_file)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Aishell Data download")
+    parser.add_argument("--data_root", default='./', type=str)
+    parser.add_argument("--output_manifest_path", default='aishell_diar_manifest.json', type=str)
+    parser.add_argument("--skip_train", help="skip downloading the training dataset", action="store_true")
+    args = parser.parse_args()
+    data_root = Path(args.data_root)
+    data_root.mkdir(exist_ok=True, parents=True)
+
+    if not args.skip_train:
+        for tag in train_datasets:
+            dataset_url = train_url.format(tag)
+            dataset_path = data_root / f'{tag}/'
+            manifest_output_path = data_root / f'train_{tag}_manifest.json'
+            __process_data(
+                dataset_url=dataset_url, dataset_path=dataset_path, manifest_output_path=manifest_output_path
+            )
+    # create test dataset
+    dataset_path = data_root / f'eval/'
+    manifest_output_path = data_root / f'eval_manifest.json'
+    __process_data(dataset_url=eval_url, dataset_path=dataset_path, manifest_output_path=manifest_output_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/speaker_tasks/get_ami_data.py

@@ -0,0 +1,102 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Download the AMI test dataset used to evaluate Speaker Diarization
+# More information here: https://groups.inf.ed.ac.uk/ami/corpus/
+# USAGE: python get_ami_data.py
+import argparse
+import os
+import subprocess
+
+from nemo.collections.asr.parts.utils.manifest_utils import create_manifest
+
+rttm_url = "https://raw.githubusercontent.com/BUTSpeechFIT/AMI-diarization-setup/main/only_words/rttms/{}/{}.rttm"
+uem_url = "https://raw.githubusercontent.com/BUTSpeechFIT/AMI-diarization-setup/main/uems/{}/{}.uem"
+list_url = "https://raw.githubusercontent.com/BUTSpeechFIT/AMI-diarization-setup/main/lists/{}.meetings.txt"
+
+
+audio_types = ['Mix-Headset', 'Array1-01']
+
+# these two IDs in the train set are missing download links for Array1-01.
+# We exclude them as a result.
+not_found_ids = ['IS1007d', 'IS1003b']
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download the AMI Corpus Dataset for Speaker Diarization")
+    parser.add_argument(
+        "--test_manifest_filepath",
+        help="path to output test manifest file",
+        type=str,
+        default='AMI_test_manifest.json',
+    )
+    parser.add_argument(
+        "--dev_manifest_filepath",
+        help="path to output dev manifest file",
+        type=str,
+        default='AMI_dev_manifest.json',
+    )
+    parser.add_argument(
+        "--train_manifest_filepath",
+        help="path to output train manifest file",
+        type=str,
+        default='AMI_train_manifest.json',
+    )
+    parser.add_argument("--data_root", help="path to output data directory", type=str, default="ami_dataset")
+    args = parser.parse_args()
+
+    data_path = os.path.abspath(args.data_root)
+    os.makedirs(data_path, exist_ok=True)
+
+    for manifest_path, split in (
+        (args.test_manifest_filepath, 'test'),
+        (args.dev_manifest_filepath, 'dev'),
+        (args.train_manifest_filepath, 'train'),
+    ):
+        split_path = os.path.join(data_path, split)
+        audio_path = os.path.join(split_path, "audio")
+        os.makedirs(split_path, exist_ok=True)
+        rttm_path = os.path.join(split_path, "rttm")
+        uem_path = os.path.join(split_path, "uem")
+
+        subprocess.run(["wget", "-P", split_path, list_url.format(split)])
+        with open(os.path.join(split_path, f"{split}.meetings.txt")) as f:
+            ids = f.read().strip().split('\n')
+        for id in [file_id for file_id in ids if file_id not in not_found_ids]:
+            for audio_type in audio_types:
+                audio_type_path = os.path.join(audio_path, audio_type)
+                os.makedirs(audio_type_path, exist_ok=True)
+                audio_download = (
+                    f"https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus/{id}/audio/" f"{id}.{audio_type}.wav"
+                )
+                subprocess.run(["wget", "-P", audio_type_path, audio_download])
+            rttm_download = rttm_url.format(split, id)
+            subprocess.run(["wget", "-P", rttm_path, rttm_download])
+            uem_download = uem_url.format(split, id)
+            subprocess.run(["wget", "-P", uem_path, uem_download])
+
+        rttm_files_path = os.path.join(split_path, 'rttm_files.txt')
+        with open(rttm_files_path, 'w') as f:
+            f.write('\n'.join(os.path.join(rttm_path, p) for p in os.listdir(rttm_path)))
+        uem_files_path = os.path.join(split_path, 'uem_files.txt')
+        with open(uem_files_path, 'w') as f:
+            f.write('\n'.join(os.path.join(uem_path, p) for p in os.listdir(uem_path)))
+        for audio_type in audio_types:
+            audio_type_path = os.path.join(audio_path, audio_type)
+            audio_files_path = os.path.join(split_path, f'audio_files_{audio_type}.txt')
+            with open(audio_files_path, 'w') as f:
+                f.write('\n'.join(os.path.join(audio_type_path, p) for p in os.listdir(audio_type_path)))
+            audio_type_manifest_path = manifest_path.replace('.json', f'.{audio_type}.json')
+            create_manifest(
+                audio_files_path, audio_type_manifest_path, rttm_path=rttm_files_path, uem_path=uem_files_path
+            )

scripts/dataset_processing/speaker_tasks/get_hi-mia_data.py

@@ -0,0 +1,208 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python get_hi-mia_data.py --data_root=<where to put data>
+
+import argparse
+import json
+import logging as _logging
+import os
+import tarfile
+import urllib.request
+from glob import glob
+
+import librosa as l
+from sklearn.model_selection import StratifiedShuffleSplit
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+parser = argparse.ArgumentParser(description="HI-MIA Data download")
+parser.add_argument("--data_root", required=True, default=None, type=str)
+parser.add_argument("--log_level", default=20, type=int)
+args = parser.parse_args()
+logging = _logging.getLogger(__name__)
+logging.addHandler(_logging.StreamHandler())
+logging.setLevel(args.log_level)
+
+URL = {
+    "dev": "http://www.openslr.org/resources/85/dev.tar.gz",
+    "test": "http://www.openslr.org/resources/85/test.tar.gz",
+    "train": "http://www.openslr.org/resources/85/train.tar.gz",
+}
+
+
+def __retrieve_with_progress(source: str, filename: str):
+    """
+    Downloads source to destination
+    Displays progress bar
+    Args:
+        source: url of resource
+        destination: local filepath
+    Returns:
+    """
+    with open(filename, "wb") as f:
+        response = urllib.request.urlopen(source)
+        total = response.length
+
+        if total is None:
+            f.write(response.content)
+        else:
+            with tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024) as pbar:
+                for data in response:
+                    f.write(data)
+                    pbar.update(len(data))
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+
+    """
+    source = URL[source]
+    if not os.path.exists(destination) and not os.path.exists(os.path.splitext(destination)[0]):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        __retrieve_with_progress(source, filename=destination + ".tmp")
+        os.rename(destination + ".tmp", destination)
+        logging.info("Downloaded {0}.".format(destination))
+    elif os.path.exists(destination):
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    elif os.path.exists(os.path.splitext(destination)[0]):
+        logging.warning(
+            "Assuming extracted folder %s contains the extracted files from %s. Will not download.",
+            os.path.basename(destination),
+            destination,
+        )
+    return destination
+
+
+def __extract_all_files(filepath: str, data_root: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_root)
+        audio_dir = os.path.join(data_dir, "wav")
+        for subfolder, _, filelist in os.walk(audio_dir):
+            for ftar in filelist:
+                extract_file(os.path.join(subfolder, ftar), subfolder)
+    else:
+        logging.info("Skipping extracting. Data already there %s" % data_dir)
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath, encoding='utf-8') as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def __remove_tarred_files(filepath: str, data_dir: str):
+    if os.path.exists(data_dir) and os.path.isfile(filepath):
+        logging.info("Deleting %s" % filepath)
+        os.remove(filepath)
+
+
+def write_file(name, lines, idx):
+    with open(name, "w") as fout:
+        for i in idx:
+            dic = lines[i]
+            json.dump(dic, fout)
+            fout.write("\n")
+    logging.info("wrote %s", name)
+
+
+def __process_data(data_folder: str, data_set: str):
+    """
+    To generate manifest
+    Args:
+        data_folder: source with wav files
+    Returns:
+
+    """
+    fullpath = os.path.abspath(data_folder)
+    filelist = glob(fullpath + "/**/*.wav", recursive=True)
+    out = os.path.join(fullpath, data_set + "_all.json")
+    utt2spk = os.path.join(fullpath, "utt2spk")
+    utt2spk_file = open(utt2spk, "w")
+    id = -2  # speaker id
+
+    if os.path.exists(out):
+        logging.warning(
+            "%s already exists and is assumed to be processed. If not, please delete %s and rerun this script",
+            out,
+            out,
+        )
+        return
+
+    speakers = []
+    lines = []
+    with open(out, "w") as outfile:
+        for line in tqdm(filelist):
+            line = line.strip()
+            y, sr = l.load(line, sr=None)
+            if sr != 16000:
+                y, sr = l.load(line, sr=16000)
+                l.output.write_wav(line, y, sr)
+            dur = l.get_duration(y=y, sr=sr)
+            if data_set == "test":
+                speaker = line.split("/")[-1].split(".")[0].split("_")[0]
+            else:
+                speaker = line.split("/")[id]
+            speaker = list(speaker)
+            speaker = "".join(speaker)
+            speakers.append(speaker)
+            meta = {"audio_filepath": line, "duration": float(dur), "label": speaker}
+            lines.append(meta)
+            json.dump(meta, outfile)
+            outfile.write("\n")
+            utt2spk_file.write(line.split("/")[-1] + "\t" + speaker + "\n")
+
+    utt2spk_file.close()
+
+    if data_set != "test":
+        sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=42)
+        for train_idx, test_idx in sss.split(speakers, speakers):
+            print(len(train_idx))
+
+        out = os.path.join(fullpath, "train.json")
+        write_file(out, lines, train_idx)
+        out = os.path.join(fullpath, "dev.json")
+        write_file(out, lines, test_idx)
+
+
+def main():
+    data_root = args.data_root
+    for data_set in URL.keys():
+
+        # data_set = 'data_aishell'
+        logging.info("\n\nWorking on: {0}".format(data_set))
+        file_path = os.path.join(data_root, data_set + ".tgz")
+        logging.info("Getting {0}".format(data_set))
+        __maybe_download_file(file_path, data_set)
+        logging.info("Extracting {0}".format(data_set))
+        data_folder = os.path.join(data_root, data_set)
+        __extract_all_files(file_path, data_root, data_folder)
+        __remove_tarred_files(file_path, data_folder)
+        logging.info("Processing {0}".format(data_set))
+        __process_data(data_folder, data_set)
+        logging.info("Done!")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/speaker_tasks/get_voxconverse.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# downloads the training/eval set for VoxConverse.
+
+import argparse
+import logging
+import os
+import urllib.request
+import zipfile
+from pathlib import Path
+
+from nemo.collections.asr.parts.utils.manifest_utils import create_manifest
+
+dev_url = "https://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_dev_wav.zip"
+test_url = "https://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_test_wav.zip"
+rttm_annotations_url = "https://github.com/joonson/voxconverse/archive/refs/heads/master.zip"
+
+
+def extract_file(filepath: Path, data_dir: Path):
+    try:
+        with zipfile.ZipFile(str(filepath), 'r') as zip_ref:
+            zip_ref.extractall(str(data_dir))
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def download_file(url: str, destination: Path) -> Path:
+    urllib.request.urlretrieve(url, filename=str(destination))
+    return destination
+
+
+def _generate_manifest(data_root: Path, audio_path: Path, rttm_path: Path, manifest_output_path: Path):
+    audio_list = str(data_root / 'audio_file.txt')
+    rttm_list = str(data_root / 'rttm_file.txt')
+    with open(audio_list, 'w') as f:
+        f.write('\n'.join([str(os.path.join(rttm_path, x)) for x in os.listdir(audio_path)]))
+    with open(rttm_list, 'w') as f:
+        f.write('\n'.join([str(os.path.join(rttm_path, x)) for x in os.listdir(rttm_path)]))
+    create_manifest(
+        audio_list,
+        str(manifest_output_path),
+        rttm_path=rttm_list,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description="VoxConverse Data download")
+    parser.add_argument("--data_root", default='./', type=str)
+    args = parser.parse_args()
+    data_root = Path(args.data_root)
+    data_root.mkdir(exist_ok=True, parents=True)
+
+    test_path = data_root / os.path.basename(test_url)
+    dev_path = data_root / os.path.basename(dev_url)
+    rttm_path = data_root / os.path.basename(rttm_annotations_url)
+
+    if not os.path.exists(test_path):
+        test_path = download_file(test_url, test_path)
+    if not os.path.exists(dev_path):
+        dev_path = download_file(dev_url, dev_path)
+    if not os.path.exists(rttm_path):
+        rttm_path = download_file(rttm_annotations_url, rttm_path)
+
+    extract_file(test_path, data_root / 'test/')
+    extract_file(dev_path, data_root / 'dev/')
+    extract_file(rttm_path, data_root)
+
+    _generate_manifest(
+        data_root=data_root,
+        audio_path=os.path.abspath(data_root / 'test/voxconverse_test_wav/'),
+        rttm_path=os.path.abspath(data_root / 'voxconverse-master/test/'),
+        manifest_output_path=data_root / 'test_manifest.json',
+    )
+    _generate_manifest(
+        data_root=data_root,
+        audio_path=os.path.abspath(data_root / 'dev/audio/'),
+        rttm_path=os.path.abspath(data_root / 'voxconverse-master/dev/'),
+        manifest_output_path=data_root / 'dev_manifest.json',
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/spoken_wikipedia/preprocess.py

@@ -0,0 +1,204 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+This script can be used to preprocess Spoken Wikipedia corpus before running ctc-segmentation.
+The input folder consists of subfolders with following stricture
+  ├── <Name of Wikipedia article>
+  │   ├── aligned.swc
+  │   ├── audiometa.txt
+  │   ├── audio.ogg
+  │   ├── info.json
+  │   ├── wiki.html
+  │   ├── wiki.txt
+  │   └── wiki.xml
+
+
+## The destination folder will contain look enumerated .ogg and .txt files like this:
+  ├── audio
+  |   ├── 1.ogg
+  |   ├── 2.ogg
+  |   ...
+  └── text
+      ├── 1.txt     
+      ├── 2.txt     
+      ...
+"""
+
+import argparse
+import os
+import re
+import shutil
+import subprocess
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--input_folder", required=True, type=str, help="Input folder in which each subfolder contains an article"
+)
+parser.add_argument(
+    "--destination_folder", required=True, type=str, help="Destination folder with audio and text subfolder"
+)
+args = parser.parse_args()
+
+
+def replace_diacritics(text):
+    text = re.sub(r"[éèëēêęěė]", "e", text)
+    text = re.sub(r"[ãâāáäăâàąåạả]", "a", text)
+    text = re.sub(r"[úūüùưûů]", "u", text)
+    text = re.sub(r"[ôōóöõòő]", "o", text)
+    text = re.sub(r"[ćçč]", "c", text)
+    text = re.sub(r"[ïīíîıì]", "i", text)
+    text = re.sub(r"[ñńňņ]", "n", text)
+    text = re.sub(r"[țť]", "t", text)
+    text = re.sub(r"[łľ]", "l", text)
+    text = re.sub(r"[żžź]", "z", text)
+    text = re.sub(r"[ğ]", "g", text)
+    text = re.sub(r"[ř]", "r", text)
+    text = re.sub(r"[ý]", "y", text)
+    text = re.sub(r"[æ]", "ae", text)
+    text = re.sub(r"[œ]", "oe", text)
+    text = re.sub(r"[șşšś]", "s", text)
+    return text
+
+
+def get_audio(name, n):
+    """
+    Copies .ogg file. If there are several .ogg files, concatenates them.
+
+    Args:
+        name - name of folder within Spoken Wikipedia
+        n - integer that will serve as output file name, e.g. if n=1, file 1.ogg will be created
+    """
+    audio_path = os.path.join(args.input_folder, name, "audio.ogg")
+    if not os.path.exists(audio_path):
+        ##  Some folders have multiple .ogg files, so we need to first combine them into one file. Example:
+        ##  |── Universe
+        ##  │   ├── aligned.swc
+        ##  │   ├── audio1.ogg
+        ##  │   ├── audio2.ogg
+        ##  │   ├── audio3.ogg
+        ##  │   ├── audio4.ogg
+        ##  │   ├── audiometa.txt
+        ##  │   ├── info.json
+        ##  │   ├── wiki.html
+        ##  │   ├── wiki.txt
+        ##  │   └── wiki.xml
+
+        multiple_ogg_files = []
+        for i in range(1, 5):
+            path = os.path.join(args.input_folder, name, "audio" + str(i) + ".ogg")
+            if os.path.exists(path):
+                multiple_ogg_files.append(path)
+            else:
+                break
+        if len(multiple_ogg_files) == 0:
+            return
+        elif len(multiple_ogg_files) == 1:
+            shutil.copy(multiple_ogg_files[0], audio_path)
+        else:
+            tmp_file_name = "ffmeg_inputs.txt"
+            print("tmp_file_name=", tmp_file_name)
+            with open(tmp_file_name, "w", encoding="utf-8") as tmp_file:
+                for path in multiple_ogg_files:
+                    tmp_file.write("file '" + path + "'\n")
+            ffmpeg_cmd = ["ffmpeg", "-f", "concat", "-i", tmp_file_name, "-c", "copy", audio_path]
+            print("ffmpeg command:", ffmpeg_cmd)
+            subprocess.run(ffmpeg_cmd, check=True)
+
+    output_audio_path = args.destination_folder + "/audio/" + str(n) + ".ogg"
+    shutil.copy(audio_path, output_audio_path)
+
+
+def get_text(name, n):
+    """
+    Cleans wiki.txt.
+
+    Args:
+        name - name of folder within Spoken Wikipedia
+        n - integer that will serve as output file name, e.g. if n=1, file 1.txt will be created
+    """
+
+    # Then we need to clean the text
+    out_text = open(args.destination_folder + "/text/" + str(n) + ".txt", "w", encoding="utf-8")
+    with open(args.input_folder + "/" + name + "/wiki.txt", "r", encoding="utf-8") as f:
+        for line in f:
+            do_break = False
+            line2 = line.strip()
+            ref_parts = line2.split("<ref")
+            for idx, s in enumerate(ref_parts):
+                if idx != 0:
+                    s = "<ref" + s
+                if s.startswith("[[Image") and s.endswith("]]"):
+                    continue
+                if s.startswith("[[File") and s.endswith("]]"):
+                    continue
+                if s.startswith(":"):
+                    continue
+                if s.startswith("{|") or s.startswith("|}") or s.startswith("|") or s.startswith("!"):
+                    continue
+                if s.startswith("{{") and (s.endswith("}}") or "}}" not in s):
+                    continue
+                if s.startswith("{{pp-move"):
+                    continue
+                s = re.sub(r"\[\[Image\:[^\]]+\]\]", r"", s)
+                s = re.sub(r"\[\[File\:[^\]]+\]\]", r"", s)
+                s = re.sub(r"\[http[^\]]+\]", r"", s)
+                s = re.sub(r"<math>[^<>]+</math>", r"", s)
+                s = re.sub(r"<!\-\-.+\-\->", r"", s)  # <!--DashBot-->    can be inside <ref>
+                s = re.sub(r"<ref>.+</ref>", r"", s)
+                s = re.sub(r"<ref .+</ref>", r"", s)
+                s = re.sub(r"<ref[^<>]+/>", r"", s)
+                s = re.sub(r"<[^ <>]+>", r"", s)  # <sub>, <sup>, </u>
+                if (
+                    re.match(r"== *Notes *==", s)
+                    or re.match(r"== *References *==", s)
+                    or re.match(r"== *External links *==", s)
+                    or re.match(r"== *See also *==", s)
+                ):
+                    do_break = True
+                    break
+                s = re.sub(r"{{convert\|(\d+)\|(\w+)\|[^}]+}}", r"\g<1> \g<2>", s)  # {{convert|7600|lb|kg}}
+                s = re.sub(r"{{cquote\|", r"", s)
+                s = re.sub(r"{{[^{}]+}}", r"", s)
+                s = s.replace("{{", "").replace("}}", "")
+                s = re.sub(r"(lang[^()]+)", r"", s)  # (lang-bn...)
+                s = re.sub(r"==+", r"", s)
+                s = re.sub(r"''+", r" ", s)  # remove multiple quotes
+                s = re.sub(r" '", r" ", s)  # remove quote at the beginning
+                s = re.sub(r"' ", r" ", s)  # remove quote at the end
+                s = re.sub(r"[…\*]", r" ", s)
+                s = re.sub(r"\\u....", r" ", s)  # remove unicode
+                s = re.sub(r"&[^ ;&]+;", r"", s)  # &nbsp; &mdash;
+
+                s = replace_diacritics(s)
+
+                s = re.sub(r"\[\[[^\]]+\|([^\]]+)\]\]", r"\g<1>", s)  # if several variants, take the last one
+                s = re.sub(r"\[\[([^\]]+)\]\]", r"\g<1>", s)
+
+                out_text.write(s + "\n")
+            if do_break:
+                break
+    out_text.close()
+
+
+if __name__ == "__main__":
+    n = 0
+    for name in os.listdir(args.input_folder):
+        n += 1
+        if not os.path.exists(args.input_folder + "/" + name + "/wiki.txt"):
+            print("wiki.txt does not exist in " + name)
+            continue
+        get_audio(name, n)
+        get_text(name, n)

scripts/dataset_processing/spoken_wikipedia/run.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Download the Spoken Wikipedia corpus for English
+## Note, that there are some other languages available
+## @InProceedings{KHN16.518,
+##  author = {Arne K{\"o}hn and Florian Stegen and Timo Baumann},
+##  title = {Mining the Spoken Wikipedia for Speech Data and Beyond},
+##  booktitle = {Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC 2016)},
+##  year = {2016},
+##  month = {may},
+##  date = {23-28},
+##  location = {Portorož, Slovenia},
+##  editor = {Nicoletta Calzolari (Conference Chair) and Khalid Choukri and Thierry Declerck and Marko Grobelnik and Bente Maegaard and Joseph Mariani and Asuncion Moreno and Jan Odijk and Stelios Piperidis},
+##  publisher = {European Language Resources Association (ELRA)},
+##  address = {Paris, France},
+##  isbn = {978-2-9517408-9-1},
+##  islrn = {684-927-624-257-3/},
+##  language = {english}
+## }
+
+wget https://corpora.uni-hamburg.de/hzsk/de/islandora/object/file:swc-2.0_en-with-audio/datastream/TAR/en-with-audio.tar .
+tar -xvf en-with-audio.tar
+
+##  We get a folder English with 1339 subfolders, each subfolder corresponds to a Wikipedia article. Example:
+##  ├── Universal_suffrage
+##  │   ├── aligned.swc
+##  │   ├── audiometa.txt
+##  │   ├── audio.ogg
+##  │   ├── info.json
+##  │   ├── wiki.html
+##  │   ├── wiki.txt
+##  │   └── wiki.xml
+
+##  We will use two files: audio.ogg and wiki.txt
+
+## Some folders have multiple .ogg files, this will be handled during preprocess.py. Example:
+##  |── Universe
+##  │   ├── aligned.swc
+##  │   ├── audio1.ogg
+##  │   ├── audio2.ogg
+##  │   ├── audio3.ogg
+##  │   ├── audio4.ogg
+##  │   ├── audiometa.txt
+##  │   ├── info.json
+##  │   ├── wiki.html
+##  │   ├── wiki.txt
+##  │   └── wiki.xml
+
+## Some rare folders are incomplete, these will be skipped during preprocessing.
+
+## Rename some folders with special symbols because they cause problems to ffmpeg when concatening multiple .ogg files
+mv "english/The_Hitchhiker%27s_Guide_to_the_Galaxy" "english/The_Hitchhikers_guide_to_the_Galaxy"
+mv "english/SummerSlam_(2003)" "english/SummerSlam_2003"
+mv "english/Over_the_Edge_(1999)" "english/Over_the_Edge_1999"
+mv "english/Lost_(TV_series)" "english/Lost_TV_series"
+mv "english/S._A._Andr%c3%a9e%27s_Arctic_Balloon_Expedition_of_1897" "english/S_A_Andres_Arctic_Balloon_Expedition_of_1897"
+
+## path to NeMo repository, e.g. /home/user/NeMo
+NEMO_PATH=
+
+INPUT_DIR="english"
+OUTPUT_DIR=${INPUT_DIR}_result
+
+rm -rf $OUTPUT_DIR
+rm -rf ${INPUT_DIR}_prepared
+mkdir ${INPUT_DIR}_prepared
+mkdir ${INPUT_DIR}_prepared/audio
+mkdir ${INPUT_DIR}_prepared/text
+python ${NEMO_PATH}/scripts/dataset_processing/spoken_wikipedia/preprocess.py --input_folder ${INPUT_DIR} --destination_folder ${INPUT_DIR}_prepared
+
+## Now we have ${INPUT_DIR}_prepared folder with the following structure:
+##  ├── audio
+##  |   ├── 1.ogg
+##  |   ├── 2.ogg
+##  |   ...
+##  └── text
+##      ├── 1.txt     
+##      ├── 2.txt     
+##      ...
+
+MODEL_FOR_SEGMENTATION="stt_en_fastconformer_ctc_large"
+MODEL_FOR_RECOGNITION="stt_en_conformer_ctc_large"
+## We set this threshold as very permissive, later we will use other metrics for filtering
+THRESHOLD=-10
+
+${NEMO_PATH}/tools/ctc_segmentation/run_segmentation.sh \
+--SCRIPTS_DIR=${NEMO_PATH}/tools/ctc_segmentation/scripts \
+--MODEL_NAME_OR_PATH=${MODEL_FOR_SEGMENTATION} \
+--DATA_DIR=${INPUT_DIR}_prepared \
+--OUTPUT_DIR=${OUTPUT_DIR} \
+--MIN_SCORE=${THRESHOLD}
+
+# Thresholds for filtering
+CER_THRESHOLD=20
+WER_THRESHOLD=30
+CER_EDGE_THRESHOLD=30
+LEN_DIFF_RATIO_THRESHOLD=0.15
+EDGE_LEN=25
+BATCH_SIZE=1
+
+${NEMO_PATH}/tools/ctc_segmentation/run_filter.sh \
+--SCRIPTS_DIR=${NEMO_PATH}/tools/ctc_segmentation/scripts \
+--MODEL_NAME_OR_PATH=${MODEL_FOR_RECOGNITION} \
+--BATCH_SIZE=${BATCH_SIZE} \
+--MANIFEST=$OUTPUT_DIR/manifests/manifest.json \
+--INPUT_AUDIO_DIR=${INPUT_DIR}_prepared/audio/ \
+--EDGE_LEN=${EDGE_LEN} \
+--CER_THRESHOLD=${CER_THRESHOLD} \
+--WER_THRESHOLD=${WER_THRESHOLD} \
+--CER_EDGE_THRESHOLD=${CER_EDGE_THRESHOLD} \
+--LEN_DIFF_RATIO_THRESHOLD=${LEN_DIFF_RATIO_THRESHOLD}
+
+python ${NEMO_PATH}/examples/asr/speech_to_text_eval.py \
+  dataset_manifest=${OUTPUT_DIR}/manifests/manifest_transcribed_metrics_filtered.json \
+  use_cer=True \
+  only_score_manifest=True
+
+python ${NEMO_PATH}/examples/asr/speech_to_text_eval.py \
+  dataset_manifest=${OUTPUT_DIR}/manifests/manifest_transcribed_metrics_filtered.json \
+  use_cer=False \
+  only_score_manifest=True

scripts/dataset_processing/tts/aishell3/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch", "speaker_id"]
+phoneme_dict_path: "scripts/tts_dataset_files/zh/24finals/pinyin_dict_nv_22.10.txt"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: true
+  trim_top_db: 50
+  trim_frame_length: 1024
+  trim_hop_length: 256
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: zh
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.ChinesePhonemesTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.zh_cn_pinyin.ChineseG2p
+      phoneme_dict: ${phoneme_dict_path}
+      word_segmenter: jieba # Only jieba is supported now.

scripts/dataset_processing/tts/aishell3/get_data.py

@@ -0,0 +1,176 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Disclaimer:
+#   Each user is responsible for checking the content of datasets and the applicable licenses and determining if suitable for the intended use.
+
+import argparse
+import json
+import os
+import random
+import subprocess
+import tarfile
+import urllib.request
+from pathlib import Path
+
+import numpy as np
+from nemo_text_processing.text_normalization.normalize import Normalizer
+from opencc import OpenCC
+
+from nemo.utils.tar_utils import safe_extract
+
+URL = "https://www.openslr.org/resources/93/data_aishell3.tgz"
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description='Prepare SF_bilingual dataset and create manifests with predefined split'
+    )
+
+    parser.add_argument(
+        "--data-root",
+        type=Path,
+        help="where the dataset will reside",
+        default="./DataChinese/sf_bilingual_speech_zh_en_vv1/SF_bilingual/",
+    )
+    parser.add_argument(
+        "--manifests-path", type=Path, help="where the resulting manifests files will reside", default="./"
+    )
+    parser.add_argument("--val-size", default=0.01, type=float, help="eval set split")
+    parser.add_argument("--test-size", default=0.01, type=float, help="test set split")
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_transcript(file_path: str):
+    # Create directory for processed wav files
+    Path(file_path / "processed").mkdir(parents=True, exist_ok=True)
+    # Create zh-TW to zh-simplify converter
+    cc = OpenCC('t2s')
+    # Create normalizer
+    text_normalizer = Normalizer(
+        lang="zh",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(file_path / "cache_dir"),
+    )
+    text_normalizer_call_kwargs = {"punct_pre_process": True, "punct_post_process": True}
+    normalizer_call = lambda x: text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+    entries = []
+    SPEAKER_LEN = 7
+
+    candidates = []
+    speakers = set()
+    with open(file_path / "train" / "content.txt", encoding="utf-8") as fin:
+        for line in fin:
+            content = line.split()
+            wav_name, text = content[0], "".join(content[1::2]) + "。"
+            wav_name = wav_name.replace(u'\ufeff', '')
+            speaker = wav_name[:SPEAKER_LEN]
+            speakers.add(speaker)
+            wav_file = file_path / "train" / "wav" / speaker / wav_name
+            assert os.path.exists(wav_file), f"{wav_file} not found!"
+            duration = subprocess.check_output(["soxi", "-D", str(wav_file)])
+            if float(duration) <= 3.0:  # filter out wav files shorter than 3 seconds
+                continue
+            processed_file = file_path / "processed" / wav_name
+            # convert wav to mono 22050HZ, 16 bit (as SFSpeech dataset)
+            subprocess.run(["sox", str(wav_file), "-r", "22050", "-c", "1", "-b", "16", str(processed_file)])
+            candidates.append((processed_file, duration, text, speaker))
+
+    # remapping the speakder to speaker_id (start from 1)
+    remapping = {}
+    for index, speaker in enumerate(sorted(speakers)):
+        remapping[speaker] = index + 1
+
+    for processed_file, duration, text, speaker in candidates:
+        simplified_text = cc.convert(text)
+        normalized_text = normalizer_call(simplified_text)
+        entry = {
+            'audio_filepath': os.path.abspath(processed_file),
+            'duration': float(duration),
+            'text': text,
+            'normalized_text': normalized_text,
+            'speaker_raw': speaker,
+            'speaker': remapping[speaker],
+        }
+
+        entries.append(entry)
+
+    return entries
+
+
+def __process_data(dataset_path, val_size, test_size, seed_for_ds_split, manifests_dir):
+    entries = __process_transcript(dataset_path)
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+
+    train_size = 1.0 - val_size - test_size
+    train_entries, validate_entries, test_entries = np.split(
+        entries, [int(len(entries) * train_size), int(len(entries) * (train_size + val_size))]
+    )
+
+    assert len(train_entries) > 0, "Not enough data for train, val and test"
+
+    def save(p, data):
+        with open(p, 'w') as f:
+            for d in data:
+                f.write(json.dumps(d) + '\n')
+
+    save(manifests_dir / "train_manifest.json", train_entries)
+    save(manifests_dir / "val_manifest.json", validate_entries)
+    save(manifests_dir / "test_manifest.json", test_entries)
+
+
+def main():
+    args = get_args()
+
+    tarred_data_path = args.data_root / "data_aishell3.tgz"
+
+    __maybe_download_file(URL, tarred_data_path)
+    __extract_file(str(tarred_data_path), str(args.data_root))
+
+    __process_data(
+        args.data_root,
+        args.val_size,
+        args.test_size,
+        args.seed_for_ds_split,
+        args.manifests_path,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/compute_feature_stats.py

@@ -0,0 +1,230 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to compute global and speaker-level feature statistics for a given TTS training manifest.
+
+This script should be run after compute_features.py as it loads the precomputed feature data.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/compute_feature_stats.py \
+    --feature_config_path=<nemo_root_path>/examples/tts/conf/features/feature_22050.yaml
+    --manifest_path=<data_root_path>/manifest1.json \
+    --manifest_path=<data_root_path>/manifest2.json \
+    --audio_dir=<data_root_path>/audio1 \
+    --audio_dir=<data_root_path>/audio2 \
+    --feature_dir=<data_root_path>/features1 \
+    --feature_dir=<data_root_path>/features2 \
+    --stats_path=<data_root_path>/feature_stats.json
+
+The output dictionary will contain the feature statistics for every speaker, as well as a "default" entry
+with the global statistics.
+
+For example:
+
+{
+    "default": {
+        "pitch_mean": 100.0,
+        "pitch_std": 50.0,
+        "energy_mean": 7.5,
+        "energy_std": 4.5
+    },
+    "speaker1": {
+        "pitch_mean": 105.0,
+        "pitch_std": 45.0,
+        "energy_mean": 7.0,
+        "energy_std": 5.0
+    },
+    "speaker2": {
+        "pitch_mean": 110.0,
+        "pitch_std": 30.0,
+        "energy_mean": 5.0,
+        "energy_std": 2.5
+    }
+}
+
+"""
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import List, Tuple
+
+import torch
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from nemo.core.classes.common import safe_instantiate
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute TTS feature statistics.",
+    )
+    parser.add_argument(
+        "--feature_config_path",
+        required=True,
+        type=Path,
+        help="Path to feature config file.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path(s) to training manifest.",
+    )
+    parser.add_argument(
+        "--audio_dir",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path(s) to base directory with audio data.",
+    )
+    parser.add_argument(
+        "--feature_dir",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path(s) to directory where feature data was stored.",
+    )
+    parser.add_argument(
+        "--feature_names",
+        default="pitch,energy",
+        type=str,
+        help="Comma separated list of features to process.",
+    )
+    parser.add_argument(
+        "--mask_field",
+        default="voiced_mask",
+        type=str,
+        help="If provided, stat computation will ignore non-masked frames.",
+    )
+    parser.add_argument(
+        "--stats_path",
+        default=Path("feature_stats.json"),
+        type=Path,
+        help="Path to output JSON file with dataset feature statistics.",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output stats file if it exists.",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def _compute_stats(values: List[torch.Tensor]) -> Tuple[float, float]:
+    values_tensor = torch.cat(values, dim=0)
+    mean = values_tensor.mean().item()
+    std = values_tensor.std(dim=0).item()
+    return mean, std
+
+
+def main():
+    args = get_args()
+
+    feature_config_path = args.feature_config_path
+    manifest_paths = args.manifest_path
+    audio_dirs = args.audio_dir
+    feature_dirs = args.feature_dir
+    feature_name_str = args.feature_names
+    mask_field = args.mask_field
+    stats_path = args.stats_path
+    overwrite = args.overwrite
+
+    if not (len(manifest_paths) == len(audio_dirs) == len(feature_dirs)):
+        raise ValueError(
+            f"Need same number of manifest, audio_dir, and feature_dir. Received: "
+            f"{len(manifest_paths)}, "
+            f"{len(audio_dirs)}, "
+            f"{len(feature_dirs)}"
+        )
+
+    for manifest_path, audio_dir, feature_dir in zip(manifest_paths, audio_dirs, feature_dirs):
+        if not manifest_path.exists():
+            raise ValueError(f"Manifest {manifest_path} does not exist.")
+
+        if not audio_dir.exists():
+            raise ValueError(f"Audio directory {audio_dir} does not exist.")
+
+        if not feature_dir.exists():
+            raise ValueError(
+                f"Feature directory {feature_dir} does not exist. "
+                f"Please check that the path is correct and that you ran compute_features.py"
+            )
+
+    if stats_path.exists():
+        if overwrite:
+            print(f"Will overwrite existing stats path: {stats_path}")
+        else:
+            raise ValueError(f"Stats path already exists: {stats_path}")
+
+    feature_config = OmegaConf.load(feature_config_path)
+    feature_config = safe_instantiate(feature_config)
+    featurizer_dict = feature_config.featurizers
+
+    print(f"Found featurizers for {list(featurizer_dict.keys())}.")
+    featurizers = featurizer_dict.values()
+
+    feature_names = feature_name_str.split(",")
+    # For each feature, we have a dictionary mapping speaker IDs to a list containing all features
+    # for that speaker
+    feature_stats = {name: defaultdict(list) for name in feature_names}
+
+    for manifest_path, audio_dir, feature_dir in zip(manifest_paths, audio_dirs, feature_dirs):
+        entries = read_manifest(manifest_path)
+
+        for entry in tqdm(entries):
+            speaker = entry["speaker"]
+
+            entry_dict = {}
+            for featurizer in featurizers:
+                feature_dict = featurizer.load(manifest_entry=entry, audio_dir=audio_dir, feature_dir=feature_dir)
+                entry_dict.update(feature_dict)
+
+            if mask_field:
+                mask = entry_dict[mask_field]
+            else:
+                mask = None
+
+            for feature_name in feature_names:
+                values = entry_dict[feature_name]
+                if mask is not None:
+                    values = values[mask]
+
+                feature_stat_dict = feature_stats[feature_name]
+                feature_stat_dict["default"].append(values)
+                feature_stat_dict[speaker].append(values)
+
+    stat_dict = defaultdict(dict)
+    for feature_name in feature_names:
+        mean_key = f"{feature_name}_mean"
+        std_key = f"{feature_name}_std"
+        feature_stat_dict = feature_stats[feature_name]
+        for speaker_id, values in feature_stat_dict.items():
+            speaker_mean, speaker_std = _compute_stats(values)
+            stat_dict[speaker_id][mean_key] = speaker_mean
+            stat_dict[speaker_id][std_key] = speaker_std
+
+    with open(stats_path, 'w', encoding="utf-8") as stats_f:
+        json.dump(stat_dict, stats_f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/compute_features.py

@@ -0,0 +1,130 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script computes features for TTS models prior to training, such as pitch and energy.
+The resulting features will be stored in the provided 'feature_dir'.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/compute_features.py \
+    --feature_config_path=<nemo_root_path>/examples/tts/conf/features/feature_22050.yaml \
+    --manifest_path=<data_root_path>/manifest.json \
+    --audio_dir=<data_root_path>/audio \
+    --feature_dir=<data_root_path>/features \
+    --overwrite \
+    --num_workers=1
+"""
+
+import argparse
+from pathlib import Path
+
+from joblib import Parallel, delayed
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from nemo.core.classes.common import safe_instantiate
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute TTS features.",
+    )
+    parser.add_argument(
+        "--feature_config_path",
+        required=True,
+        type=Path,
+        help="Path to feature config file.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        help="Path to training manifest.",
+    )
+    parser.add_argument(
+        "--audio_dir",
+        required=True,
+        type=Path,
+        help="Path to base directory with audio data.",
+    )
+    parser.add_argument(
+        "--feature_dir",
+        required=True,
+        type=Path,
+        help="Path to directory where feature data will be stored.",
+    )
+    parser.add_argument(
+        "--dedupe_files",
+        action=argparse.BooleanOptionalAction,
+        help="If given, will only process the first manifest entry found for each audio file.",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite existing feature files.",
+    )
+    parser.add_argument(
+        "--num_workers", default=1, type=int, help="Number of parallel threads to use. If -1 all CPUs are used."
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+    feature_config_path = args.feature_config_path
+    manifest_path = args.manifest_path
+    audio_dir = args.audio_dir
+    feature_dir = args.feature_dir
+    dedupe_files = args.dedupe_files
+    overwrite = args.overwrite
+    num_workers = args.num_workers
+
+    if not manifest_path.exists():
+        raise ValueError(f"Manifest {manifest_path} does not exist.")
+
+    if not audio_dir.exists():
+        raise ValueError(f"Audio directory {audio_dir} does not exist.")
+
+    feature_config = OmegaConf.load(feature_config_path)
+    feature_config = safe_instantiate(feature_config)
+    featurizers = feature_config.featurizers
+
+    entries = read_manifest(manifest_path)
+
+    if dedupe_files:
+        final_entries = []
+        audio_filepath_set = set()
+        for entry in entries:
+            audio_filepath = entry["audio_filepath"]
+            if audio_filepath in audio_filepath_set:
+                continue
+            final_entries.append(entry)
+            audio_filepath_set.add(audio_filepath)
+        entries = final_entries
+
+    for feature_name, featurizer in featurizers.items():
+        print(f"Computing: {feature_name}")
+        Parallel(n_jobs=num_workers)(
+            delayed(featurizer.save)(
+                manifest_entry=entry, audio_dir=audio_dir, feature_dir=feature_dir, overwrite=overwrite
+            )
+            for entry in tqdm(entries)
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/compute_speaker_stats.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to compute speaker-level statistics, such as pitch mean & standard deviation, for a given
+TTS training manifest.
+
+This script should be run after extract_sup_data.py as it uses the precomputed supplemental features.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/compute_speaker_stats.py \
+    --manifest_path=<data_root_path>/fastpitch_manifest.json \
+    --sup_data_path=<data_root_path>/sup_data \
+    --pitch_stats_path=<data_root_path>/pitch_stats.json
+"""
+
+import argparse
+import json
+import os
+from collections import defaultdict
+from pathlib import Path
+from typing import List, Tuple
+
+import torch
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from nemo.collections.tts.parts.utils.tts_dataset_utils import get_base_dir
+from nemo.collections.tts.torch.tts_data_types import Pitch
+from nemo.utils import logging
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute speaker level pitch statistics.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        help="Path to training manifest.",
+    )
+    parser.add_argument(
+        "--sup_data_path",
+        default=Path("sup_data"),
+        type=Path,
+        help="Path to base directory with supplementary data.",
+    )
+    parser.add_argument(
+        "--pitch_stats_path",
+        default=Path("pitch_stats.json"),
+        type=Path,
+        help="Path to output JSON file with speaker pitch statistics.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def _compute_stats(values: List[torch.Tensor]) -> Tuple[float, float]:
+    values_tensor = torch.cat(values, dim=0)
+    mean = values_tensor.mean().item()
+    std = values_tensor.std(dim=0).item()
+    return mean, std
+
+
+def _get_sup_data_filepath(manifest_entry: dict, audio_dir: Path, sup_data_dir: Path) -> Path:
+    """
+    Get the absolute path of a supplementary data type for the input manifest entry.
+
+    Example: audio_filepath "<audio_dir>/speaker1/audio1.wav" becomes "<sup_data_dir>/speaker1_audio1.pt"
+
+    Args:
+        manifest_entry: Manifest entry dictionary.
+        audio_dir: base directory where audio is stored.
+        sup_data_dir: base directory where supplementary data is stored.
+
+    Returns:
+        Path to the supplementary data file.
+    """
+    audio_path = Path(manifest_entry["audio_filepath"])
+    rel_audio_path = audio_path.relative_to(audio_dir)
+    rel_sup_data_path = rel_audio_path.with_suffix(".pt")
+    sup_data_filename = str(rel_sup_data_path).replace(os.sep, "_")
+    sup_data_filepath = sup_data_dir / sup_data_filename
+    return sup_data_filepath
+
+
+def main():
+    args = get_args()
+    manifest_path = args.manifest_path
+    sup_data_path = args.sup_data_path
+    pitch_stats_path = args.pitch_stats_path
+
+    pitch_data_path = Path(os.path.join(sup_data_path, Pitch.name))
+    if not os.path.exists(pitch_data_path):
+        raise ValueError(
+            f"Pitch directory {pitch_data_path} does not exist. Make sure 'sup_data_path' is correct "
+            f"and that you have computed the pitch using extract_sup_data.py"
+        )
+
+    entries = read_manifest(manifest_path)
+
+    audio_paths = [entry["audio_filepath"] for entry in entries]
+    base_dir = get_base_dir(audio_paths)
+
+    global_pitch_values = []
+    speaker_pitch_values = defaultdict(list)
+    for entry in tqdm(entries):
+        pitch_path = _get_sup_data_filepath(manifest_entry=entry, audio_dir=base_dir, sup_data_dir=pitch_data_path)
+        if not os.path.exists(pitch_path):
+            logging.warning(f"Unable to find pitch file for {entry}")
+            continue
+
+        pitch = torch.load(pitch_path)
+        # Filter out non-speech frames
+        pitch = pitch[pitch != 0]
+        global_pitch_values.append(pitch)
+        if "speaker" in entry:
+            speaker_id = entry["speaker"]
+            speaker_pitch_values[speaker_id].append(pitch)
+
+    global_pitch_mean, global_pitch_std = _compute_stats(global_pitch_values)
+    pitch_stats = {"default": {"pitch_mean": global_pitch_mean, "pitch_std": global_pitch_std}}
+    for speaker_id, pitch_values in speaker_pitch_values.items():
+        pitch_mean, pitch_std = _compute_stats(pitch_values)
+        pitch_stats[speaker_id] = {"pitch_mean": pitch_mean, "pitch_std": pitch_std}
+
+    with open(pitch_stats_path, 'w', encoding="utf-8") as stats_f:
+        json.dump(pitch_stats, stats_f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/create_speaker_map.py

@@ -0,0 +1,104 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script takes a list of TTS manifests and creates a JSON mapping the input speaker names to
+unique indices for multi-speaker TTS training.
+
+To ensure that speaker names are unique across datasets, it is recommended that you prepend the speaker
+names in your manifest with the name of the dataset.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/create_speaker_map.py \
+    --manifest_path=manifest1.json \
+    --manifest_path=manifest2.json \
+    --speaker_map_path=speakers.json
+
+Example output:
+
+{
+    "vctk_p225": 0,
+    "vctk_p226": 1,
+    "vctk_p227": 2,
+    ...
+}
+
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Create mapping from speaker names to numerical speaker indices.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path to training manifest(s).",
+    )
+    parser.add_argument(
+        "--speaker_map_path",
+        required=True,
+        type=Path,
+        help="Path for output speaker index JSON",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output speaker file if it exists.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+    manifest_paths = args.manifest_path
+    speaker_map_path = args.speaker_map_path
+    overwrite = args.overwrite
+
+    for manifest_path in manifest_paths:
+        if not manifest_path.exists():
+            raise ValueError(f"Manifest {manifest_path} does not exist.")
+
+    if speaker_map_path.exists():
+        if overwrite:
+            print(f"Will overwrite existing speaker path: {speaker_map_path}")
+        else:
+            raise ValueError(f"Speaker path already exists: {speaker_map_path}")
+
+    speaker_set = set()
+    for manifest_path in manifest_paths:
+        entries = read_manifest(manifest_path)
+        for entry in entries:
+            speaker = str(entry["speaker"])
+            speaker_set.add(speaker)
+
+    speaker_list = list(speaker_set)
+    speaker_list.sort()
+    speaker_index_map = {speaker_list[i]: i for i in range(len(speaker_list))}
+
+    with open(speaker_map_path, 'w', encoding="utf-8") as stats_f:
+        json.dump(speaker_index_map, stats_f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/extract_sup_data.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch
+from tqdm import tqdm
+
+from nemo.core.classes.common import safe_instantiate
+from nemo.core.config import hydra_runner
+
+
+def get_pitch_stats(pitch_list):
+    pitch_tensor = torch.cat(pitch_list)
+    pitch_mean, pitch_std = pitch_tensor.mean().item(), pitch_tensor.std().item()
+    pitch_min, pitch_max = pitch_tensor.min().item(), pitch_tensor.max().item()
+    print(f"PITCH_MEAN={pitch_mean}, PITCH_STD={pitch_std}")
+    print(f"PITCH_MIN={pitch_min}, PITCH_MAX={pitch_max}")
+
+
+def preprocess_ds_for_fastpitch_align(dataloader):
+    pitch_list = []
+    for batch in tqdm(dataloader, total=len(dataloader)):
+        audios, audio_lengths, tokens, tokens_lengths, align_prior_matrices, pitches, pitches_lengths, *_ = batch
+        pitch = pitches.squeeze(0)
+        pitch_list.append(pitch[pitch != 0])
+
+    get_pitch_stats(pitch_list)
+
+
+CFG_NAME2FUNC = {
+    "ds_for_fastpitch_align": preprocess_ds_for_fastpitch_align,
+    "ds_for_mixer_tts": preprocess_ds_for_fastpitch_align,
+}
+
+
+@hydra_runner(config_path='ljspeech/ds_conf', config_name='ds_for_fastpitch_align')
+def main(cfg):
+    dataset = safe_instantiate(cfg.dataset)
+    dataloader = torch.utils.data.DataLoader(
+        dataset=dataset,
+        batch_size=1,
+        collate_fn=dataset._collate_fn,
+        num_workers=cfg.get("dataloader_params", {}).get("num_workers", 4),
+    )
+
+    print(f"Processing {cfg.manifest_filepath}:")
+    CFG_NAME2FUNC[cfg.name](dataloader)
+
+
+if __name__ == '__main__':
+    main()  # noqa pylint: disable=no-value-for-parameter

scripts/dataset_processing/tts/generate_mels.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to generate mel spectrograms from a Fastpitch model checkpoint. Please see general usage below. It runs
+on GPUs by default, but you can add `--num-workers 5 --cpu` as an option to run on CPUs.
+
+$ python scripts/dataset_processing/tts/generate_mels.py \
+    --fastpitch-model-ckpt ./models/fastpitch/multi_spk/FastPitch--val_loss\=1.4473-epoch\=209.ckpt \
+    --input-json-manifests /home/xueyang/HUI-Audio-Corpus-German-clean/test_manifest_text_normed_phonemes.json
+    --output-json-manifest-root /home/xueyang/experiments/multi_spk_tts_de
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+import torch
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+from nemo.collections.tts.models import FastPitchModel
+from nemo.collections.tts.parts.utils.tts_dataset_utils import (
+    BetaBinomialInterpolator,
+    beta_binomial_prior_distribution,
+)
+from nemo.utils import logging
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Generate mel spectrograms with pretrained FastPitch model, and create manifests for finetuning Hifigan.",
+    )
+    parser.add_argument(
+        "--fastpitch-model-ckpt",
+        required=True,
+        type=Path,
+        help="Specify a full path of a fastpitch model checkpoint with the suffix of either .ckpt or .nemo.",
+    )
+    parser.add_argument(
+        "--input-json-manifests",
+        nargs="+",
+        required=True,
+        type=Path,
+        help="Specify a full path of a JSON manifest. You could add multiple manifests.",
+    )
+    parser.add_argument(
+        "--output-json-manifest-root",
+        required=True,
+        type=Path,
+        help="Specify a full path of output root that would contain new manifests.",
+    )
+    parser.add_argument(
+        "--num-workers",
+        default=-1,
+        type=int,
+        help="Specify the max number of concurrently Python workers processes. "
+        "If -1 all CPUs are used. If 1 no parallel computing is used.",
+    )
+    parser.add_argument("--cpu", action='store_true', default=False, help="Generate mel spectrograms using CPUs.")
+    args = parser.parse_args()
+    return args
+
+
+def __load_wav(audio_file):
+    with sf.SoundFile(audio_file, 'r') as f:
+        samples = f.read(dtype='float32')
+    return samples.transpose()
+
+
+def __generate_mels(entry, spec_model, device, use_beta_binomial_interpolator, mel_root):
+    # Generate a spectrograms (we need to use ground truth alignment for correct matching between audio and mels)
+    audio = __load_wav(entry["audio_filepath"])
+    audio = torch.from_numpy(audio).unsqueeze(0).to(device)
+    audio_len = torch.tensor(audio.shape[1], dtype=torch.long, device=device).unsqueeze(0)
+
+    if spec_model.fastpitch.speaker_emb is not None and "speaker" in entry:
+        speaker = torch.tensor([entry['speaker']]).to(device)
+    else:
+        speaker = None
+
+    with torch.no_grad():
+        if "normalized_text" in entry:
+            text = spec_model.parse(entry["normalized_text"], normalize=False)
+        else:
+            text = spec_model.parse(entry['text'])
+
+        text_len = torch.tensor(text.shape[-1], dtype=torch.long, device=device).unsqueeze(0)
+        spect, spect_len = spec_model.preprocessor(input_signal=audio, length=audio_len)
+
+        # Generate attention prior and spectrogram inputs for HiFi-GAN
+        if use_beta_binomial_interpolator:
+            beta_binomial_interpolator = BetaBinomialInterpolator()
+            attn_prior = (
+                torch.from_numpy(beta_binomial_interpolator(spect_len.item(), text_len.item()))
+                .unsqueeze(0)
+                .to(text.device)
+            )
+        else:
+            attn_prior = (
+                torch.from_numpy(beta_binomial_prior_distribution(text_len.item(), spect_len.item()))
+                .unsqueeze(0)
+                .to(text.device)
+            )
+
+        spectrogram = spec_model.forward(
+            text=text,
+            input_lens=text_len,
+            spec=spect,
+            mel_lens=spect_len,
+            attn_prior=attn_prior,
+            speaker=speaker,
+        )[0]
+
+        save_path = mel_root / f"{Path(entry['audio_filepath']).stem}.npy"
+        np.save(save_path, spectrogram[0].to('cpu').numpy())
+        entry["mel_filepath"] = str(save_path)
+
+    return entry
+
+
+def main():
+    args = get_args()
+    ckpt_path = args.fastpitch_model_ckpt
+    input_manifest_filepaths = args.input_json_manifests
+    output_json_manifest_root = args.output_json_manifest_root
+
+    mel_root = output_json_manifest_root / "mels"
+    mel_root.mkdir(exist_ok=True, parents=True)
+
+    # load pretrained FastPitch model checkpoint
+    suffix = ckpt_path.suffix
+    if suffix == ".nemo":
+        spec_model = FastPitchModel.restore_from(ckpt_path).eval()
+    elif suffix == ".ckpt":
+        spec_model = FastPitchModel.load_from_checkpoint(ckpt_path).eval()
+    else:
+        raise ValueError(f"Unsupported suffix: {suffix}")
+    if not args.cpu:
+        spec_model.cuda()
+    device = spec_model.device
+
+    use_beta_binomial_interpolator = spec_model.cfg.train_ds.dataset.get("use_beta_binomial_interpolator", False)
+
+    for manifest in input_manifest_filepaths:
+        logging.info(f"Processing {manifest}.")
+        entries = []
+        with open(manifest, "r") as fjson:
+            for line in fjson:
+                entries.append(json.loads(line.strip()))
+
+        if device == "cpu":
+            new_entries = Parallel(n_jobs=args.num_workers)(
+                delayed(__generate_mels)(entry, spec_model, device, use_beta_binomial_interpolator, mel_root)
+                for entry in entries
+            )
+        else:
+            new_entries = []
+            for entry in tqdm(entries):
+                new_entry = __generate_mels(entry, spec_model, device, use_beta_binomial_interpolator, mel_root)
+                new_entries.append(new_entry)
+
+        mel_manifest_path = output_json_manifest_root / f"{manifest.stem}_mel{manifest.suffix}"
+        with open(mel_manifest_path, "w") as fmel:
+            for entry in new_entries:
+                fmel.write(json.dumps(entry) + "\n")
+        logging.info(f"Processing {manifest} is complete --> {mel_manifest_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/hifitts/get_data.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import glob
+import json
+import re
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Download HiFiTTS and create manifests with predefined split')
+    parser.add_argument(
+        "--data-root",
+        required=True,
+        type=Path,
+        help='Directory into which to download and extract dataset. \{data-root\}/hi_fi_tts_v0 will be created.',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='all',
+        help='Choose to generate manifest for all or one of (train, test, split), note that this will still download the full dataset.',
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+URL = "https://us.openslr.org/resources/109/hi_fi_tts_v0.tar.gz"
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_data(data_root, filelists):
+    # Create manifests (based on predefined NVIDIA's split)
+    for split in tqdm(filelists):
+        manifest_target = data_root / f"{split}_manifest.json"
+        print(f"Creating manifest for {split}.")
+
+        entries = []
+        for manifest_src in glob.glob(str(data_root / f"*_{split}.json")):
+            try:
+                search_res = re.search('.*\/([0-9]+)_manifest_([a-z]+)_.*.json', manifest_src)
+                speaker_id = search_res.group(1)
+                audio_quality = search_res.group(2)
+            except Exception:
+                print(f"Failed to find speaker id or audio quality for {manifest_src}, check formatting.")
+                continue
+
+            with open(manifest_src, 'r') as f_in:
+                for input_json_entry in f_in:
+                    data = json.loads(input_json_entry)
+
+                    # Make sure corresponding wavfile exists
+                    wav_path = data_root / data['audio_filepath']
+                    assert wav_path.exists(), f"{wav_path} does not exist!"
+
+                    entry = {
+                        'audio_filepath': data['audio_filepath'],
+                        'duration': data['duration'],
+                        'text': data['text'],
+                        'normalized_text': data['text_normalized'],
+                        'speaker': int(speaker_id),
+                        # Audio_quality is either clean or other.
+                        # The clean set includes recordings with high sound-to-noise ratio and wide bandwidth.
+                        # The books with noticeable noise or narrow bandwidth are included in the other subset.
+                        # Note: some speaker_id's have both clean and other audio quality.
+                        'audio_quality': audio_quality,
+                    }
+                    entries.append(entry)
+
+        with open(manifest_target, 'w') as f_out:
+            for m in entries:
+                f_out.write(json.dumps(m) + '\n')
+
+
+def main():
+    args = get_args()
+
+    split = ['train', 'dev', 'test'] if args.split == 'all' else list(args.split)
+
+    tarred_data_path = args.data_root / "hi_fi_tts_v0.tar.gz"
+
+    __maybe_download_file(URL, tarred_data_path)
+    __extract_file(str(tarred_data_path), str(args.data_root))
+
+    data_root = args.data_root / "hi_fi_tts_v0"
+    __process_data(data_root, split)
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/tts/hui_acg/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,45 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: ???
+sup_data_path: ???
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 44100
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 2048
+  win_length: 2048
+  hop_length: 512
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: 15
+  min_duration: 0.1
+  ignore_file: null
+  trim: false
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+  use_beta_binomial_interpolator: false
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: de
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.GermanPhonemesTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true
+
+dataloader_params:
+  num_workers: 12

scripts/dataset_processing/tts/hui_acg/get_data.py

@@ -0,0 +1,334 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import random
+import shutil
+import urllib.request
+from pathlib import Path
+
+import pandas as pd
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+from nemo.utils import logging
+
+# full corpus.
+URLS_FULL = {
+    "Bernd_Ungerer": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Bernd_Ungerer.zip",
+    "Eva_K": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Eva_K.zip",
+    "Friedrich": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Friedrich.zip",
+    "Hokuspokus": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Hokuspokus.zip",
+    "Karlsson": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Karlsson.zip",
+    "others": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/others.zip",
+}
+URL_STATS_FULL = "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/datasetStatistic.zip"
+
+# the clean subset of the full corpus.
+URLS_CLEAN = {
+    "Bernd_Ungerer": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Bernd_Ungerer_Clean.zip",
+    "Eva_K": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Eva_K_Clean.zip",
+    "Friedrich": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Friedrich_Clean.zip",
+    "Hokuspokus": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Hokuspokus_Clean.zip",
+    "Karlsson": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Karlsson_Clean.zip",
+    "others": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/others_Clean.zip",
+}
+URL_STATS_CLEAN = "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/datasetStatisticClean.zip"
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Download HUI-Audio-Corpus-German and create manifests with predefined split. "
+        "Please check details about the corpus in https://github.com/iisys-hof/HUI-Audio-Corpus-German.",
+    )
+    parser.add_argument("--data-root", required=True, type=Path, help="where the resulting dataset will reside.")
+    parser.add_argument("--manifests-root", required=True, type=Path, help="where the manifests files will reside.")
+    parser.add_argument("--set-type", default="clean", choices=["full", "clean"], type=str)
+    parser.add_argument("--min-duration", default=0.1, type=float)
+    parser.add_argument("--max-duration", default=15, type=float)
+    parser.add_argument(
+        "--num-workers",
+        default=-1,
+        type=int,
+        help="Specify the max number of concurrently Python workers processes. "
+        "If -1 all CPUs are used. If 1 no parallel computing is used.",
+    )
+    parser.add_argument(
+        "--normalize-text",
+        default=False,
+        action='store_true',
+        help="Normalize original text and add a new entry 'normalized_text' to .json file if True.",
+    )
+    parser.add_argument(
+        "--val-num-utts-per-speaker",
+        default=1,
+        type=int,
+        help="Specify the number of utterances for each speaker in val split. All speakers are covered.",
+    )
+    parser.add_argument(
+        "--test-num-utts-per-speaker",
+        default=1,
+        type=int,
+        help="Specify the number of utterances for each speaker in test split. All speakers are covered.",
+    )
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        logging.info(f"Downloading data: {source_url} --> {destination_path}")
+        tmp_file_path = destination_path.with_suffix(".tmp")
+        urllib.request.urlretrieve(source_url, filename=tmp_file_path)
+        tmp_file_path.rename(destination_path)
+    else:
+        logging.info(f"Skipped downloading data because it exists: {destination_path}")
+
+
+def __extract_file(filepath, data_dir):
+    logging.info(f"Unzipping data: {filepath} --> {data_dir}")
+    shutil.unpack_archive(filepath, data_dir)
+    logging.info(f"Unzipping data is complete: {filepath}.")
+
+
+def __save_json(json_file, dict_list):
+    logging.info(f"Saving JSON split to {json_file}.")
+    with open(json_file, "w") as f:
+        for d in dict_list:
+            f.write(json.dumps(d) + "\n")
+
+
+def __process_data(
+    dataset_path,
+    stat_path_root,
+    speaker_id,
+    min_duration,
+    max_duration,
+    val_size,
+    test_size,
+    seed_for_ds_split,
+):
+    logging.info(f"Preparing JSON split for speaker {speaker_id}.")
+    # parse statistic.txt
+    stat_path = stat_path_root / "statistic.txt"
+    with open(stat_path, 'r') as fstat:
+        lines = fstat.readlines()
+        num_utts = int(lines[4].strip().split()[-1])
+        hours = round(float(lines[9].strip().split()[-1]) / 3600.0, 2)
+
+    # parse overview.csv to generate JSON splits.
+    overview_path = stat_path_root / "overview.csv"
+    entries = []
+    with open(overview_path, 'r') as foverview:
+        # Let's skip the header
+        foverview.readline()
+        for line in tqdm(foverview):
+            file_stem, duration, *_, text = line.strip().split("|")
+            duration = float(duration)
+
+            # file_stem -> dir_name (e.g. maerchen_01_f000051 -> maerchen)
+            dir_name = "_".join(file_stem.split("_")[:-2])
+            audio_path = dataset_path / dir_name / "wavs" / f"{file_stem}.wav"
+
+            if min_duration <= duration <= max_duration:
+                entry = {
+                    "audio_filepath": str(audio_path),
+                    "duration": duration,
+                    "text": text,
+                    "speaker": speaker_id,
+                }
+                entries.append(entry)
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+    train_size = len(entries) - val_size - test_size
+    if train_size <= 0:
+        logging.warning(f"Skipped speaker {speaker_id}. Not enough data for train, val and test.")
+        train, val, test, is_skipped = [], [], [], True
+    else:
+        logging.info(f"Preparing JSON split for speaker {speaker_id} is complete.")
+        train, val, test, is_skipped = (
+            entries[:train_size],
+            entries[train_size : train_size + val_size],
+            entries[train_size + val_size :],
+            False,
+        )
+
+    return {
+        "train": train,
+        "val": val,
+        "test": test,
+        "is_skipped": is_skipped,
+        "hours": hours,
+        "num_utts": num_utts,
+    }
+
+
+def __text_normalization(json_file, num_workers=-1):
+    text_normalizer_call_kwargs = {
+        "punct_pre_process": True,
+        "punct_post_process": True,
+    }
+    text_normalizer = Normalizer(
+        lang="de",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(json_file.parent / "cache_dir"),
+    )
+
+    def normalizer_call(x):
+        return text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+
+    def add_normalized_text(line_dict):
+        normalized_text = normalizer_call(line_dict["text"])
+        line_dict.update({"normalized_text": normalized_text})
+        return line_dict
+
+    logging.info(f"Normalizing text for {json_file}.")
+    with open(json_file, 'r', encoding='utf-8') as fjson:
+        lines = fjson.readlines()
+        # Note: you need to verify which backend works well on your cluster.
+        # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+        dict_list = Parallel(n_jobs=num_workers)(
+            delayed(add_normalized_text)(json.loads(line)) for line in tqdm(lines)
+        )
+
+    json_file_text_normed = json_file.parent / f"{json_file.stem}_text_normed{json_file.suffix}"
+    with open(json_file_text_normed, 'w', encoding="utf-8") as fjson_norm:
+        for dct in dict_list:
+            fjson_norm.write(json.dumps(dct) + "\n")
+    logging.info(f"Normalizing text is complete: {json_file} --> {json_file_text_normed}")
+
+
+def main():
+    args = get_args()
+    data_root = args.data_root
+    manifests_root = args.manifests_root
+    set_type = args.set_type
+
+    dataset_root = data_root / f"HUI-Audio-Corpus-German-{set_type}"
+    dataset_root.mkdir(parents=True, exist_ok=True)
+
+    if set_type == "full":
+        data_source = URLS_FULL
+        stats_source = URL_STATS_FULL
+    elif set_type == "clean":
+        data_source = URLS_CLEAN
+        stats_source = URL_STATS_CLEAN
+    else:
+        raise ValueError(f"Unknown {set_type}. Please choose either clean or full.")
+
+    # download and unzip dataset stats
+    zipped_stats_path = dataset_root / Path(stats_source).name
+    __maybe_download_file(stats_source, zipped_stats_path)
+    __extract_file(zipped_stats_path, dataset_root)
+
+    # download datasets
+    # Note: you need to verify which backend works well on your cluster.
+    # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+    Parallel(n_jobs=args.num_workers)(
+        delayed(__maybe_download_file)(data_url, dataset_root / Path(data_url).name)
+        for _, data_url in data_source.items()
+    )
+
+    # unzip datasets
+    # Note: you need to verify which backend works well on your cluster.
+    # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+    Parallel(n_jobs=args.num_workers)(
+        delayed(__extract_file)(dataset_root / Path(data_url).name, dataset_root)
+        for _, data_url in data_source.items()
+    )
+
+    # generate json files for train/val/test splits
+    stats_path_root = dataset_root / Path(stats_source).stem / "speacker"
+    entries_train, entries_val, entries_test = [], [], []
+    speaker_entries = []
+    num_speakers = 0
+    for child in stats_path_root.iterdir():
+        if child.is_dir():
+            speaker = child.name
+            num_speakers += 1
+            speaker_stats_root = stats_path_root / speaker
+            speaker_data_path = dataset_root / speaker
+
+            logging.info(f"Processing Speaker: {speaker}")
+            results = __process_data(
+                speaker_data_path,
+                speaker_stats_root,
+                num_speakers,
+                args.min_duration,
+                args.max_duration,
+                args.val_num_utts_per_speaker,
+                args.test_num_utts_per_speaker,
+                args.seed_for_ds_split,
+            )
+
+            entries_train.extend(results["train"])
+            entries_val.extend(results["val"])
+            entries_test.extend(results["test"])
+
+            speaker_entry = {
+                "speaker_name": speaker,
+                "speaker_id": num_speakers,
+                "hours": results["hours"],
+                "num_utts": results["num_utts"],
+                "is_skipped": results["is_skipped"],
+            }
+            speaker_entries.append(speaker_entry)
+
+    # shuffle in place across multiple speakers
+    random.Random(args.seed_for_ds_split).shuffle(entries_train)
+    random.Random(args.seed_for_ds_split).shuffle(entries_val)
+    random.Random(args.seed_for_ds_split).shuffle(entries_test)
+
+    # save speaker stats.
+    df = pd.DataFrame.from_records(speaker_entries)
+    df.sort_values(by="hours", ascending=False, inplace=True)
+    spk2id_file_path = manifests_root / "spk2id.csv"
+    df.to_csv(spk2id_file_path, index=False)
+    logging.info(f"Saving Speaker to ID mapping to {spk2id_file_path}.")
+
+    # save json splits.
+    train_json = manifests_root / "train_manifest.json"
+    val_json = manifests_root / "val_manifest.json"
+    test_json = manifests_root / "test_manifest.json"
+    __save_json(train_json, entries_train)
+    __save_json(val_json, entries_val)
+    __save_json(test_json, entries_test)
+
+    # normalize text if requested. New json file, train_manifest_text_normed.json, will be generated.
+    if args.normalize_text:
+        __text_normalization(train_json, args.num_workers)
+        __text_normalization(val_json, args.num_workers)
+        __text_normalization(test_json, args.num_workers)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/libritts/get_data.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# USAGE: python get_data.py --data-root=<where to put data> --data-set=<datasets_to_download> --num-workers=<number of parallel workers>
+# where <datasets_to_download> can be: dev_clean, dev_other, test_clean,
+# test_other, train_clean_100, train_clean_360, train_other_500 or ALL
+# You can also put more than one data_set comma-separated:
+# --data-set=dev_clean,train_clean_100
+import argparse
+import fnmatch
+import functools
+import json
+import multiprocessing
+import os
+import subprocess
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+parser = argparse.ArgumentParser(description='Download LibriTTS and create manifests')
+parser.add_argument("--data-root", required=True, type=Path)
+parser.add_argument("--data-sets", default="dev_clean", type=str)
+parser.add_argument("--num-workers", default=4, type=int)
+args = parser.parse_args()
+
+URLS = {
+    'TRAIN_CLEAN_100': "https://www.openslr.org/resources/60/train-clean-100.tar.gz",
+    'TRAIN_CLEAN_360': "https://www.openslr.org/resources/60/train-clean-360.tar.gz",
+    'TRAIN_OTHER_500': "https://www.openslr.org/resources/60/train-other-500.tar.gz",
+    'DEV_CLEAN': "https://www.openslr.org/resources/60/dev-clean.tar.gz",
+    'DEV_OTHER': "https://www.openslr.org/resources/60/dev-other.tar.gz",
+    'TEST_CLEAN': "https://www.openslr.org/resources/60/test-clean.tar.gz",
+    'TEST_OTHER': "https://www.openslr.org/resources/60/test-other.tar.gz",
+}
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_transcript(file_path: str):
+    entries = []
+    with open(file_path, encoding="utf-8") as fin:
+        text = fin.readlines()[0].strip()
+
+        # TODO(oktai15): add normalized text via Normalizer/NormalizerWithAudio
+        wav_file = file_path.replace(".normalized.txt", ".wav")
+        speaker_id = file_path.split('/')[-3]
+        assert os.path.exists(wav_file), f"{wav_file} not found!"
+        duration = subprocess.check_output(["soxi", "-D", wav_file])
+        entry = {
+            'audio_filepath': os.path.abspath(wav_file),
+            'duration': float(duration),
+            'text': text,
+            'speaker': int(speaker_id),
+        }
+
+        entries.append(entry)
+
+    return entries
+
+
+def __process_data(data_folder, manifest_file, num_workers):
+    files = []
+    entries = []
+
+    for root, dirnames, filenames in os.walk(data_folder):
+        # we will use normalized text provided by the original dataset
+        for filename in fnmatch.filter(filenames, '*.normalized.txt'):
+            files.append(os.path.join(root, filename))
+
+    with multiprocessing.Pool(num_workers) as p:
+        processing_func = functools.partial(__process_transcript)
+        results = p.imap(processing_func, files)
+        for result in tqdm(results, total=len(files)):
+            entries.extend(result)
+
+    with open(manifest_file, 'w') as fout:
+        for m in entries:
+            fout.write(json.dumps(m) + '\n')
+
+
+def main():
+    data_root = args.data_root
+    data_sets = args.data_sets
+    num_workers = args.num_workers
+
+    if data_sets == "ALL":
+        data_sets = "dev_clean,dev_other,train_clean_100,train_clean_360,train_other_500,test_clean,test_other"
+    if data_sets == "mini":
+        data_sets = "dev_clean,train_clean_100"
+    for data_set in data_sets.split(','):
+        filepath = data_root / f"{data_set}.tar.gz"
+        print(f"Downloading data for {data_set}...")
+        __maybe_download_file(URLS[data_set.upper()], filepath)
+        print("Extracting...")
+        __extract_file(str(filepath), str(data_root))
+
+        print("Processing and building manifest.")
+        __process_data(
+            str(data_root / "LibriTTS" / data_set.replace("_", "-")),
+            str(data_root / "LibriTTS" / f"{data_set}.json"),
+            num_workers=num_workers,
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.10"
+heteronyms_path: "scripts/tts_dataset_files/heteronyms-052722"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: 8000
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: false
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: en
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.EnglishPhonemesTokenizer
+    punct: true
+    stresses: true
+    chars: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.en_us_arpabet.EnglishG2p
+      phoneme_dict: ${phoneme_dict_path}
+      heteronyms: ${heteronyms_path}

scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_mixer_tts.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_mixer_tts"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.10"
+heteronyms_path: "scripts/tts_dataset_files/heteronyms-052722"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: 8000
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: false
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: en
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.EnglishPhonemesTokenizer
+    punct: true
+    stresses: true
+    chars: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.en_us_arpabet.EnglishG2p
+      phoneme_dict: ${phoneme_dict_path}
+      heteronyms: ${heteronyms_path}

scripts/dataset_processing/tts/ljspeech/get_data.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Download LJSpeech and create manifests with predefined split')
+    parser.add_argument("--data-root", required=True, type=Path)
+
+    args = parser.parse_args()
+    return args
+
+
+URL = "https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2"
+FILELIST_BASE = 'https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists'
+
+
+def _load_sox():
+    try:
+        import sox
+    except ImportError:
+        raise ImportError(
+            "Optional dependency 'sox' is required by this script. Install it with: pip install sox"
+        ) from None
+
+    return sox
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_data(data_root):
+    sox = _load_sox()
+    text_normalizer = Normalizer(
+        lang="en",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=data_root / "cache_dir",
+    )
+    text_normalizer_call_kwargs = {"punct_pre_process": True, "punct_post_process": True}
+    normalizer_call = lambda x: text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+
+    # Create manifests (based on predefined NVIDIA's split)
+    filelists = ['train', 'val', 'test']
+    for split in tqdm(filelists):
+        # Download file list if necessary
+        filelist_path = data_root / f"ljs_audio_text_{split}_filelist.txt"
+
+        if not filelist_path.exists():
+            urllib.request.urlretrieve(
+                f"{FILELIST_BASE}/ljs_audio_text_{split}_filelist.txt",
+                filename=str(filelist_path),
+            )
+
+        manifest_target = data_root / f"{split}_manifest.json"
+        with open(manifest_target, 'w') as f_out:
+            with open(filelist_path, 'r') as filelist:
+                print(f"\nCreating {manifest_target}...")
+                for line in tqdm(filelist):
+                    basename = line[6:16]
+
+                    text = line[21:].strip()
+                    norm_text = normalizer_call(text)
+
+                    # Make sure corresponding wavfile exists
+                    wav_path = data_root / 'wavs' / f"{basename}.wav"
+                    assert wav_path.exists(), f"{wav_path} does not exist!"
+
+                    entry = {
+                        'audio_filepath': str(wav_path),
+                        'duration': sox.file_info.duration(wav_path),
+                        'text': text,
+                        'normalized_text': norm_text,
+                    }
+
+                    f_out.write(json.dumps(entry) + '\n')
+
+
+def main():
+    args = get_args()
+
+    tarred_data_path = args.data_root / "LJSpeech-1.1.tar.bz2"
+
+    __maybe_download_file(URL, tarred_data_path)
+    __extract_file(str(tarred_data_path), str(args.data_root))
+
+    data_root = args.data_root / "LJSpeech-1.1"
+
+    __process_data(data_root)
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/tts/ljspeech/lj_speech.tsv

@@ -0,0 +1,21 @@
+Mr.	mister
+Mrs.	misses
+Dr.	doctor
+Drs.	doctors
+Co.	company
+Lt.	lieutenant
+Sgt.	sergeant
+St.	saint
+Jr.	junior
+Maj.	major
+Hon.	honorable
+Gov.	governor
+Capt.	captain
+Esq.	esquire
+Gen.	general
+Ltd.	limited
+Rev.	reverend
+Col.	colonel
+Mt.	mount
+Ft.	fort
+etc.	et cetera

scripts/dataset_processing/tts/preprocess_audio.py

@@ -0,0 +1,280 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to preprocess audio before TTS model training.
+
+It can be configured to do several processing steps such as silence trimming, volume normalization,
+and duration filtering.
+
+These can be done separately through multiple executions of the script, or all at once to avoid saving
+too many copies of the same audio.
+
+Most of these can also be done by the TTS data loader at training time, but doing them ahead of time
+lets us implement more complex processing, validate the correctness of the output, and save on compute time.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/preprocess_audio.py \
+    --input_manifest="<data_root_path>/manifest.json" \
+    --output_manifest="<data_root_path>/manifest_processed.json" \
+    --input_audio_dir="<data_root_path>/audio" \
+    --output_audio_dir="<data_root_path>/audio_processed" \
+    --num_workers=1 \
+    --trim_config_path="<nemo_root_path>/examples/tts/conf/trim/energy.yaml" \
+    --output_sample_rate=22050 \
+    --output_format=flac \
+    --volume_level=0.95 \
+    --min_duration=0.5 \
+    --max_duration=20.0 \
+    --filter_file="filtered.txt"
+"""
+
+import argparse
+import os
+from pathlib import Path
+from typing import Tuple
+
+import librosa
+import soundfile as sf
+from joblib import Parallel, delayed
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+from nemo.collections.tts.parts.preprocessing.audio_trimming import AudioTrimmer
+from nemo.collections.tts.parts.utils.tts_dataset_utils import get_abs_rel_paths, normalize_volume
+from nemo.core.classes.common import safe_instantiate
+from nemo.utils import logging
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute speaker level pitch statistics.",
+    )
+    parser.add_argument(
+        "--input_manifest",
+        required=True,
+        type=Path,
+        help="Path to input training manifest.",
+    )
+    parser.add_argument(
+        "--input_audio_dir",
+        required=True,
+        type=Path,
+        help="Path to base directory with audio files.",
+    )
+    parser.add_argument(
+        "--output_manifest",
+        required=True,
+        type=Path,
+        help="Path to output training manifest with processed audio.",
+    )
+    parser.add_argument(
+        "--output_audio_dir",
+        required=True,
+        type=Path,
+        help="Path to output directory for audio files.",
+    )
+    parser.add_argument(
+        "--overwrite_audio",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to reprocess and overwrite existing audio files in output_audio_dir.",
+    )
+    parser.add_argument(
+        "--overwrite_manifest",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output manifest file if it exists.",
+    )
+    parser.add_argument(
+        "--num_workers", default=1, type=int, help="Number of parallel threads to use. If -1 all CPUs are used."
+    )
+    parser.add_argument(
+        "--trim_config_path",
+        required=False,
+        type=Path,
+        help="Path to config file for nemo.collections.tts.data.audio_trimming.AudioTrimmer",
+    )
+    parser.add_argument(
+        "--max_entries", default=0, type=int, help="If provided, maximum number of entries in the manifest to process."
+    )
+    parser.add_argument(
+        "--output_sample_rate", default=0, type=int, help="If provided, rate to resample the audio to."
+    )
+    parser.add_argument(
+        "--output_format",
+        default="wav",
+        type=str,
+        help="If provided, format output audio will be saved as. If not provided, will keep original format.",
+    )
+    parser.add_argument(
+        "--volume_level", default=0.0, type=float, help="If provided, peak volume to normalize audio to."
+    )
+    parser.add_argument(
+        "--min_duration", default=0.0, type=float, help="If provided, filter out utterances shorter than min_duration."
+    )
+    parser.add_argument(
+        "--max_duration", default=0.0, type=float, help="If provided, filter out utterances longer than max_duration."
+    )
+    parser.add_argument(
+        "--filter_file",
+        required=False,
+        type=Path,
+        help="If provided, output filter_file will contain list of " "utterances filtered out.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def _process_entry(
+    entry: dict,
+    input_audio_dir: Path,
+    output_audio_dir: Path,
+    overwrite_audio: bool,
+    audio_trimmer: AudioTrimmer,
+    output_sample_rate: int,
+    output_format: str,
+    volume_level: float,
+) -> Tuple[dict, float, float]:
+    audio_filepath = Path(entry["audio_filepath"])
+
+    audio_path, audio_path_rel = get_abs_rel_paths(input_path=audio_filepath, base_path=input_audio_dir)
+
+    if not output_format:
+        output_format = audio_path.suffix
+
+    output_path = output_audio_dir / audio_path_rel
+    output_path = output_path.with_suffix(output_format)
+    output_path.parent.mkdir(exist_ok=True, parents=True)
+
+    if output_path.exists() and not overwrite_audio:
+        original_duration = librosa.get_duration(path=audio_path)
+        output_duration = librosa.get_duration(path=output_path)
+    else:
+        audio, sample_rate = librosa.load(audio_path, sr=None)
+        original_duration = librosa.get_duration(y=audio, sr=sample_rate)
+        if audio_trimmer is not None:
+            audio, start_i, end_i = audio_trimmer.trim_audio(
+                audio=audio, sample_rate=int(sample_rate), audio_id=str(audio_path)
+            )
+
+        if output_sample_rate:
+            audio = librosa.resample(y=audio, orig_sr=sample_rate, target_sr=output_sample_rate)
+            sample_rate = output_sample_rate
+
+        if volume_level:
+            audio = normalize_volume(audio, volume_level=volume_level)
+
+        if audio.size > 0:
+            sf.write(file=output_path, data=audio, samplerate=sample_rate)
+            output_duration = librosa.get_duration(y=audio, sr=sample_rate)
+        else:
+            output_duration = 0.0
+
+    entry["duration"] = round(output_duration, 2)
+
+    if os.path.isabs(audio_filepath):
+        entry["audio_filepath"] = str(output_path)
+    else:
+        output_filepath = audio_path_rel.with_suffix(output_format)
+        entry["audio_filepath"] = str(output_filepath)
+
+    return entry, original_duration, output_duration
+
+
+def main():
+    args = get_args()
+
+    input_manifest_path = args.input_manifest
+    output_manifest_path = args.output_manifest
+    input_audio_dir = args.input_audio_dir
+    output_audio_dir = args.output_audio_dir
+    overwrite_audio = args.overwrite_audio
+    overwrite_manifest = args.overwrite_manifest
+    num_workers = args.num_workers
+    max_entries = args.max_entries
+    output_sample_rate = args.output_sample_rate
+    output_format = args.output_format
+    volume_level = args.volume_level
+    min_duration = args.min_duration
+    max_duration = args.max_duration
+    filter_file = args.filter_file
+
+    if output_manifest_path.exists():
+        if overwrite_manifest:
+            print(f"Will overwrite existing manifest path: {output_manifest_path}")
+        else:
+            raise ValueError(f"Manifest path already exists: {output_manifest_path}")
+
+    if args.trim_config_path:
+        audio_trimmer_config = OmegaConf.load(args.trim_config_path)
+        audio_trimmer = safe_instantiate(audio_trimmer_config)
+    else:
+        audio_trimmer = None
+
+    if output_format:
+        if output_format.upper() not in sf.available_formats():
+            raise ValueError(f"Unsupported output audio format: {output_format}")
+        output_format = f".{output_format}"
+
+    output_audio_dir.mkdir(exist_ok=True, parents=True)
+
+    entries = read_manifest(input_manifest_path)
+    if max_entries:
+        entries = entries[:max_entries]
+
+    # 'threading' backend is required when parallelizing torch models.
+    job_outputs = Parallel(n_jobs=num_workers, backend='threading')(
+        delayed(_process_entry)(
+            entry=entry,
+            input_audio_dir=input_audio_dir,
+            output_audio_dir=output_audio_dir,
+            overwrite_audio=overwrite_audio,
+            audio_trimmer=audio_trimmer,
+            output_sample_rate=output_sample_rate,
+            output_format=output_format,
+            volume_level=volume_level,
+        )
+        for entry in tqdm(entries)
+    )
+
+    output_entries = []
+    filtered_entries = []
+    original_durations = 0.0
+    output_durations = 0.0
+    for output_entry, original_duration, output_duration in job_outputs:
+        original_durations += original_duration
+
+        if (
+            output_duration == 0.0
+            or (min_duration and output_duration < min_duration)
+            or (max_duration and output_duration > max_duration)
+        ):
+            if output_duration != original_duration:
+                output_entry["original_duration"] = original_duration
+            filtered_entries.append(output_entry)
+            continue
+
+        output_durations += output_duration
+        output_entries.append(output_entry)
+
+    write_manifest(output_path=output_manifest_path, target_manifest=output_entries, ensure_ascii=False)
+    if filter_file:
+        write_manifest(output_path=str(filter_file), target_manifest=filtered_entries, ensure_ascii=False)
+
+    logging.info(f"Duration of original audio: {original_durations / 3600:.2f} hours")
+    logging.info(f"Duration of processed audio: {output_durations / 3600:.2f} hours")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/preprocess_text.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to preprocess text before TTS model training. This is needed mainly for text normalization,
+which is slow to rerun during training.
+
+The output manifest will be the same as the input manifest but with final text stored in the 'normalized_text' field.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/preprocess_text.py \
+    --input_manifest="<data_root_path>/manifest.json" \
+    --output_manifest="<data_root_path>/manifest_processed.json" \
+    --normalizer_config_path="<nemo_root_path>/examples/tts/conf/text/normalizer_en.yaml" \
+    --lower_case \
+    --num_workers=4 \
+    --joblib_batch_size=16
+"""
+
+import argparse
+from pathlib import Path
+
+from joblib import Parallel, delayed
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.core.classes.common import safe_instantiate
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Process and normalize text data.",
+    )
+    parser.add_argument(
+        "--input_manifest",
+        required=True,
+        type=Path,
+        help="Path to input training manifest.",
+    )
+    parser.add_argument(
+        "--output_manifest",
+        required=True,
+        type=Path,
+        help="Path to output training manifest with processed text.",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output manifest file if it exists.",
+    )
+    parser.add_argument(
+        "--text_key",
+        default="text",
+        type=str,
+        help="Input text field to normalize.",
+    )
+    parser.add_argument(
+        "--normalized_text_key",
+        default="normalized_text",
+        type=str,
+        help="Output field to save normalized text to.",
+    )
+    parser.add_argument(
+        "--lower_case",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to convert the final text to lower case.",
+    )
+    parser.add_argument(
+        "--normalizer_config_path",
+        required=False,
+        type=Path,
+        help="Path to config file for nemo_text_processing.text_normalization.normalize.Normalizer.",
+    )
+    parser.add_argument(
+        "--num_workers", default=1, type=int, help="Number of parallel threads to use. If -1 all CPUs are used."
+    )
+    parser.add_argument(
+        "--joblib_batch_size", type=int, help="Batch size for joblib workers. Defaults to 'auto' if not provided."
+    )
+    parser.add_argument(
+        "--max_entries", default=0, type=int, help="If provided, maximum number of entries in the manifest to process."
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def _process_entry(
+    entry: dict,
+    normalizer: Normalizer,
+    text_key: str,
+    normalized_text_key: str,
+    lower_case: bool,
+    lower_case_norm: bool,
+) -> dict:
+    text = entry[text_key]
+
+    if normalizer is not None:
+        if lower_case_norm:
+            text = text.lower()
+        text = normalizer.normalize(text, punct_pre_process=True, punct_post_process=True)
+
+    if lower_case:
+        text = text.lower()
+
+    entry[normalized_text_key] = text
+
+    return entry
+
+
+def main():
+    args = get_args()
+
+    input_manifest_path = args.input_manifest
+    output_manifest_path = args.output_manifest
+    text_key = args.text_key
+    normalized_text_key = args.normalized_text_key
+    lower_case = args.lower_case
+    num_workers = args.num_workers
+    batch_size = args.joblib_batch_size
+    max_entries = args.max_entries
+    overwrite = args.overwrite
+
+    if output_manifest_path.exists():
+        if overwrite:
+            print(f"Will overwrite existing manifest path: {output_manifest_path}")
+        else:
+            raise ValueError(f"Manifest path already exists: {output_manifest_path}")
+
+    if args.normalizer_config_path:
+        normalizer_config = OmegaConf.load(args.normalizer_config_path)
+        normalizer = safe_instantiate(normalizer_config)
+        lower_case_norm = normalizer.input_case == "lower_cased"
+    else:
+        normalizer = None
+        lower_case_norm = False
+
+    entries = read_manifest(input_manifest_path)
+    if max_entries:
+        entries = entries[:max_entries]
+
+    if not batch_size:
+        batch_size = 'auto'
+
+    output_entries = Parallel(n_jobs=num_workers, batch_size=batch_size)(
+        delayed(_process_entry)(
+            entry=entry,
+            normalizer=normalizer,
+            text_key=text_key,
+            normalized_text_key=normalized_text_key,
+            lower_case=lower_case,
+            lower_case_norm=lower_case_norm,
+        )
+        for entry in tqdm(entries)
+    )
+
+    write_manifest(output_path=output_manifest_path, target_manifest=output_entries, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/resynthesize_dataset.py

@@ -0,0 +1,252 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is a helper for resynthesizing TTS dataset using a pretrained text-to-spectrogram model.
+Goal of resynthesis (as opposed to text-to-speech) is to use the largest amount of ground-truth features from existing speech data.
+For example, for resynthesis we want to have the same pitch and durations instead of ones predicted by the model.
+The results are to be used for some other task: vocoder finetuning, spectrogram enhancer training, etc.
+
+Let's say we have the following toy dataset:
+/dataset/manifest.json
+/dataset/1/foo.wav
+/dataset/2/bar.wav
+/dataset/sup_data/pitch/1_foo.pt
+/dataset/sup_data/pitch/2_bar.pt
+
+manifest.json has two entries for "/dataset/1/foo.wav" and "/dataset/2/bar.wav"
+(sup_data folder contains pitch files precomputed during training a FastPitch model on this dataset.)
+(If you lost your sup_data - don't worry, we use TTSDataset class so they would be created on-the-fly)
+
+Our script call is
+$ python scripts/dataset_processing/tts/resynthesize_dataset.py \
+    --model-path ./models/fastpitch/multi_spk/FastPitch--val_loss\=1.4473-epoch\=209.ckpt \
+    --input-json-manifest "/dataset/manifest.json" \
+    --input-sup-data-path "/dataset/sup_data/" \
+    --output-folder "/output/" \
+    --device "cuda:0" \
+    --batch-size 1 \
+    --num-workers 1
+
+Then we get output dataset with following directory structure:
+/output/manifest_mel.json
+/output/mels/foo.npy
+/output/mels/foo_gt.npy
+/output/mels/bar.npy
+/output/mels/bar_gt.npy
+
+/output/manifest_mel.json has the same entries as /dataset/manifest.json but with new fields for spectrograms.
+"mel_filepath" is path to the resynthesized spectrogram .npy, "mel_gt_filepath" is path to ground-truth spectrogram .npy
+
+The output structure is similar to generate_mels.py script for compatibility reasons.
+"""
+
+import argparse
+import itertools
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterable, Iterator, List
+
+import numpy as np
+import torch
+from omegaconf import DictConfig, OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+from nemo.collections.tts.models import FastPitchModel
+from nemo.collections.tts.models.base import SpectrogramGenerator
+from nemo.collections.tts.parts.utils.helpers import process_batch, to_device_recursive
+
+
+def chunks(iterable: Iterable, size: int) -> Iterator[List]:
+    # chunks([1, 2, 3, 4, 5], size=2) -> [[1, 2], [3, 4], [5]]
+    # assumes iterable does not have any `None`s
+    args = [iter(iterable)] * size
+    for chunk in itertools.zip_longest(*args, fillvalue=None):
+        chunk = list(item for item in chunk if item is not None)
+        if chunk:
+            yield chunk
+
+
+def load_model(path: Path, device: torch.device) -> SpectrogramGenerator:
+    model = None
+    if path.suffix == ".nemo":
+        model = SpectrogramGenerator.restore_from(path, map_location=device)
+    elif path.suffix == ".ckpt":
+        model = SpectrogramGenerator.load_from_checkpoint(path, map_location=device)
+    else:
+        raise ValueError(f"Unknown checkpoint type {path.suffix} ({path})")
+
+    return model.eval().to(device)
+
+
+@dataclass
+class TTSDatasetResynthesizer:
+    """
+    Reuses internals of a SpectrogramGenerator to resynthesize dataset using ground truth features.
+    Default setup is FastPitch with learned alignment.
+    If your use case requires different setup, you can either contribute to this script or subclass this class.
+    """
+
+    model: SpectrogramGenerator
+    device: torch.device
+
+    @torch.no_grad()
+    def resynthesize_batch(self, batch: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Resynthesizes a single batch.
+        Takes a dict with main data and sup data.
+        Outputs a dict with model outputs.
+        """
+        if not isinstance(self.model, FastPitchModel):
+            raise NotImplementedError(
+                "This script supports only FastPitch. Please implement resynthesizing routine for your desired model."
+            )
+
+        batch = to_device_recursive(batch, self.device)
+
+        mels, mel_lens = self.model.preprocessor(input_signal=batch["audio"], length=batch["audio_lens"])
+
+        reference_audio = batch.get("reference_audio", None)
+        reference_audio_len = batch.get("reference_audio_lens", None)
+        reference_spec, reference_spec_len = None, None
+        if reference_audio is not None:
+            reference_spec, reference_spec_len = self.model.preprocessor(
+                input_signal=reference_audio, length=reference_audio_len
+            )
+
+        outputs_tuple = self.model.forward(
+            text=batch["text"],
+            durs=None,
+            pitch=batch["pitch"],
+            speaker=batch.get("speaker"),
+            pace=1.0,
+            spec=mels,
+            attn_prior=batch.get("attn_prior"),
+            mel_lens=mel_lens,
+            input_lens=batch["text_lens"],
+            reference_spec=reference_spec,
+            reference_spec_lens=reference_spec_len,
+        )
+        names = self.model.fastpitch.output_types.keys()
+        return {"spec": mels, "mel_lens": mel_lens, **dict(zip(names, outputs_tuple))}
+
+    def resynthesized_batches(self) -> Iterator[Dict[str, Any]]:
+        """
+        Returns a generator of resynthesized batches.
+        Each returned batch is a dict containing main data, sup data, and model output
+        """
+        self.model.setup_training_data(self.model._cfg["train_ds"])
+
+        for batch_tuple in iter(self.model._train_dl):
+            batch = process_batch(batch_tuple, sup_data_types_set=self.model._train_dl.dataset.sup_data_types)
+            yield self.resynthesize_batch(batch)
+
+
+def prepare_paired_mel_spectrograms(
+    model_path: Path,
+    input_json_manifest: Path,
+    input_sup_data_path: Path,
+    output_folder: Path,
+    device: torch.device,
+    batch_size: int,
+    num_workers: int,
+):
+    model = load_model(model_path, device)
+
+    dataset_config_overrides = {
+        "dataset": {
+            "manifest_filepath": str(input_json_manifest.absolute()),
+            "sup_data_path": str(input_sup_data_path.absolute()),
+        },
+        "dataloader_params": {"batch_size": batch_size, "num_workers": num_workers, "shuffle": False},
+    }
+    model._cfg.train_ds = OmegaConf.merge(model._cfg.train_ds, DictConfig(dataset_config_overrides))
+    resynthesizer = TTSDatasetResynthesizer(model, device)
+
+    input_manifest = read_manifest(input_json_manifest)
+
+    output_manifest = []
+    output_json_manifest = output_folder / f"{input_json_manifest.stem}_mel{input_json_manifest.suffix}"
+    output_mels_folder = output_folder / "mels"
+    output_mels_folder.mkdir(exist_ok=True, parents=True)
+    for batch, batch_manifest in tqdm(
+        zip(resynthesizer.resynthesized_batches(), chunks(input_manifest, size=batch_size)), desc="Batch #"
+    ):
+        pred_mels = batch["spect"].cpu()  # key from fastpitch.output_types
+        true_mels = batch["spec"].cpu()  # key from code above
+        mel_lens = batch["mel_lens"].cpu().flatten()  # key from code above
+
+        for i, (manifest_entry, length) in enumerate(zip(batch_manifest, mel_lens.tolist())):
+            print(manifest_entry["audio_filepath"])
+            filename = Path(manifest_entry["audio_filepath"]).stem
+
+            # note that lengths match
+            pred_mel = pred_mels[i, :, :length].clone().numpy()
+            true_mel = true_mels[i, :, :length].clone().numpy()
+
+            pred_mel_path = output_mels_folder / f"{filename}.npy"
+            true_mel_path = output_mels_folder / f"{filename}_gt.npy"
+
+            np.save(pred_mel_path, pred_mel)
+            np.save(true_mel_path, true_mel)
+
+            new_manifest_entry = {
+                **manifest_entry,
+                "mel_filepath": str(pred_mel_path),
+                "mel_gt_filepath": str(true_mel_path),
+            }
+            output_manifest.append(new_manifest_entry)
+
+    write_manifest(output_json_manifest, output_manifest, ensure_ascii=False)
+
+
+def argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Resynthesize TTS dataset using a pretrained text-to-spectrogram model",
+    )
+    parser.add_argument(
+        "--model-path",
+        required=True,
+        type=Path,
+        help="Path to a checkpoint (either .nemo or .ckpt)",
+    )
+    parser.add_argument(
+        "--input-json-manifest",
+        required=True,
+        type=Path,
+        help="Path to the input JSON manifest",
+    )
+    parser.add_argument(
+        "--input-sup-data-path",
+        required=True,
+        type=Path,
+        help="sup_data_path for the JSON manifest",
+    )
+    parser.add_argument(
+        "--output-folder",
+        required=True,
+        type=Path,
+        help="Path to the output folder. Will contain updated manifest and mels/ folder with spectrograms in .npy files",
+    )
+    parser.add_argument("--device", required=True, type=torch.device, help="Device ('cpu', 'cuda:0', ...)")
+    parser.add_argument("--batch-size", required=True, type=int, help="Batch size in the DataLoader")
+    parser.add_argument("--num-workers", required=True, type=int, help="Num workers in the DataLoader")
+    return parser
+
+
+if __name__ == "__main__":
+    arguments = argument_parser().parse_args()
+    prepare_paired_mel_spectrograms(**vars(arguments))

scripts/dataset_processing/tts/sfbilingual/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+phoneme_dict_path: "scripts/tts_dataset_files/zh/24finals/pinyin_dict_nv_22.10.txt"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: true
+  trim_top_db: 50
+  trim_frame_length: 1024
+  trim_hop_length: 256
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: zh
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.ChinesePhonemesTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.zh_cn_pinyin.ChineseG2p
+      phoneme_dict: ${phoneme_dict_path}
+      word_segmenter: jieba # Only jieba is supported now.

scripts/dataset_processing/tts/sfbilingual/get_data.py

@@ -0,0 +1,137 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import random
+import subprocess
+from pathlib import Path
+
+import numpy as np
+from opencc import OpenCC
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description='Prepare SF_bilingual dataset and create manifests with predefined split'
+    )
+
+    parser.add_argument(
+        "--data-root",
+        type=Path,
+        help="where the dataset will reside",
+        default="./DataChinese/sf_bilingual_speech_zh_en_vv1/SF_bilingual/",
+    )
+    parser.add_argument(
+        "--manifests-path", type=Path, help="where the resulting manifests files will reside", default="./"
+    )
+    parser.add_argument("--val-size", default=0.01, type=float, help="eval set split")
+    parser.add_argument("--test-size", default=0.01, type=float, help="test set split")
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def __process_transcript(file_path: str):
+    # Create zh-TW to zh-simplify converter
+    cc = OpenCC('t2s')
+    # Create normalizer
+    text_normalizer = Normalizer(
+        lang="zh",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(file_path / "cache_dir"),
+    )
+    text_normalizer_call_kwargs = {"punct_pre_process": True, "punct_post_process": True}
+    normalizer_call = lambda x: text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+    entries = []
+    i = 0
+    with open(file_path / "text_SF.txt", encoding="utf-8") as fin:
+        for line in fin:
+            content = line.split()
+            wav_name, text = content[0], "".join(content[1:])
+            wav_name = wav_name.replace(u'\ufeff', '')
+            # WAR: change DL to SF, e.g. real wave file com_SF_ce2727.wav, wav name in text_SF
+            # com_DL_ce2727. It would be fixed through the dataset in the future.
+            wav_name = wav_name.replace('DL', 'SF')
+            wav_file = file_path / "wavs" / (wav_name + ".wav")
+            assert os.path.exists(wav_file), f"{wav_file} not found!"
+            duration = subprocess.check_output(["soxi", "-D", str(wav_file)])
+            simplified_text = cc.convert(text)
+            normalized_text = normalizer_call(simplified_text)
+            entry = {
+                'audio_filepath': os.path.abspath(wav_file),
+                'duration': float(duration),
+                'text': text,
+                'normalized_text': normalized_text,
+            }
+
+            i += 1
+            entries.append(entry)
+    return entries
+
+
+def __process_data(dataset_path, val_size, test_size, seed_for_ds_split, manifests_dir):
+    entries = __process_transcript(dataset_path)
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+
+    train_size = 1.0 - val_size - test_size
+    train_entries, validate_entries, test_entries = np.split(
+        entries, [int(len(entries) * train_size), int(len(entries) * (train_size + val_size))]
+    )
+
+    assert len(train_entries) > 0, "Not enough data for train, val and test"
+
+    def save(p, data):
+        with open(p, 'w') as f:
+            for d in data:
+                f.write(json.dumps(d) + '\n')
+
+    save(manifests_dir / "train_manifest.json", train_entries)
+    save(manifests_dir / "val_manifest.json", validate_entries)
+    save(manifests_dir / "test_manifest.json", test_entries)
+
+
+def main():
+    args = get_args()
+    dataset_root = args.data_root
+    dataset_root.mkdir(parents=True, exist_ok=True)
+    __process_data(
+        dataset_root,
+        args.val_size,
+        args.test_size,
+        args.seed_for_ds_split,
+        args.manifests_path,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/thorsten_neutral/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,44 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: true
+  trim_top_db: 50
+  trim_frame_length: ${dataset.win_length}
+  trim_hop_length: ${dataset.hop_length}
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: de
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.GermanCharsTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true

scripts/dataset_processing/tts/thorsten_neutral/get_data.py

@@ -0,0 +1,274 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to generate JSON manifests for mel-generator model training. The usage is below.
+
+$ python scripts/dataset_processing/tts/thorsten_neutral/get_data.py \
+    --data-root ~/experiments/thorsten_neutral \
+    --manifests-root ~/experiments/thorsten_neutral \
+    --data-version "22_10" \
+    --min-duration 0.1 \
+    --normalize-text
+"""
+
+import argparse
+import json
+import random
+import shutil
+import subprocess
+import urllib.request
+from pathlib import Path
+
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+from nemo.utils import logging
+
+# Thorsten Müller published two neural voice datasets, 21.02 and 22.10.
+THORSTEN_NEUTRAL = {
+    "21_02": {
+        "url": "https://zenodo.org/record/5525342/files/thorsten-neutral_v03.tgz?download=1",
+        "dir_name": "thorsten-de_v03",
+        "metadata": ["metadata.csv"],
+    },
+    "22_10": {
+        "url": "https://zenodo.org/record/7265581/files/ThorstenVoice-Dataset_2022.10.zip?download=1",
+        "dir_name": "ThorstenVoice-Dataset_2022.10",
+        "metadata": ["metadata_train.csv", "metadata_dev.csv", "metadata_test.csv"],
+    },
+}
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Download Thorsten Müller's neutral voice dataset and create manifests with predefined split. "
+        "Thorsten Müller published two neural voice datasets, 21.02 and 22.10, where 22.10 provides better "
+        "audio quality. Please choose one of the two for your TTS models. Details about the dataset are "
+        "in https://github.com/thorstenMueller/Thorsten-Voice.",
+    )
+    parser.add_argument("--data-root", required=True, type=Path, help="where the resulting dataset will reside.")
+    parser.add_argument("--manifests-root", required=True, type=Path, help="where the manifests files will reside.")
+    parser.add_argument("--data-version", default="22_10", choices=["21_02", "22_10"], type=str)
+    parser.add_argument("--min-duration", default=0.1, type=float)
+    parser.add_argument("--max-duration", default=float('inf'), type=float)
+    parser.add_argument("--val-size", default=100, type=int)
+    parser.add_argument("--test-size", default=100, type=int)
+    parser.add_argument(
+        "--num-workers",
+        default=-1,
+        type=int,
+        help="Specify the max number of concurrent Python worker processes. "
+        "If -1 all CPUs are used. If 1 no parallel computing is used.",
+    )
+    parser.add_argument(
+        "--normalize-text",
+        default=False,
+        action='store_true',
+        help="Normalize original text and add a new entry 'normalized_text' to .json file if True.",
+    )
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        logging.info(f"Downloading data: {source_url} --> {destination_path}")
+        tmp_file_path = destination_path.with_suffix(".tmp")
+        urllib.request.urlretrieve(source_url, filename=tmp_file_path)
+        tmp_file_path.rename(destination_path)
+    else:
+        logging.info(f"Skipped downloading data because it exists: {destination_path}")
+
+
+def __extract_file(filepath, data_dir):
+    logging.info(f"Unzipping data: {filepath} --> {data_dir}")
+    shutil.unpack_archive(filepath, data_dir)
+    logging.info(f"Unzipping data is complete: {filepath}.")
+
+
+def __save_json(json_file, dict_list):
+    logging.info(f"Saving JSON split to {json_file}.")
+    with open(json_file, "w") as f:
+        for d in dict_list:
+            f.write(json.dumps(d) + "\n")
+
+
+def __text_normalization(json_file, num_workers=-1):
+    text_normalizer_call_kwargs = {
+        "punct_pre_process": True,
+        "punct_post_process": True,
+    }
+    text_normalizer = Normalizer(
+        lang="de",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(json_file.parent / "cache_dir"),
+    )
+
+    def normalizer_call(x):
+        return text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+
+    def add_normalized_text(line_dict):
+        normalized_text = normalizer_call(line_dict["text"])
+        line_dict.update({"normalized_text": normalized_text})
+        return line_dict
+
+    logging.info(f"Normalizing text for {json_file}.")
+    with open(json_file, 'r', encoding='utf-8') as fjson:
+        lines = fjson.readlines()
+        # Note: you need to verify which backend works well on your cluster.
+        # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+        dict_list = Parallel(n_jobs=num_workers)(
+            delayed(add_normalized_text)(json.loads(line)) for line in tqdm(lines)
+        )
+
+    json_file_text_normed = json_file.parent / f"{json_file.stem}_text_normed{json_file.suffix}"
+    with open(json_file_text_normed, 'w', encoding="utf-8") as fjson_norm:
+        for dct in dict_list:
+            fjson_norm.write(json.dumps(dct) + "\n")
+    logging.info(f"Normalizing text is complete: {json_file} --> {json_file_text_normed}")
+
+
+def __process_data(
+    unzipped_dataset_path, metadata, min_duration, max_duration, val_size, test_size, seed_for_ds_split
+):
+    logging.info("Preparing JSON train/val/test splits.")
+
+    entries = list()
+    not_found_wavs = list()
+    wrong_duration_wavs = list()
+
+    for metadata_fname in metadata:
+        meta_file = unzipped_dataset_path / metadata_fname
+        with open(meta_file, 'r') as fmeta:
+            for line in tqdm(fmeta):
+                items = line.strip().split('|')
+                wav_file_stem, text = items[0], items[1]
+                wav_file = unzipped_dataset_path / "wavs" / f"{wav_file_stem}.wav"
+
+                # skip audios if they do not exist.
+                if not wav_file.exists():
+                    not_found_wavs.append(wav_file)
+                    logging.warning(f"Skipping {wav_file}: it is not found.")
+                    continue
+
+                # skip audios if their duration is out of range.
+                duration = subprocess.check_output(["soxi", "-D", str(wav_file)])
+                duration = float(duration)
+                if min_duration <= duration <= max_duration:
+                    entry = {
+                        'audio_filepath': str(wav_file),
+                        'duration': duration,
+                        'text': text,
+                    }
+                    entries.append(entry)
+                elif duration < min_duration:
+                    wrong_duration_wavs.append(wav_file)
+                    logging.warning(f"Skipping {wav_file}: it is too short, less than {min_duration} seconds.")
+                    continue
+                else:
+                    wrong_duration_wavs.append(wav_file)
+                    logging.warning(f"Skipping {wav_file}: it is too long, greater than {max_duration} seconds.")
+                    continue
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+    train_size = len(entries) - val_size - test_size
+    if train_size <= 0:
+        raise ValueError("Not enough data for the train split.")
+
+    logging.info("Preparing JSON train/val/test splits is complete.")
+    train, val, test = (
+        entries[:train_size],
+        entries[train_size : train_size + val_size],
+        entries[train_size + val_size :],
+    )
+
+    return train, val, test, not_found_wavs, wrong_duration_wavs
+
+
+def main():
+    args = get_args()
+    data_root = args.data_root
+    manifests_root = args.manifests_root
+    data_version = args.data_version
+
+    dataset_root = data_root / f"ThorstenVoice-Dataset-{data_version}"
+    dataset_root.mkdir(parents=True, exist_ok=True)
+
+    # download and extract dataset
+    dataset_url = THORSTEN_NEUTRAL[data_version]["url"]
+    zipped_dataset_path = dataset_root / Path(dataset_url).name.split("?")[0]
+    __maybe_download_file(dataset_url, zipped_dataset_path)
+    __extract_file(zipped_dataset_path, dataset_root)
+
+    # generate train/dev/test splits
+    unzipped_dataset_path = dataset_root / THORSTEN_NEUTRAL[data_version]["dir_name"]
+    entries_train, entries_val, entries_test, not_found_wavs, wrong_duration_wavs = __process_data(
+        unzipped_dataset_path=unzipped_dataset_path,
+        metadata=THORSTEN_NEUTRAL[data_version]["metadata"],
+        min_duration=args.min_duration,
+        max_duration=args.max_duration,
+        val_size=args.val_size,
+        test_size=args.test_size,
+        seed_for_ds_split=args.seed_for_ds_split,
+    )
+
+    # save json splits.
+    train_json = manifests_root / "train_manifest.json"
+    val_json = manifests_root / "val_manifest.json"
+    test_json = manifests_root / "test_manifest.json"
+    __save_json(train_json, entries_train)
+    __save_json(val_json, entries_val)
+    __save_json(test_json, entries_test)
+
+    # save skipped audios that are not found into a file.
+    if len(not_found_wavs) > 0:
+        skipped_not_found_file = manifests_root / "skipped_not_found_wavs.list"
+        with open(skipped_not_found_file, "w") as f_notfound:
+            for line in not_found_wavs:
+                f_notfound.write(f"{line}\n")
+
+    # save skipped audios that are too short or too long into a file.
+    if len(wrong_duration_wavs) > 0:
+        skipped_wrong_duration_file = manifests_root / "skipped_wrong_duration_wavs.list"
+        with open(skipped_wrong_duration_file, "w") as f_wrong_dur:
+            for line in wrong_duration_wavs:
+                f_wrong_dur.write(f"{line}\n")
+
+    # normalize text if requested. New json file, train_manifest_text_normed.json, will be generated.
+    if args.normalize_text:
+        __text_normalization(train_json, args.num_workers)
+        __text_normalization(val_json, args.num_workers)
+        __text_normalization(test_json, args.num_workers)
+
+
+if __name__ == "__main__":
+    main()