chore: import upstream snapshot with attribution

scripts/dataset_processing/tts/aishell3/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch", "speaker_id"]
+phoneme_dict_path: "scripts/tts_dataset_files/zh/24finals/pinyin_dict_nv_22.10.txt"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: true
+  trim_top_db: 50
+  trim_frame_length: 1024
+  trim_hop_length: 256
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: zh
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.ChinesePhonemesTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.zh_cn_pinyin.ChineseG2p
+      phoneme_dict: ${phoneme_dict_path}
+      word_segmenter: jieba # Only jieba is supported now.

scripts/dataset_processing/tts/aishell3/get_data.py

@@ -0,0 +1,176 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Disclaimer:
+#   Each user is responsible for checking the content of datasets and the applicable licenses and determining if suitable for the intended use.
+
+import argparse
+import json
+import os
+import random
+import subprocess
+import tarfile
+import urllib.request
+from pathlib import Path
+
+import numpy as np
+from nemo_text_processing.text_normalization.normalize import Normalizer
+from opencc import OpenCC
+
+from nemo.utils.tar_utils import safe_extract
+
+URL = "https://www.openslr.org/resources/93/data_aishell3.tgz"
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description='Prepare SF_bilingual dataset and create manifests with predefined split'
+    )
+
+    parser.add_argument(
+        "--data-root",
+        type=Path,
+        help="where the dataset will reside",
+        default="./DataChinese/sf_bilingual_speech_zh_en_vv1/SF_bilingual/",
+    )
+    parser.add_argument(
+        "--manifests-path", type=Path, help="where the resulting manifests files will reside", default="./"
+    )
+    parser.add_argument("--val-size", default=0.01, type=float, help="eval set split")
+    parser.add_argument("--test-size", default=0.01, type=float, help="test set split")
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_transcript(file_path: str):
+    # Create directory for processed wav files
+    Path(file_path / "processed").mkdir(parents=True, exist_ok=True)
+    # Create zh-TW to zh-simplify converter
+    cc = OpenCC('t2s')
+    # Create normalizer
+    text_normalizer = Normalizer(
+        lang="zh",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(file_path / "cache_dir"),
+    )
+    text_normalizer_call_kwargs = {"punct_pre_process": True, "punct_post_process": True}
+    normalizer_call = lambda x: text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+    entries = []
+    SPEAKER_LEN = 7
+
+    candidates = []
+    speakers = set()
+    with open(file_path / "train" / "content.txt", encoding="utf-8") as fin:
+        for line in fin:
+            content = line.split()
+            wav_name, text = content[0], "".join(content[1::2]) + "。"
+            wav_name = wav_name.replace(u'\ufeff', '')
+            speaker = wav_name[:SPEAKER_LEN]
+            speakers.add(speaker)
+            wav_file = file_path / "train" / "wav" / speaker / wav_name
+            assert os.path.exists(wav_file), f"{wav_file} not found!"
+            duration = subprocess.check_output(["soxi", "-D", str(wav_file)])
+            if float(duration) <= 3.0:  # filter out wav files shorter than 3 seconds
+                continue
+            processed_file = file_path / "processed" / wav_name
+            # convert wav to mono 22050HZ, 16 bit (as SFSpeech dataset)
+            subprocess.run(["sox", str(wav_file), "-r", "22050", "-c", "1", "-b", "16", str(processed_file)])
+            candidates.append((processed_file, duration, text, speaker))
+
+    # remapping the speakder to speaker_id (start from 1)
+    remapping = {}
+    for index, speaker in enumerate(sorted(speakers)):
+        remapping[speaker] = index + 1
+
+    for processed_file, duration, text, speaker in candidates:
+        simplified_text = cc.convert(text)
+        normalized_text = normalizer_call(simplified_text)
+        entry = {
+            'audio_filepath': os.path.abspath(processed_file),
+            'duration': float(duration),
+            'text': text,
+            'normalized_text': normalized_text,
+            'speaker_raw': speaker,
+            'speaker': remapping[speaker],
+        }
+
+        entries.append(entry)
+
+    return entries
+
+
+def __process_data(dataset_path, val_size, test_size, seed_for_ds_split, manifests_dir):
+    entries = __process_transcript(dataset_path)
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+
+    train_size = 1.0 - val_size - test_size
+    train_entries, validate_entries, test_entries = np.split(
+        entries, [int(len(entries) * train_size), int(len(entries) * (train_size + val_size))]
+    )
+
+    assert len(train_entries) > 0, "Not enough data for train, val and test"
+
+    def save(p, data):
+        with open(p, 'w') as f:
+            for d in data:
+                f.write(json.dumps(d) + '\n')
+
+    save(manifests_dir / "train_manifest.json", train_entries)
+    save(manifests_dir / "val_manifest.json", validate_entries)
+    save(manifests_dir / "test_manifest.json", test_entries)
+
+
+def main():
+    args = get_args()
+
+    tarred_data_path = args.data_root / "data_aishell3.tgz"
+
+    __maybe_download_file(URL, tarred_data_path)
+    __extract_file(str(tarred_data_path), str(args.data_root))
+
+    __process_data(
+        args.data_root,
+        args.val_size,
+        args.test_size,
+        args.seed_for_ds_split,
+        args.manifests_path,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/compute_feature_stats.py

@@ -0,0 +1,230 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to compute global and speaker-level feature statistics for a given TTS training manifest.
+
+This script should be run after compute_features.py as it loads the precomputed feature data.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/compute_feature_stats.py \
+    --feature_config_path=<nemo_root_path>/examples/tts/conf/features/feature_22050.yaml
+    --manifest_path=<data_root_path>/manifest1.json \
+    --manifest_path=<data_root_path>/manifest2.json \
+    --audio_dir=<data_root_path>/audio1 \
+    --audio_dir=<data_root_path>/audio2 \
+    --feature_dir=<data_root_path>/features1 \
+    --feature_dir=<data_root_path>/features2 \
+    --stats_path=<data_root_path>/feature_stats.json
+
+The output dictionary will contain the feature statistics for every speaker, as well as a "default" entry
+with the global statistics.
+
+For example:
+
+{
+    "default": {
+        "pitch_mean": 100.0,
+        "pitch_std": 50.0,
+        "energy_mean": 7.5,
+        "energy_std": 4.5
+    },
+    "speaker1": {
+        "pitch_mean": 105.0,
+        "pitch_std": 45.0,
+        "energy_mean": 7.0,
+        "energy_std": 5.0
+    },
+    "speaker2": {
+        "pitch_mean": 110.0,
+        "pitch_std": 30.0,
+        "energy_mean": 5.0,
+        "energy_std": 2.5
+    }
+}
+
+"""
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import List, Tuple
+
+import torch
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from nemo.core.classes.common import safe_instantiate
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute TTS feature statistics.",
+    )
+    parser.add_argument(
+        "--feature_config_path",
+        required=True,
+        type=Path,
+        help="Path to feature config file.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path(s) to training manifest.",
+    )
+    parser.add_argument(
+        "--audio_dir",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path(s) to base directory with audio data.",
+    )
+    parser.add_argument(
+        "--feature_dir",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path(s) to directory where feature data was stored.",
+    )
+    parser.add_argument(
+        "--feature_names",
+        default="pitch,energy",
+        type=str,
+        help="Comma separated list of features to process.",
+    )
+    parser.add_argument(
+        "--mask_field",
+        default="voiced_mask",
+        type=str,
+        help="If provided, stat computation will ignore non-masked frames.",
+    )
+    parser.add_argument(
+        "--stats_path",
+        default=Path("feature_stats.json"),
+        type=Path,
+        help="Path to output JSON file with dataset feature statistics.",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output stats file if it exists.",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def _compute_stats(values: List[torch.Tensor]) -> Tuple[float, float]:
+    values_tensor = torch.cat(values, dim=0)
+    mean = values_tensor.mean().item()
+    std = values_tensor.std(dim=0).item()
+    return mean, std
+
+
+def main():
+    args = get_args()
+
+    feature_config_path = args.feature_config_path
+    manifest_paths = args.manifest_path
+    audio_dirs = args.audio_dir
+    feature_dirs = args.feature_dir
+    feature_name_str = args.feature_names
+    mask_field = args.mask_field
+    stats_path = args.stats_path
+    overwrite = args.overwrite
+
+    if not (len(manifest_paths) == len(audio_dirs) == len(feature_dirs)):
+        raise ValueError(
+            f"Need same number of manifest, audio_dir, and feature_dir. Received: "
+            f"{len(manifest_paths)}, "
+            f"{len(audio_dirs)}, "
+            f"{len(feature_dirs)}"
+        )
+
+    for manifest_path, audio_dir, feature_dir in zip(manifest_paths, audio_dirs, feature_dirs):
+        if not manifest_path.exists():
+            raise ValueError(f"Manifest {manifest_path} does not exist.")
+
+        if not audio_dir.exists():
+            raise ValueError(f"Audio directory {audio_dir} does not exist.")
+
+        if not feature_dir.exists():
+            raise ValueError(
+                f"Feature directory {feature_dir} does not exist. "
+                f"Please check that the path is correct and that you ran compute_features.py"
+            )
+
+    if stats_path.exists():
+        if overwrite:
+            print(f"Will overwrite existing stats path: {stats_path}")
+        else:
+            raise ValueError(f"Stats path already exists: {stats_path}")
+
+    feature_config = OmegaConf.load(feature_config_path)
+    feature_config = safe_instantiate(feature_config)
+    featurizer_dict = feature_config.featurizers
+
+    print(f"Found featurizers for {list(featurizer_dict.keys())}.")
+    featurizers = featurizer_dict.values()
+
+    feature_names = feature_name_str.split(",")
+    # For each feature, we have a dictionary mapping speaker IDs to a list containing all features
+    # for that speaker
+    feature_stats = {name: defaultdict(list) for name in feature_names}
+
+    for manifest_path, audio_dir, feature_dir in zip(manifest_paths, audio_dirs, feature_dirs):
+        entries = read_manifest(manifest_path)
+
+        for entry in tqdm(entries):
+            speaker = entry["speaker"]
+
+            entry_dict = {}
+            for featurizer in featurizers:
+                feature_dict = featurizer.load(manifest_entry=entry, audio_dir=audio_dir, feature_dir=feature_dir)
+                entry_dict.update(feature_dict)
+
+            if mask_field:
+                mask = entry_dict[mask_field]
+            else:
+                mask = None
+
+            for feature_name in feature_names:
+                values = entry_dict[feature_name]
+                if mask is not None:
+                    values = values[mask]
+
+                feature_stat_dict = feature_stats[feature_name]
+                feature_stat_dict["default"].append(values)
+                feature_stat_dict[speaker].append(values)
+
+    stat_dict = defaultdict(dict)
+    for feature_name in feature_names:
+        mean_key = f"{feature_name}_mean"
+        std_key = f"{feature_name}_std"
+        feature_stat_dict = feature_stats[feature_name]
+        for speaker_id, values in feature_stat_dict.items():
+            speaker_mean, speaker_std = _compute_stats(values)
+            stat_dict[speaker_id][mean_key] = speaker_mean
+            stat_dict[speaker_id][std_key] = speaker_std
+
+    with open(stats_path, 'w', encoding="utf-8") as stats_f:
+        json.dump(stat_dict, stats_f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/compute_features.py

@@ -0,0 +1,130 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script computes features for TTS models prior to training, such as pitch and energy.
+The resulting features will be stored in the provided 'feature_dir'.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/compute_features.py \
+    --feature_config_path=<nemo_root_path>/examples/tts/conf/features/feature_22050.yaml \
+    --manifest_path=<data_root_path>/manifest.json \
+    --audio_dir=<data_root_path>/audio \
+    --feature_dir=<data_root_path>/features \
+    --overwrite \
+    --num_workers=1
+"""
+
+import argparse
+from pathlib import Path
+
+from joblib import Parallel, delayed
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from nemo.core.classes.common import safe_instantiate
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute TTS features.",
+    )
+    parser.add_argument(
+        "--feature_config_path",
+        required=True,
+        type=Path,
+        help="Path to feature config file.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        help="Path to training manifest.",
+    )
+    parser.add_argument(
+        "--audio_dir",
+        required=True,
+        type=Path,
+        help="Path to base directory with audio data.",
+    )
+    parser.add_argument(
+        "--feature_dir",
+        required=True,
+        type=Path,
+        help="Path to directory where feature data will be stored.",
+    )
+    parser.add_argument(
+        "--dedupe_files",
+        action=argparse.BooleanOptionalAction,
+        help="If given, will only process the first manifest entry found for each audio file.",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite existing feature files.",
+    )
+    parser.add_argument(
+        "--num_workers", default=1, type=int, help="Number of parallel threads to use. If -1 all CPUs are used."
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+    feature_config_path = args.feature_config_path
+    manifest_path = args.manifest_path
+    audio_dir = args.audio_dir
+    feature_dir = args.feature_dir
+    dedupe_files = args.dedupe_files
+    overwrite = args.overwrite
+    num_workers = args.num_workers
+
+    if not manifest_path.exists():
+        raise ValueError(f"Manifest {manifest_path} does not exist.")
+
+    if not audio_dir.exists():
+        raise ValueError(f"Audio directory {audio_dir} does not exist.")
+
+    feature_config = OmegaConf.load(feature_config_path)
+    feature_config = safe_instantiate(feature_config)
+    featurizers = feature_config.featurizers
+
+    entries = read_manifest(manifest_path)
+
+    if dedupe_files:
+        final_entries = []
+        audio_filepath_set = set()
+        for entry in entries:
+            audio_filepath = entry["audio_filepath"]
+            if audio_filepath in audio_filepath_set:
+                continue
+            final_entries.append(entry)
+            audio_filepath_set.add(audio_filepath)
+        entries = final_entries
+
+    for feature_name, featurizer in featurizers.items():
+        print(f"Computing: {feature_name}")
+        Parallel(n_jobs=num_workers)(
+            delayed(featurizer.save)(
+                manifest_entry=entry, audio_dir=audio_dir, feature_dir=feature_dir, overwrite=overwrite
+            )
+            for entry in tqdm(entries)
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/compute_speaker_stats.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to compute speaker-level statistics, such as pitch mean & standard deviation, for a given
+TTS training manifest.
+
+This script should be run after extract_sup_data.py as it uses the precomputed supplemental features.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/compute_speaker_stats.py \
+    --manifest_path=<data_root_path>/fastpitch_manifest.json \
+    --sup_data_path=<data_root_path>/sup_data \
+    --pitch_stats_path=<data_root_path>/pitch_stats.json
+"""
+
+import argparse
+import json
+import os
+from collections import defaultdict
+from pathlib import Path
+from typing import List, Tuple
+
+import torch
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from nemo.collections.tts.parts.utils.tts_dataset_utils import get_base_dir
+from nemo.collections.tts.torch.tts_data_types import Pitch
+from nemo.utils import logging
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute speaker level pitch statistics.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        help="Path to training manifest.",
+    )
+    parser.add_argument(
+        "--sup_data_path",
+        default=Path("sup_data"),
+        type=Path,
+        help="Path to base directory with supplementary data.",
+    )
+    parser.add_argument(
+        "--pitch_stats_path",
+        default=Path("pitch_stats.json"),
+        type=Path,
+        help="Path to output JSON file with speaker pitch statistics.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def _compute_stats(values: List[torch.Tensor]) -> Tuple[float, float]:
+    values_tensor = torch.cat(values, dim=0)
+    mean = values_tensor.mean().item()
+    std = values_tensor.std(dim=0).item()
+    return mean, std
+
+
+def _get_sup_data_filepath(manifest_entry: dict, audio_dir: Path, sup_data_dir: Path) -> Path:
+    """
+    Get the absolute path of a supplementary data type for the input manifest entry.
+
+    Example: audio_filepath "<audio_dir>/speaker1/audio1.wav" becomes "<sup_data_dir>/speaker1_audio1.pt"
+
+    Args:
+        manifest_entry: Manifest entry dictionary.
+        audio_dir: base directory where audio is stored.
+        sup_data_dir: base directory where supplementary data is stored.
+
+    Returns:
+        Path to the supplementary data file.
+    """
+    audio_path = Path(manifest_entry["audio_filepath"])
+    rel_audio_path = audio_path.relative_to(audio_dir)
+    rel_sup_data_path = rel_audio_path.with_suffix(".pt")
+    sup_data_filename = str(rel_sup_data_path).replace(os.sep, "_")
+    sup_data_filepath = sup_data_dir / sup_data_filename
+    return sup_data_filepath
+
+
+def main():
+    args = get_args()
+    manifest_path = args.manifest_path
+    sup_data_path = args.sup_data_path
+    pitch_stats_path = args.pitch_stats_path
+
+    pitch_data_path = Path(os.path.join(sup_data_path, Pitch.name))
+    if not os.path.exists(pitch_data_path):
+        raise ValueError(
+            f"Pitch directory {pitch_data_path} does not exist. Make sure 'sup_data_path' is correct "
+            f"and that you have computed the pitch using extract_sup_data.py"
+        )
+
+    entries = read_manifest(manifest_path)
+
+    audio_paths = [entry["audio_filepath"] for entry in entries]
+    base_dir = get_base_dir(audio_paths)
+
+    global_pitch_values = []
+    speaker_pitch_values = defaultdict(list)
+    for entry in tqdm(entries):
+        pitch_path = _get_sup_data_filepath(manifest_entry=entry, audio_dir=base_dir, sup_data_dir=pitch_data_path)
+        if not os.path.exists(pitch_path):
+            logging.warning(f"Unable to find pitch file for {entry}")
+            continue
+
+        pitch = torch.load(pitch_path)
+        # Filter out non-speech frames
+        pitch = pitch[pitch != 0]
+        global_pitch_values.append(pitch)
+        if "speaker" in entry:
+            speaker_id = entry["speaker"]
+            speaker_pitch_values[speaker_id].append(pitch)
+
+    global_pitch_mean, global_pitch_std = _compute_stats(global_pitch_values)
+    pitch_stats = {"default": {"pitch_mean": global_pitch_mean, "pitch_std": global_pitch_std}}
+    for speaker_id, pitch_values in speaker_pitch_values.items():
+        pitch_mean, pitch_std = _compute_stats(pitch_values)
+        pitch_stats[speaker_id] = {"pitch_mean": pitch_mean, "pitch_std": pitch_std}
+
+    with open(pitch_stats_path, 'w', encoding="utf-8") as stats_f:
+        json.dump(pitch_stats, stats_f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/create_speaker_map.py

@@ -0,0 +1,104 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script takes a list of TTS manifests and creates a JSON mapping the input speaker names to
+unique indices for multi-speaker TTS training.
+
+To ensure that speaker names are unique across datasets, it is recommended that you prepend the speaker
+names in your manifest with the name of the dataset.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/create_speaker_map.py \
+    --manifest_path=manifest1.json \
+    --manifest_path=manifest2.json \
+    --speaker_map_path=speakers.json
+
+Example output:
+
+{
+    "vctk_p225": 0,
+    "vctk_p226": 1,
+    "vctk_p227": 2,
+    ...
+}
+
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Create mapping from speaker names to numerical speaker indices.",
+    )
+    parser.add_argument(
+        "--manifest_path",
+        required=True,
+        type=Path,
+        action="append",
+        help="Path to training manifest(s).",
+    )
+    parser.add_argument(
+        "--speaker_map_path",
+        required=True,
+        type=Path,
+        help="Path for output speaker index JSON",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output speaker file if it exists.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+    manifest_paths = args.manifest_path
+    speaker_map_path = args.speaker_map_path
+    overwrite = args.overwrite
+
+    for manifest_path in manifest_paths:
+        if not manifest_path.exists():
+            raise ValueError(f"Manifest {manifest_path} does not exist.")
+
+    if speaker_map_path.exists():
+        if overwrite:
+            print(f"Will overwrite existing speaker path: {speaker_map_path}")
+        else:
+            raise ValueError(f"Speaker path already exists: {speaker_map_path}")
+
+    speaker_set = set()
+    for manifest_path in manifest_paths:
+        entries = read_manifest(manifest_path)
+        for entry in entries:
+            speaker = str(entry["speaker"])
+            speaker_set.add(speaker)
+
+    speaker_list = list(speaker_set)
+    speaker_list.sort()
+    speaker_index_map = {speaker_list[i]: i for i in range(len(speaker_list))}
+
+    with open(speaker_map_path, 'w', encoding="utf-8") as stats_f:
+        json.dump(speaker_index_map, stats_f, indent=4)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/extract_sup_data.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch
+from tqdm import tqdm
+
+from nemo.core.classes.common import safe_instantiate
+from nemo.core.config import hydra_runner
+
+
+def get_pitch_stats(pitch_list):
+    pitch_tensor = torch.cat(pitch_list)
+    pitch_mean, pitch_std = pitch_tensor.mean().item(), pitch_tensor.std().item()
+    pitch_min, pitch_max = pitch_tensor.min().item(), pitch_tensor.max().item()
+    print(f"PITCH_MEAN={pitch_mean}, PITCH_STD={pitch_std}")
+    print(f"PITCH_MIN={pitch_min}, PITCH_MAX={pitch_max}")
+
+
+def preprocess_ds_for_fastpitch_align(dataloader):
+    pitch_list = []
+    for batch in tqdm(dataloader, total=len(dataloader)):
+        audios, audio_lengths, tokens, tokens_lengths, align_prior_matrices, pitches, pitches_lengths, *_ = batch
+        pitch = pitches.squeeze(0)
+        pitch_list.append(pitch[pitch != 0])
+
+    get_pitch_stats(pitch_list)
+
+
+CFG_NAME2FUNC = {
+    "ds_for_fastpitch_align": preprocess_ds_for_fastpitch_align,
+    "ds_for_mixer_tts": preprocess_ds_for_fastpitch_align,
+}
+
+
+@hydra_runner(config_path='ljspeech/ds_conf', config_name='ds_for_fastpitch_align')
+def main(cfg):
+    dataset = safe_instantiate(cfg.dataset)
+    dataloader = torch.utils.data.DataLoader(
+        dataset=dataset,
+        batch_size=1,
+        collate_fn=dataset._collate_fn,
+        num_workers=cfg.get("dataloader_params", {}).get("num_workers", 4),
+    )
+
+    print(f"Processing {cfg.manifest_filepath}:")
+    CFG_NAME2FUNC[cfg.name](dataloader)
+
+
+if __name__ == '__main__':
+    main()  # noqa pylint: disable=no-value-for-parameter

scripts/dataset_processing/tts/generate_mels.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is to generate mel spectrograms from a Fastpitch model checkpoint. Please see general usage below. It runs
+on GPUs by default, but you can add `--num-workers 5 --cpu` as an option to run on CPUs.
+
+$ python scripts/dataset_processing/tts/generate_mels.py \
+    --fastpitch-model-ckpt ./models/fastpitch/multi_spk/FastPitch--val_loss\=1.4473-epoch\=209.ckpt \
+    --input-json-manifests /home/xueyang/HUI-Audio-Corpus-German-clean/test_manifest_text_normed_phonemes.json
+    --output-json-manifest-root /home/xueyang/experiments/multi_spk_tts_de
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+import torch
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+from nemo.collections.tts.models import FastPitchModel
+from nemo.collections.tts.parts.utils.tts_dataset_utils import (
+    BetaBinomialInterpolator,
+    beta_binomial_prior_distribution,
+)
+from nemo.utils import logging
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Generate mel spectrograms with pretrained FastPitch model, and create manifests for finetuning Hifigan.",
+    )
+    parser.add_argument(
+        "--fastpitch-model-ckpt",
+        required=True,
+        type=Path,
+        help="Specify a full path of a fastpitch model checkpoint with the suffix of either .ckpt or .nemo.",
+    )
+    parser.add_argument(
+        "--input-json-manifests",
+        nargs="+",
+        required=True,
+        type=Path,
+        help="Specify a full path of a JSON manifest. You could add multiple manifests.",
+    )
+    parser.add_argument(
+        "--output-json-manifest-root",
+        required=True,
+        type=Path,
+        help="Specify a full path of output root that would contain new manifests.",
+    )
+    parser.add_argument(
+        "--num-workers",
+        default=-1,
+        type=int,
+        help="Specify the max number of concurrently Python workers processes. "
+        "If -1 all CPUs are used. If 1 no parallel computing is used.",
+    )
+    parser.add_argument("--cpu", action='store_true', default=False, help="Generate mel spectrograms using CPUs.")
+    args = parser.parse_args()
+    return args
+
+
+def __load_wav(audio_file):
+    with sf.SoundFile(audio_file, 'r') as f:
+        samples = f.read(dtype='float32')
+    return samples.transpose()
+
+
+def __generate_mels(entry, spec_model, device, use_beta_binomial_interpolator, mel_root):
+    # Generate a spectrograms (we need to use ground truth alignment for correct matching between audio and mels)
+    audio = __load_wav(entry["audio_filepath"])
+    audio = torch.from_numpy(audio).unsqueeze(0).to(device)
+    audio_len = torch.tensor(audio.shape[1], dtype=torch.long, device=device).unsqueeze(0)
+
+    if spec_model.fastpitch.speaker_emb is not None and "speaker" in entry:
+        speaker = torch.tensor([entry['speaker']]).to(device)
+    else:
+        speaker = None
+
+    with torch.no_grad():
+        if "normalized_text" in entry:
+            text = spec_model.parse(entry["normalized_text"], normalize=False)
+        else:
+            text = spec_model.parse(entry['text'])
+
+        text_len = torch.tensor(text.shape[-1], dtype=torch.long, device=device).unsqueeze(0)
+        spect, spect_len = spec_model.preprocessor(input_signal=audio, length=audio_len)
+
+        # Generate attention prior and spectrogram inputs for HiFi-GAN
+        if use_beta_binomial_interpolator:
+            beta_binomial_interpolator = BetaBinomialInterpolator()
+            attn_prior = (
+                torch.from_numpy(beta_binomial_interpolator(spect_len.item(), text_len.item()))
+                .unsqueeze(0)
+                .to(text.device)
+            )
+        else:
+            attn_prior = (
+                torch.from_numpy(beta_binomial_prior_distribution(text_len.item(), spect_len.item()))
+                .unsqueeze(0)
+                .to(text.device)
+            )
+
+        spectrogram = spec_model.forward(
+            text=text,
+            input_lens=text_len,
+            spec=spect,
+            mel_lens=spect_len,
+            attn_prior=attn_prior,
+            speaker=speaker,
+        )[0]
+
+        save_path = mel_root / f"{Path(entry['audio_filepath']).stem}.npy"
+        np.save(save_path, spectrogram[0].to('cpu').numpy())
+        entry["mel_filepath"] = str(save_path)
+
+    return entry
+
+
+def main():
+    args = get_args()
+    ckpt_path = args.fastpitch_model_ckpt
+    input_manifest_filepaths = args.input_json_manifests
+    output_json_manifest_root = args.output_json_manifest_root
+
+    mel_root = output_json_manifest_root / "mels"
+    mel_root.mkdir(exist_ok=True, parents=True)
+
+    # load pretrained FastPitch model checkpoint
+    suffix = ckpt_path.suffix
+    if suffix == ".nemo":
+        spec_model = FastPitchModel.restore_from(ckpt_path).eval()
+    elif suffix == ".ckpt":
+        spec_model = FastPitchModel.load_from_checkpoint(ckpt_path).eval()
+    else:
+        raise ValueError(f"Unsupported suffix: {suffix}")
+    if not args.cpu:
+        spec_model.cuda()
+    device = spec_model.device
+
+    use_beta_binomial_interpolator = spec_model.cfg.train_ds.dataset.get("use_beta_binomial_interpolator", False)
+
+    for manifest in input_manifest_filepaths:
+        logging.info(f"Processing {manifest}.")
+        entries = []
+        with open(manifest, "r") as fjson:
+            for line in fjson:
+                entries.append(json.loads(line.strip()))
+
+        if device == "cpu":
+            new_entries = Parallel(n_jobs=args.num_workers)(
+                delayed(__generate_mels)(entry, spec_model, device, use_beta_binomial_interpolator, mel_root)
+                for entry in entries
+            )
+        else:
+            new_entries = []
+            for entry in tqdm(entries):
+                new_entry = __generate_mels(entry, spec_model, device, use_beta_binomial_interpolator, mel_root)
+                new_entries.append(new_entry)
+
+        mel_manifest_path = output_json_manifest_root / f"{manifest.stem}_mel{manifest.suffix}"
+        with open(mel_manifest_path, "w") as fmel:
+            for entry in new_entries:
+                fmel.write(json.dumps(entry) + "\n")
+        logging.info(f"Processing {manifest} is complete --> {mel_manifest_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/hifitts/get_data.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import glob
+import json
+import re
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Download HiFiTTS and create manifests with predefined split')
+    parser.add_argument(
+        "--data-root",
+        required=True,
+        type=Path,
+        help='Directory into which to download and extract dataset. \{data-root\}/hi_fi_tts_v0 will be created.',
+    )
+    parser.add_argument(
+        '--split',
+        type=str,
+        default='all',
+        help='Choose to generate manifest for all or one of (train, test, split), note that this will still download the full dataset.',
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+URL = "https://us.openslr.org/resources/109/hi_fi_tts_v0.tar.gz"
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_data(data_root, filelists):
+    # Create manifests (based on predefined NVIDIA's split)
+    for split in tqdm(filelists):
+        manifest_target = data_root / f"{split}_manifest.json"
+        print(f"Creating manifest for {split}.")
+
+        entries = []
+        for manifest_src in glob.glob(str(data_root / f"*_{split}.json")):
+            try:
+                search_res = re.search('.*\/([0-9]+)_manifest_([a-z]+)_.*.json', manifest_src)
+                speaker_id = search_res.group(1)
+                audio_quality = search_res.group(2)
+            except Exception:
+                print(f"Failed to find speaker id or audio quality for {manifest_src}, check formatting.")
+                continue
+
+            with open(manifest_src, 'r') as f_in:
+                for input_json_entry in f_in:
+                    data = json.loads(input_json_entry)
+
+                    # Make sure corresponding wavfile exists
+                    wav_path = data_root / data['audio_filepath']
+                    assert wav_path.exists(), f"{wav_path} does not exist!"
+
+                    entry = {
+                        'audio_filepath': data['audio_filepath'],
+                        'duration': data['duration'],
+                        'text': data['text'],
+                        'normalized_text': data['text_normalized'],
+                        'speaker': int(speaker_id),
+                        # Audio_quality is either clean or other.
+                        # The clean set includes recordings with high sound-to-noise ratio and wide bandwidth.
+                        # The books with noticeable noise or narrow bandwidth are included in the other subset.
+                        # Note: some speaker_id's have both clean and other audio quality.
+                        'audio_quality': audio_quality,
+                    }
+                    entries.append(entry)
+
+        with open(manifest_target, 'w') as f_out:
+            for m in entries:
+                f_out.write(json.dumps(m) + '\n')
+
+
+def main():
+    args = get_args()
+
+    split = ['train', 'dev', 'test'] if args.split == 'all' else list(args.split)
+
+    tarred_data_path = args.data_root / "hi_fi_tts_v0.tar.gz"
+
+    __maybe_download_file(URL, tarred_data_path)
+    __extract_file(str(tarred_data_path), str(args.data_root))
+
+    data_root = args.data_root / "hi_fi_tts_v0"
+    __process_data(data_root, split)
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/tts/hui_acg/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,45 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: ???
+sup_data_path: ???
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 44100
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 2048
+  win_length: 2048
+  hop_length: 512
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: 15
+  min_duration: 0.1
+  ignore_file: null
+  trim: false
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+  use_beta_binomial_interpolator: false
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: de
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.GermanPhonemesTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true
+
+dataloader_params:
+  num_workers: 12

scripts/dataset_processing/tts/hui_acg/get_data.py

@@ -0,0 +1,334 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import random
+import shutil
+import urllib.request
+from pathlib import Path
+
+import pandas as pd
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+from nemo.utils import logging
+
+# full corpus.
+URLS_FULL = {
+    "Bernd_Ungerer": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Bernd_Ungerer.zip",
+    "Eva_K": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Eva_K.zip",
+    "Friedrich": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Friedrich.zip",
+    "Hokuspokus": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Hokuspokus.zip",
+    "Karlsson": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/Karlsson.zip",
+    "others": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_full/others.zip",
+}
+URL_STATS_FULL = "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/datasetStatistic.zip"
+
+# the clean subset of the full corpus.
+URLS_CLEAN = {
+    "Bernd_Ungerer": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Bernd_Ungerer_Clean.zip",
+    "Eva_K": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Eva_K_Clean.zip",
+    "Friedrich": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Friedrich_Clean.zip",
+    "Hokuspokus": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Hokuspokus_Clean.zip",
+    "Karlsson": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/Karlsson_Clean.zip",
+    "others": "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/dataset_clean/others_Clean.zip",
+}
+URL_STATS_CLEAN = "https://opendata.iisys.de/opendata/Datasets/HUI-Audio-Corpus-German/datasetStatisticClean.zip"
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Download HUI-Audio-Corpus-German and create manifests with predefined split. "
+        "Please check details about the corpus in https://github.com/iisys-hof/HUI-Audio-Corpus-German.",
+    )
+    parser.add_argument("--data-root", required=True, type=Path, help="where the resulting dataset will reside.")
+    parser.add_argument("--manifests-root", required=True, type=Path, help="where the manifests files will reside.")
+    parser.add_argument("--set-type", default="clean", choices=["full", "clean"], type=str)
+    parser.add_argument("--min-duration", default=0.1, type=float)
+    parser.add_argument("--max-duration", default=15, type=float)
+    parser.add_argument(
+        "--num-workers",
+        default=-1,
+        type=int,
+        help="Specify the max number of concurrently Python workers processes. "
+        "If -1 all CPUs are used. If 1 no parallel computing is used.",
+    )
+    parser.add_argument(
+        "--normalize-text",
+        default=False,
+        action='store_true',
+        help="Normalize original text and add a new entry 'normalized_text' to .json file if True.",
+    )
+    parser.add_argument(
+        "--val-num-utts-per-speaker",
+        default=1,
+        type=int,
+        help="Specify the number of utterances for each speaker in val split. All speakers are covered.",
+    )
+    parser.add_argument(
+        "--test-num-utts-per-speaker",
+        default=1,
+        type=int,
+        help="Specify the number of utterances for each speaker in test split. All speakers are covered.",
+    )
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        logging.info(f"Downloading data: {source_url} --> {destination_path}")
+        tmp_file_path = destination_path.with_suffix(".tmp")
+        urllib.request.urlretrieve(source_url, filename=tmp_file_path)
+        tmp_file_path.rename(destination_path)
+    else:
+        logging.info(f"Skipped downloading data because it exists: {destination_path}")
+
+
+def __extract_file(filepath, data_dir):
+    logging.info(f"Unzipping data: {filepath} --> {data_dir}")
+    shutil.unpack_archive(filepath, data_dir)
+    logging.info(f"Unzipping data is complete: {filepath}.")
+
+
+def __save_json(json_file, dict_list):
+    logging.info(f"Saving JSON split to {json_file}.")
+    with open(json_file, "w") as f:
+        for d in dict_list:
+            f.write(json.dumps(d) + "\n")
+
+
+def __process_data(
+    dataset_path,
+    stat_path_root,
+    speaker_id,
+    min_duration,
+    max_duration,
+    val_size,
+    test_size,
+    seed_for_ds_split,
+):
+    logging.info(f"Preparing JSON split for speaker {speaker_id}.")
+    # parse statistic.txt
+    stat_path = stat_path_root / "statistic.txt"
+    with open(stat_path, 'r') as fstat:
+        lines = fstat.readlines()
+        num_utts = int(lines[4].strip().split()[-1])
+        hours = round(float(lines[9].strip().split()[-1]) / 3600.0, 2)
+
+    # parse overview.csv to generate JSON splits.
+    overview_path = stat_path_root / "overview.csv"
+    entries = []
+    with open(overview_path, 'r') as foverview:
+        # Let's skip the header
+        foverview.readline()
+        for line in tqdm(foverview):
+            file_stem, duration, *_, text = line.strip().split("|")
+            duration = float(duration)
+
+            # file_stem -> dir_name (e.g. maerchen_01_f000051 -> maerchen)
+            dir_name = "_".join(file_stem.split("_")[:-2])
+            audio_path = dataset_path / dir_name / "wavs" / f"{file_stem}.wav"
+
+            if min_duration <= duration <= max_duration:
+                entry = {
+                    "audio_filepath": str(audio_path),
+                    "duration": duration,
+                    "text": text,
+                    "speaker": speaker_id,
+                }
+                entries.append(entry)
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+    train_size = len(entries) - val_size - test_size
+    if train_size <= 0:
+        logging.warning(f"Skipped speaker {speaker_id}. Not enough data for train, val and test.")
+        train, val, test, is_skipped = [], [], [], True
+    else:
+        logging.info(f"Preparing JSON split for speaker {speaker_id} is complete.")
+        train, val, test, is_skipped = (
+            entries[:train_size],
+            entries[train_size : train_size + val_size],
+            entries[train_size + val_size :],
+            False,
+        )
+
+    return {
+        "train": train,
+        "val": val,
+        "test": test,
+        "is_skipped": is_skipped,
+        "hours": hours,
+        "num_utts": num_utts,
+    }
+
+
+def __text_normalization(json_file, num_workers=-1):
+    text_normalizer_call_kwargs = {
+        "punct_pre_process": True,
+        "punct_post_process": True,
+    }
+    text_normalizer = Normalizer(
+        lang="de",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(json_file.parent / "cache_dir"),
+    )
+
+    def normalizer_call(x):
+        return text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+
+    def add_normalized_text(line_dict):
+        normalized_text = normalizer_call(line_dict["text"])
+        line_dict.update({"normalized_text": normalized_text})
+        return line_dict
+
+    logging.info(f"Normalizing text for {json_file}.")
+    with open(json_file, 'r', encoding='utf-8') as fjson:
+        lines = fjson.readlines()
+        # Note: you need to verify which backend works well on your cluster.
+        # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+        dict_list = Parallel(n_jobs=num_workers)(
+            delayed(add_normalized_text)(json.loads(line)) for line in tqdm(lines)
+        )
+
+    json_file_text_normed = json_file.parent / f"{json_file.stem}_text_normed{json_file.suffix}"
+    with open(json_file_text_normed, 'w', encoding="utf-8") as fjson_norm:
+        for dct in dict_list:
+            fjson_norm.write(json.dumps(dct) + "\n")
+    logging.info(f"Normalizing text is complete: {json_file} --> {json_file_text_normed}")
+
+
+def main():
+    args = get_args()
+    data_root = args.data_root
+    manifests_root = args.manifests_root
+    set_type = args.set_type
+
+    dataset_root = data_root / f"HUI-Audio-Corpus-German-{set_type}"
+    dataset_root.mkdir(parents=True, exist_ok=True)
+
+    if set_type == "full":
+        data_source = URLS_FULL
+        stats_source = URL_STATS_FULL
+    elif set_type == "clean":
+        data_source = URLS_CLEAN
+        stats_source = URL_STATS_CLEAN
+    else:
+        raise ValueError(f"Unknown {set_type}. Please choose either clean or full.")
+
+    # download and unzip dataset stats
+    zipped_stats_path = dataset_root / Path(stats_source).name
+    __maybe_download_file(stats_source, zipped_stats_path)
+    __extract_file(zipped_stats_path, dataset_root)
+
+    # download datasets
+    # Note: you need to verify which backend works well on your cluster.
+    # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+    Parallel(n_jobs=args.num_workers)(
+        delayed(__maybe_download_file)(data_url, dataset_root / Path(data_url).name)
+        for _, data_url in data_source.items()
+    )
+
+    # unzip datasets
+    # Note: you need to verify which backend works well on your cluster.
+    # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+    Parallel(n_jobs=args.num_workers)(
+        delayed(__extract_file)(dataset_root / Path(data_url).name, dataset_root)
+        for _, data_url in data_source.items()
+    )
+
+    # generate json files for train/val/test splits
+    stats_path_root = dataset_root / Path(stats_source).stem / "speacker"
+    entries_train, entries_val, entries_test = [], [], []
+    speaker_entries = []
+    num_speakers = 0
+    for child in stats_path_root.iterdir():
+        if child.is_dir():
+            speaker = child.name
+            num_speakers += 1
+            speaker_stats_root = stats_path_root / speaker
+            speaker_data_path = dataset_root / speaker
+
+            logging.info(f"Processing Speaker: {speaker}")
+            results = __process_data(
+                speaker_data_path,
+                speaker_stats_root,
+                num_speakers,
+                args.min_duration,
+                args.max_duration,
+                args.val_num_utts_per_speaker,
+                args.test_num_utts_per_speaker,
+                args.seed_for_ds_split,
+            )
+
+            entries_train.extend(results["train"])
+            entries_val.extend(results["val"])
+            entries_test.extend(results["test"])
+
+            speaker_entry = {
+                "speaker_name": speaker,
+                "speaker_id": num_speakers,
+                "hours": results["hours"],
+                "num_utts": results["num_utts"],
+                "is_skipped": results["is_skipped"],
+            }
+            speaker_entries.append(speaker_entry)
+
+    # shuffle in place across multiple speakers
+    random.Random(args.seed_for_ds_split).shuffle(entries_train)
+    random.Random(args.seed_for_ds_split).shuffle(entries_val)
+    random.Random(args.seed_for_ds_split).shuffle(entries_test)
+
+    # save speaker stats.
+    df = pd.DataFrame.from_records(speaker_entries)
+    df.sort_values(by="hours", ascending=False, inplace=True)
+    spk2id_file_path = manifests_root / "spk2id.csv"
+    df.to_csv(spk2id_file_path, index=False)
+    logging.info(f"Saving Speaker to ID mapping to {spk2id_file_path}.")
+
+    # save json splits.
+    train_json = manifests_root / "train_manifest.json"
+    val_json = manifests_root / "val_manifest.json"
+    test_json = manifests_root / "test_manifest.json"
+    __save_json(train_json, entries_train)
+    __save_json(val_json, entries_val)
+    __save_json(test_json, entries_test)
+
+    # normalize text if requested. New json file, train_manifest_text_normed.json, will be generated.
+    if args.normalize_text:
+        __text_normalization(train_json, args.num_workers)
+        __text_normalization(val_json, args.num_workers)
+        __text_normalization(test_json, args.num_workers)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/libritts/get_data.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# USAGE: python get_data.py --data-root=<where to put data> --data-set=<datasets_to_download> --num-workers=<number of parallel workers>
+# where <datasets_to_download> can be: dev_clean, dev_other, test_clean,
+# test_other, train_clean_100, train_clean_360, train_other_500 or ALL
+# You can also put more than one data_set comma-separated:
+# --data-set=dev_clean,train_clean_100
+import argparse
+import fnmatch
+import functools
+import json
+import multiprocessing
+import os
+import subprocess
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+parser = argparse.ArgumentParser(description='Download LibriTTS and create manifests')
+parser.add_argument("--data-root", required=True, type=Path)
+parser.add_argument("--data-sets", default="dev_clean", type=str)
+parser.add_argument("--num-workers", default=4, type=int)
+args = parser.parse_args()
+
+URLS = {
+    'TRAIN_CLEAN_100': "https://www.openslr.org/resources/60/train-clean-100.tar.gz",
+    'TRAIN_CLEAN_360': "https://www.openslr.org/resources/60/train-clean-360.tar.gz",
+    'TRAIN_OTHER_500': "https://www.openslr.org/resources/60/train-other-500.tar.gz",
+    'DEV_CLEAN': "https://www.openslr.org/resources/60/dev-clean.tar.gz",
+    'DEV_OTHER': "https://www.openslr.org/resources/60/dev-other.tar.gz",
+    'TEST_CLEAN': "https://www.openslr.org/resources/60/test-clean.tar.gz",
+    'TEST_OTHER': "https://www.openslr.org/resources/60/test-other.tar.gz",
+}
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_transcript(file_path: str):
+    entries = []
+    with open(file_path, encoding="utf-8") as fin:
+        text = fin.readlines()[0].strip()
+
+        # TODO(oktai15): add normalized text via Normalizer/NormalizerWithAudio
+        wav_file = file_path.replace(".normalized.txt", ".wav")
+        speaker_id = file_path.split('/')[-3]
+        assert os.path.exists(wav_file), f"{wav_file} not found!"
+        duration = subprocess.check_output(["soxi", "-D", wav_file])
+        entry = {
+            'audio_filepath': os.path.abspath(wav_file),
+            'duration': float(duration),
+            'text': text,
+            'speaker': int(speaker_id),
+        }
+
+        entries.append(entry)
+
+    return entries
+
+
+def __process_data(data_folder, manifest_file, num_workers):
+    files = []
+    entries = []
+
+    for root, dirnames, filenames in os.walk(data_folder):
+        # we will use normalized text provided by the original dataset
+        for filename in fnmatch.filter(filenames, '*.normalized.txt'):
+            files.append(os.path.join(root, filename))
+
+    with multiprocessing.Pool(num_workers) as p:
+        processing_func = functools.partial(__process_transcript)
+        results = p.imap(processing_func, files)
+        for result in tqdm(results, total=len(files)):
+            entries.extend(result)
+
+    with open(manifest_file, 'w') as fout:
+        for m in entries:
+            fout.write(json.dumps(m) + '\n')
+
+
+def main():
+    data_root = args.data_root
+    data_sets = args.data_sets
+    num_workers = args.num_workers
+
+    if data_sets == "ALL":
+        data_sets = "dev_clean,dev_other,train_clean_100,train_clean_360,train_other_500,test_clean,test_other"
+    if data_sets == "mini":
+        data_sets = "dev_clean,train_clean_100"
+    for data_set in data_sets.split(','):
+        filepath = data_root / f"{data_set}.tar.gz"
+        print(f"Downloading data for {data_set}...")
+        __maybe_download_file(URLS[data_set.upper()], filepath)
+        print("Extracting...")
+        __extract_file(str(filepath), str(data_root))
+
+        print("Processing and building manifest.")
+        __process_data(
+            str(data_root / "LibriTTS" / data_set.replace("_", "-")),
+            str(data_root / "LibriTTS" / f"{data_set}.json"),
+            num_workers=num_workers,
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.10"
+heteronyms_path: "scripts/tts_dataset_files/heteronyms-052722"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: 8000
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: false
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: en
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.EnglishPhonemesTokenizer
+    punct: true
+    stresses: true
+    chars: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.en_us_arpabet.EnglishG2p
+      phoneme_dict: ${phoneme_dict_path}
+      heteronyms: ${heteronyms_path}

scripts/dataset_processing/tts/ljspeech/ds_conf/ds_for_mixer_tts.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_mixer_tts"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+phoneme_dict_path: "scripts/tts_dataset_files/cmudict-0.7b_nv22.10"
+heteronyms_path: "scripts/tts_dataset_files/heteronyms-052722"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: 8000
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: false
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: en
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.EnglishPhonemesTokenizer
+    punct: true
+    stresses: true
+    chars: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.en_us_arpabet.EnglishG2p
+      phoneme_dict: ${phoneme_dict_path}
+      heteronyms: ${heteronyms_path}

scripts/dataset_processing/tts/ljspeech/get_data.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description='Download LJSpeech and create manifests with predefined split')
+    parser.add_argument("--data-root", required=True, type=Path)
+
+    args = parser.parse_args()
+    return args
+
+
+URL = "https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2"
+FILELIST_BASE = 'https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists'
+
+
+def _load_sox():
+    try:
+        import sox
+    except ImportError:
+        raise ImportError(
+            "Optional dependency 'sox' is required by this script. Install it with: pip install sox"
+        ) from None
+
+    return sox
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        tmp_file_path = destination_path.with_suffix('.tmp')
+        urllib.request.urlretrieve(source_url, filename=str(tmp_file_path))
+        tmp_file_path.rename(destination_path)
+
+
+def __extract_file(filepath, data_dir):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, str(data_dir))
+    except Exception:
+        print(f"Error while extracting {filepath}. Already extracted?")
+
+
+def __process_data(data_root):
+    sox = _load_sox()
+    text_normalizer = Normalizer(
+        lang="en",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=data_root / "cache_dir",
+    )
+    text_normalizer_call_kwargs = {"punct_pre_process": True, "punct_post_process": True}
+    normalizer_call = lambda x: text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+
+    # Create manifests (based on predefined NVIDIA's split)
+    filelists = ['train', 'val', 'test']
+    for split in tqdm(filelists):
+        # Download file list if necessary
+        filelist_path = data_root / f"ljs_audio_text_{split}_filelist.txt"
+
+        if not filelist_path.exists():
+            urllib.request.urlretrieve(
+                f"{FILELIST_BASE}/ljs_audio_text_{split}_filelist.txt",
+                filename=str(filelist_path),
+            )
+
+        manifest_target = data_root / f"{split}_manifest.json"
+        with open(manifest_target, 'w') as f_out:
+            with open(filelist_path, 'r') as filelist:
+                print(f"\nCreating {manifest_target}...")
+                for line in tqdm(filelist):
+                    basename = line[6:16]
+
+                    text = line[21:].strip()
+                    norm_text = normalizer_call(text)
+
+                    # Make sure corresponding wavfile exists
+                    wav_path = data_root / 'wavs' / f"{basename}.wav"
+                    assert wav_path.exists(), f"{wav_path} does not exist!"
+
+                    entry = {
+                        'audio_filepath': str(wav_path),
+                        'duration': sox.file_info.duration(wav_path),
+                        'text': text,
+                        'normalized_text': norm_text,
+                    }
+
+                    f_out.write(json.dumps(entry) + '\n')
+
+
+def main():
+    args = get_args()
+
+    tarred_data_path = args.data_root / "LJSpeech-1.1.tar.bz2"
+
+    __maybe_download_file(URL, tarred_data_path)
+    __extract_file(str(tarred_data_path), str(args.data_root))
+
+    data_root = args.data_root / "LJSpeech-1.1"
+
+    __process_data(data_root)
+
+
+if __name__ == '__main__':
+    main()

scripts/dataset_processing/tts/ljspeech/lj_speech.tsv

@@ -0,0 +1,21 @@
+Mr.	mister
+Mrs.	misses
+Dr.	doctor
+Drs.	doctors
+Co.	company
+Lt.	lieutenant
+Sgt.	sergeant
+St.	saint
+Jr.	junior
+Maj.	major
+Hon.	honorable
+Gov.	governor
+Capt.	captain
+Esq.	esquire
+Gen.	general
+Ltd.	limited
+Rev.	reverend
+Col.	colonel
+Mt.	mount
+Ft.	fort
+etc.	et cetera

scripts/dataset_processing/tts/preprocess_audio.py

@@ -0,0 +1,280 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to preprocess audio before TTS model training.
+
+It can be configured to do several processing steps such as silence trimming, volume normalization,
+and duration filtering.
+
+These can be done separately through multiple executions of the script, or all at once to avoid saving
+too many copies of the same audio.
+
+Most of these can also be done by the TTS data loader at training time, but doing them ahead of time
+lets us implement more complex processing, validate the correctness of the output, and save on compute time.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/preprocess_audio.py \
+    --input_manifest="<data_root_path>/manifest.json" \
+    --output_manifest="<data_root_path>/manifest_processed.json" \
+    --input_audio_dir="<data_root_path>/audio" \
+    --output_audio_dir="<data_root_path>/audio_processed" \
+    --num_workers=1 \
+    --trim_config_path="<nemo_root_path>/examples/tts/conf/trim/energy.yaml" \
+    --output_sample_rate=22050 \
+    --output_format=flac \
+    --volume_level=0.95 \
+    --min_duration=0.5 \
+    --max_duration=20.0 \
+    --filter_file="filtered.txt"
+"""
+
+import argparse
+import os
+from pathlib import Path
+from typing import Tuple
+
+import librosa
+import soundfile as sf
+from joblib import Parallel, delayed
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+from nemo.collections.tts.parts.preprocessing.audio_trimming import AudioTrimmer
+from nemo.collections.tts.parts.utils.tts_dataset_utils import get_abs_rel_paths, normalize_volume
+from nemo.core.classes.common import safe_instantiate
+from nemo.utils import logging
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Compute speaker level pitch statistics.",
+    )
+    parser.add_argument(
+        "--input_manifest",
+        required=True,
+        type=Path,
+        help="Path to input training manifest.",
+    )
+    parser.add_argument(
+        "--input_audio_dir",
+        required=True,
+        type=Path,
+        help="Path to base directory with audio files.",
+    )
+    parser.add_argument(
+        "--output_manifest",
+        required=True,
+        type=Path,
+        help="Path to output training manifest with processed audio.",
+    )
+    parser.add_argument(
+        "--output_audio_dir",
+        required=True,
+        type=Path,
+        help="Path to output directory for audio files.",
+    )
+    parser.add_argument(
+        "--overwrite_audio",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to reprocess and overwrite existing audio files in output_audio_dir.",
+    )
+    parser.add_argument(
+        "--overwrite_manifest",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output manifest file if it exists.",
+    )
+    parser.add_argument(
+        "--num_workers", default=1, type=int, help="Number of parallel threads to use. If -1 all CPUs are used."
+    )
+    parser.add_argument(
+        "--trim_config_path",
+        required=False,
+        type=Path,
+        help="Path to config file for nemo.collections.tts.data.audio_trimming.AudioTrimmer",
+    )
+    parser.add_argument(
+        "--max_entries", default=0, type=int, help="If provided, maximum number of entries in the manifest to process."
+    )
+    parser.add_argument(
+        "--output_sample_rate", default=0, type=int, help="If provided, rate to resample the audio to."
+    )
+    parser.add_argument(
+        "--output_format",
+        default="wav",
+        type=str,
+        help="If provided, format output audio will be saved as. If not provided, will keep original format.",
+    )
+    parser.add_argument(
+        "--volume_level", default=0.0, type=float, help="If provided, peak volume to normalize audio to."
+    )
+    parser.add_argument(
+        "--min_duration", default=0.0, type=float, help="If provided, filter out utterances shorter than min_duration."
+    )
+    parser.add_argument(
+        "--max_duration", default=0.0, type=float, help="If provided, filter out utterances longer than max_duration."
+    )
+    parser.add_argument(
+        "--filter_file",
+        required=False,
+        type=Path,
+        help="If provided, output filter_file will contain list of " "utterances filtered out.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def _process_entry(
+    entry: dict,
+    input_audio_dir: Path,
+    output_audio_dir: Path,
+    overwrite_audio: bool,
+    audio_trimmer: AudioTrimmer,
+    output_sample_rate: int,
+    output_format: str,
+    volume_level: float,
+) -> Tuple[dict, float, float]:
+    audio_filepath = Path(entry["audio_filepath"])
+
+    audio_path, audio_path_rel = get_abs_rel_paths(input_path=audio_filepath, base_path=input_audio_dir)
+
+    if not output_format:
+        output_format = audio_path.suffix
+
+    output_path = output_audio_dir / audio_path_rel
+    output_path = output_path.with_suffix(output_format)
+    output_path.parent.mkdir(exist_ok=True, parents=True)
+
+    if output_path.exists() and not overwrite_audio:
+        original_duration = librosa.get_duration(path=audio_path)
+        output_duration = librosa.get_duration(path=output_path)
+    else:
+        audio, sample_rate = librosa.load(audio_path, sr=None)
+        original_duration = librosa.get_duration(y=audio, sr=sample_rate)
+        if audio_trimmer is not None:
+            audio, start_i, end_i = audio_trimmer.trim_audio(
+                audio=audio, sample_rate=int(sample_rate), audio_id=str(audio_path)
+            )
+
+        if output_sample_rate:
+            audio = librosa.resample(y=audio, orig_sr=sample_rate, target_sr=output_sample_rate)
+            sample_rate = output_sample_rate
+
+        if volume_level:
+            audio = normalize_volume(audio, volume_level=volume_level)
+
+        if audio.size > 0:
+            sf.write(file=output_path, data=audio, samplerate=sample_rate)
+            output_duration = librosa.get_duration(y=audio, sr=sample_rate)
+        else:
+            output_duration = 0.0
+
+    entry["duration"] = round(output_duration, 2)
+
+    if os.path.isabs(audio_filepath):
+        entry["audio_filepath"] = str(output_path)
+    else:
+        output_filepath = audio_path_rel.with_suffix(output_format)
+        entry["audio_filepath"] = str(output_filepath)
+
+    return entry, original_duration, output_duration
+
+
+def main():
+    args = get_args()
+
+    input_manifest_path = args.input_manifest
+    output_manifest_path = args.output_manifest
+    input_audio_dir = args.input_audio_dir
+    output_audio_dir = args.output_audio_dir
+    overwrite_audio = args.overwrite_audio
+    overwrite_manifest = args.overwrite_manifest
+    num_workers = args.num_workers
+    max_entries = args.max_entries
+    output_sample_rate = args.output_sample_rate
+    output_format = args.output_format
+    volume_level = args.volume_level
+    min_duration = args.min_duration
+    max_duration = args.max_duration
+    filter_file = args.filter_file
+
+    if output_manifest_path.exists():
+        if overwrite_manifest:
+            print(f"Will overwrite existing manifest path: {output_manifest_path}")
+        else:
+            raise ValueError(f"Manifest path already exists: {output_manifest_path}")
+
+    if args.trim_config_path:
+        audio_trimmer_config = OmegaConf.load(args.trim_config_path)
+        audio_trimmer = safe_instantiate(audio_trimmer_config)
+    else:
+        audio_trimmer = None
+
+    if output_format:
+        if output_format.upper() not in sf.available_formats():
+            raise ValueError(f"Unsupported output audio format: {output_format}")
+        output_format = f".{output_format}"
+
+    output_audio_dir.mkdir(exist_ok=True, parents=True)
+
+    entries = read_manifest(input_manifest_path)
+    if max_entries:
+        entries = entries[:max_entries]
+
+    # 'threading' backend is required when parallelizing torch models.
+    job_outputs = Parallel(n_jobs=num_workers, backend='threading')(
+        delayed(_process_entry)(
+            entry=entry,
+            input_audio_dir=input_audio_dir,
+            output_audio_dir=output_audio_dir,
+            overwrite_audio=overwrite_audio,
+            audio_trimmer=audio_trimmer,
+            output_sample_rate=output_sample_rate,
+            output_format=output_format,
+            volume_level=volume_level,
+        )
+        for entry in tqdm(entries)
+    )
+
+    output_entries = []
+    filtered_entries = []
+    original_durations = 0.0
+    output_durations = 0.0
+    for output_entry, original_duration, output_duration in job_outputs:
+        original_durations += original_duration
+
+        if (
+            output_duration == 0.0
+            or (min_duration and output_duration < min_duration)
+            or (max_duration and output_duration > max_duration)
+        ):
+            if output_duration != original_duration:
+                output_entry["original_duration"] = original_duration
+            filtered_entries.append(output_entry)
+            continue
+
+        output_durations += output_duration
+        output_entries.append(output_entry)
+
+    write_manifest(output_path=output_manifest_path, target_manifest=output_entries, ensure_ascii=False)
+    if filter_file:
+        write_manifest(output_path=str(filter_file), target_manifest=filtered_entries, ensure_ascii=False)
+
+    logging.info(f"Duration of original audio: {original_durations / 3600:.2f} hours")
+    logging.info(f"Duration of processed audio: {output_durations / 3600:.2f} hours")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/preprocess_text.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to preprocess text before TTS model training. This is needed mainly for text normalization,
+which is slow to rerun during training.
+
+The output manifest will be the same as the input manifest but with final text stored in the 'normalized_text' field.
+
+$ python <nemo_root_path>/scripts/dataset_processing/tts/preprocess_text.py \
+    --input_manifest="<data_root_path>/manifest.json" \
+    --output_manifest="<data_root_path>/manifest_processed.json" \
+    --normalizer_config_path="<nemo_root_path>/examples/tts/conf/text/normalizer_en.yaml" \
+    --lower_case \
+    --num_workers=4 \
+    --joblib_batch_size=16
+"""
+
+import argparse
+from pathlib import Path
+
+from joblib import Parallel, delayed
+from omegaconf import OmegaConf
+from tqdm import tqdm
+
+from nemo.core.classes.common import safe_instantiate
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Process and normalize text data.",
+    )
+    parser.add_argument(
+        "--input_manifest",
+        required=True,
+        type=Path,
+        help="Path to input training manifest.",
+    )
+    parser.add_argument(
+        "--output_manifest",
+        required=True,
+        type=Path,
+        help="Path to output training manifest with processed text.",
+    )
+    parser.add_argument(
+        "--overwrite",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to overwrite the output manifest file if it exists.",
+    )
+    parser.add_argument(
+        "--text_key",
+        default="text",
+        type=str,
+        help="Input text field to normalize.",
+    )
+    parser.add_argument(
+        "--normalized_text_key",
+        default="normalized_text",
+        type=str,
+        help="Output field to save normalized text to.",
+    )
+    parser.add_argument(
+        "--lower_case",
+        action=argparse.BooleanOptionalAction,
+        help="Whether to convert the final text to lower case.",
+    )
+    parser.add_argument(
+        "--normalizer_config_path",
+        required=False,
+        type=Path,
+        help="Path to config file for nemo_text_processing.text_normalization.normalize.Normalizer.",
+    )
+    parser.add_argument(
+        "--num_workers", default=1, type=int, help="Number of parallel threads to use. If -1 all CPUs are used."
+    )
+    parser.add_argument(
+        "--joblib_batch_size", type=int, help="Batch size for joblib workers. Defaults to 'auto' if not provided."
+    )
+    parser.add_argument(
+        "--max_entries", default=0, type=int, help="If provided, maximum number of entries in the manifest to process."
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def _process_entry(
+    entry: dict,
+    normalizer: Normalizer,
+    text_key: str,
+    normalized_text_key: str,
+    lower_case: bool,
+    lower_case_norm: bool,
+) -> dict:
+    text = entry[text_key]
+
+    if normalizer is not None:
+        if lower_case_norm:
+            text = text.lower()
+        text = normalizer.normalize(text, punct_pre_process=True, punct_post_process=True)
+
+    if lower_case:
+        text = text.lower()
+
+    entry[normalized_text_key] = text
+
+    return entry
+
+
+def main():
+    args = get_args()
+
+    input_manifest_path = args.input_manifest
+    output_manifest_path = args.output_manifest
+    text_key = args.text_key
+    normalized_text_key = args.normalized_text_key
+    lower_case = args.lower_case
+    num_workers = args.num_workers
+    batch_size = args.joblib_batch_size
+    max_entries = args.max_entries
+    overwrite = args.overwrite
+
+    if output_manifest_path.exists():
+        if overwrite:
+            print(f"Will overwrite existing manifest path: {output_manifest_path}")
+        else:
+            raise ValueError(f"Manifest path already exists: {output_manifest_path}")
+
+    if args.normalizer_config_path:
+        normalizer_config = OmegaConf.load(args.normalizer_config_path)
+        normalizer = safe_instantiate(normalizer_config)
+        lower_case_norm = normalizer.input_case == "lower_cased"
+    else:
+        normalizer = None
+        lower_case_norm = False
+
+    entries = read_manifest(input_manifest_path)
+    if max_entries:
+        entries = entries[:max_entries]
+
+    if not batch_size:
+        batch_size = 'auto'
+
+    output_entries = Parallel(n_jobs=num_workers, batch_size=batch_size)(
+        delayed(_process_entry)(
+            entry=entry,
+            normalizer=normalizer,
+            text_key=text_key,
+            normalized_text_key=normalized_text_key,
+            lower_case=lower_case,
+            lower_case_norm=lower_case_norm,
+        )
+        for entry in tqdm(entries)
+    )
+
+    write_manifest(output_path=output_manifest_path, target_manifest=output_entries, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/resynthesize_dataset.py

@@ -0,0 +1,252 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is a helper for resynthesizing TTS dataset using a pretrained text-to-spectrogram model.
+Goal of resynthesis (as opposed to text-to-speech) is to use the largest amount of ground-truth features from existing speech data.
+For example, for resynthesis we want to have the same pitch and durations instead of ones predicted by the model.
+The results are to be used for some other task: vocoder finetuning, spectrogram enhancer training, etc.
+
+Let's say we have the following toy dataset:
+/dataset/manifest.json
+/dataset/1/foo.wav
+/dataset/2/bar.wav
+/dataset/sup_data/pitch/1_foo.pt
+/dataset/sup_data/pitch/2_bar.pt
+
+manifest.json has two entries for "/dataset/1/foo.wav" and "/dataset/2/bar.wav"
+(sup_data folder contains pitch files precomputed during training a FastPitch model on this dataset.)
+(If you lost your sup_data - don't worry, we use TTSDataset class so they would be created on-the-fly)
+
+Our script call is
+$ python scripts/dataset_processing/tts/resynthesize_dataset.py \
+    --model-path ./models/fastpitch/multi_spk/FastPitch--val_loss\=1.4473-epoch\=209.ckpt \
+    --input-json-manifest "/dataset/manifest.json" \
+    --input-sup-data-path "/dataset/sup_data/" \
+    --output-folder "/output/" \
+    --device "cuda:0" \
+    --batch-size 1 \
+    --num-workers 1
+
+Then we get output dataset with following directory structure:
+/output/manifest_mel.json
+/output/mels/foo.npy
+/output/mels/foo_gt.npy
+/output/mels/bar.npy
+/output/mels/bar_gt.npy
+
+/output/manifest_mel.json has the same entries as /dataset/manifest.json but with new fields for spectrograms.
+"mel_filepath" is path to the resynthesized spectrogram .npy, "mel_gt_filepath" is path to ground-truth spectrogram .npy
+
+The output structure is similar to generate_mels.py script for compatibility reasons.
+"""
+
+import argparse
+import itertools
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterable, Iterator, List
+
+import numpy as np
+import torch
+from omegaconf import DictConfig, OmegaConf
+from tqdm import tqdm
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+from nemo.collections.tts.models import FastPitchModel
+from nemo.collections.tts.models.base import SpectrogramGenerator
+from nemo.collections.tts.parts.utils.helpers import process_batch, to_device_recursive
+
+
+def chunks(iterable: Iterable, size: int) -> Iterator[List]:
+    # chunks([1, 2, 3, 4, 5], size=2) -> [[1, 2], [3, 4], [5]]
+    # assumes iterable does not have any `None`s
+    args = [iter(iterable)] * size
+    for chunk in itertools.zip_longest(*args, fillvalue=None):
+        chunk = list(item for item in chunk if item is not None)
+        if chunk:
+            yield chunk
+
+
+def load_model(path: Path, device: torch.device) -> SpectrogramGenerator:
+    model = None
+    if path.suffix == ".nemo":
+        model = SpectrogramGenerator.restore_from(path, map_location=device)
+    elif path.suffix == ".ckpt":
+        model = SpectrogramGenerator.load_from_checkpoint(path, map_location=device)
+    else:
+        raise ValueError(f"Unknown checkpoint type {path.suffix} ({path})")
+
+    return model.eval().to(device)
+
+
+@dataclass
+class TTSDatasetResynthesizer:
+    """
+    Reuses internals of a SpectrogramGenerator to resynthesize dataset using ground truth features.
+    Default setup is FastPitch with learned alignment.
+    If your use case requires different setup, you can either contribute to this script or subclass this class.
+    """
+
+    model: SpectrogramGenerator
+    device: torch.device
+
+    @torch.no_grad()
+    def resynthesize_batch(self, batch: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Resynthesizes a single batch.
+        Takes a dict with main data and sup data.
+        Outputs a dict with model outputs.
+        """
+        if not isinstance(self.model, FastPitchModel):
+            raise NotImplementedError(
+                "This script supports only FastPitch. Please implement resynthesizing routine for your desired model."
+            )
+
+        batch = to_device_recursive(batch, self.device)
+
+        mels, mel_lens = self.model.preprocessor(input_signal=batch["audio"], length=batch["audio_lens"])
+
+        reference_audio = batch.get("reference_audio", None)
+        reference_audio_len = batch.get("reference_audio_lens", None)
+        reference_spec, reference_spec_len = None, None
+        if reference_audio is not None:
+            reference_spec, reference_spec_len = self.model.preprocessor(
+                input_signal=reference_audio, length=reference_audio_len
+            )
+
+        outputs_tuple = self.model.forward(
+            text=batch["text"],
+            durs=None,
+            pitch=batch["pitch"],
+            speaker=batch.get("speaker"),
+            pace=1.0,
+            spec=mels,
+            attn_prior=batch.get("attn_prior"),
+            mel_lens=mel_lens,
+            input_lens=batch["text_lens"],
+            reference_spec=reference_spec,
+            reference_spec_lens=reference_spec_len,
+        )
+        names = self.model.fastpitch.output_types.keys()
+        return {"spec": mels, "mel_lens": mel_lens, **dict(zip(names, outputs_tuple))}
+
+    def resynthesized_batches(self) -> Iterator[Dict[str, Any]]:
+        """
+        Returns a generator of resynthesized batches.
+        Each returned batch is a dict containing main data, sup data, and model output
+        """
+        self.model.setup_training_data(self.model._cfg["train_ds"])
+
+        for batch_tuple in iter(self.model._train_dl):
+            batch = process_batch(batch_tuple, sup_data_types_set=self.model._train_dl.dataset.sup_data_types)
+            yield self.resynthesize_batch(batch)
+
+
+def prepare_paired_mel_spectrograms(
+    model_path: Path,
+    input_json_manifest: Path,
+    input_sup_data_path: Path,
+    output_folder: Path,
+    device: torch.device,
+    batch_size: int,
+    num_workers: int,
+):
+    model = load_model(model_path, device)
+
+    dataset_config_overrides = {
+        "dataset": {
+            "manifest_filepath": str(input_json_manifest.absolute()),
+            "sup_data_path": str(input_sup_data_path.absolute()),
+        },
+        "dataloader_params": {"batch_size": batch_size, "num_workers": num_workers, "shuffle": False},
+    }
+    model._cfg.train_ds = OmegaConf.merge(model._cfg.train_ds, DictConfig(dataset_config_overrides))
+    resynthesizer = TTSDatasetResynthesizer(model, device)
+
+    input_manifest = read_manifest(input_json_manifest)
+
+    output_manifest = []
+    output_json_manifest = output_folder / f"{input_json_manifest.stem}_mel{input_json_manifest.suffix}"
+    output_mels_folder = output_folder / "mels"
+    output_mels_folder.mkdir(exist_ok=True, parents=True)
+    for batch, batch_manifest in tqdm(
+        zip(resynthesizer.resynthesized_batches(), chunks(input_manifest, size=batch_size)), desc="Batch #"
+    ):
+        pred_mels = batch["spect"].cpu()  # key from fastpitch.output_types
+        true_mels = batch["spec"].cpu()  # key from code above
+        mel_lens = batch["mel_lens"].cpu().flatten()  # key from code above
+
+        for i, (manifest_entry, length) in enumerate(zip(batch_manifest, mel_lens.tolist())):
+            print(manifest_entry["audio_filepath"])
+            filename = Path(manifest_entry["audio_filepath"]).stem
+
+            # note that lengths match
+            pred_mel = pred_mels[i, :, :length].clone().numpy()
+            true_mel = true_mels[i, :, :length].clone().numpy()
+
+            pred_mel_path = output_mels_folder / f"{filename}.npy"
+            true_mel_path = output_mels_folder / f"{filename}_gt.npy"
+
+            np.save(pred_mel_path, pred_mel)
+            np.save(true_mel_path, true_mel)
+
+            new_manifest_entry = {
+                **manifest_entry,
+                "mel_filepath": str(pred_mel_path),
+                "mel_gt_filepath": str(true_mel_path),
+            }
+            output_manifest.append(new_manifest_entry)
+
+    write_manifest(output_json_manifest, output_manifest, ensure_ascii=False)
+
+
+def argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Resynthesize TTS dataset using a pretrained text-to-spectrogram model",
+    )
+    parser.add_argument(
+        "--model-path",
+        required=True,
+        type=Path,
+        help="Path to a checkpoint (either .nemo or .ckpt)",
+    )
+    parser.add_argument(
+        "--input-json-manifest",
+        required=True,
+        type=Path,
+        help="Path to the input JSON manifest",
+    )
+    parser.add_argument(
+        "--input-sup-data-path",
+        required=True,
+        type=Path,
+        help="sup_data_path for the JSON manifest",
+    )
+    parser.add_argument(
+        "--output-folder",
+        required=True,
+        type=Path,
+        help="Path to the output folder. Will contain updated manifest and mels/ folder with spectrograms in .npy files",
+    )
+    parser.add_argument("--device", required=True, type=torch.device, help="Device ('cpu', 'cuda:0', ...)")
+    parser.add_argument("--batch-size", required=True, type=int, help="Batch size in the DataLoader")
+    parser.add_argument("--num-workers", required=True, type=int, help="Num workers in the DataLoader")
+    return parser
+
+
+if __name__ == "__main__":
+    arguments = argument_parser().parse_args()
+    prepare_paired_mel_spectrograms(**vars(arguments))

scripts/dataset_processing/tts/sfbilingual/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,49 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+phoneme_dict_path: "scripts/tts_dataset_files/zh/24finals/pinyin_dict_nv_22.10.txt"
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: true
+  trim_top_db: 50
+  trim_frame_length: 1024
+  trim_hop_length: 256
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: zh
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.ChinesePhonemesTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true
+    g2p:
+      _target_: nemo.collections.tts.g2p.models.zh_cn_pinyin.ChineseG2p
+      phoneme_dict: ${phoneme_dict_path}
+      word_segmenter: jieba # Only jieba is supported now.

scripts/dataset_processing/tts/sfbilingual/get_data.py

@@ -0,0 +1,137 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import random
+import subprocess
+from pathlib import Path
+
+import numpy as np
+from opencc import OpenCC
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description='Prepare SF_bilingual dataset and create manifests with predefined split'
+    )
+
+    parser.add_argument(
+        "--data-root",
+        type=Path,
+        help="where the dataset will reside",
+        default="./DataChinese/sf_bilingual_speech_zh_en_vv1/SF_bilingual/",
+    )
+    parser.add_argument(
+        "--manifests-path", type=Path, help="where the resulting manifests files will reside", default="./"
+    )
+    parser.add_argument("--val-size", default=0.01, type=float, help="eval set split")
+    parser.add_argument("--test-size", default=0.01, type=float, help="test set split")
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def __process_transcript(file_path: str):
+    # Create zh-TW to zh-simplify converter
+    cc = OpenCC('t2s')
+    # Create normalizer
+    text_normalizer = Normalizer(
+        lang="zh",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(file_path / "cache_dir"),
+    )
+    text_normalizer_call_kwargs = {"punct_pre_process": True, "punct_post_process": True}
+    normalizer_call = lambda x: text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+    entries = []
+    i = 0
+    with open(file_path / "text_SF.txt", encoding="utf-8") as fin:
+        for line in fin:
+            content = line.split()
+            wav_name, text = content[0], "".join(content[1:])
+            wav_name = wav_name.replace(u'\ufeff', '')
+            # WAR: change DL to SF, e.g. real wave file com_SF_ce2727.wav, wav name in text_SF
+            # com_DL_ce2727. It would be fixed through the dataset in the future.
+            wav_name = wav_name.replace('DL', 'SF')
+            wav_file = file_path / "wavs" / (wav_name + ".wav")
+            assert os.path.exists(wav_file), f"{wav_file} not found!"
+            duration = subprocess.check_output(["soxi", "-D", str(wav_file)])
+            simplified_text = cc.convert(text)
+            normalized_text = normalizer_call(simplified_text)
+            entry = {
+                'audio_filepath': os.path.abspath(wav_file),
+                'duration': float(duration),
+                'text': text,
+                'normalized_text': normalized_text,
+            }
+
+            i += 1
+            entries.append(entry)
+    return entries
+
+
+def __process_data(dataset_path, val_size, test_size, seed_for_ds_split, manifests_dir):
+    entries = __process_transcript(dataset_path)
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+
+    train_size = 1.0 - val_size - test_size
+    train_entries, validate_entries, test_entries = np.split(
+        entries, [int(len(entries) * train_size), int(len(entries) * (train_size + val_size))]
+    )
+
+    assert len(train_entries) > 0, "Not enough data for train, val and test"
+
+    def save(p, data):
+        with open(p, 'w') as f:
+            for d in data:
+                f.write(json.dumps(d) + '\n')
+
+    save(manifests_dir / "train_manifest.json", train_entries)
+    save(manifests_dir / "val_manifest.json", validate_entries)
+    save(manifests_dir / "test_manifest.json", test_entries)
+
+
+def main():
+    args = get_args()
+    dataset_root = args.data_root
+    dataset_root.mkdir(parents=True, exist_ok=True)
+    __process_data(
+        dataset_root,
+        args.val_size,
+        args.test_size,
+        args.seed_for_ds_split,
+        args.manifests_path,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/tts/thorsten_neutral/ds_conf/ds_for_fastpitch_align.yaml

@@ -0,0 +1,44 @@
+name: "ds_for_fastpitch_align"
+
+manifest_filepath: "train_manifest.json"
+sup_data_path: "sup_data"
+sup_data_types: [ "align_prior_matrix", "pitch" ]
+
+dataset:
+  _target_: nemo.collections.tts.data.dataset.TTSDataset
+  manifest_filepath: ${manifest_filepath}
+  sample_rate: 22050
+  sup_data_path: ${sup_data_path}
+  sup_data_types: ${sup_data_types}
+  n_fft: 1024
+  win_length: 1024
+  hop_length: 256
+  window: "hann"
+  n_mels: 80
+  lowfreq: 0
+  highfreq: null
+  max_duration: null
+  min_duration: 0.1
+  ignore_file: null
+  trim: true
+  trim_top_db: 50
+  trim_frame_length: ${dataset.win_length}
+  trim_hop_length: ${dataset.hop_length}
+  pitch_fmin: 65.40639132514966
+  pitch_fmax: 2093.004522404789
+
+  text_normalizer:
+    _target_: nemo_text_processing.text_normalization.normalize.Normalizer
+    lang: de
+    input_case: cased
+
+  text_normalizer_call_kwargs:
+    verbose: false
+    punct_pre_process: true
+    punct_post_process: true
+
+  text_tokenizer:
+    _target_: nemo.collections.common.tokenizers.text_to_speech.tts_tokenizers.GermanCharsTokenizer
+    punct: true
+    apostrophe: true
+    pad_with_space: true

scripts/dataset_processing/tts/thorsten_neutral/get_data.py

@@ -0,0 +1,274 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to generate JSON manifests for mel-generator model training. The usage is below.
+
+$ python scripts/dataset_processing/tts/thorsten_neutral/get_data.py \
+    --data-root ~/experiments/thorsten_neutral \
+    --manifests-root ~/experiments/thorsten_neutral \
+    --data-version "22_10" \
+    --min-duration 0.1 \
+    --normalize-text
+"""
+
+import argparse
+import json
+import random
+import shutil
+import subprocess
+import urllib.request
+from pathlib import Path
+
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+try:
+    from nemo_text_processing.text_normalization.normalize import Normalizer
+except (ImportError, ModuleNotFoundError):
+    raise ModuleNotFoundError(
+        "The package `nemo_text_processing` was not installed in this environment. Please refer to"
+        " https://github.com/NVIDIA/NeMo-text-processing and install this package before using "
+        "this script"
+    )
+
+from nemo.utils import logging
+
+# Thorsten Müller published two neural voice datasets, 21.02 and 22.10.
+THORSTEN_NEUTRAL = {
+    "21_02": {
+        "url": "https://zenodo.org/record/5525342/files/thorsten-neutral_v03.tgz?download=1",
+        "dir_name": "thorsten-de_v03",
+        "metadata": ["metadata.csv"],
+    },
+    "22_10": {
+        "url": "https://zenodo.org/record/7265581/files/ThorstenVoice-Dataset_2022.10.zip?download=1",
+        "dir_name": "ThorstenVoice-Dataset_2022.10",
+        "metadata": ["metadata_train.csv", "metadata_dev.csv", "metadata_test.csv"],
+    },
+}
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Download Thorsten Müller's neutral voice dataset and create manifests with predefined split. "
+        "Thorsten Müller published two neural voice datasets, 21.02 and 22.10, where 22.10 provides better "
+        "audio quality. Please choose one of the two for your TTS models. Details about the dataset are "
+        "in https://github.com/thorstenMueller/Thorsten-Voice.",
+    )
+    parser.add_argument("--data-root", required=True, type=Path, help="where the resulting dataset will reside.")
+    parser.add_argument("--manifests-root", required=True, type=Path, help="where the manifests files will reside.")
+    parser.add_argument("--data-version", default="22_10", choices=["21_02", "22_10"], type=str)
+    parser.add_argument("--min-duration", default=0.1, type=float)
+    parser.add_argument("--max-duration", default=float('inf'), type=float)
+    parser.add_argument("--val-size", default=100, type=int)
+    parser.add_argument("--test-size", default=100, type=int)
+    parser.add_argument(
+        "--num-workers",
+        default=-1,
+        type=int,
+        help="Specify the max number of concurrent Python worker processes. "
+        "If -1 all CPUs are used. If 1 no parallel computing is used.",
+    )
+    parser.add_argument(
+        "--normalize-text",
+        default=False,
+        action='store_true',
+        help="Normalize original text and add a new entry 'normalized_text' to .json file if True.",
+    )
+    parser.add_argument(
+        "--seed-for-ds-split",
+        default=100,
+        type=float,
+        help="Seed for deterministic split of train/dev/test, NVIDIA's default is 100.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def __maybe_download_file(source_url, destination_path):
+    if not destination_path.exists():
+        logging.info(f"Downloading data: {source_url} --> {destination_path}")
+        tmp_file_path = destination_path.with_suffix(".tmp")
+        urllib.request.urlretrieve(source_url, filename=tmp_file_path)
+        tmp_file_path.rename(destination_path)
+    else:
+        logging.info(f"Skipped downloading data because it exists: {destination_path}")
+
+
+def __extract_file(filepath, data_dir):
+    logging.info(f"Unzipping data: {filepath} --> {data_dir}")
+    shutil.unpack_archive(filepath, data_dir)
+    logging.info(f"Unzipping data is complete: {filepath}.")
+
+
+def __save_json(json_file, dict_list):
+    logging.info(f"Saving JSON split to {json_file}.")
+    with open(json_file, "w") as f:
+        for d in dict_list:
+            f.write(json.dumps(d) + "\n")
+
+
+def __text_normalization(json_file, num_workers=-1):
+    text_normalizer_call_kwargs = {
+        "punct_pre_process": True,
+        "punct_post_process": True,
+    }
+    text_normalizer = Normalizer(
+        lang="de",
+        input_case="cased",
+        overwrite_cache=True,
+        cache_dir=str(json_file.parent / "cache_dir"),
+    )
+
+    def normalizer_call(x):
+        return text_normalizer.normalize(x, **text_normalizer_call_kwargs)
+
+    def add_normalized_text(line_dict):
+        normalized_text = normalizer_call(line_dict["text"])
+        line_dict.update({"normalized_text": normalized_text})
+        return line_dict
+
+    logging.info(f"Normalizing text for {json_file}.")
+    with open(json_file, 'r', encoding='utf-8') as fjson:
+        lines = fjson.readlines()
+        # Note: you need to verify which backend works well on your cluster.
+        # backend="loky" is fine on multi-core Ubuntu OS; backend="threading" on Slurm.
+        dict_list = Parallel(n_jobs=num_workers)(
+            delayed(add_normalized_text)(json.loads(line)) for line in tqdm(lines)
+        )
+
+    json_file_text_normed = json_file.parent / f"{json_file.stem}_text_normed{json_file.suffix}"
+    with open(json_file_text_normed, 'w', encoding="utf-8") as fjson_norm:
+        for dct in dict_list:
+            fjson_norm.write(json.dumps(dct) + "\n")
+    logging.info(f"Normalizing text is complete: {json_file} --> {json_file_text_normed}")
+
+
+def __process_data(
+    unzipped_dataset_path, metadata, min_duration, max_duration, val_size, test_size, seed_for_ds_split
+):
+    logging.info("Preparing JSON train/val/test splits.")
+
+    entries = list()
+    not_found_wavs = list()
+    wrong_duration_wavs = list()
+
+    for metadata_fname in metadata:
+        meta_file = unzipped_dataset_path / metadata_fname
+        with open(meta_file, 'r') as fmeta:
+            for line in tqdm(fmeta):
+                items = line.strip().split('|')
+                wav_file_stem, text = items[0], items[1]
+                wav_file = unzipped_dataset_path / "wavs" / f"{wav_file_stem}.wav"
+
+                # skip audios if they do not exist.
+                if not wav_file.exists():
+                    not_found_wavs.append(wav_file)
+                    logging.warning(f"Skipping {wav_file}: it is not found.")
+                    continue
+
+                # skip audios if their duration is out of range.
+                duration = subprocess.check_output(["soxi", "-D", str(wav_file)])
+                duration = float(duration)
+                if min_duration <= duration <= max_duration:
+                    entry = {
+                        'audio_filepath': str(wav_file),
+                        'duration': duration,
+                        'text': text,
+                    }
+                    entries.append(entry)
+                elif duration < min_duration:
+                    wrong_duration_wavs.append(wav_file)
+                    logging.warning(f"Skipping {wav_file}: it is too short, less than {min_duration} seconds.")
+                    continue
+                else:
+                    wrong_duration_wavs.append(wav_file)
+                    logging.warning(f"Skipping {wav_file}: it is too long, greater than {max_duration} seconds.")
+                    continue
+
+    random.Random(seed_for_ds_split).shuffle(entries)
+    train_size = len(entries) - val_size - test_size
+    if train_size <= 0:
+        raise ValueError("Not enough data for the train split.")
+
+    logging.info("Preparing JSON train/val/test splits is complete.")
+    train, val, test = (
+        entries[:train_size],
+        entries[train_size : train_size + val_size],
+        entries[train_size + val_size :],
+    )
+
+    return train, val, test, not_found_wavs, wrong_duration_wavs
+
+
+def main():
+    args = get_args()
+    data_root = args.data_root
+    manifests_root = args.manifests_root
+    data_version = args.data_version
+
+    dataset_root = data_root / f"ThorstenVoice-Dataset-{data_version}"
+    dataset_root.mkdir(parents=True, exist_ok=True)
+
+    # download and extract dataset
+    dataset_url = THORSTEN_NEUTRAL[data_version]["url"]
+    zipped_dataset_path = dataset_root / Path(dataset_url).name.split("?")[0]
+    __maybe_download_file(dataset_url, zipped_dataset_path)
+    __extract_file(zipped_dataset_path, dataset_root)
+
+    # generate train/dev/test splits
+    unzipped_dataset_path = dataset_root / THORSTEN_NEUTRAL[data_version]["dir_name"]
+    entries_train, entries_val, entries_test, not_found_wavs, wrong_duration_wavs = __process_data(
+        unzipped_dataset_path=unzipped_dataset_path,
+        metadata=THORSTEN_NEUTRAL[data_version]["metadata"],
+        min_duration=args.min_duration,
+        max_duration=args.max_duration,
+        val_size=args.val_size,
+        test_size=args.test_size,
+        seed_for_ds_split=args.seed_for_ds_split,
+    )
+
+    # save json splits.
+    train_json = manifests_root / "train_manifest.json"
+    val_json = manifests_root / "val_manifest.json"
+    test_json = manifests_root / "test_manifest.json"
+    __save_json(train_json, entries_train)
+    __save_json(val_json, entries_val)
+    __save_json(test_json, entries_test)
+
+    # save skipped audios that are not found into a file.
+    if len(not_found_wavs) > 0:
+        skipped_not_found_file = manifests_root / "skipped_not_found_wavs.list"
+        with open(skipped_not_found_file, "w") as f_notfound:
+            for line in not_found_wavs:
+                f_notfound.write(f"{line}\n")
+
+    # save skipped audios that are too short or too long into a file.
+    if len(wrong_duration_wavs) > 0:
+        skipped_wrong_duration_file = manifests_root / "skipped_wrong_duration_wavs.list"
+        with open(skipped_wrong_duration_file, "w") as f_wrong_dur:
+            for line in wrong_duration_wavs:
+                f_wrong_dur.write(f"{line}\n")
+
+    # normalize text if requested. New json file, train_manifest_text_normed.json, will be generated.
+    if args.normalize_text:
+        __text_normalization(train_json, args.num_workers)
+        __text_normalization(val_json, args.num_workers)
+        __text_normalization(test_json, args.num_workers)
+
+
+if __name__ == "__main__":
+    main()