chore: import upstream snapshot with attribution

scripts/speech_recognition/code_switching/README.md

@@ -0,0 +1,17 @@
+# Scripts for creation of synthetic code-switched data from monolingual sources
+Follow the 2 steps listed below in order - 
+
+
+1. Create the (intermediate) manifest file using `code_switching_manifest_creation.py`.  It's usage is as follows:
+
+    `python code_switching_manifest_creation.py --manifest_language1 <absolute path of Language 1's manifest file> --manifest_language2 <absolute path of Language 2's manifest file> --manifest_save_path <absolute path to save the created manifest> --id_language1 <language code for language 1 (e.g. en)> --id_language2 <language code for language 2 (e.g. es)> --max_sample_duration_sec <maximum duration of generated sample in seconds> --min_sample_duration_sec <maximum duration of generated sample in seconds> --dataset_size_required_hrs <size of generated synthetic dataset required in hrs>`
+
+    Estimated runtime for dataset_size_required_hrs=10,000 is ~2 mins
+
+2. Create the synthetic audio data and the corresponding manifest file using `code_switching_audio_data_creation.py` It's usage is as follows: 
+
+    `python code_switching_audio_data_creation.py --manifest_path <absolute path to intermediate CS manifest generated in step 1> --audio_save_folder_path <absolute path to directory where you want to save the synthesized audios> --manifest_save_path <absolute path to save the created manifest> --audio_normalized_amplitude <scaled normalized amplitude desired> --cs_data_sampling_rate <desired sampling rate for generated audios> --sample_beginning_pause_msec <pause to be added to the beginning of the generated sample in milli seconds> --sample_joining_pause_msec <pause to be added between segments while joining, in milli seconds> --sample_end_pause_msec <pause to be added to the end of the generated sample in milli seconds> --is_lid_manifest <boolean to create manifest in the multi-sample lid format for the text field, true by default> --workers <number of worker processes>`
+
+    Example of the multi-sample LID format: ```[{“str”:“esta muestra ” “lang”:”es”},{“str”:“was generated synthetically”: “lang”:”en”}]```
+    
+    Estimated runtime for generating a 10,000 hour corpus is ~40 hrs with a single worker

scripts/speech_recognition/code_switching/code_switching_audio_data_creation.py

@@ -0,0 +1,288 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import logging
+import os
+
+import librosa
+import numpy as np
+from joblib import Parallel, delayed
+from scipy.io import wavfile
+from tqdm import tqdm
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+
+parser = argparse.ArgumentParser(description='Create synthetic code-switching data audio data from monolingual data')
+parser.add_argument("--manifest_path", default=None, type=str, help='Path to CS indermediate manifest', required=True)
+parser.add_argument(
+    "--audio_save_folder_path",
+    default=None,
+    type=str,
+    help='Path to directory where created synthetic set would be saved',
+    required=True,
+)
+parser.add_argument(
+    "--manifest_save_path", default=None, type=str, help='Path to save the created manifest', required=True
+)
+parser.add_argument(
+    "--audio_normalized_amplitude", default=15000, type=int, help='Normalized amplitdue of audio samples'
+)
+parser.add_argument(
+    "--cs_data_sampling_rate",
+    default=16000,
+    type=int,
+    help='Desired sampling rate for the audios in the generated dataset',
+)
+parser.add_argument(
+    "--sample_beginning_pause_msec",
+    default=20,
+    type=int,
+    help='Pause to be added at the beginning of the sample (msec)',
+)
+parser.add_argument(
+    "--sample_joining_pause_msec",
+    default=100,
+    type=int,
+    help='Pause to be added between different phrases of the sample (msec)',
+)
+parser.add_argument(
+    "--sample_end_pause_msec", default=20, type=int, help='Pause to be added at the end of the sample (msec)'
+)
+parser.add_argument(
+    "--is_lid_manifest",
+    default=True,
+    type=bool,
+    help='If true, generate manifest in the multi-sample lid format, else the standard manifest format',
+)
+parser.add_argument("--workers", default=1, type=int, help='Number of worker processes')
+
+args = parser.parse_args()
+
+
+def split_list(input_list: list, num_splits: int):
+    """
+    Args:
+        input_list: the input list to split
+        num_splits: number of splits required
+
+    Returns:
+        iterator of split lists
+
+    """
+    k, m = divmod(len(input_list), num_splits)
+    return (input_list[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(num_splits))
+
+
+def combine_manifests(manifest_save_path: str, num_split: int):
+    """
+    Args:
+        manifest_save_path: absolute path to save the combined manifest
+        num_splits: number of splits of manifest
+
+    Returns:
+        num_samples_combined: the total number of samples in the generated dataset
+    """
+    num_samples_combined = 0
+    base_directory = os.path.dirname(manifest_save_path)
+
+    with open(manifest_save_path, 'w') as outfile:
+        for i in range(num_split):
+            split_manifest_path = base_directory + '/temp_' + str(i) + '.json'
+            data_split = read_manifest(split_manifest_path)
+
+            for elem in data_split:
+                s = json.dumps(elem)
+                outfile.write(s + '\n')
+                num_samples_combined += 1
+
+            # removing the intermediate file
+            os.remove(split_manifest_path)
+
+    return num_samples_combined
+
+
+def create_cs_data(
+    intermediate_cs_manifest_list: list,
+    audio_save_folder: str,
+    manfest_save_path: str,
+    audio_amplitude_normalization: int,
+    pause_beg_msec: int,
+    pause_join_msec: int,
+    pause_end_msec: int,
+    cs_data_sampling_rate: int,
+    is_lid_manifest: bool,
+):
+    """
+    Args:
+        intermediate_cs_manifest_list: the intermediate cs manifest obtained from code_switching_manifest_creation.py as a list
+        audio_save_folder: Absolute path to save the generated audio samples
+        manfest_save_path: Absolute path to save the corresponding manifest
+        audio_amplitude_normalization: The amplitude to scale to after normalization
+        pause_beg_msec: Pause to be added at the beginning of the sample (msec)
+        pause_join_msec: Pause to be added between different phrases of the sample (msec)
+        pause_end_msec: Pause to be added at the end of the sample (msec)
+        cs_data_sampling_rate: Desired sampling rate of the generated samples
+        is_lid_manifest: If true, generate manifest in the multi-sample lid format, else the standard manifest format
+
+    Returns:
+
+    """
+
+    fs = cs_data_sampling_rate
+    incorrect_sample_flag = 0
+
+    with open(manfest_save_path, 'w') as outfile:
+        for data in tqdm(intermediate_cs_manifest_list):
+
+            combined_audio = []
+
+            staring_pause = np.zeros(int(pause_beg_msec * fs / 1000))
+            combined_audio += list(staring_pause)
+
+            text_entry_list = []
+            for index in range(len(data['lang_ids'])):
+
+                phrase_entry = {}
+                # dictionary to store the phrase information which will be added to the complete sentence
+
+                data_sample, fs_sample = librosa.load(data['paths'][index], sr=fs)
+                # Alternative-  fs_sample, data_sample = wavfile.read(data['paths'][index])
+
+                if fs_sample != fs:
+                    logging.error('Sampling rate error inside create_cs_data function')
+                    exit
+
+                # Remove leading and trailing zeros
+                data_sample = np.trim_zeros(data_sample)
+
+                # take care of empty arrays: rare
+                if data_sample.size == 0:
+                    incorrect_sample_flag = 1
+                    continue
+
+                # normalizing data
+                data_sample_norm = (
+                    data_sample
+                    / np.maximum(np.abs(data_sample.max()), np.abs(data_sample.min()))
+                    * audio_amplitude_normalization
+                )
+
+                combined_audio += list(data_sample_norm)
+
+                phrase_entry['str'] = data['texts'][index]
+                phrase_entry['lang'] = data['lang_ids'][index]
+
+                text_entry_list.append(phrase_entry)
+
+                # adding small pause between semgments
+                if index != (len(data['lang_ids']) - 1):
+                    pause = np.zeros(int(pause_join_msec * fs / 1000))
+                    combined_audio += list(pause)
+
+            if incorrect_sample_flag == 1:
+                incorrect_sample_flag = 0
+                continue
+
+            ending_pause = np.zeros(int(pause_end_msec * fs / 1000))
+            combined_audio += list(ending_pause)
+
+            sample_id = data['uid']
+            audio_file_path = audio_save_folder + '/' + str(sample_id) + ".wav"
+
+            # saving audio
+            wavfile.write(audio_file_path, fs, np.array(combined_audio).astype(np.int16))
+            # Alternative-  librosa.output.write_wav(audio_file_path, combined_audio, fs)
+
+            metadata_json = {}
+            metadata_json['audio_filepath'] = audio_file_path
+            metadata_json['duration'] = float(len(combined_audio) / fs)
+            if is_lid_manifest:
+                metadata_json['text'] = text_entry_list
+            else:
+                metadata_json['text'] = ' '.join(data['texts'])
+
+            metadata_json['language_ids'] = data['lang_ids']
+            metadata_json['original_texts'] = data['texts']
+            metadata_json['original_paths'] = data['paths']
+            metadata_json['original_durations'] = data['durations']
+
+            s = json.dumps(metadata_json)
+            outfile.write(s + '\n')
+
+
+def main():
+
+    cs_intermediate_manifest_path = args.manifest_path
+    audio_save_folder = args.audio_save_folder_path
+    manifest_save_path = args.manifest_save_path
+    audio_amplitude_normalization = args.audio_normalized_amplitude
+    pause_beg_msec = args.sample_beginning_pause_msec
+    pause_join_msec = args.sample_joining_pause_msec
+    pause_end_msec = args.sample_end_pause_msec
+    cs_data_sampling_rate = args.cs_data_sampling_rate
+    is_lid_manifest = args.is_lid_manifest
+    num_process = args.workers
+
+    # Sanity Checks
+    if (cs_intermediate_manifest_path is None) or (not os.path.exists(cs_intermediate_manifest_path)):
+        logging.error('Please provide correct CS manifest (obtained from code_switching_manifest_creation.py)')
+        exit
+
+    if (audio_save_folder is None) or (not os.path.exists(audio_save_folder)):
+        logging.error('audio_save_folder_path is incorrect or does not exist')
+        exit
+
+    if manifest_save_path is None:
+        logging.error('Please provide valid manifest_save_path')
+        exit
+
+    # Reading data
+    logging.info('Reading manifests')
+    intermediate_cs_manifest = read_manifest(cs_intermediate_manifest_path)
+
+    # Spliting the data
+    data_split = split_list(intermediate_cs_manifest, num_process)
+
+    # Creating Audio data
+    logging.info('Creating synthetic audio data')
+    base_directory = os.path.dirname(manifest_save_path)
+
+    Parallel(n_jobs=num_process)(
+        delayed(create_cs_data)(
+            split_manifest,
+            audio_save_folder,
+            base_directory + '/temp_' + str(idx) + '.json',
+            audio_amplitude_normalization,
+            pause_beg_msec,
+            pause_join_msec,
+            pause_end_msec,
+            cs_data_sampling_rate,
+            is_lid_manifest,
+        )
+        for idx, split_manifest in enumerate(data_split)
+    )
+
+    # Combining manifests
+    num_samples_combined = combine_manifests(manifest_save_path, num_process)
+
+    print("Synthetic CS audio data saved at :", audio_save_folder)
+    print("Synthetic CS manifest saved at :", manifest_save_path)
+    print("Total number of samples in the generated dataset :", str(num_samples_combined))
+
+    logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()

scripts/speech_recognition/code_switching/code_switching_manifest_creation.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+import random
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+
+# Checks -
+# (Recommendation) Please normalize the text for each language (avoid numbers, special characters, punctuation)
+# Please ensure that the audio_filepaths are absolute locations
+
+
+parser = argparse.ArgumentParser(description='Create synthetic code-switching data manifest from monolingual data')
+
+parser.add_argument("--manifest_language1", default=None, type=str, help='Manifest file for language 1', required=True)
+parser.add_argument("--manifest_language2", default=None, type=str, help='Manifest file for language 2', required=True)
+parser.add_argument(
+    "--manifest_save_path", default=None, type=str, help='Path to save created CS indermediate manifest', required=True
+)
+parser.add_argument(
+    "--id_language1", default=None, type=str, help='Identifier for language 1, eg: en, es, hi', required=True
+)
+parser.add_argument(
+    "--id_language2", default=None, type=str, help='Identifier for language 2, eg: en, es, hi', required=True
+)
+parser.add_argument("--max_sample_duration_sec", default=19, type=int, help='Maximum duration of sample (sec)')
+parser.add_argument("--min_sample_duration_sec", default=16, type=int, help='Minimum duration of sample (sec)')
+parser.add_argument("--dataset_size_required_hrs", default=1, type=int, help='Duration of dataset required (hrs)')
+
+args = parser.parse_args()
+
+
+def create_cs_manifest(
+    data_lang_0: list,
+    data_lang_1: list,
+    lid_lang_0: str,
+    lid_lang_1: str,
+    max_sample_duration_sec: int,
+    min_sample_duration_sec: int,
+    data_requirement_hrs: int,
+):
+    """
+    Args:
+        data_lang_0: Manifest entries from first langauge
+        data_lang_1: Manifest entries from second langauge
+        lid_lang_0: Language ID marker for first langauge
+        lid_lang_1: Language ID marker for second langauge
+        max_sample_duration_sec: Maximum permissible duration of generated CS sample in sec
+        min_sample_duration_sec: Minimum permissible duration of generated CS sample in sec
+        data_requirement_hrs: Required size of generated corpus
+
+    Returns:
+        Created synthetic CS manifest as list
+
+    """
+
+    total_duration = 0
+    constructed_data = []
+    sample_id = 0
+
+    num_samples_lang0 = len(data_lang_0)
+    num_samples_lang1 = len(data_lang_1)
+
+    while total_duration < (data_requirement_hrs * 3600):
+
+        created_sample_duration_sec = 0
+        created_sample_dict = {}
+        created_sample_dict['lang_ids'] = []
+        created_sample_dict['texts'] = []
+        created_sample_dict['paths'] = []
+        created_sample_dict['durations'] = []
+
+        while created_sample_duration_sec < min_sample_duration_sec:
+
+            lang_selection = random.randint(0, 1)
+
+            if lang_selection == 0:
+                index = random.randint(0, num_samples_lang0 - 1)
+                sample = data_lang_0[index]
+                lang_id = lid_lang_0
+            else:
+                index = random.randint(0, num_samples_lang1 - 1)
+                sample = data_lang_1[index]
+                lang_id = lid_lang_1
+
+            if (created_sample_duration_sec + sample['duration']) > max_sample_duration_sec:
+                continue
+            else:
+                created_sample_duration_sec += sample['duration']
+                created_sample_dict['lang_ids'].append(lang_id)
+                created_sample_dict['texts'].append(sample['text'])
+                created_sample_dict['paths'].append(sample['audio_filepath'])
+                created_sample_dict['durations'].append(sample['duration'])
+
+        created_sample_dict['total_duration'] = created_sample_duration_sec
+
+        # adding a uid which will be used to save the generated audio file later
+        created_sample_dict['uid'] = sample_id
+        sample_id += 1
+
+        constructed_data.append(created_sample_dict)
+        total_duration += created_sample_duration_sec
+
+    return constructed_data
+
+
+def main():
+
+    manifest0 = args.manifest_language1
+    manifest1 = args.manifest_language2
+    lid0 = args.id_language1
+    lid1 = args.id_language2
+    min_sample_duration = args.min_sample_duration_sec
+    max_sample_duration = args.max_sample_duration_sec
+    dataset_requirement = args.dataset_size_required_hrs
+    manifest_save_path = args.manifest_save_path
+
+    # Sanity Checks
+    if (manifest0 is None) or (not os.path.exists(manifest0)):
+        logging.error('Manifest for language 1 is incorrect')
+        exit
+
+    if (manifest1 is None) or (not os.path.exists(manifest1)):
+        logging.error('Manifest for language 2 is incorrect')
+        exit
+
+    if lid0 is None:
+        logging.error('Please provide correct language code for language 1')
+        exit
+
+    if lid1 is None:
+        logging.error('Please provide correct language code for language 2')
+        exit
+
+    if manifest_save_path is None:
+        logging.error('Please provide correct manifest save path')
+        exit
+
+    if min_sample_duration >= max_sample_duration:
+        logging.error('Please ensure max_sample_duration > min_sample_duration')
+        exit
+
+    # Reading data
+    logging.info('Reading manifests')
+    data_language0 = read_manifest(manifest0)
+    data_language1 = read_manifest(manifest1)
+
+    # Creating the CS data Manifest
+    logging.info('Creating CS manifest')
+    constructed_data = create_cs_manifest(
+        data_language0, data_language1, lid0, lid1, max_sample_duration, min_sample_duration, dataset_requirement
+    )
+
+    # Saving Manifest
+    logging.info('saving manifest')
+    write_manifest(manifest_save_path, constructed_data)
+
+    print("Synthetic CS manifest saved at :", manifest_save_path)
+
+    logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()