chore: import upstream snapshot with attribution

scripts/dataset_processing/speaker_tasks/README.md

@@ -0,0 +1,5 @@
+# Speaker Tasks Dataset Scripts
+
+In this folder are scripts to download speaker tasks (mainly for diarization) datasets. These scripts will return NeMo format manifest files to use with Diarization.
+
+We also have scripts for CallHome and DIHARD3, however the data has to be downloaded separately. If you require the scripts please leave an issue.

scripts/dataset_processing/speaker_tasks/get_aishell_diarization_data.py

@@ -0,0 +1,118 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# downloads the training/eval set for AISHELL Diarization.
+# the training dataset is around 170GiB, to skip pass the --skip_train flag.
+
+import argparse
+import glob
+import logging
+import os
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from nemo.collections.asr.parts.utils.manifest_utils import create_manifest
+from nemo.utils.tar_utils import safe_extract
+
+train_url = "https://www.openslr.org/resources/111/train_{}.tar.gz"
+train_datasets = ["S", "M", "L"]
+
+eval_url = "https://www.openslr.org/resources/111/test.tar.gz"
+
+
+def _load_sox_transformer():
+    try:
+        from sox import Transformer
+    except ImportError:
+        raise ImportError(
+            "Optional dependency 'sox' is required by this script. Install it with: pip install sox"
+        ) from None
+
+    return Transformer
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath) as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def __process_data(dataset_url: str, dataset_path: Path, manifest_output_path: Path):
+    os.makedirs(dataset_path, exist_ok=True)
+    tar_file_path = os.path.join(dataset_path, os.path.basename(dataset_url))
+    if not os.path.exists(tar_file_path):
+        urllib.request.urlretrieve(dataset_url, filename=tar_file_path)
+    extract_file(tar_file_path, str(dataset_path))
+    wav_path = dataset_path / 'converted_wav/'
+    extracted_dir = Path(tar_file_path).stem.replace('.tar', '')
+    flac_path = dataset_path / (extracted_dir + '/wav/')
+    __process_flac_audio(flac_path, wav_path)
+
+    audio_files = [os.path.join(os.path.abspath(wav_path), file) for file in os.listdir(str(wav_path))]
+    rttm_files = glob.glob(str(dataset_path / (extracted_dir + '/TextGrid/*.rttm')))
+    rttm_files = [os.path.abspath(file) for file in rttm_files]
+
+    audio_list = dataset_path / 'audio_files.txt'
+    rttm_list = dataset_path / 'rttm_files.txt'
+    with open(audio_list, 'w') as f:
+        f.write('\n'.join(audio_files))
+    with open(rttm_list, 'w') as f:
+        f.write('\n'.join(rttm_files))
+    create_manifest(
+        str(audio_list),
+        manifest_output_path,
+        rttm_path=str(rttm_list),
+    )
+
+
+def __process_flac_audio(flac_path, wav_path):
+    Transformer = _load_sox_transformer()
+    os.makedirs(wav_path, exist_ok=True)
+    flac_files = os.listdir(flac_path)
+    for flac_file in flac_files:
+        # Convert FLAC file to WAV
+        id = Path(flac_file).stem
+        wav_file = os.path.join(wav_path, id + ".wav")
+        if not os.path.exists(wav_file):
+            Transformer().build(os.path.join(flac_path, flac_file), wav_file)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Aishell Data download")
+    parser.add_argument("--data_root", default='./', type=str)
+    parser.add_argument("--output_manifest_path", default='aishell_diar_manifest.json', type=str)
+    parser.add_argument("--skip_train", help="skip downloading the training dataset", action="store_true")
+    args = parser.parse_args()
+    data_root = Path(args.data_root)
+    data_root.mkdir(exist_ok=True, parents=True)
+
+    if not args.skip_train:
+        for tag in train_datasets:
+            dataset_url = train_url.format(tag)
+            dataset_path = data_root / f'{tag}/'
+            manifest_output_path = data_root / f'train_{tag}_manifest.json'
+            __process_data(
+                dataset_url=dataset_url, dataset_path=dataset_path, manifest_output_path=manifest_output_path
+            )
+    # create test dataset
+    dataset_path = data_root / f'eval/'
+    manifest_output_path = data_root / f'eval_manifest.json'
+    __process_data(dataset_url=eval_url, dataset_path=dataset_path, manifest_output_path=manifest_output_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/speaker_tasks/get_ami_data.py

@@ -0,0 +1,102 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Download the AMI test dataset used to evaluate Speaker Diarization
+# More information here: https://groups.inf.ed.ac.uk/ami/corpus/
+# USAGE: python get_ami_data.py
+import argparse
+import os
+import subprocess
+
+from nemo.collections.asr.parts.utils.manifest_utils import create_manifest
+
+rttm_url = "https://raw.githubusercontent.com/BUTSpeechFIT/AMI-diarization-setup/main/only_words/rttms/{}/{}.rttm"
+uem_url = "https://raw.githubusercontent.com/BUTSpeechFIT/AMI-diarization-setup/main/uems/{}/{}.uem"
+list_url = "https://raw.githubusercontent.com/BUTSpeechFIT/AMI-diarization-setup/main/lists/{}.meetings.txt"
+
+
+audio_types = ['Mix-Headset', 'Array1-01']
+
+# these two IDs in the train set are missing download links for Array1-01.
+# We exclude them as a result.
+not_found_ids = ['IS1007d', 'IS1003b']
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download the AMI Corpus Dataset for Speaker Diarization")
+    parser.add_argument(
+        "--test_manifest_filepath",
+        help="path to output test manifest file",
+        type=str,
+        default='AMI_test_manifest.json',
+    )
+    parser.add_argument(
+        "--dev_manifest_filepath",
+        help="path to output dev manifest file",
+        type=str,
+        default='AMI_dev_manifest.json',
+    )
+    parser.add_argument(
+        "--train_manifest_filepath",
+        help="path to output train manifest file",
+        type=str,
+        default='AMI_train_manifest.json',
+    )
+    parser.add_argument("--data_root", help="path to output data directory", type=str, default="ami_dataset")
+    args = parser.parse_args()
+
+    data_path = os.path.abspath(args.data_root)
+    os.makedirs(data_path, exist_ok=True)
+
+    for manifest_path, split in (
+        (args.test_manifest_filepath, 'test'),
+        (args.dev_manifest_filepath, 'dev'),
+        (args.train_manifest_filepath, 'train'),
+    ):
+        split_path = os.path.join(data_path, split)
+        audio_path = os.path.join(split_path, "audio")
+        os.makedirs(split_path, exist_ok=True)
+        rttm_path = os.path.join(split_path, "rttm")
+        uem_path = os.path.join(split_path, "uem")
+
+        subprocess.run(["wget", "-P", split_path, list_url.format(split)])
+        with open(os.path.join(split_path, f"{split}.meetings.txt")) as f:
+            ids = f.read().strip().split('\n')
+        for id in [file_id for file_id in ids if file_id not in not_found_ids]:
+            for audio_type in audio_types:
+                audio_type_path = os.path.join(audio_path, audio_type)
+                os.makedirs(audio_type_path, exist_ok=True)
+                audio_download = (
+                    f"https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus/{id}/audio/" f"{id}.{audio_type}.wav"
+                )
+                subprocess.run(["wget", "-P", audio_type_path, audio_download])
+            rttm_download = rttm_url.format(split, id)
+            subprocess.run(["wget", "-P", rttm_path, rttm_download])
+            uem_download = uem_url.format(split, id)
+            subprocess.run(["wget", "-P", uem_path, uem_download])
+
+        rttm_files_path = os.path.join(split_path, 'rttm_files.txt')
+        with open(rttm_files_path, 'w') as f:
+            f.write('\n'.join(os.path.join(rttm_path, p) for p in os.listdir(rttm_path)))
+        uem_files_path = os.path.join(split_path, 'uem_files.txt')
+        with open(uem_files_path, 'w') as f:
+            f.write('\n'.join(os.path.join(uem_path, p) for p in os.listdir(uem_path)))
+        for audio_type in audio_types:
+            audio_type_path = os.path.join(audio_path, audio_type)
+            audio_files_path = os.path.join(split_path, f'audio_files_{audio_type}.txt')
+            with open(audio_files_path, 'w') as f:
+                f.write('\n'.join(os.path.join(audio_type_path, p) for p in os.listdir(audio_type_path)))
+            audio_type_manifest_path = manifest_path.replace('.json', f'.{audio_type}.json')
+            create_manifest(
+                audio_files_path, audio_type_manifest_path, rttm_path=rttm_files_path, uem_path=uem_files_path
+            )

scripts/dataset_processing/speaker_tasks/get_hi-mia_data.py

@@ -0,0 +1,208 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python get_hi-mia_data.py --data_root=<where to put data>
+
+import argparse
+import json
+import logging as _logging
+import os
+import tarfile
+import urllib.request
+from glob import glob
+
+import librosa as l
+from sklearn.model_selection import StratifiedShuffleSplit
+from tqdm import tqdm
+
+from nemo.utils.tar_utils import safe_extract
+
+parser = argparse.ArgumentParser(description="HI-MIA Data download")
+parser.add_argument("--data_root", required=True, default=None, type=str)
+parser.add_argument("--log_level", default=20, type=int)
+args = parser.parse_args()
+logging = _logging.getLogger(__name__)
+logging.addHandler(_logging.StreamHandler())
+logging.setLevel(args.log_level)
+
+URL = {
+    "dev": "http://www.openslr.org/resources/85/dev.tar.gz",
+    "test": "http://www.openslr.org/resources/85/test.tar.gz",
+    "train": "http://www.openslr.org/resources/85/train.tar.gz",
+}
+
+
+def __retrieve_with_progress(source: str, filename: str):
+    """
+    Downloads source to destination
+    Displays progress bar
+    Args:
+        source: url of resource
+        destination: local filepath
+    Returns:
+    """
+    with open(filename, "wb") as f:
+        response = urllib.request.urlopen(source)
+        total = response.length
+
+        if total is None:
+            f.write(response.content)
+        else:
+            with tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024) as pbar:
+                for data in response:
+                    f.write(data)
+                    pbar.update(len(data))
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+
+    """
+    source = URL[source]
+    if not os.path.exists(destination) and not os.path.exists(os.path.splitext(destination)[0]):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        __retrieve_with_progress(source, filename=destination + ".tmp")
+        os.rename(destination + ".tmp", destination)
+        logging.info("Downloaded {0}.".format(destination))
+    elif os.path.exists(destination):
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    elif os.path.exists(os.path.splitext(destination)[0]):
+        logging.warning(
+            "Assuming extracted folder %s contains the extracted files from %s. Will not download.",
+            os.path.basename(destination),
+            destination,
+        )
+    return destination
+
+
+def __extract_all_files(filepath: str, data_root: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_root)
+        audio_dir = os.path.join(data_dir, "wav")
+        for subfolder, _, filelist in os.walk(audio_dir):
+            for ftar in filelist:
+                extract_file(os.path.join(subfolder, ftar), subfolder)
+    else:
+        logging.info("Skipping extracting. Data already there %s" % data_dir)
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        with tarfile.open(filepath, encoding='utf-8') as tar:
+            safe_extract(tar, data_dir)
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def __remove_tarred_files(filepath: str, data_dir: str):
+    if os.path.exists(data_dir) and os.path.isfile(filepath):
+        logging.info("Deleting %s" % filepath)
+        os.remove(filepath)
+
+
+def write_file(name, lines, idx):
+    with open(name, "w") as fout:
+        for i in idx:
+            dic = lines[i]
+            json.dump(dic, fout)
+            fout.write("\n")
+    logging.info("wrote %s", name)
+
+
+def __process_data(data_folder: str, data_set: str):
+    """
+    To generate manifest
+    Args:
+        data_folder: source with wav files
+    Returns:
+
+    """
+    fullpath = os.path.abspath(data_folder)
+    filelist = glob(fullpath + "/**/*.wav", recursive=True)
+    out = os.path.join(fullpath, data_set + "_all.json")
+    utt2spk = os.path.join(fullpath, "utt2spk")
+    utt2spk_file = open(utt2spk, "w")
+    id = -2  # speaker id
+
+    if os.path.exists(out):
+        logging.warning(
+            "%s already exists and is assumed to be processed. If not, please delete %s and rerun this script",
+            out,
+            out,
+        )
+        return
+
+    speakers = []
+    lines = []
+    with open(out, "w") as outfile:
+        for line in tqdm(filelist):
+            line = line.strip()
+            y, sr = l.load(line, sr=None)
+            if sr != 16000:
+                y, sr = l.load(line, sr=16000)
+                l.output.write_wav(line, y, sr)
+            dur = l.get_duration(y=y, sr=sr)
+            if data_set == "test":
+                speaker = line.split("/")[-1].split(".")[0].split("_")[0]
+            else:
+                speaker = line.split("/")[id]
+            speaker = list(speaker)
+            speaker = "".join(speaker)
+            speakers.append(speaker)
+            meta = {"audio_filepath": line, "duration": float(dur), "label": speaker}
+            lines.append(meta)
+            json.dump(meta, outfile)
+            outfile.write("\n")
+            utt2spk_file.write(line.split("/")[-1] + "\t" + speaker + "\n")
+
+    utt2spk_file.close()
+
+    if data_set != "test":
+        sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=42)
+        for train_idx, test_idx in sss.split(speakers, speakers):
+            print(len(train_idx))
+
+        out = os.path.join(fullpath, "train.json")
+        write_file(out, lines, train_idx)
+        out = os.path.join(fullpath, "dev.json")
+        write_file(out, lines, test_idx)
+
+
+def main():
+    data_root = args.data_root
+    for data_set in URL.keys():
+
+        # data_set = 'data_aishell'
+        logging.info("\n\nWorking on: {0}".format(data_set))
+        file_path = os.path.join(data_root, data_set + ".tgz")
+        logging.info("Getting {0}".format(data_set))
+        __maybe_download_file(file_path, data_set)
+        logging.info("Extracting {0}".format(data_set))
+        data_folder = os.path.join(data_root, data_set)
+        __extract_all_files(file_path, data_root, data_folder)
+        __remove_tarred_files(file_path, data_folder)
+        logging.info("Processing {0}".format(data_set))
+        __process_data(data_folder, data_set)
+        logging.info("Done!")
+
+
+if __name__ == "__main__":
+    main()

scripts/dataset_processing/speaker_tasks/get_voxconverse.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# downloads the training/eval set for VoxConverse.
+
+import argparse
+import logging
+import os
+import urllib.request
+import zipfile
+from pathlib import Path
+
+from nemo.collections.asr.parts.utils.manifest_utils import create_manifest
+
+dev_url = "https://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_dev_wav.zip"
+test_url = "https://www.robots.ox.ac.uk/~vgg/data/voxconverse/data/voxconverse_test_wav.zip"
+rttm_annotations_url = "https://github.com/joonson/voxconverse/archive/refs/heads/master.zip"
+
+
+def extract_file(filepath: Path, data_dir: Path):
+    try:
+        with zipfile.ZipFile(str(filepath), 'r') as zip_ref:
+            zip_ref.extractall(str(data_dir))
+    except Exception:
+        logging.info("Not extracting. Maybe already there?")
+
+
+def download_file(url: str, destination: Path) -> Path:
+    urllib.request.urlretrieve(url, filename=str(destination))
+    return destination
+
+
+def _generate_manifest(data_root: Path, audio_path: Path, rttm_path: Path, manifest_output_path: Path):
+    audio_list = str(data_root / 'audio_file.txt')
+    rttm_list = str(data_root / 'rttm_file.txt')
+    with open(audio_list, 'w') as f:
+        f.write('\n'.join([str(os.path.join(rttm_path, x)) for x in os.listdir(audio_path)]))
+    with open(rttm_list, 'w') as f:
+        f.write('\n'.join([str(os.path.join(rttm_path, x)) for x in os.listdir(rttm_path)]))
+    create_manifest(
+        audio_list,
+        str(manifest_output_path),
+        rttm_path=rttm_list,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description="VoxConverse Data download")
+    parser.add_argument("--data_root", default='./', type=str)
+    args = parser.parse_args()
+    data_root = Path(args.data_root)
+    data_root.mkdir(exist_ok=True, parents=True)
+
+    test_path = data_root / os.path.basename(test_url)
+    dev_path = data_root / os.path.basename(dev_url)
+    rttm_path = data_root / os.path.basename(rttm_annotations_url)
+
+    if not os.path.exists(test_path):
+        test_path = download_file(test_url, test_path)
+    if not os.path.exists(dev_path):
+        dev_path = download_file(dev_url, dev_path)
+    if not os.path.exists(rttm_path):
+        rttm_path = download_file(rttm_annotations_url, rttm_path)
+
+    extract_file(test_path, data_root / 'test/')
+    extract_file(dev_path, data_root / 'dev/')
+    extract_file(rttm_path, data_root)
+
+    _generate_manifest(
+        data_root=data_root,
+        audio_path=os.path.abspath(data_root / 'test/voxconverse_test_wav/'),
+        rttm_path=os.path.abspath(data_root / 'voxconverse-master/test/'),
+        manifest_output_path=data_root / 'test_manifest.json',
+    )
+    _generate_manifest(
+        data_root=data_root,
+        audio_path=os.path.abspath(data_root / 'dev/audio/'),
+        rttm_path=os.path.abspath(data_root / 'voxconverse-master/dev/'),
+        manifest_output_path=data_root / 'dev_manifest.json',
+    )
+
+
+if __name__ == "__main__":
+    main()