wehub-resource-sync
ba4be087d5
Create PR to main with cherry-pick from release / cherry-pick (push) Failing after 0s
CICD NeMo / pre-flight (push) Failing after 0s
CICD NeMo / configure (push) Has been skipped
Build, validate, and release Neural Modules / pre-flight (push) Failing after 1s
CICD NeMo / code-linting (push) Has been skipped
Build, validate, and release Neural Modules / release (push) Has been skipped
Build, validate, and release Neural Modules / release-summary (push) Has been cancelled
CICD NeMo / cicd-test-container-build (push) Has been cancelled
CICD NeMo / cicd-import-tests (push) Has been cancelled
CICD NeMo / L0_Setup_Test_Data_And_Models (push) Has been cancelled
CICD NeMo / cicd-main-unit-tests (push) Has been cancelled
CICD NeMo / cicd-main-speech (push) Has been cancelled
CICD NeMo / Nemo_CICD_Test (push) Has been cancelled
CICD NeMo / Coverage (e2e) (push) Has been cancelled
CICD NeMo / Coverage (unit-test) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
CICD NeMo / cicd-wait-in-queue (push) Has been cancelled
1667 lines
77 KiB
Python
1667 lines
77 KiB
Python
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import concurrent
|
|
import os
|
|
import warnings
|
|
from typing import Dict, List, Tuple
|
|
|
|
import numpy as np
|
|
import soundfile as sf
|
|
import torch
|
|
from omegaconf import OmegaConf
|
|
from scipy.signal import convolve
|
|
from scipy.signal.windows import cosine, hamming, hann
|
|
from tqdm import tqdm
|
|
|
|
from nemo.collections.asr.parts.preprocessing.perturb import process_augmentations
|
|
from nemo.collections.asr.parts.utils.data_simulation_utils import (
|
|
DataAnnotator,
|
|
SpeechSampler,
|
|
build_speaker_samples_map,
|
|
get_background_noise,
|
|
get_cleaned_base_path,
|
|
get_random_offset_index,
|
|
get_speaker_ids,
|
|
get_speaker_samples,
|
|
get_split_points_in_alignments,
|
|
load_speaker_sample,
|
|
normalize_audio,
|
|
per_speaker_normalize,
|
|
perturb_audio,
|
|
read_audio_from_buffer,
|
|
read_noise_manifest,
|
|
)
|
|
from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
|
|
from nemo.collections.asr.parts.utils.speaker_utils import get_overlap_range, is_overlap, merge_float_intervals
|
|
from nemo.utils import logging
|
|
|
|
try:
|
|
import pyroomacoustics as pra
|
|
from pyroomacoustics.directivities import CardioidFamily, DirectionVector, DirectivityPattern
|
|
|
|
PRA = True
|
|
except ImportError:
|
|
PRA = False
|
|
try:
|
|
from gpuRIR import att2t_SabineEstimator, beta_SabineEstimation, simulateRIR, t2n
|
|
|
|
GPURIR = True
|
|
except ImportError:
|
|
GPURIR = False
|
|
|
|
|
|
class MultiSpeakerSimulator(object):
|
|
"""
|
|
Multispeaker Audio Session Simulator - Simulates multispeaker audio sessions using single-speaker audio files and
|
|
corresponding word alignments.
|
|
|
|
Args:
|
|
cfg: OmegaConf configuration loaded from yaml file.
|
|
|
|
Configuration parameters (YAML)::
|
|
|
|
Parameters:
|
|
manifest_filepath (str): Manifest file with paths to single speaker audio files
|
|
sr (int): Sampling rate of the input audio files from the manifest
|
|
random_seed (int): Seed to random number generator
|
|
|
|
session_config:
|
|
num_speakers (int): Number of unique speakers per multispeaker audio session
|
|
num_sessions (int): Number of sessions to simulate
|
|
session_length (int): Length of each simulated multispeaker audio session (seconds)
|
|
|
|
session_params:
|
|
max_audio_read_sec (int): Max audio length in seconds when loading an audio file
|
|
sentence_length_params (list): k,p values for a negative_binomial distribution
|
|
dominance_var (float): Variance in speaker dominance
|
|
min_dominance (float): Minimum percentage of speaking time per speaker
|
|
turn_prob (float): Probability of switching speakers after each utterance
|
|
mean_silence (float): Mean proportion of silence to speaking time [0, 1)
|
|
mean_silence_var (float): Variance for mean silence in all audio sessions
|
|
per_silence_var (float): Variance for each silence in an audio session
|
|
per_silence_min (float): Minimum duration for each silence (default: 0)
|
|
per_silence_max (float): Maximum duration for each silence (default: -1, no max)
|
|
mean_overlap (float): Mean proportion of overlap in non-silence duration [0, 1)
|
|
mean_overlap_var (float): Variance for mean overlap in all audio sessions
|
|
per_overlap_var (float): Variance for per overlap in each session
|
|
per_overlap_min (float): Minimum per overlap duration in seconds
|
|
per_overlap_max (float): Maximum per overlap duration in seconds (-1 for no max)
|
|
start_window (bool): Whether to window the start of sentences
|
|
window_type (str): Type of windowing ("hamming", "hann", "cosine")
|
|
window_size (float): Length of window at start/end of segmented utterance (seconds)
|
|
start_buffer (float): Buffer of silence before the start of the sentence
|
|
split_buffer (float): Split RTTM labels if greater than twice this amount of silence
|
|
release_buffer (float): Buffer before window at end of sentence
|
|
normalize (bool): Normalize speaker volumes
|
|
normalization_type (str): "equal" or "var" volume per speaker
|
|
normalization_var (str): Variance in speaker volume
|
|
min_volume (float): Minimum speaker volume (variable normalization only)
|
|
max_volume (float): Maximum speaker volume (variable normalization only)
|
|
end_buffer (float): Buffer at the end of the session to leave blank
|
|
|
|
outputs:
|
|
output_dir (str): Output directory for audio sessions and label files
|
|
output_filename (str): Output filename for the wav and RTTM files
|
|
overwrite_output (bool): If true, delete the output directory if it exists
|
|
output_precision (int): Number of decimal places in output files
|
|
|
|
background_noise:
|
|
add_bg (bool): Add ambient background noise if true
|
|
background_manifest (str): Path to background noise manifest file
|
|
snr (int): SNR for background noise (using average speaker power)
|
|
snr_min (int): Min random SNR (set null to use fixed SNR)
|
|
snr_max (int): Max random SNR (set null to use fixed SNR)
|
|
|
|
segment_augmentor:
|
|
add_seg_aug (bool): Enable augmentation on each speech segment (Default: False)
|
|
segmentor.gain:
|
|
prob (float): Probability of gain augmentation
|
|
min_gain_dbfs (float): minimum gain in dB
|
|
max_gain_dbfs (float): maximum gain in dB
|
|
|
|
session_augmentor:
|
|
add_sess_aug (bool): Enable audio augmentation on the whole session (Default: False)
|
|
segmentor.white_noise:
|
|
prob (float): Probability of adding white noise (Default: 1.0)
|
|
min_level (float): minimum gain in dB
|
|
max_level (float): maximum gain in dB
|
|
|
|
speaker_enforcement:
|
|
enforce_num_speakers (bool): Enforce all requested speakers are present
|
|
enforce_time (list): Percentage through session that enforcement triggers
|
|
|
|
segment_manifest:
|
|
window (float): Window length for segmentation
|
|
shift (float): Shift length for segmentation
|
|
step_count (int): Number of unit segments per utterance
|
|
deci (int): Rounding decimals for segment manifest file
|
|
"""
|
|
|
|
def __init__(self, cfg):
|
|
self._params = cfg
|
|
self.annotator = DataAnnotator(cfg)
|
|
self.sampler = SpeechSampler(cfg)
|
|
# internal params
|
|
self._manifest = read_manifest(self._params.data_simulator.manifest_filepath)
|
|
self._speaker_samples = build_speaker_samples_map(self._manifest)
|
|
self._noise_samples = []
|
|
self._sentence = None
|
|
self._text = ""
|
|
self._words = []
|
|
self._alignments = []
|
|
# minimum number of alignments for a manifest to be considered valid
|
|
self._min_alignment_count = 2
|
|
self._merged_speech_intervals = []
|
|
# keep track of furthest sample per speaker to avoid overlapping same speaker
|
|
self._furthest_sample = [0 for n in range(self._params.data_simulator.session_config.num_speakers)]
|
|
# use to ensure overlap percentage is correct
|
|
self._missing_overlap = 0
|
|
# creating manifests during online data simulation
|
|
self.base_manifest_filepath = None
|
|
self.segment_manifest_filepath = None
|
|
self._max_audio_read_sec = self._params.data_simulator.session_params.max_audio_read_sec
|
|
self._turn_prob_min = self._params.data_simulator.session_params.get("turn_prob_min", 0.5)
|
|
# variable speaker volume
|
|
self._volume = None
|
|
self._speaker_ids = None
|
|
self._device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
|
|
self._audio_read_buffer_dict = {}
|
|
self.add_missing_overlap = self._params.data_simulator.session_params.get("add_missing_overlap", False)
|
|
|
|
if (
|
|
self._params.data_simulator.segment_augmentor.get("augmentor", None)
|
|
and self._params.data_simulator.segment_augmentor.add_seg_aug
|
|
):
|
|
self.segment_augmentor = process_augmentations(
|
|
augmenter=self._params.data_simulator.segment_augmentor.augmentor
|
|
)
|
|
else:
|
|
self.segment_augmentor = None
|
|
|
|
if (
|
|
self._params.data_simulator.session_augmentor.get("augmentor", None)
|
|
and self._params.data_simulator.session_augmentor.add_sess_aug
|
|
):
|
|
self.session_augmentor = process_augmentations(
|
|
augmenter=self._params.data_simulator.session_augmentor.augmentor
|
|
)
|
|
else:
|
|
self.session_augmentor = None
|
|
|
|
# Error check the input arguments for simulation
|
|
self._check_args()
|
|
|
|
# Initialize speaker permutations to maximize the number of speakers in the created dataset
|
|
self._permutated_speaker_inds = self._init_speaker_permutations(
|
|
num_sess=self._params.data_simulator.session_config.num_sessions,
|
|
num_speakers=self._params.data_simulator.session_config.num_speakers,
|
|
all_speaker_ids=self._speaker_samples.keys(),
|
|
random_seed=self._params.data_simulator.random_seed,
|
|
)
|
|
|
|
# Intialize multiprocessing related variables
|
|
self.num_workers = self._params.get("num_workers", 1)
|
|
self.multiprocessing_chunksize = self._params.data_simulator.get('multiprocessing_chunksize', 10000)
|
|
self.chunk_count = self._init_chunk_count()
|
|
|
|
def _init_speaker_permutations(self, num_sess: int, num_speakers: int, all_speaker_ids: List, random_seed: int):
|
|
"""
|
|
Initialize the speaker permutations for the number of speakers in the session.
|
|
When generating the simulated sessions, we want to include as many speakers as possible.
|
|
This function generates a set of permutations that can be used to sweep all speakers in
|
|
the source dataset to make sure we maximize the total number of speakers included in
|
|
the simulated sessions.
|
|
|
|
Args:
|
|
num_sess (int): Number of sessions to generate
|
|
num_speakers (int): Number of speakers in each session
|
|
all_speaker_ids (list): List of all speaker IDs
|
|
|
|
Returns:
|
|
permuted_inds (np.array):
|
|
Array of permuted speaker indices to use for each session
|
|
Dimensions: (num_sess, num_speakers)
|
|
"""
|
|
np.random.seed(random_seed)
|
|
all_speaker_id_counts = len(list(all_speaker_ids))
|
|
|
|
# Calculate how many permutations are needed
|
|
perm_set_count = int(np.ceil(num_speakers * num_sess / all_speaker_id_counts))
|
|
|
|
target_count = num_speakers * num_sess
|
|
for count in range(perm_set_count):
|
|
if target_count < all_speaker_id_counts:
|
|
seq_len = target_count
|
|
else:
|
|
seq_len = all_speaker_id_counts
|
|
if seq_len <= 0:
|
|
raise ValueError(f"seq_len is {seq_len} at count {count} and should be greater than 0")
|
|
|
|
if count == 0:
|
|
permuted_inds = np.random.permutation(len(all_speaker_ids))[:seq_len]
|
|
else:
|
|
permuted_inds = np.hstack((permuted_inds, np.random.permutation(len(all_speaker_ids))[:seq_len]))
|
|
target_count -= seq_len
|
|
|
|
logging.info(f"Total {all_speaker_id_counts} speakers in the source dataset.")
|
|
logging.info(f"Initialized speaker permutations for {num_sess} sessions with {num_speakers} speakers each.")
|
|
return permuted_inds.reshape(num_sess, num_speakers)
|
|
|
|
def _init_chunk_count(self):
|
|
"""
|
|
Initialize the chunk count for multi-processing to prevent over-flow of job counts.
|
|
The multi-processing pipeline can freeze if there are more than approximately 10,000 jobs
|
|
in the pipeline at the same time.
|
|
"""
|
|
return int(np.ceil(self._params.data_simulator.session_config.num_sessions / self.multiprocessing_chunksize))
|
|
|
|
def _check_args(self):
|
|
"""
|
|
Checks YAML arguments to ensure they are within valid ranges.
|
|
"""
|
|
if self._params.data_simulator.session_config.num_speakers < 1:
|
|
raise Exception("At least one speaker is required for making audio sessions (num_speakers < 1)")
|
|
if (
|
|
self._params.data_simulator.session_params.turn_prob < 0
|
|
or self._params.data_simulator.session_params.turn_prob > 1
|
|
):
|
|
raise Exception("Turn probability is outside of [0,1]")
|
|
if (
|
|
self._params.data_simulator.session_params.turn_prob < 0
|
|
or self._params.data_simulator.session_params.turn_prob > 1
|
|
):
|
|
raise Exception("Turn probability is outside of [0,1]")
|
|
elif (
|
|
self._params.data_simulator.session_params.turn_prob < self._turn_prob_min
|
|
and self._params.data_simulator.speaker_enforcement.enforce_num_speakers == True
|
|
):
|
|
logging.warning(
|
|
"Turn probability is less than {self._turn_prob_min} while enforce_num_speakers=True, which may result in excessive session lengths. Forcing turn_prob to 0.5."
|
|
)
|
|
self._params.data_simulator.session_params.turn_prob = self._turn_prob_min
|
|
if self._params.data_simulator.session_params.max_audio_read_sec < 2.5:
|
|
raise Exception("Max audio read time must be greater than 2.5 seconds")
|
|
|
|
if self._params.data_simulator.session_params.sentence_length_params[0] <= 0:
|
|
raise Exception(
|
|
"k (number of success until the exp. ends) in Sentence length parameter value must be a positive number"
|
|
)
|
|
|
|
if not (0 < self._params.data_simulator.session_params.sentence_length_params[1] <= 1):
|
|
raise Exception("p (success probability) value in sentence length parameter must be in range (0,1]")
|
|
|
|
if (
|
|
self._params.data_simulator.session_params.mean_overlap < 0
|
|
or self._params.data_simulator.session_params.mean_overlap > 1
|
|
):
|
|
raise Exception("Mean overlap is outside of [0,1]")
|
|
if (
|
|
self._params.data_simulator.session_params.mean_silence < 0
|
|
or self._params.data_simulator.session_params.mean_silence > 1
|
|
):
|
|
raise Exception("Mean silence is outside of [0,1]")
|
|
if self._params.data_simulator.session_params.mean_silence_var < 0:
|
|
raise Exception("Mean silence variance is not below 0")
|
|
if (
|
|
self._params.data_simulator.session_params.mean_silence > 0
|
|
and self._params.data_simulator.session_params.mean_silence_var
|
|
>= self._params.data_simulator.session_params.mean_silence
|
|
* (1 - self._params.data_simulator.session_params.mean_silence)
|
|
):
|
|
raise Exception("Mean silence variance should be lower than mean_silence * (1-mean_silence)")
|
|
if self._params.data_simulator.session_params.per_silence_var < 0:
|
|
raise Exception("Per silence variance is below 0")
|
|
|
|
if self._params.data_simulator.session_params.mean_overlap_var < 0:
|
|
raise Exception("Mean overlap variance is not larger than 0")
|
|
if (
|
|
self._params.data_simulator.session_params.mean_overlap > 0
|
|
and self._params.data_simulator.session_params.mean_overlap_var
|
|
>= self._params.data_simulator.session_params.mean_overlap
|
|
* (1 - self._params.data_simulator.session_params.mean_overlap)
|
|
):
|
|
raise Exception("Mean overlap variance should be lower than mean_overlap * (1-mean_overlap)")
|
|
if self._params.data_simulator.session_params.per_overlap_var < 0:
|
|
raise Exception("Per overlap variance is not larger than 0")
|
|
|
|
if (
|
|
self._params.data_simulator.session_params.min_dominance < 0
|
|
or self._params.data_simulator.session_params.min_dominance > 1
|
|
):
|
|
raise Exception("Minimum dominance is outside of [0,1]")
|
|
if (
|
|
self._params.data_simulator.speaker_enforcement.enforce_time[0] < 0
|
|
or self._params.data_simulator.speaker_enforcement.enforce_time[0] > 1
|
|
):
|
|
raise Exception("Speaker enforcement start is outside of [0,1]")
|
|
if (
|
|
self._params.data_simulator.speaker_enforcement.enforce_time[1] < 0
|
|
or self._params.data_simulator.speaker_enforcement.enforce_time[1] > 1
|
|
):
|
|
raise Exception("Speaker enforcement end is outside of [0,1]")
|
|
|
|
if (
|
|
self._params.data_simulator.session_params.min_dominance
|
|
* self._params.data_simulator.session_config.num_speakers
|
|
> 1
|
|
):
|
|
raise Exception("Number of speakers times minimum dominance is greater than 1")
|
|
|
|
if (
|
|
self._params.data_simulator.session_params.window_type not in ['hamming', 'hann', 'cosine']
|
|
and self._params.data_simulator.session_params.window_type is not None
|
|
):
|
|
raise Exception("Incorrect window type provided")
|
|
|
|
if len(self._manifest) == 0:
|
|
raise Exception("Manifest file is empty. Check that the source path is correct.")
|
|
|
|
def clean_up(self):
|
|
"""
|
|
Clear the system memory. Cache data for audio files and alignments are removed.
|
|
"""
|
|
self._sentence = None
|
|
self._words = []
|
|
self._alignments = []
|
|
self._audio_read_buffer_dict = {}
|
|
torch.cuda.empty_cache()
|
|
|
|
def _get_speaker_dominance(self) -> List[float]:
|
|
"""
|
|
Get the dominance value for each speaker, accounting for the dominance variance and
|
|
the minimum per-speaker dominance.
|
|
|
|
Returns:
|
|
dominance (list): Per-speaker dominance
|
|
"""
|
|
dominance_mean = 1.0 / self._params.data_simulator.session_config.num_speakers
|
|
dominance = np.random.normal(
|
|
loc=dominance_mean,
|
|
scale=self._params.data_simulator.session_params.dominance_var,
|
|
size=self._params.data_simulator.session_config.num_speakers,
|
|
)
|
|
dominance = np.clip(dominance, a_min=0, a_max=np.inf)
|
|
# normalize while maintaining minimum dominance
|
|
total = np.sum(dominance)
|
|
if total == 0:
|
|
for i in range(len(dominance)):
|
|
dominance[i] += self._params.data_simulator.session_params.min_dominance
|
|
# scale accounting for min_dominance which has to be added after
|
|
dominance = (dominance / total) * (
|
|
1
|
|
- self._params.data_simulator.session_params.min_dominance
|
|
* self._params.data_simulator.session_config.num_speakers
|
|
)
|
|
for i in range(len(dominance)):
|
|
dominance[i] += self._params.data_simulator.session_params.min_dominance
|
|
if (
|
|
i > 0
|
|
): # dominance values are cumulative to make it easy to select the speaker using a random value in [0,1]
|
|
dominance[i] = dominance[i] + dominance[i - 1]
|
|
return dominance
|
|
|
|
def _increase_speaker_dominance(
|
|
self, base_speaker_dominance: List[float], factor: int
|
|
) -> Tuple[List[float], bool]:
|
|
"""
|
|
Increase speaker dominance for unrepresented speakers (used only in enforce mode).
|
|
Increases the dominance for these speakers by the input factor (and then re-normalizes the probabilities to 1).
|
|
|
|
Args:
|
|
base_speaker_dominance (list): Dominance values for each speaker.
|
|
factor (int): Factor to increase dominance of unrepresented speakers by.
|
|
Returns:
|
|
dominance (list): Per-speaker dominance
|
|
enforce (bool): Whether to keep enforce mode turned on
|
|
"""
|
|
increase_percent = []
|
|
for i in range(self._params.data_simulator.session_config.num_speakers):
|
|
if self._furthest_sample[i] == 0:
|
|
increase_percent.append(i)
|
|
# ramp up enforce counter until speaker is sampled, then reset once all speakers have spoken
|
|
if len(increase_percent) > 0:
|
|
# extract original per-speaker probabilities
|
|
dominance = np.copy(base_speaker_dominance)
|
|
for i in range(len(dominance) - 1, 0, -1):
|
|
dominance[i] = dominance[i] - dominance[i - 1]
|
|
# increase specified speakers by the desired factor
|
|
for i in increase_percent:
|
|
dominance[i] = dominance[i] * factor
|
|
# renormalize
|
|
dominance = dominance / np.sum(dominance)
|
|
for i in range(1, len(dominance)):
|
|
dominance[i] = dominance[i] + dominance[i - 1]
|
|
enforce = True
|
|
else: # no unrepresented speakers, so enforce mode can be turned off
|
|
dominance = base_speaker_dominance
|
|
enforce = False
|
|
return dominance, enforce
|
|
|
|
def _set_speaker_volume(self):
|
|
"""
|
|
Set the volume for each speaker (either equal volume or variable speaker volume).
|
|
"""
|
|
if self._params.data_simulator.session_params.normalization_type == 'equal':
|
|
self._volume = np.ones(self._params.data_simulator.session_config.num_speakers)
|
|
elif self._params.data_simulator.session_params.normalization_type == 'variable':
|
|
self._volume = np.random.normal(
|
|
loc=1.0,
|
|
scale=self._params.data_simulator.session_params.normalization_var,
|
|
size=self._params.data_simulator.session_config.num_speakers,
|
|
)
|
|
self._volume = np.clip(
|
|
np.array(self._volume),
|
|
a_min=self._params.data_simulator.session_params.min_volume,
|
|
a_max=self._params.data_simulator.session_params.max_volume,
|
|
).tolist()
|
|
|
|
def _get_next_speaker(self, prev_speaker: int, dominance: List[float]) -> int:
|
|
"""
|
|
Get the next speaker (accounting for turn probability and dominance distribution).
|
|
|
|
Args:
|
|
prev_speaker (int): Previous speaker turn.
|
|
dominance (list): Dominance values for each speaker.
|
|
Returns:
|
|
prev_speaker/speaker_turn (int): Speaker turn
|
|
"""
|
|
if self._params.data_simulator.session_config.num_speakers == 1:
|
|
prev_speaker = 0 if prev_speaker is None else prev_speaker
|
|
return prev_speaker
|
|
else:
|
|
if (
|
|
np.random.uniform(0, 1) > self._params.data_simulator.session_params.turn_prob
|
|
and prev_speaker is not None
|
|
):
|
|
return prev_speaker
|
|
else:
|
|
speaker_turn = prev_speaker
|
|
while speaker_turn == prev_speaker: # ensure another speaker goes next
|
|
rand = np.random.uniform(0, 1)
|
|
speaker_turn = 0
|
|
while rand > dominance[speaker_turn]:
|
|
speaker_turn += 1
|
|
return speaker_turn
|
|
|
|
def _get_window(self, window_amount: int, start: bool = False):
|
|
"""
|
|
Get window curve to alleviate abrupt change of time-series signal when segmenting audio samples.
|
|
|
|
Args:
|
|
window_amount (int): Window length (in terms of number of samples).
|
|
start (bool): If true, return the first half of the window.
|
|
|
|
Returns:
|
|
window (tensor): Half window (either first half or second half)
|
|
"""
|
|
if self._params.data_simulator.session_params.window_type == 'hamming':
|
|
window = hamming(window_amount * 2)
|
|
elif self._params.data_simulator.session_params.window_type == 'hann':
|
|
window = hann(window_amount * 2)
|
|
elif self._params.data_simulator.session_params.window_type == 'cosine':
|
|
window = cosine(window_amount * 2)
|
|
else:
|
|
raise Exception("Incorrect window type provided")
|
|
|
|
window = torch.from_numpy(window).to(self._device)
|
|
|
|
# return the first half or second half of the window
|
|
if start:
|
|
return window[:window_amount]
|
|
else:
|
|
return window[window_amount:]
|
|
|
|
def _get_start_buffer_and_window(self, first_alignment: int) -> Tuple[int, int]:
|
|
"""
|
|
Get the start cutoff and window length for smoothing the start of the sentence.
|
|
|
|
Args:
|
|
first_alignment (int): Start of the first word (in terms of number of samples).
|
|
Returns:
|
|
start_cutoff (int): Amount into the audio clip to start
|
|
window_amount (int): Window length
|
|
"""
|
|
window_amount = int(self._params.data_simulator.session_params.window_size * self._params.data_simulator.sr)
|
|
start_buffer = int(self._params.data_simulator.session_params.start_buffer * self._params.data_simulator.sr)
|
|
|
|
if first_alignment < start_buffer:
|
|
window_amount = 0
|
|
start_cutoff = 0
|
|
elif first_alignment < start_buffer + window_amount:
|
|
window_amount = first_alignment - start_buffer
|
|
start_cutoff = 0
|
|
else:
|
|
start_cutoff = first_alignment - start_buffer - window_amount
|
|
|
|
return start_cutoff, window_amount
|
|
|
|
def _get_end_buffer_and_window(
|
|
self, current_sample_cursor: int, remaining_dur_samples: int, remaining_len_audio_file: int
|
|
) -> Tuple[int, int]:
|
|
"""
|
|
Get the end buffer and window length for smoothing the end of the sentence.
|
|
|
|
Args:
|
|
current_sample_cursor (int): Current location in the target file (in terms of number of samples).
|
|
remaining_dur_samples (int): Remaining duration in the target file (in terms of number of samples).
|
|
remaining_len_audio_file (int): Length remaining in audio file (in terms of number of samples).
|
|
Returns:
|
|
release_buffer (int): Amount after the end of the last alignment to include
|
|
window_amount (int): Window length
|
|
"""
|
|
window_amount = int(self._params.data_simulator.session_params.window_size * self._params.data_simulator.sr)
|
|
release_buffer = int(
|
|
self._params.data_simulator.session_params.release_buffer * self._params.data_simulator.sr
|
|
)
|
|
|
|
if current_sample_cursor + release_buffer > remaining_dur_samples:
|
|
release_buffer = remaining_dur_samples - current_sample_cursor
|
|
window_amount = 0
|
|
elif current_sample_cursor + window_amount + release_buffer > remaining_dur_samples:
|
|
window_amount = remaining_dur_samples - current_sample_cursor - release_buffer
|
|
|
|
if remaining_len_audio_file < release_buffer:
|
|
release_buffer = remaining_len_audio_file
|
|
window_amount = 0
|
|
elif remaining_len_audio_file < release_buffer + window_amount:
|
|
window_amount = remaining_len_audio_file - release_buffer
|
|
|
|
return release_buffer, window_amount
|
|
|
|
def _check_missing_speakers(self, num_missing: int = 0):
|
|
"""
|
|
Check if any speakers were not included in the clip and display a warning.
|
|
|
|
Args:
|
|
num_missing (int): Number of missing speakers.
|
|
"""
|
|
for k in range(len(self._furthest_sample)):
|
|
if self._furthest_sample[k] == 0:
|
|
num_missing += 1
|
|
if num_missing != 0:
|
|
warnings.warn(
|
|
f"{self._params.data_simulator.session_config.num_speakers - num_missing}"
|
|
"speakers were included in the clip instead of the requested amount of "
|
|
f"{self._params.data_simulator.session_config.num_speakers}"
|
|
)
|
|
|
|
def _add_file(
|
|
self,
|
|
audio_manifest: dict,
|
|
audio_file,
|
|
sentence_word_count: int,
|
|
max_word_count_in_sentence: int,
|
|
max_samples_in_sentence: int,
|
|
random_offset: bool = False,
|
|
) -> Tuple[int, torch.Tensor]:
|
|
"""
|
|
Add audio file to current sentence (up to the desired number of words).
|
|
Uses the alignments to segment the audio file.
|
|
NOTE: 0 index is always silence in `audio_manifest['words']`, so we choose `offset_idx=1` as the first word
|
|
|
|
Args:
|
|
audio_manifest (dict): Line from manifest file for current audio file
|
|
audio_file (tensor): Current loaded audio file
|
|
sentence_word_count (int): Running count for number of words in sentence
|
|
max_word_count_in_sentence (int): Maximum count for number of words in sentence
|
|
max_samples_in_sentence (int): Maximum length for sentence in terms of samples
|
|
|
|
Returns:
|
|
sentence_word_count+current_word_count (int): Running word count
|
|
len(self._sentence) (tensor): Current length of the audio file
|
|
"""
|
|
# In general, random offset is not needed since random silence index has already been chosen
|
|
if random_offset:
|
|
offset_idx = np.random.randint(low=1, high=len(audio_manifest['words']))
|
|
else:
|
|
offset_idx = 1
|
|
|
|
first_alignment = int(audio_manifest['alignments'][offset_idx - 1] * self._params.data_simulator.sr)
|
|
start_cutoff, start_window_amount = self._get_start_buffer_and_window(first_alignment)
|
|
if not self._params.data_simulator.session_params.start_window: # cut off the start of the sentence
|
|
start_window_amount = 0
|
|
|
|
# Ensure the desired number of words are added and the length of the output session isn't exceeded
|
|
sentence_samples = len(self._sentence)
|
|
|
|
remaining_dur_samples = max_samples_in_sentence - sentence_samples
|
|
remaining_duration = max_word_count_in_sentence - sentence_word_count
|
|
prev_dur_samples, dur_samples, curr_dur_samples = 0, 0, 0
|
|
current_word_count = 0
|
|
word_idx = offset_idx
|
|
silence_count = 1
|
|
while (
|
|
current_word_count < remaining_duration
|
|
and dur_samples < remaining_dur_samples
|
|
and word_idx < len(audio_manifest['words'])
|
|
):
|
|
dur_samples = int(audio_manifest['alignments'][word_idx] * self._params.data_simulator.sr) - start_cutoff
|
|
|
|
# check the length of the generated sentence in terms of sample count (int).
|
|
if curr_dur_samples + dur_samples > remaining_dur_samples:
|
|
# if the upcoming loop will exceed the remaining sample count, break out of the loop.
|
|
break
|
|
|
|
word = audio_manifest['words'][word_idx]
|
|
|
|
if silence_count > 0 and word == "":
|
|
break
|
|
|
|
self._words.append(word)
|
|
self._alignments.append(
|
|
float(sentence_samples * 1.0 / self._params.data_simulator.sr)
|
|
- float(start_cutoff * 1.0 / self._params.data_simulator.sr)
|
|
+ audio_manifest['alignments'][word_idx]
|
|
)
|
|
|
|
if word == "":
|
|
word_idx += 1
|
|
silence_count += 1
|
|
continue
|
|
elif self._text == "":
|
|
self._text += word
|
|
else:
|
|
self._text += " " + word
|
|
|
|
word_idx += 1
|
|
current_word_count += 1
|
|
prev_dur_samples = dur_samples
|
|
curr_dur_samples += dur_samples
|
|
|
|
# add audio clip up to the final alignment
|
|
if self._params.data_simulator.session_params.window_type is not None: # cut off the start of the sentence
|
|
if start_window_amount > 0: # include window
|
|
window = self._get_window(start_window_amount, start=True)
|
|
self._sentence = self._sentence.to(self._device)
|
|
self._sentence = torch.cat(
|
|
(
|
|
self._sentence,
|
|
torch.multiply(audio_file[start_cutoff : start_cutoff + start_window_amount], window),
|
|
),
|
|
0,
|
|
)
|
|
self._sentence = torch.cat(
|
|
(
|
|
self._sentence,
|
|
audio_file[start_cutoff + start_window_amount : start_cutoff + prev_dur_samples],
|
|
),
|
|
0,
|
|
).to(self._device)
|
|
|
|
else:
|
|
self._sentence = torch.cat(
|
|
(self._sentence, audio_file[start_cutoff : start_cutoff + prev_dur_samples]), 0
|
|
).to(self._device)
|
|
|
|
# windowing at the end of the sentence
|
|
if (
|
|
word_idx < len(audio_manifest['words'])
|
|
) and self._params.data_simulator.session_params.window_type is not None:
|
|
release_buffer, end_window_amount = self._get_end_buffer_and_window(
|
|
prev_dur_samples,
|
|
remaining_dur_samples,
|
|
len(audio_file[start_cutoff + prev_dur_samples :]),
|
|
)
|
|
self._sentence = torch.cat(
|
|
(
|
|
self._sentence,
|
|
audio_file[start_cutoff + prev_dur_samples : start_cutoff + prev_dur_samples + release_buffer],
|
|
),
|
|
0,
|
|
).to(self._device)
|
|
|
|
if end_window_amount > 0: # include window
|
|
window = self._get_window(end_window_amount, start=False)
|
|
sig_start = start_cutoff + prev_dur_samples + release_buffer
|
|
sig_end = start_cutoff + prev_dur_samples + release_buffer + end_window_amount
|
|
windowed_audio_file = torch.multiply(audio_file[sig_start:sig_end], window)
|
|
self._sentence = torch.cat((self._sentence, windowed_audio_file), 0).to(self._device)
|
|
|
|
del audio_file
|
|
return sentence_word_count + current_word_count, len(self._sentence)
|
|
|
|
def _build_sentence(
|
|
self,
|
|
speaker_turn: int,
|
|
speaker_ids: List[str],
|
|
speaker_wav_align_map: Dict[str, list],
|
|
max_samples_in_sentence: int,
|
|
):
|
|
"""
|
|
Build a new sentence by attaching utterance samples together until the sentence has reached a desired length.
|
|
While generating the sentence, alignment information is used to segment the audio.
|
|
|
|
Args:
|
|
speaker_turn (int): Current speaker turn.
|
|
speaker_ids (list): LibriSpeech speaker IDs for each speaker in the current session.
|
|
speaker_wav_align_map (dict): Dictionary containing speaker IDs and their corresponding wav filepath and alignments.
|
|
max_samples_in_sentence (int): Maximum length for sentence in terms of samples
|
|
"""
|
|
# select speaker length
|
|
sl = (
|
|
np.random.negative_binomial(
|
|
self._params.data_simulator.session_params.sentence_length_params[0],
|
|
self._params.data_simulator.session_params.sentence_length_params[1],
|
|
)
|
|
+ 1
|
|
)
|
|
|
|
# initialize sentence, text, words, alignments
|
|
self._sentence = torch.zeros(0, dtype=torch.float64, device=self._device)
|
|
self._text = ""
|
|
self._words, self._alignments = [], []
|
|
sentence_word_count, sentence_samples = 0, 0
|
|
|
|
# build sentence
|
|
while sentence_word_count < sl and sentence_samples < max_samples_in_sentence:
|
|
audio_manifest = load_speaker_sample(
|
|
speaker_wav_align_map=speaker_wav_align_map,
|
|
speaker_ids=speaker_ids,
|
|
speaker_turn=speaker_turn,
|
|
min_alignment_count=self._min_alignment_count,
|
|
)
|
|
|
|
offset_index = get_random_offset_index(
|
|
audio_manifest=audio_manifest,
|
|
audio_read_buffer_dict=self._audio_read_buffer_dict,
|
|
offset_min=0,
|
|
max_audio_read_sec=self._max_audio_read_sec,
|
|
min_alignment_count=self._min_alignment_count,
|
|
)
|
|
|
|
audio_file, sr, audio_manifest = read_audio_from_buffer(
|
|
audio_manifest=audio_manifest,
|
|
buffer_dict=self._audio_read_buffer_dict,
|
|
offset_index=offset_index,
|
|
device=self._device,
|
|
max_audio_read_sec=self._max_audio_read_sec,
|
|
min_alignment_count=self._min_alignment_count,
|
|
read_subset=True,
|
|
)
|
|
|
|
# Step 6-2: Add optional perturbations to the specific audio segment (i.e. to `self._sentnece`)
|
|
if self._params.data_simulator.segment_augmentor.add_seg_aug:
|
|
audio_file = perturb_audio(audio_file, sr, self.segment_augmentor, device=self._device)
|
|
|
|
sentence_word_count, sentence_samples = self._add_file(
|
|
audio_manifest, audio_file, sentence_word_count, sl, max_samples_in_sentence
|
|
)
|
|
|
|
# per-speaker normalization (accounting for active speaker time)
|
|
if self._params.data_simulator.session_params.normalize and torch.max(torch.abs(self._sentence)) > 0:
|
|
splits = get_split_points_in_alignments(
|
|
words=self._words,
|
|
alignments=self._alignments,
|
|
split_buffer=self._params.data_simulator.session_params.split_buffer,
|
|
sr=self._params.data_simulator.sr,
|
|
sentence_audio_len=len(self._sentence),
|
|
)
|
|
self._sentence = per_speaker_normalize(
|
|
sentence_audio=self._sentence,
|
|
splits=splits,
|
|
speaker_turn=speaker_turn,
|
|
volume=self._volume,
|
|
device=self._device,
|
|
)
|
|
|
|
def _add_silence_or_overlap(
|
|
self,
|
|
speaker_turn: int,
|
|
prev_speaker: int,
|
|
start: int,
|
|
length: int,
|
|
session_len_samples: int,
|
|
prev_len_samples: int,
|
|
enforce: bool,
|
|
) -> int:
|
|
"""
|
|
Returns new overlapped (or shifted) start position after inserting overlap or silence.
|
|
|
|
Args:
|
|
speaker_turn (int): The integer index of the current speaker turn.
|
|
prev_speaker (int): The integer index of the previous speaker turn.
|
|
start (int): Current start of the audio file being inserted.
|
|
length (int): Length of the audio file being inserted.
|
|
session_len_samples (int): Maximum length of the session in terms of number of samples
|
|
prev_len_samples (int): Length of previous sentence (in terms of number of samples)
|
|
enforce (bool): Whether speaker enforcement mode is being used
|
|
Returns:
|
|
new_start (int): New starting position in the session accounting for overlap or silence
|
|
"""
|
|
running_len_samples = start + length
|
|
# `length` is the length of the current sentence to be added, so not included in self.sampler.running_speech_len_samples
|
|
non_silence_len_samples = self.sampler.running_speech_len_samples + length
|
|
|
|
# compare silence and overlap ratios
|
|
add_overlap = self.sampler.silence_vs_overlap_selector(running_len_samples, non_silence_len_samples)
|
|
|
|
# choose overlap if this speaker is not the same as the previous speaker and add_overlap is True.
|
|
if prev_speaker != speaker_turn and prev_speaker is not None and add_overlap:
|
|
desired_overlap_amount = self.sampler.sample_from_overlap_model(non_silence_len_samples)
|
|
new_start = start - desired_overlap_amount
|
|
|
|
# avoid overlap at start of clip
|
|
if new_start < 0:
|
|
desired_overlap_amount -= 0 - new_start
|
|
self._missing_overlap += 0 - new_start
|
|
new_start = 0
|
|
|
|
# if same speaker ends up overlapping from any previous clip, pad with silence instead
|
|
if new_start < self._furthest_sample[speaker_turn]:
|
|
desired_overlap_amount -= self._furthest_sample[speaker_turn] - new_start
|
|
self._missing_overlap += self._furthest_sample[speaker_turn] - new_start
|
|
new_start = self._furthest_sample[speaker_turn]
|
|
|
|
prev_start = start - prev_len_samples
|
|
prev_end = start
|
|
new_end = new_start + length
|
|
|
|
# check overlap amount to calculate the actual amount of generated overlaps
|
|
overlap_amount = 0
|
|
if is_overlap([prev_start, prev_end], [new_start, new_end]):
|
|
overlap_range = get_overlap_range([prev_start, prev_end], [new_start, new_end])
|
|
overlap_amount = max(overlap_range[1] - overlap_range[0], 0)
|
|
|
|
if overlap_amount < desired_overlap_amount:
|
|
self._missing_overlap += desired_overlap_amount - overlap_amount
|
|
self.sampler.running_overlap_len_samples += overlap_amount
|
|
|
|
# if we are not adding overlap, add silence
|
|
else:
|
|
silence_amount = self.sampler.sample_from_silence_model(running_len_samples)
|
|
if start + length + silence_amount > session_len_samples and not enforce:
|
|
new_start = max(session_len_samples - length, start)
|
|
else:
|
|
new_start = start + silence_amount
|
|
return new_start
|
|
|
|
def _get_session_meta_data(self, array: np.ndarray, snr: float) -> dict:
|
|
"""
|
|
Get meta data for the current session.
|
|
|
|
Args:
|
|
array (np.ndarray): audio array
|
|
snr (float): signal-to-noise ratio
|
|
|
|
Returns:
|
|
dict: meta data
|
|
"""
|
|
meta_data = {
|
|
"duration": array.shape[0] / self._params.data_simulator.sr,
|
|
"silence_mean": self.sampler.sess_silence_mean,
|
|
"overlap_mean": self.sampler.sess_overlap_mean,
|
|
"bg_snr": snr,
|
|
"speaker_ids": self._speaker_ids,
|
|
"speaker_volumes": list(self._volume),
|
|
}
|
|
return meta_data
|
|
|
|
def _get_session_silence_from_rttm(self, rttm_list: List[str], running_len_samples: int):
|
|
"""
|
|
Calculate the total speech and silence duration in the current session using RTTM file.
|
|
|
|
Args:
|
|
rttm_list (list):
|
|
List of RTTM timestamps
|
|
running_len_samples (int):
|
|
Total number of samples generated so far in the current session
|
|
|
|
Returns:
|
|
sess_speech_len_rttm (int):
|
|
The total number of speech samples in the current session
|
|
sess_silence_len_rttm (int):
|
|
The total number of silence samples in the current session
|
|
"""
|
|
all_sample_list = []
|
|
for x_raw in rttm_list:
|
|
x = [token for token in x_raw.split()]
|
|
all_sample_list.append([float(x[0]), float(x[1])])
|
|
|
|
self._merged_speech_intervals = merge_float_intervals(all_sample_list)
|
|
total_speech_in_secs = sum([x[1] - x[0] for x in self._merged_speech_intervals])
|
|
total_silence_in_secs = running_len_samples / self._params.data_simulator.sr - total_speech_in_secs
|
|
sess_speech_len = int(total_speech_in_secs * self._params.data_simulator.sr)
|
|
sess_silence_len = int(total_silence_in_secs * self._params.data_simulator.sr)
|
|
return sess_speech_len, sess_silence_len
|
|
|
|
def _add_sentence_to_array(
|
|
self, start: int, length: int, array: torch.Tensor, is_speech: torch.Tensor
|
|
) -> Tuple[torch.Tensor, torch.Tensor, int]:
|
|
"""
|
|
Add a sentence to the session array containing time-series signal.
|
|
|
|
Args:
|
|
start (int): Starting position in the session
|
|
length (int): Length of the sentence
|
|
array (torch.Tensor): Session array
|
|
is_speech (torch.Tensor): Session array containing speech/non-speech labels
|
|
|
|
Returns:
|
|
array (torch.Tensor): Session array in torch.Tensor format
|
|
is_speech (torch.Tensor): Session array containing speech/non-speech labels in torch.Tensor format
|
|
"""
|
|
end = start + length
|
|
if end > len(array): # only occurs in enforce mode
|
|
array = torch.nn.functional.pad(array, (0, end - len(array)))
|
|
is_speech = torch.nn.functional.pad(is_speech, (0, end - len(is_speech)))
|
|
array[start:end] += self._sentence
|
|
is_speech[start:end] = 1
|
|
return array, is_speech, end
|
|
|
|
def _generate_session(
|
|
self,
|
|
idx: int,
|
|
basepath: str,
|
|
filename: str,
|
|
speaker_ids: List[str],
|
|
speaker_wav_align_map: Dict[str, list],
|
|
noise_samples: list,
|
|
device: torch.device,
|
|
enforce_counter: int = 2,
|
|
):
|
|
"""
|
|
_generate_session function without RIR simulation.
|
|
Generate a multispeaker audio session and corresponding label files.
|
|
|
|
Args:
|
|
idx (int): Index for current session (out of total number of sessions).
|
|
basepath (str): Path to output directory.
|
|
filename (str): Filename for output files.
|
|
speaker_ids (list): List of speaker IDs that will be used in this session.
|
|
speaker_wav_align_map (dict): Dictionary containing speaker IDs and their corresponding wav filepath and alignments.
|
|
noise_samples (list): List of randomly sampled noise source files that will be used for generating this session.
|
|
device (torch.device): Device to use for generating this session.
|
|
enforce_counter (int): In enforcement mode, dominance is increased by a factor of enforce_counter for unrepresented speakers
|
|
"""
|
|
random_seed = self._params.data_simulator.random_seed
|
|
np.random.seed(random_seed + idx)
|
|
|
|
self._device = device
|
|
speaker_dominance = self._get_speaker_dominance() # randomly determine speaker dominance
|
|
base_speaker_dominance = np.copy(speaker_dominance)
|
|
self._set_speaker_volume()
|
|
|
|
running_len_samples, prev_len_samples = 0, 0
|
|
prev_speaker = None
|
|
self.annotator.init_annotation_lists()
|
|
self._noise_samples = noise_samples
|
|
self._furthest_sample = [0 for n in range(self._params.data_simulator.session_config.num_speakers)]
|
|
self._missing_silence = 0
|
|
|
|
# hold enforce until all speakers have spoken
|
|
enforce_time = np.random.uniform(
|
|
self._params.data_simulator.speaker_enforcement.enforce_time[0],
|
|
self._params.data_simulator.speaker_enforcement.enforce_time[1],
|
|
)
|
|
enforce = self._params.data_simulator.speaker_enforcement.enforce_num_speakers
|
|
|
|
session_len_samples = int(
|
|
(self._params.data_simulator.session_config.session_length * self._params.data_simulator.sr)
|
|
)
|
|
array = torch.zeros(session_len_samples).to(self._device)
|
|
is_speech = torch.zeros(session_len_samples).to(self._device)
|
|
|
|
self.sampler.get_session_silence_mean()
|
|
self.sampler.get_session_overlap_mean()
|
|
|
|
while running_len_samples < session_len_samples or enforce:
|
|
# Step 1: Prepare parameters for sentence generation
|
|
# Enforce speakers depending on running length
|
|
if running_len_samples > enforce_time * session_len_samples and enforce:
|
|
speaker_dominance, enforce = self._increase_speaker_dominance(base_speaker_dominance, enforce_counter)
|
|
if enforce:
|
|
enforce_counter += 1
|
|
|
|
# Step 2: Select a speaker
|
|
speaker_turn = self._get_next_speaker(prev_speaker, speaker_dominance)
|
|
|
|
# Calculate parameters for building a sentence (only add if remaining length > specific time)
|
|
max_samples_in_sentence = session_len_samples - running_len_samples
|
|
if enforce:
|
|
max_samples_in_sentence = float('inf')
|
|
elif (
|
|
max_samples_in_sentence
|
|
< self._params.data_simulator.session_params.end_buffer * self._params.data_simulator.sr
|
|
):
|
|
break
|
|
|
|
# Step 3: Generate a sentence
|
|
self._build_sentence(speaker_turn, speaker_ids, speaker_wav_align_map, max_samples_in_sentence)
|
|
length = len(self._sentence)
|
|
|
|
# Step 4: Generate a timestamp for either silence or overlap
|
|
start = self._add_silence_or_overlap(
|
|
speaker_turn=speaker_turn,
|
|
prev_speaker=prev_speaker,
|
|
start=running_len_samples,
|
|
length=length,
|
|
session_len_samples=session_len_samples,
|
|
prev_len_samples=prev_len_samples,
|
|
enforce=enforce,
|
|
)
|
|
# step 5: add sentence to array
|
|
array, is_speech, end = self._add_sentence_to_array(
|
|
start=start,
|
|
length=length,
|
|
array=array,
|
|
is_speech=is_speech,
|
|
)
|
|
|
|
# Step 6: Build entries for output files
|
|
new_rttm_entries = self.annotator.create_new_rttm_entry(
|
|
words=self._words,
|
|
alignments=self._alignments,
|
|
start=start / self._params.data_simulator.sr,
|
|
end=end / self._params.data_simulator.sr,
|
|
speaker_id=speaker_ids[speaker_turn],
|
|
)
|
|
|
|
self.annotator.annote_lists['rttm'].extend(new_rttm_entries)
|
|
|
|
new_json_entry = self.annotator.create_new_json_entry(
|
|
text=self._text,
|
|
wav_filename=os.path.join(basepath, filename + '.wav'),
|
|
start=start / self._params.data_simulator.sr,
|
|
length=length / self._params.data_simulator.sr,
|
|
speaker_id=speaker_ids[speaker_turn],
|
|
rttm_filepath=os.path.join(basepath, filename + '.rttm'),
|
|
ctm_filepath=os.path.join(basepath, filename + '.ctm'),
|
|
)
|
|
self.annotator.annote_lists['json'].append(new_json_entry)
|
|
|
|
new_ctm_entries, _ = self.annotator.create_new_ctm_entry(
|
|
words=self._words,
|
|
alignments=self._alignments,
|
|
session_name=filename,
|
|
speaker_id=speaker_ids[speaker_turn],
|
|
start=float(start / self._params.data_simulator.sr),
|
|
)
|
|
|
|
self.annotator.annote_lists['ctm'].extend(new_ctm_entries)
|
|
|
|
running_len_samples = np.maximum(running_len_samples, end)
|
|
(
|
|
self.sampler.running_speech_len_samples,
|
|
self.sampler.running_silence_len_samples,
|
|
) = self._get_session_silence_from_rttm(
|
|
rttm_list=self.annotator.annote_lists['rttm'], running_len_samples=running_len_samples
|
|
)
|
|
|
|
self._furthest_sample[speaker_turn] = running_len_samples
|
|
prev_speaker = speaker_turn
|
|
prev_len_samples = length
|
|
|
|
# Step 7-1: Add optional perturbations to the whole session, such as white noise.
|
|
if self._params.data_simulator.session_augmentor.add_sess_aug:
|
|
# NOTE: This perturbation is not reflected in the session SNR in meta dictionary.
|
|
array = perturb_audio(array, self._params.data_simulator.sr, self.session_augmentor, device=array.device)
|
|
|
|
# Step 7-2: Additive background noise from noise manifest files
|
|
if self._params.data_simulator.background_noise.add_bg:
|
|
if len(self._noise_samples) > 0:
|
|
avg_power_array = torch.mean(array[is_speech == 1] ** 2)
|
|
bg, snr, _ = get_background_noise(
|
|
len_array=len(array),
|
|
power_array=avg_power_array,
|
|
noise_samples=self._noise_samples,
|
|
audio_read_buffer_dict=self._audio_read_buffer_dict,
|
|
snr_min=self._params.data_simulator.background_noise.snr_min,
|
|
snr_max=self._params.data_simulator.background_noise.snr_max,
|
|
background_noise_snr=self._params.data_simulator.background_noise.snr,
|
|
seed=(random_seed + idx),
|
|
device=self._device,
|
|
)
|
|
array += bg
|
|
else:
|
|
raise ValueError('No background noise samples found in self._noise_samples.')
|
|
else:
|
|
snr = "N/A"
|
|
|
|
# Step 7: Normalize and write to disk
|
|
array = normalize_audio(array)
|
|
|
|
if torch.is_tensor(array):
|
|
array = array.cpu().numpy()
|
|
sf.write(os.path.join(basepath, filename + '.wav'), array, self._params.data_simulator.sr)
|
|
|
|
self.annotator.write_annotation_files(
|
|
basepath=basepath,
|
|
filename=filename,
|
|
meta_data=self._get_session_meta_data(array=array, snr=snr),
|
|
)
|
|
|
|
# Step 8: Clean up memory
|
|
del array
|
|
self.clean_up()
|
|
return basepath, filename
|
|
|
|
def generate_sessions(self, random_seed: int = None):
|
|
"""
|
|
Generate several multispeaker audio sessions and corresponding list files.
|
|
|
|
Args:
|
|
random_seed (int): random seed for reproducibility
|
|
"""
|
|
logging.info("Generating Diarization Sessions")
|
|
if random_seed is None:
|
|
random_seed = self._params.data_simulator.random_seed
|
|
np.random.seed(random_seed)
|
|
|
|
output_dir = self._params.data_simulator.outputs.output_dir
|
|
|
|
basepath = get_cleaned_base_path(
|
|
output_dir, overwrite_output=self._params.data_simulator.outputs.overwrite_output
|
|
)
|
|
OmegaConf.save(self._params, os.path.join(output_dir, "params.yaml"))
|
|
|
|
tp = concurrent.futures.ProcessPoolExecutor(max_workers=self.num_workers)
|
|
futures = []
|
|
|
|
num_sessions = self._params.data_simulator.session_config.num_sessions
|
|
source_noise_manifest = read_noise_manifest(
|
|
add_bg=self._params.data_simulator.background_noise.add_bg,
|
|
background_manifest=self._params.data_simulator.background_noise.background_manifest,
|
|
)
|
|
queue = []
|
|
|
|
# add radomly sampled arguments to a list(queue) for multiprocessing
|
|
for sess_idx in range(num_sessions):
|
|
filename = self._params.data_simulator.outputs.output_filename + f"_{sess_idx}"
|
|
speaker_ids = get_speaker_ids(
|
|
sess_idx=sess_idx,
|
|
speaker_samples=self._speaker_samples,
|
|
permutated_speaker_inds=self._permutated_speaker_inds,
|
|
)
|
|
speaker_wav_align_map = get_speaker_samples(speaker_ids=speaker_ids, speaker_samples=self._speaker_samples)
|
|
noise_samples = self.sampler.sample_noise_manifest(noise_manifest=source_noise_manifest)
|
|
|
|
if torch.cuda.is_available():
|
|
device = torch.device(f"cuda:{sess_idx % torch.cuda.device_count()}")
|
|
else:
|
|
device = self._device
|
|
queue.append((sess_idx, basepath, filename, speaker_ids, speaker_wav_align_map, noise_samples, device))
|
|
|
|
# for multiprocessing speed, we avoid loading potentially huge manifest list and speaker sample files into each process.
|
|
if self.num_workers > 1:
|
|
self._manifest = None
|
|
self._speaker_samples = None
|
|
|
|
# Chunk the sessions into smaller chunks for very large number of sessions (10K+ sessions)
|
|
for chunk_idx in range(self.chunk_count):
|
|
futures = []
|
|
stt_idx, end_idx = (
|
|
chunk_idx * self.multiprocessing_chunksize,
|
|
min((chunk_idx + 1) * self.multiprocessing_chunksize, num_sessions),
|
|
)
|
|
for sess_idx in range(stt_idx, end_idx):
|
|
self._furthest_sample = [0 for n in range(self._params.data_simulator.session_config.num_speakers)]
|
|
self._audio_read_buffer_dict = {}
|
|
if self.num_workers > 1:
|
|
futures.append(tp.submit(self._generate_session, *queue[sess_idx]))
|
|
else:
|
|
futures.append(queue[sess_idx])
|
|
|
|
if self.num_workers > 1:
|
|
generator = concurrent.futures.as_completed(futures)
|
|
else:
|
|
generator = futures
|
|
|
|
for future in tqdm(
|
|
generator,
|
|
desc=f"[{chunk_idx+1}/{self.chunk_count}] Waiting jobs from {stt_idx+1: 2} to {end_idx: 2}",
|
|
unit="jobs",
|
|
total=len(futures),
|
|
):
|
|
if self.num_workers > 1:
|
|
basepath, filename = future.result()
|
|
else:
|
|
self._noise_samples = self.sampler.sample_noise_manifest(
|
|
noise_manifest=source_noise_manifest,
|
|
)
|
|
basepath, filename = self._generate_session(*future)
|
|
|
|
self.annotator.add_to_filename_lists(basepath=basepath, filename=filename)
|
|
|
|
# throw warning if number of speakers is less than requested
|
|
self._check_missing_speakers()
|
|
|
|
tp.shutdown()
|
|
self.annotator.write_filelist_files(basepath=basepath)
|
|
logging.info(f"Data simulation has been completed, results saved at: {basepath}")
|
|
|
|
|
|
class RIRMultiSpeakerSimulator(MultiSpeakerSimulator):
|
|
"""
|
|
RIR Augmented Multispeaker Audio Session Simulator - simulates multispeaker audio sessions using single-speaker
|
|
audio files and corresponding word alignments, as well as simulated RIRs for augmentation.
|
|
|
|
Args:
|
|
cfg: OmegaConf configuration loaded from yaml file.
|
|
|
|
Additional configuration parameters (on top of ``MultiSpeakerSimulator``)::
|
|
|
|
rir_generation:
|
|
use_rir (bool): Whether to generate synthetic RIR
|
|
toolkit (str): Which toolkit to use ("pyroomacoustics", "gpuRIR")
|
|
room_config:
|
|
room_sz (list): Size of the shoebox room environment
|
|
pos_src (list): Positions of the speakers in the simulated room
|
|
noise_src_pos (list): Position in room for background noise source
|
|
mic_config:
|
|
num_channels (int): Number of output audio channels
|
|
pos_rcv (list): Microphone positions in the simulated room
|
|
orV_rcv (list or null): Microphone orientations
|
|
mic_pattern (str): Microphone type ("omni")
|
|
absorbtion_params:
|
|
abs_weights (list): Absorption coefficient ratios for each surface
|
|
T60 (float): Room reverberation time (decay by 60dB)
|
|
att_diff (float): Starting attenuation for diffuse reverberation model
|
|
att_max (float): End attenuation for diffuse reverberation model (gpuRIR)
|
|
"""
|
|
|
|
def __init__(self, cfg):
|
|
super().__init__(cfg)
|
|
self._check_args_rir()
|
|
|
|
def _check_args_rir(self):
|
|
"""
|
|
Checks RIR YAML arguments to ensure they are within valid ranges
|
|
"""
|
|
|
|
if not (self._params.data_simulator.rir_generation.toolkit in ['pyroomacoustics', 'gpuRIR']):
|
|
raise Exception("Toolkit must be pyroomacoustics or gpuRIR")
|
|
if self._params.data_simulator.rir_generation.toolkit == 'pyroomacoustics' and not PRA:
|
|
raise ImportError("pyroomacoustics should be installed to run this simulator with RIR augmentation")
|
|
|
|
if self._params.data_simulator.rir_generation.toolkit == 'gpuRIR' and not GPURIR:
|
|
raise ImportError("gpuRIR should be installed to run this simulator with RIR augmentation")
|
|
|
|
if len(self._params.data_simulator.rir_generation.room_config.room_sz) != 3:
|
|
raise Exception("Incorrect room dimensions provided")
|
|
if self._params.data_simulator.rir_generation.mic_config.num_channels == 0:
|
|
raise Exception("Number of channels should be greater or equal to 1")
|
|
if len(self._params.data_simulator.rir_generation.room_config.pos_src) < 2:
|
|
raise Exception("Less than 2 provided source positions")
|
|
for sublist in self._params.data_simulator.rir_generation.room_config.pos_src:
|
|
if len(sublist) != 3:
|
|
raise Exception("Three coordinates must be provided for sources positions")
|
|
if len(self._params.data_simulator.rir_generation.mic_config.pos_rcv) == 0:
|
|
raise Exception("No provided mic positions")
|
|
for sublist in self._params.data_simulator.rir_generation.room_config.pos_src:
|
|
if len(sublist) != 3:
|
|
raise Exception("Three coordinates must be provided for mic positions")
|
|
|
|
if self._params.data_simulator.session_config.num_speakers != len(
|
|
self._params.data_simulator.rir_generation.room_config.pos_src
|
|
):
|
|
raise Exception("Number of speakers is not equal to the number of provided source positions")
|
|
if self._params.data_simulator.rir_generation.mic_config.num_channels != len(
|
|
self._params.data_simulator.rir_generation.mic_config.pos_rcv
|
|
):
|
|
raise Exception("Number of channels is not equal to the number of provided microphone positions")
|
|
|
|
if (
|
|
not self._params.data_simulator.rir_generation.mic_config.orV_rcv
|
|
and self._params.data_simulator.rir_generation.mic_config.mic_pattern != 'omni'
|
|
):
|
|
raise Exception("Microphone orientations must be provided if mic_pattern != omni")
|
|
if self._params.data_simulator.rir_generation.mic_config.orV_rcv is not None:
|
|
if len(self._params.data_simulator.rir_generation.mic_config.orV_rcv) != len(
|
|
self._params.data_simulator.rir_generation.mic_config.pos_rcv
|
|
):
|
|
raise Exception("A different number of microphone orientations and microphone positions were provided")
|
|
for sublist in self._params.data_simulator.rir_generation.mic_config.orV_rcv:
|
|
if len(sublist) != 3:
|
|
raise Exception("Three coordinates must be provided for orientations")
|
|
|
|
def _generate_rir_gpuRIR(self):
|
|
"""
|
|
Create simulated RIR using the gpuRIR library
|
|
|
|
Returns:
|
|
RIR (tensor): Generated RIR
|
|
RIR_pad (int): Length of padding added when convolving the RIR with an audio file
|
|
"""
|
|
room_sz_tmp = np.array(self._params.data_simulator.rir_generation.room_config.room_sz)
|
|
if room_sz_tmp.ndim == 2: # randomize
|
|
room_sz = np.zeros(room_sz_tmp.shape[0])
|
|
for i in range(room_sz_tmp.shape[0]):
|
|
room_sz[i] = np.random.uniform(room_sz_tmp[i, 0], room_sz_tmp[i, 1])
|
|
else:
|
|
room_sz = room_sz_tmp
|
|
|
|
pos_src_tmp = np.array(self._params.data_simulator.rir_generation.room_config.pos_src)
|
|
if pos_src_tmp.ndim == 3: # randomize
|
|
pos_src = np.zeros((pos_src_tmp.shape[0], pos_src_tmp.shape[1]))
|
|
for i in range(pos_src_tmp.shape[0]):
|
|
for j in range(pos_src_tmp.shape[1]):
|
|
pos_src[i] = np.random.uniform(pos_src_tmp[i, j, 0], pos_src_tmp[i, j, 1])
|
|
else:
|
|
pos_src = pos_src_tmp
|
|
|
|
if self._params.data_simulator.background_noise.add_bg:
|
|
pos_src = np.vstack((pos_src, self._params.data_simulator.rir_generation.room_config.noise_src_pos))
|
|
|
|
mic_pos_tmp = np.array(self._params.data_simulator.rir_generation.mic_config.pos_rcv)
|
|
if mic_pos_tmp.ndim == 3: # randomize
|
|
mic_pos = np.zeros((mic_pos_tmp.shape[0], mic_pos_tmp.shape[1]))
|
|
for i in range(mic_pos_tmp.shape[0]):
|
|
for j in range(mic_pos_tmp.shape[1]):
|
|
mic_pos[i] = np.random.uniform(mic_pos_tmp[i, j, 0], mic_pos_tmp[i, j, 1])
|
|
else:
|
|
mic_pos = mic_pos_tmp
|
|
|
|
orV_rcv = self._params.data_simulator.rir_generation.mic_config.orV_rcv
|
|
if orV_rcv: # not needed for omni mics
|
|
orV_rcv = np.array(orV_rcv)
|
|
mic_pattern = self._params.data_simulator.rir_generation.mic_config.mic_pattern
|
|
abs_weights = self._params.data_simulator.rir_generation.absorbtion_params.abs_weights
|
|
T60 = self._params.data_simulator.rir_generation.absorbtion_params.T60
|
|
att_diff = self._params.data_simulator.rir_generation.absorbtion_params.att_diff
|
|
att_max = self._params.data_simulator.rir_generation.absorbtion_params.att_max
|
|
sr = self._params.data_simulator.sr
|
|
|
|
beta = beta_SabineEstimation(room_sz, T60, abs_weights=abs_weights) # Reflection coefficients
|
|
Tdiff = att2t_SabineEstimator(att_diff, T60) # Time to start the diffuse reverberation model [s]
|
|
Tmax = att2t_SabineEstimator(att_max, T60) # Time to stop the simulation [s]
|
|
nb_img = t2n(Tdiff, room_sz) # Number of image sources in each dimension
|
|
RIR = simulateRIR(
|
|
room_sz, beta, pos_src, mic_pos, nb_img, Tmax, sr, Tdiff=Tdiff, orV_rcv=orV_rcv, mic_pattern=mic_pattern
|
|
)
|
|
RIR_pad = RIR.shape[2] - 1
|
|
return RIR, RIR_pad
|
|
|
|
def _generate_rir_pyroomacoustics(self) -> Tuple[torch.Tensor, int]:
|
|
"""
|
|
Create simulated RIR using the pyroomacoustics library
|
|
|
|
Returns:
|
|
RIR (tensor): Generated RIR
|
|
RIR_pad (int): Length of padding added when convolving the RIR with an audio file
|
|
"""
|
|
|
|
rt60 = self._params.data_simulator.rir_generation.absorbtion_params.T60 # The desired reverberation time
|
|
sr = self._params.data_simulator.sr
|
|
|
|
room_sz_tmp = np.array(self._params.data_simulator.rir_generation.room_config.room_sz)
|
|
if room_sz_tmp.ndim == 2: # randomize
|
|
room_sz = np.zeros(room_sz_tmp.shape[0])
|
|
for i in range(room_sz_tmp.shape[0]):
|
|
room_sz[i] = np.random.uniform(room_sz_tmp[i, 0], room_sz_tmp[i, 1])
|
|
else:
|
|
room_sz = room_sz_tmp
|
|
|
|
pos_src_tmp = np.array(self._params.data_simulator.rir_generation.room_config.pos_src)
|
|
if pos_src_tmp.ndim == 3: # randomize
|
|
pos_src = np.zeros((pos_src_tmp.shape[0], pos_src_tmp.shape[1]))
|
|
for i in range(pos_src_tmp.shape[0]):
|
|
for j in range(pos_src_tmp.shape[1]):
|
|
pos_src[i] = np.random.uniform(pos_src_tmp[i, j, 0], pos_src_tmp[i, j, 1])
|
|
else:
|
|
pos_src = pos_src_tmp
|
|
|
|
# We invert Sabine's formula to obtain the parameters for the ISM simulator
|
|
e_absorption, max_order = pra.inverse_sabine(rt60, room_sz)
|
|
room = pra.ShoeBox(room_sz, fs=sr, materials=pra.Material(e_absorption), max_order=max_order)
|
|
|
|
if self._params.data_simulator.background_noise.add_bg:
|
|
pos_src = np.vstack((pos_src, self._params.data_simulator.rir_generation.room_config.noise_src_pos))
|
|
for pos in pos_src:
|
|
room.add_source(pos)
|
|
|
|
# currently only supports omnidirectional microphones
|
|
mic_pattern = self._params.data_simulator.rir_generation.mic_config.mic_pattern
|
|
if self._params.data_simulator.rir_generation.mic_config.mic_pattern == 'omni':
|
|
mic_pattern = DirectivityPattern.OMNI
|
|
dir_vec = DirectionVector(azimuth=0, colatitude=90, degrees=True)
|
|
else:
|
|
raise Exception("Currently, microphone pattern must be omni. Aborting RIR generation.")
|
|
dir_obj = CardioidFamily(
|
|
orientation=dir_vec,
|
|
pattern_enum=mic_pattern,
|
|
)
|
|
|
|
mic_pos_tmp = np.array(self._params.data_simulator.rir_generation.mic_config.pos_rcv)
|
|
if mic_pos_tmp.ndim == 3: # randomize
|
|
mic_pos = np.zeros((mic_pos_tmp.shape[0], mic_pos_tmp.shape[1]))
|
|
for i in range(mic_pos_tmp.shape[0]):
|
|
for j in range(mic_pos_tmp.shape[1]):
|
|
mic_pos[i] = np.random.uniform(mic_pos_tmp[i, j, 0], mic_pos_tmp[i, j, 1])
|
|
else:
|
|
mic_pos = mic_pos_tmp
|
|
|
|
room.add_microphone_array(mic_pos.T, directivity=dir_obj)
|
|
|
|
room.compute_rir()
|
|
rir_pad = 0
|
|
for channel in room.rir:
|
|
for pos in channel:
|
|
if pos.shape[0] - 1 > rir_pad:
|
|
rir_pad = pos.shape[0] - 1
|
|
return room.rir, rir_pad
|
|
|
|
def _convolve_rir(self, input, speaker_turn: int, RIR: torch.Tensor) -> Tuple[list, int]:
|
|
"""
|
|
Augment one sentence (or background noise segment) using a synthetic RIR.
|
|
|
|
Args:
|
|
input (torch.tensor): Input audio.
|
|
speaker_turn (int): Current speaker turn.
|
|
RIR (torch.tensor): Room Impulse Response.
|
|
Returns:
|
|
output_sound (list): List of tensors containing augmented audio
|
|
length (int): Length of output audio channels (or of the longest if they have different lengths)
|
|
"""
|
|
output_sound = []
|
|
length = 0
|
|
for channel in range(self._params.data_simulator.rir_generation.mic_config.num_channels):
|
|
if self._params.data_simulator.rir_generation.toolkit == 'gpuRIR':
|
|
out_channel = convolve(input, RIR[speaker_turn, channel, : len(input)]).tolist()
|
|
elif self._params.data_simulator.rir_generation.toolkit == 'pyroomacoustics':
|
|
out_channel = convolve(input, RIR[channel][speaker_turn][: len(input)]).tolist()
|
|
else:
|
|
raise Exception("Toolkit must be pyroomacoustics or gpuRIR. Aborting RIR convolution.")
|
|
if len(out_channel) > length:
|
|
length = len(out_channel)
|
|
output_sound.append(torch.tensor(out_channel))
|
|
return output_sound, length
|
|
|
|
def _generate_session(
|
|
self,
|
|
idx: int,
|
|
basepath: str,
|
|
filename: str,
|
|
speaker_ids: list,
|
|
speaker_wav_align_map: dict,
|
|
noise_samples: list,
|
|
device: torch.device,
|
|
enforce_counter: int = 2,
|
|
):
|
|
"""
|
|
Generate a multispeaker audio session and corresponding label files.
|
|
|
|
Args:
|
|
idx (int): Index for current session (out of total number of sessions).
|
|
basepath (str): Path to output directory.
|
|
filename (str): Filename for output files.
|
|
speaker_ids (list): List of speaker IDs that will be used in this session.
|
|
speaker_wav_align_map (dict): Dictionary containing speaker IDs and their corresponding wav filepath and alignments.
|
|
noise_samples (list): List of randomly sampled noise source files that will be used for generating this session.
|
|
device (torch.device): Device to use for generating this session.
|
|
enforce_counter (int): In enforcement mode, dominance is increased by a factor of enforce_counter for unrepresented speakers
|
|
"""
|
|
random_seed = self._params.data_simulator.random_seed
|
|
np.random.seed(random_seed + idx)
|
|
|
|
self._device = device
|
|
speaker_dominance = self._get_speaker_dominance() # randomly determine speaker dominance
|
|
base_speaker_dominance = np.copy(speaker_dominance)
|
|
self._set_speaker_volume()
|
|
|
|
running_len_samples, prev_len_samples = 0, 0 # starting point for each sentence
|
|
prev_speaker = None
|
|
self.annotator.init_annotation_lists()
|
|
self._noise_samples = noise_samples
|
|
self._furthest_sample = [0 for n in range(self._params.data_simulator.session_config.num_speakers)]
|
|
|
|
# Room Impulse Response Generation (performed once per batch of sessions)
|
|
if self._params.data_simulator.rir_generation.toolkit == 'gpuRIR':
|
|
RIR, RIR_pad = self._generate_rir_gpuRIR()
|
|
elif self._params.data_simulator.rir_generation.toolkit == 'pyroomacoustics':
|
|
RIR, RIR_pad = self._generate_rir_pyroomacoustics()
|
|
else:
|
|
raise Exception("Toolkit must be pyroomacoustics or gpuRIR")
|
|
|
|
# hold enforce until all speakers have spoken
|
|
enforce_time = np.random.uniform(
|
|
self._params.data_simulator.speaker_enforcement.enforce_time[0],
|
|
self._params.data_simulator.speaker_enforcement.enforce_time[1],
|
|
)
|
|
enforce = self._params.data_simulator.speaker_enforcement.enforce_num_speakers
|
|
|
|
session_len_samples = int(
|
|
(self._params.data_simulator.session_config.session_length * self._params.data_simulator.sr)
|
|
)
|
|
array = torch.zeros((session_len_samples, self._params.data_simulator.rir_generation.mic_config.num_channels))
|
|
is_speech = torch.zeros(session_len_samples)
|
|
|
|
while running_len_samples < session_len_samples or enforce:
|
|
# Step 1: Prepare parameters for sentence generation
|
|
# Enforce speakers depending on running length
|
|
if running_len_samples > enforce_time * session_len_samples and enforce:
|
|
speaker_dominance, enforce = self._increase_speaker_dominance(base_speaker_dominance, enforce_counter)
|
|
if enforce:
|
|
enforce_counter += 1
|
|
|
|
# Step 2: Select a speaker
|
|
speaker_turn = self._get_next_speaker(prev_speaker, speaker_dominance)
|
|
|
|
# Calculate parameters for building a sentence (only add if remaining length > specific time)
|
|
max_samples_in_sentence = (
|
|
session_len_samples - running_len_samples - RIR_pad
|
|
) # sentence will be RIR_len - 1 longer than the audio was pre-augmentation
|
|
if enforce:
|
|
max_samples_in_sentence = float('inf')
|
|
elif (
|
|
max_samples_in_sentence
|
|
< self._params.data_simulator.session_params.end_buffer * self._params.data_simulator.sr
|
|
):
|
|
break
|
|
|
|
# Step 3: Generate a sentence
|
|
self._build_sentence(speaker_turn, speaker_ids, speaker_wav_align_map, max_samples_in_sentence)
|
|
augmented_sentence, length = self._convolve_rir(self._sentence, speaker_turn, RIR)
|
|
|
|
# Step 4: Generate a time-stamp for either silence or overlap
|
|
start = self._add_silence_or_overlap(
|
|
speaker_turn=speaker_turn,
|
|
prev_speaker=prev_speaker,
|
|
start=running_len_samples,
|
|
length=length,
|
|
session_len_samples=session_len_samples,
|
|
prev_len_samples=prev_len_samples,
|
|
enforce=enforce,
|
|
)
|
|
# step 5: add sentence to array
|
|
end = start + length
|
|
if end > len(array):
|
|
array = torch.nn.functional.pad(array, (0, 0, 0, end - len(array)))
|
|
is_speech = torch.nn.functional.pad(is_speech, (0, end - len(is_speech)))
|
|
is_speech[start:end] = 1
|
|
|
|
for channel in range(self._params.data_simulator.rir_generation.mic_config.num_channels):
|
|
len_ch = len(augmented_sentence[channel]) # accounts for how channels are slightly different lengths
|
|
array[start : start + len_ch, channel] += augmented_sentence[channel]
|
|
|
|
# Step 6: Build entries for output files
|
|
new_rttm_entries = self.annotator.create_new_rttm_entry(
|
|
self._words,
|
|
self._alignments,
|
|
start / self._params.data_simulator.sr,
|
|
end / self._params.data_simulator.sr,
|
|
speaker_ids[speaker_turn],
|
|
)
|
|
|
|
self.annotator.annote_lists['rttm'].extend(new_rttm_entries)
|
|
|
|
new_json_entry = self.annotator.create_new_json_entry(
|
|
self._text,
|
|
os.path.join(basepath, filename + '.wav'),
|
|
start / self._params.data_simulator.sr,
|
|
length / self._params.data_simulator.sr,
|
|
speaker_ids[speaker_turn],
|
|
os.path.join(basepath, filename + '.rttm'),
|
|
os.path.join(basepath, filename + '.ctm'),
|
|
)
|
|
self.annotator.annote_lists['json'].append(new_json_entry)
|
|
|
|
new_ctm_entries, _ = self.annotator.create_new_ctm_entry(
|
|
words=self._text,
|
|
alignments=self._alignments,
|
|
session_name=filename,
|
|
speaker_id=speaker_ids[speaker_turn],
|
|
start=start / self._params.data_simulator.sr,
|
|
)
|
|
self.annotator.annote_lists['ctm'].extend(new_ctm_entries)
|
|
|
|
running_len_samples = np.maximum(running_len_samples, end)
|
|
self._furthest_sample[speaker_turn] = running_len_samples
|
|
prev_speaker = speaker_turn
|
|
prev_len_samples = length
|
|
|
|
# Step 7-1: Add optional perturbations to the whole session, such as white noise.
|
|
if self._params.data_simulator.session_augmentor.add_sess_aug:
|
|
# NOTE: This perturbation is not reflected in the session SNR in meta dictionary.
|
|
array = perturb_audio(array, self._params.data_simulator.sr, self.session_augmentor)
|
|
|
|
# Step 7-2: Additive background noise from noise manifest files
|
|
if self._params.data_simulator.background_noise.add_bg and len(self._noise_samples) > 0:
|
|
avg_power_array = torch.mean(array[is_speech == 1] ** 2)
|
|
bg, snr, _ = get_background_noise(
|
|
len_array=len(array),
|
|
power_array=avg_power_array,
|
|
noise_samples=self._noise_samples,
|
|
audio_read_buffer_dict=self._audio_read_buffer_dict,
|
|
snr_min=self._params.data_simulator.background_noise.snr_min,
|
|
snr_max=self._params.data_simulator.background_noise.snr_max,
|
|
background_noise_snr=self._params.data_simulator.background_noise.snr,
|
|
seed=(random_seed + idx),
|
|
device=self._device,
|
|
)
|
|
array += bg
|
|
length = array.shape[0]
|
|
augmented_bg, _ = self._convolve_rir(bg, -1, RIR)
|
|
for channel in range(self._params.data_simulator.rir_generation.mic_config.num_channels):
|
|
array[:, channel] += augmented_bg[channel][:length]
|
|
else:
|
|
snr = "N/A"
|
|
|
|
# Step 7: Normalize and write to disk
|
|
array = normalize_audio(array)
|
|
|
|
if torch.is_tensor(array):
|
|
array = array.cpu().numpy()
|
|
sf.write(os.path.join(basepath, filename + '.wav'), array, self._params.data_simulator.sr)
|
|
|
|
self.annotator.write_annotation_files(
|
|
basepath=basepath,
|
|
filename=filename,
|
|
meta_data=self._get_session_meta_data(array=array, snr=snr),
|
|
)
|
|
|
|
del array
|
|
self.clean_up()
|
|
return basepath, filename
|