chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,11 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from .asr_dataset import AsrDataset
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AsrDataset",
|
||||
]
|
||||
@@ -0,0 +1,122 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
from fairseq.data import FairseqDataset
|
||||
|
||||
from . import data_utils
|
||||
from .collaters import Seq2SeqCollater
|
||||
|
||||
|
||||
class AsrDataset(FairseqDataset):
|
||||
"""
|
||||
A dataset representing speech and corresponding transcription.
|
||||
|
||||
Args:
|
||||
aud_paths: (List[str]): A list of str with paths to audio files.
|
||||
aud_durations_ms (List[int]): A list of int containing the durations of
|
||||
audio files.
|
||||
tgt (List[torch.LongTensor]): A list of LongTensors containing the indices
|
||||
of target transcriptions.
|
||||
tgt_dict (~fairseq.data.Dictionary): target vocabulary.
|
||||
ids (List[str]): A list of utterance IDs.
|
||||
speakers (List[str]): A list of speakers corresponding to utterances.
|
||||
num_mel_bins (int): Number of triangular mel-frequency bins (default: 80)
|
||||
frame_length (float): Frame length in milliseconds (default: 25.0)
|
||||
frame_shift (float): Frame shift in milliseconds (default: 10.0)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
aud_paths,
|
||||
aud_durations_ms,
|
||||
tgt,
|
||||
tgt_dict,
|
||||
ids,
|
||||
speakers,
|
||||
num_mel_bins=80,
|
||||
frame_length=25.0,
|
||||
frame_shift=10.0,
|
||||
):
|
||||
assert frame_length > 0
|
||||
assert frame_shift > 0
|
||||
assert all(x > frame_length for x in aud_durations_ms)
|
||||
self.frame_sizes = [
|
||||
int(1 + (d - frame_length) / frame_shift) for d in aud_durations_ms
|
||||
]
|
||||
|
||||
assert len(aud_paths) > 0
|
||||
assert len(aud_paths) == len(aud_durations_ms)
|
||||
assert len(aud_paths) == len(tgt)
|
||||
assert len(aud_paths) == len(ids)
|
||||
assert len(aud_paths) == len(speakers)
|
||||
self.aud_paths = aud_paths
|
||||
self.tgt_dict = tgt_dict
|
||||
self.tgt = tgt
|
||||
self.ids = ids
|
||||
self.speakers = speakers
|
||||
self.num_mel_bins = num_mel_bins
|
||||
self.frame_length = frame_length
|
||||
self.frame_shift = frame_shift
|
||||
|
||||
self.s2s_collater = Seq2SeqCollater(
|
||||
0,
|
||||
1,
|
||||
pad_index=self.tgt_dict.pad(),
|
||||
eos_index=self.tgt_dict.eos(),
|
||||
move_eos_to_beginning=True,
|
||||
)
|
||||
|
||||
def __getitem__(self, index):
|
||||
import torchaudio
|
||||
import torchaudio.compliance.kaldi as kaldi
|
||||
|
||||
tgt_item = self.tgt[index] if self.tgt is not None else None
|
||||
|
||||
path = self.aud_paths[index]
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError("Audio file not found: {}".format(path))
|
||||
sound, sample_rate = torchaudio.load_wav(path)
|
||||
output = kaldi.fbank(
|
||||
sound,
|
||||
num_mel_bins=self.num_mel_bins,
|
||||
frame_length=self.frame_length,
|
||||
frame_shift=self.frame_shift,
|
||||
)
|
||||
output_cmvn = data_utils.apply_mv_norm(output)
|
||||
|
||||
return {"id": index, "data": [output_cmvn.detach(), tgt_item]}
|
||||
|
||||
def __len__(self):
|
||||
return len(self.aud_paths)
|
||||
|
||||
def collater(self, samples):
|
||||
"""Merge a list of samples to form a mini-batch.
|
||||
|
||||
Args:
|
||||
samples (List[int]): sample indices to collate
|
||||
|
||||
Returns:
|
||||
dict: a mini-batch suitable for forwarding with a Model
|
||||
"""
|
||||
return self.s2s_collater.collate(samples)
|
||||
|
||||
def num_tokens(self, index):
|
||||
return self.frame_sizes[index]
|
||||
|
||||
def size(self, index):
|
||||
"""Return an example's size as a float or tuple. This value is used when
|
||||
filtering a dataset with ``--max-positions``."""
|
||||
return (
|
||||
self.frame_sizes[index],
|
||||
len(self.tgt[index]) if self.tgt is not None else 0,
|
||||
)
|
||||
|
||||
def ordered_indices(self):
|
||||
"""Return an ordered list of indices. Batches will be constructed based
|
||||
on this order."""
|
||||
return np.arange(len(self))
|
||||
@@ -0,0 +1,131 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""
|
||||
This module contains collection of classes which implement
|
||||
collate functionalities for various tasks.
|
||||
|
||||
Collaters should know what data to expect for each sample
|
||||
and they should pack / collate them into batches
|
||||
"""
|
||||
|
||||
|
||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from fairseq.data import data_utils as fairseq_data_utils
|
||||
|
||||
|
||||
class Seq2SeqCollater(object):
|
||||
"""
|
||||
Implements collate function mainly for seq2seq tasks
|
||||
This expects each sample to contain feature (src_tokens) and
|
||||
targets.
|
||||
This collator is also used for aligned training task.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
feature_index=0,
|
||||
label_index=1,
|
||||
pad_index=1,
|
||||
eos_index=2,
|
||||
move_eos_to_beginning=True,
|
||||
):
|
||||
self.feature_index = feature_index
|
||||
self.label_index = label_index
|
||||
self.pad_index = pad_index
|
||||
self.eos_index = eos_index
|
||||
self.move_eos_to_beginning = move_eos_to_beginning
|
||||
|
||||
def _collate_frames(self, frames):
|
||||
"""Convert a list of 2d frames into a padded 3d tensor
|
||||
Args:
|
||||
frames (list): list of 2d frames of size L[i]*f_dim. Where L[i] is
|
||||
length of i-th frame and f_dim is static dimension of features
|
||||
Returns:
|
||||
3d tensor of size len(frames)*len_max*f_dim where len_max is max of L[i]
|
||||
"""
|
||||
len_max = max(frame.size(0) for frame in frames)
|
||||
f_dim = frames[0].size(1)
|
||||
res = frames[0].new(len(frames), len_max, f_dim).fill_(0.0)
|
||||
|
||||
for i, v in enumerate(frames):
|
||||
res[i, : v.size(0)] = v
|
||||
|
||||
return res
|
||||
|
||||
def collate(self, samples):
|
||||
"""
|
||||
utility function to collate samples into batch for speech recognition.
|
||||
"""
|
||||
if len(samples) == 0:
|
||||
return {}
|
||||
|
||||
# parse samples into torch tensors
|
||||
parsed_samples = []
|
||||
for s in samples:
|
||||
# skip invalid samples
|
||||
if s["data"][self.feature_index] is None:
|
||||
continue
|
||||
source = s["data"][self.feature_index]
|
||||
if isinstance(source, (np.ndarray, np.generic)):
|
||||
source = torch.from_numpy(source)
|
||||
target = s["data"][self.label_index]
|
||||
if isinstance(target, (np.ndarray, np.generic)):
|
||||
target = torch.from_numpy(target).long()
|
||||
elif isinstance(target, list):
|
||||
target = torch.LongTensor(target)
|
||||
|
||||
parsed_sample = {"id": s["id"], "source": source, "target": target}
|
||||
parsed_samples.append(parsed_sample)
|
||||
samples = parsed_samples
|
||||
|
||||
id = torch.LongTensor([s["id"] for s in samples])
|
||||
frames = self._collate_frames([s["source"] for s in samples])
|
||||
# sort samples by descending number of frames
|
||||
frames_lengths = torch.LongTensor([s["source"].size(0) for s in samples])
|
||||
frames_lengths, sort_order = frames_lengths.sort(descending=True)
|
||||
id = id.index_select(0, sort_order)
|
||||
frames = frames.index_select(0, sort_order)
|
||||
|
||||
target = None
|
||||
target_lengths = None
|
||||
prev_output_tokens = None
|
||||
if samples[0].get("target", None) is not None:
|
||||
ntokens = sum(len(s["target"]) for s in samples)
|
||||
target = fairseq_data_utils.collate_tokens(
|
||||
[s["target"] for s in samples],
|
||||
self.pad_index,
|
||||
self.eos_index,
|
||||
left_pad=False,
|
||||
move_eos_to_beginning=False,
|
||||
)
|
||||
target = target.index_select(0, sort_order)
|
||||
target_lengths = torch.LongTensor(
|
||||
[s["target"].size(0) for s in samples]
|
||||
).index_select(0, sort_order)
|
||||
prev_output_tokens = fairseq_data_utils.collate_tokens(
|
||||
[s["target"] for s in samples],
|
||||
self.pad_index,
|
||||
self.eos_index,
|
||||
left_pad=False,
|
||||
move_eos_to_beginning=self.move_eos_to_beginning,
|
||||
)
|
||||
prev_output_tokens = prev_output_tokens.index_select(0, sort_order)
|
||||
else:
|
||||
ntokens = sum(len(s["source"]) for s in samples)
|
||||
|
||||
batch = {
|
||||
"id": id,
|
||||
"ntokens": ntokens,
|
||||
"net_input": {"src_tokens": frames, "src_lengths": frames_lengths},
|
||||
"target": target,
|
||||
"target_lengths": target_lengths,
|
||||
"nsentences": len(samples),
|
||||
}
|
||||
if prev_output_tokens is not None:
|
||||
batch["net_input"]["prev_output_tokens"] = prev_output_tokens
|
||||
return batch
|
||||
@@ -0,0 +1,100 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def calc_mean_invstddev(feature):
|
||||
if len(feature.size()) != 2:
|
||||
raise ValueError("We expect the input feature to be 2-D tensor")
|
||||
mean = feature.mean(0)
|
||||
var = feature.var(0)
|
||||
# avoid division by ~zero
|
||||
eps = 1e-8
|
||||
if (var < eps).any():
|
||||
return mean, 1.0 / (torch.sqrt(var) + eps)
|
||||
return mean, 1.0 / torch.sqrt(var)
|
||||
|
||||
|
||||
def apply_mv_norm(features):
|
||||
# If there is less than 2 spectrograms, the variance cannot be computed (is NaN)
|
||||
# and normalization is not possible, so return the item as it is
|
||||
if features.size(0) < 2:
|
||||
return features
|
||||
mean, invstddev = calc_mean_invstddev(features)
|
||||
res = (features - mean) * invstddev
|
||||
return res
|
||||
|
||||
|
||||
def lengths_to_encoder_padding_mask(lengths, batch_first=False):
|
||||
"""
|
||||
convert lengths (a 1-D Long/Int tensor) to 2-D binary tensor
|
||||
|
||||
Args:
|
||||
lengths: a (B, )-shaped tensor
|
||||
|
||||
Return:
|
||||
max_length: maximum length of B sequences
|
||||
encoder_padding_mask: a (max_length, B) binary mask, where
|
||||
[t, b] = 0 for t < lengths[b] and 1 otherwise
|
||||
|
||||
TODO:
|
||||
kernelize this function if benchmarking shows this function is slow
|
||||
"""
|
||||
max_lengths = torch.max(lengths).item()
|
||||
bsz = lengths.size(0)
|
||||
encoder_padding_mask = torch.arange(
|
||||
max_lengths
|
||||
).to( # a (T, ) tensor with [0, ..., T-1]
|
||||
lengths.device
|
||||
).view( # move to the right device
|
||||
1, max_lengths
|
||||
).expand( # reshape to (1, T)-shaped tensor
|
||||
bsz, -1
|
||||
) >= lengths.view( # expand to (B, T)-shaped tensor
|
||||
bsz, 1
|
||||
).expand(
|
||||
-1, max_lengths
|
||||
)
|
||||
if not batch_first:
|
||||
return encoder_padding_mask.t(), max_lengths
|
||||
else:
|
||||
return encoder_padding_mask, max_lengths
|
||||
|
||||
|
||||
def encoder_padding_mask_to_lengths(
|
||||
encoder_padding_mask, max_lengths, batch_size, device
|
||||
):
|
||||
"""
|
||||
convert encoder_padding_mask (2-D binary tensor) to a 1-D tensor
|
||||
|
||||
Conventionally, encoder output contains a encoder_padding_mask, which is
|
||||
a 2-D mask in a shape (T, B), whose (t, b) element indicate whether
|
||||
encoder_out[t, b] is a valid output (=0) or not (=1). Occasionally, we
|
||||
need to convert this mask tensor to a 1-D tensor in shape (B, ), where
|
||||
[b] denotes the valid length of b-th sequence
|
||||
|
||||
Args:
|
||||
encoder_padding_mask: a (T, B)-shaped binary tensor or None; if None,
|
||||
indicating all are valid
|
||||
Return:
|
||||
seq_lengths: a (B,)-shaped tensor, where its (b, )-th element is the
|
||||
number of valid elements of b-th sequence
|
||||
|
||||
max_lengths: maximum length of all sequence, if encoder_padding_mask is
|
||||
not None, max_lengths must equal to encoder_padding_mask.size(0)
|
||||
|
||||
batch_size: batch size; if encoder_padding_mask is
|
||||
not None, max_lengths must equal to encoder_padding_mask.size(1)
|
||||
|
||||
device: which device to put the result on
|
||||
"""
|
||||
if encoder_padding_mask is None:
|
||||
return torch.Tensor([max_lengths] * batch_size).to(torch.int32).to(device)
|
||||
|
||||
assert encoder_padding_mask.size(0) == max_lengths, "max_lengths does not match"
|
||||
assert encoder_padding_mask.size(1) == batch_size, "batch_size does not match"
|
||||
|
||||
return max_lengths - torch.sum(encoder_padding_mask, dim=0)
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
"""
|
||||
Replabel transforms for use with flashlight's ASG criterion.
|
||||
"""
|
||||
|
||||
|
||||
def replabel_symbol(i):
|
||||
"""
|
||||
Replabel symbols used in flashlight, currently just "1", "2", ...
|
||||
This prevents training with numeral tokens, so this might change in the future
|
||||
"""
|
||||
return str(i)
|
||||
|
||||
|
||||
def pack_replabels(tokens, dictionary, max_reps):
|
||||
"""
|
||||
Pack a token sequence so that repeated symbols are replaced by replabels
|
||||
"""
|
||||
if len(tokens) == 0 or max_reps <= 0:
|
||||
return tokens
|
||||
|
||||
replabel_value_to_idx = [0] * (max_reps + 1)
|
||||
for i in range(1, max_reps + 1):
|
||||
replabel_value_to_idx[i] = dictionary.index(replabel_symbol(i))
|
||||
|
||||
result = []
|
||||
prev_token = -1
|
||||
num_reps = 0
|
||||
for token in tokens:
|
||||
if token == prev_token and num_reps < max_reps:
|
||||
num_reps += 1
|
||||
else:
|
||||
if num_reps > 0:
|
||||
result.append(replabel_value_to_idx[num_reps])
|
||||
num_reps = 0
|
||||
result.append(token)
|
||||
prev_token = token
|
||||
if num_reps > 0:
|
||||
result.append(replabel_value_to_idx[num_reps])
|
||||
return result
|
||||
|
||||
|
||||
def unpack_replabels(tokens, dictionary, max_reps):
|
||||
"""
|
||||
Unpack a token sequence so that replabels are replaced by repeated symbols
|
||||
"""
|
||||
if len(tokens) == 0 or max_reps <= 0:
|
||||
return tokens
|
||||
|
||||
replabel_idx_to_value = {}
|
||||
for i in range(1, max_reps + 1):
|
||||
replabel_idx_to_value[dictionary.index(replabel_symbol(i))] = i
|
||||
|
||||
result = []
|
||||
prev_token = -1
|
||||
for token in tokens:
|
||||
try:
|
||||
for _ in range(replabel_idx_to_value[token]):
|
||||
result.append(prev_token)
|
||||
prev_token = -1
|
||||
except KeyError:
|
||||
result.append(token)
|
||||
prev_token = token
|
||||
return result
|
||||
Reference in New Issue
Block a user