chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
from collections import namedtuple
|
||||
from itertools import chain
|
||||
|
||||
import sentencepiece as spm
|
||||
from fairseq.data import Dictionary
|
||||
|
||||
|
||||
MILLISECONDS_TO_SECONDS = 0.001
|
||||
|
||||
|
||||
def process_sample(aud_path, lable, utt_id, sp, tgt_dict):
|
||||
import torchaudio
|
||||
|
||||
input = {}
|
||||
output = {}
|
||||
si, ei = torchaudio.info(aud_path)
|
||||
input["length_ms"] = int(
|
||||
si.length / si.channels / si.rate / MILLISECONDS_TO_SECONDS
|
||||
)
|
||||
input["path"] = aud_path
|
||||
|
||||
token = " ".join(sp.EncodeAsPieces(lable))
|
||||
ids = tgt_dict.encode_line(token, append_eos=False)
|
||||
output["text"] = lable
|
||||
output["token"] = token
|
||||
output["tokenid"] = ", ".join(map(str, [t.tolist() for t in ids]))
|
||||
return {utt_id: {"input": input, "output": output}}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--audio-dirs",
|
||||
nargs="+",
|
||||
default=["-"],
|
||||
required=True,
|
||||
help="input directories with audio files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--labels",
|
||||
required=True,
|
||||
help="aggregated input labels with format <ID LABEL> per line",
|
||||
type=argparse.FileType("r", encoding="UTF-8"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--spm-model",
|
||||
required=True,
|
||||
help="sentencepiece model to use for encoding",
|
||||
type=argparse.FileType("r", encoding="UTF-8"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dictionary",
|
||||
required=True,
|
||||
help="file to load fairseq dictionary from",
|
||||
type=argparse.FileType("r", encoding="UTF-8"),
|
||||
)
|
||||
parser.add_argument("--audio-format", choices=["flac", "wav"], default="wav")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
required=True,
|
||||
type=argparse.FileType("w"),
|
||||
help="path to save json output",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
sp = spm.SentencePieceProcessor()
|
||||
sp.Load(args.spm_model.name)
|
||||
|
||||
tgt_dict = Dictionary.load(args.dictionary)
|
||||
|
||||
labels = {}
|
||||
for line in args.labels:
|
||||
(utt_id, label) = line.split(" ", 1)
|
||||
labels[utt_id] = label
|
||||
if len(labels) == 0:
|
||||
raise Exception("No labels found in ", args.labels_path)
|
||||
|
||||
Sample = namedtuple("Sample", "aud_path utt_id")
|
||||
samples = []
|
||||
for path, _, files in chain.from_iterable(
|
||||
os.walk(path) for path in args.audio_dirs
|
||||
):
|
||||
for f in files:
|
||||
if f.endswith(args.audio_format):
|
||||
if len(os.path.splitext(f)) != 2:
|
||||
raise Exception("Expect <utt_id.extension> file name. Got: ", f)
|
||||
utt_id = os.path.splitext(f)[0]
|
||||
if utt_id not in labels:
|
||||
continue
|
||||
samples.append(Sample(os.path.join(path, f), utt_id))
|
||||
|
||||
utts = {}
|
||||
num_cpu = multiprocessing.cpu_count()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=num_cpu) as executor:
|
||||
future_to_sample = {
|
||||
executor.submit(
|
||||
process_sample, s.aud_path, labels[s.utt_id], s.utt_id, sp, tgt_dict
|
||||
): s
|
||||
for s in samples
|
||||
}
|
||||
for future in concurrent.futures.as_completed(future_to_sample):
|
||||
try:
|
||||
data = future.result()
|
||||
except Exception as exc:
|
||||
print("generated an exception: ", exc)
|
||||
else:
|
||||
utts.update(data)
|
||||
json.dump({"utts": utts}, args.output, indent=4)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
# Prepare librispeech dataset
|
||||
|
||||
base_url=www.openslr.org/resources/12
|
||||
train_dir=train_960
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "Usage: $0 <download_dir> <out_dir>"
|
||||
echo "e.g.: $0 /tmp/librispeech_raw/ ~/data/librispeech_final"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
download_dir=${1%/}
|
||||
out_dir=${2%/}
|
||||
|
||||
fairseq_root=~/fairseq-py/
|
||||
mkdir -p ${out_dir}
|
||||
cd ${out_dir} || exit
|
||||
|
||||
nbpe=5000
|
||||
bpemode=unigram
|
||||
|
||||
if [ ! -d "$fairseq_root" ]; then
|
||||
echo "$0: Please set correct fairseq_root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Data Download"
|
||||
for part in dev-clean test-clean dev-other test-other train-clean-100 train-clean-360 train-other-500; do
|
||||
url=$base_url/$part.tar.gz
|
||||
if ! wget -P $download_dir $url; then
|
||||
echo "$0: wget failed for $url"
|
||||
exit 1
|
||||
fi
|
||||
if ! tar -C $download_dir -xvzf $download_dir/$part.tar.gz; then
|
||||
echo "$0: error un-tarring archive $download_dir/$part.tar.gz"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Merge all train packs into one"
|
||||
mkdir -p ${download_dir}/LibriSpeech/${train_dir}/
|
||||
for part in train-clean-100 train-clean-360 train-other-500; do
|
||||
mv ${download_dir}/LibriSpeech/${part}/* $download_dir/LibriSpeech/${train_dir}/
|
||||
done
|
||||
echo "Merge train text"
|
||||
find ${download_dir}/LibriSpeech/${train_dir}/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/${train_dir}/text
|
||||
|
||||
# Use combined dev-clean and dev-other as validation set
|
||||
find ${download_dir}/LibriSpeech/dev-clean/ ${download_dir}/LibriSpeech/dev-other/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/valid_text
|
||||
find ${download_dir}/LibriSpeech/test-clean/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/test-clean/text
|
||||
find ${download_dir}/LibriSpeech/test-other/ -name '*.txt' -exec cat {} \; >> ${download_dir}/LibriSpeech/test-other/text
|
||||
|
||||
|
||||
dict=data/lang_char/${train_dir}_${bpemode}${nbpe}_units.txt
|
||||
encoded=data/lang_char/${train_dir}_${bpemode}${nbpe}_encoded.txt
|
||||
fairseq_dict=data/lang_char/${train_dir}_${bpemode}${nbpe}_fairseq_dict.txt
|
||||
bpemodel=data/lang_char/${train_dir}_${bpemode}${nbpe}
|
||||
echo "dictionary: ${dict}"
|
||||
echo "Dictionary preparation"
|
||||
mkdir -p data/lang_char/
|
||||
echo "<unk> 3" > ${dict}
|
||||
echo "</s> 2" >> ${dict}
|
||||
echo "<pad> 1" >> ${dict}
|
||||
cut -f 2- -d" " ${download_dir}/LibriSpeech/${train_dir}/text > data/lang_char/input.txt
|
||||
spm_train --input=data/lang_char/input.txt --vocab_size=${nbpe} --model_type=${bpemode} --model_prefix=${bpemodel} --input_sentence_size=100000000 --unk_id=3 --eos_id=2 --pad_id=1 --bos_id=-1 --character_coverage=1
|
||||
spm_encode --model=${bpemodel}.model --output_format=piece < data/lang_char/input.txt > ${encoded}
|
||||
cat ${encoded} | tr ' ' '\n' | sort | uniq | awk '{print $0 " " NR+3}' >> ${dict}
|
||||
cat ${encoded} | tr ' ' '\n' | sort | uniq -c | awk '{print $2 " " $1}' > ${fairseq_dict}
|
||||
wc -l ${dict}
|
||||
|
||||
echo "Prepare train and test jsons"
|
||||
for part in train_960 test-other test-clean; do
|
||||
python ${fairseq_root}/examples/speech_recognition/datasets/asr_prep_json.py --audio-dirs ${download_dir}/LibriSpeech/${part} --labels ${download_dir}/LibriSpeech/${part}/text --spm-model ${bpemodel}.model --audio-format flac --dictionary ${fairseq_dict} --output ${part}.json
|
||||
done
|
||||
# fairseq expects to find train.json and valid.json during training
|
||||
mv train_960.json train.json
|
||||
|
||||
echo "Prepare valid json"
|
||||
python ${fairseq_root}/examples/speech_recognition/datasets/asr_prep_json.py --audio-dirs ${download_dir}/LibriSpeech/dev-clean ${download_dir}/LibriSpeech/dev-other --labels ${download_dir}/LibriSpeech/valid_text --spm-model ${bpemodel}.model --audio-format flac --dictionary ${fairseq_dict} --output valid.json
|
||||
|
||||
cp ${fairseq_dict} ./dict.txt
|
||||
cp ${bpemodel}.model ./spm.model
|
||||
Reference in New Issue
Block a user