wehub-resource-sync
34 lines
1.1 KiB
Bash
34 lines
1.1 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# usage: bash binarize_manifest <dest_dir> <train_split> <valid_split>
|
|
|
|
DEST_DIR=$1
|
|
TRAIN_SPLIT=$2
|
|
VALID_SPLIT=$3
|
|
FAIRSEQ_ROOT=$4
|
|
|
|
mkdir -p $DEST_DIR
|
|
|
|
# split file path and lengths into separate files
|
|
cut -f1 $TRAIN_SPLIT.tsv > $DEST_DIR/train_fnames.txt
|
|
cut -f1 $VALID_SPLIT.tsv > $DEST_DIR/valid_fnames.txt
|
|
cut -f2 $TRAIN_SPLIT.tsv > $DEST_DIR/train.lengths
|
|
cut -f2 $VALID_SPLIT.tsv > $DEST_DIR/valid.lengths
|
|
|
|
# copy root directory
|
|
head -1 $TRAIN_SPLIT.tsv > $DEST_DIR/train.root
|
|
head -1 $VALID_SPLIT.tsv > $DEST_DIR/valid.root
|
|
|
|
# remove root directory
|
|
sed -i '1d' $DEST_DIR/train_fnames.txt
|
|
sed -i '1d' $DEST_DIR/valid_fnames.txt
|
|
sed -i '1d' $DEST_DIR/train.lengths
|
|
sed -i '1d' $DEST_DIR/valid.lengths
|
|
|
|
# insert spaces between characters
|
|
sed -i -e 's/\(.\)/\1 /g' $DEST_DIR/train_fnames.txt
|
|
sed -i -e 's/\(.\)/\1 /g' $DEST_DIR/valid_fnames.txt
|
|
|
|
# run preprocessor
|
|
PYTHONPATH=$FAIRSEQ_ROOT python $FAIRSEQ_ROOT/fairseq_cli/preprocess.py --dataset-impl mmap --trainpref $DEST_DIR/train_fnames.txt --validpref $DEST_DIR/valid_fnames.txt --workers 60 --only-source --destdir $DEST_DIR
|