chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import os
|
||||
import re
|
||||
|
||||
import torch
|
||||
from fairseq.file_io import PathManager
|
||||
|
||||
|
||||
def average_checkpoints(inputs):
|
||||
"""Loads checkpoints from inputs and returns a model with averaged weights.
|
||||
|
||||
Args:
|
||||
inputs: An iterable of string paths of checkpoints to load from.
|
||||
|
||||
Returns:
|
||||
A dict of string keys mapping to various values. The 'model' key
|
||||
from the returned dict should correspond to an OrderedDict mapping
|
||||
string parameter names to torch Tensors.
|
||||
"""
|
||||
params_dict = collections.OrderedDict()
|
||||
params_keys = None
|
||||
new_state = None
|
||||
num_models = len(inputs)
|
||||
|
||||
for fpath in inputs:
|
||||
with PathManager.open(fpath, "rb") as f:
|
||||
state = torch.load(
|
||||
f,
|
||||
map_location=(
|
||||
lambda s, _: torch.serialization.default_restore_location(s, "cpu")
|
||||
),
|
||||
)
|
||||
# Copies over the settings from the first checkpoint
|
||||
if new_state is None:
|
||||
new_state = state
|
||||
|
||||
model_params = state["model"]
|
||||
|
||||
model_params_keys = list(model_params.keys())
|
||||
if params_keys is None:
|
||||
params_keys = model_params_keys
|
||||
elif params_keys != model_params_keys:
|
||||
raise KeyError(
|
||||
"For checkpoint {}, expected list of params: {}, "
|
||||
"but found: {}".format(f, params_keys, model_params_keys)
|
||||
)
|
||||
|
||||
for k in params_keys:
|
||||
p = model_params[k]
|
||||
if isinstance(p, torch.HalfTensor):
|
||||
p = p.float()
|
||||
if k not in params_dict:
|
||||
params_dict[k] = p.clone()
|
||||
# NOTE: clone() is needed in case of p is a shared parameter
|
||||
else:
|
||||
params_dict[k] += p
|
||||
|
||||
averaged_params = collections.OrderedDict()
|
||||
for k, v in params_dict.items():
|
||||
averaged_params[k] = v
|
||||
if averaged_params[k].is_floating_point():
|
||||
averaged_params[k].div_(num_models)
|
||||
else:
|
||||
averaged_params[k] //= num_models
|
||||
new_state["model"] = averaged_params
|
||||
return new_state
|
||||
|
||||
|
||||
def last_n_checkpoints(paths, n, update_based, upper_bound=None):
|
||||
assert len(paths) == 1
|
||||
path = paths[0]
|
||||
if update_based:
|
||||
pt_regexp = re.compile(r"checkpoint_\d+_(\d+)\.pt")
|
||||
else:
|
||||
pt_regexp = re.compile(r"checkpoint(\d+)\.pt")
|
||||
files = PathManager.ls(path)
|
||||
|
||||
entries = []
|
||||
for f in files:
|
||||
m = pt_regexp.fullmatch(f)
|
||||
if m is not None:
|
||||
sort_key = int(m.group(1))
|
||||
if upper_bound is None or sort_key <= upper_bound:
|
||||
entries.append((sort_key, m.group(0)))
|
||||
if len(entries) < n:
|
||||
raise Exception(
|
||||
"Found {} checkpoint files but need at least {}", len(entries), n
|
||||
)
|
||||
return [os.path.join(path, x[1]) for x in sorted(entries, reverse=True)[:n]]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Tool to average the params of input checkpoints to "
|
||||
"produce a new checkpoint",
|
||||
)
|
||||
# fmt: off
|
||||
parser.add_argument('--inputs', required=True, nargs='+',
|
||||
help='Input checkpoint file paths.')
|
||||
parser.add_argument('--output', required=True, metavar='FILE',
|
||||
help='Write the new checkpoint containing the averaged weights to this path.')
|
||||
num_group = parser.add_mutually_exclusive_group()
|
||||
num_group.add_argument('--num-epoch-checkpoints', type=int,
|
||||
help='if set, will try to find checkpoints with names checkpoint_xx.pt in the path specified by input, '
|
||||
'and average last this many of them.')
|
||||
num_group.add_argument('--num-update-checkpoints', type=int,
|
||||
help='if set, will try to find checkpoints with names checkpoint_ee_xx.pt in the path specified by input, '
|
||||
'and average last this many of them.')
|
||||
parser.add_argument('--checkpoint-upper-bound', type=int,
|
||||
help='when using --num-epoch-checkpoints, this will set an upper bound on which epoch to use, '
|
||||
'when using --num-update-checkpoints, this will set an upper bound on which update to use'
|
||||
'e.g., with --num-epoch-checkpoints=10 --checkpoint-upper-bound=50, checkpoints 41-50 would be averaged.'
|
||||
'e.g., with --num-update-checkpoints=10 --checkpoint-upper-bound=50000, checkpoints 40500-50000 would be averaged assuming --save-interval-updates 500'
|
||||
)
|
||||
# fmt: on
|
||||
args = parser.parse_args()
|
||||
print(args)
|
||||
|
||||
num = None
|
||||
is_update_based = False
|
||||
if args.num_update_checkpoints is not None:
|
||||
num = args.num_update_checkpoints
|
||||
is_update_based = True
|
||||
elif args.num_epoch_checkpoints is not None:
|
||||
num = args.num_epoch_checkpoints
|
||||
|
||||
assert args.checkpoint_upper_bound is None or (
|
||||
args.num_epoch_checkpoints is not None
|
||||
or args.num_update_checkpoints is not None
|
||||
), "--checkpoint-upper-bound requires --num-epoch-checkpoints or --num-update-checkpoints"
|
||||
assert (
|
||||
args.num_epoch_checkpoints is None or args.num_update_checkpoints is None
|
||||
), "Cannot combine --num-epoch-checkpoints and --num-update-checkpoints"
|
||||
|
||||
if num is not None:
|
||||
args.inputs = last_n_checkpoints(
|
||||
args.inputs,
|
||||
num,
|
||||
is_update_based,
|
||||
upper_bound=args.checkpoint_upper_bound,
|
||||
)
|
||||
print("averaging checkpoints: ", args.inputs)
|
||||
|
||||
new_state = average_checkpoints(args.inputs)
|
||||
with PathManager.open(args.output, "wb") as f:
|
||||
torch.save(new_state, f)
|
||||
print("Finished writing averaged checkpoint to {}".format(args.output))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""
|
||||
Use this script in order to build symmetric alignments for your translation
|
||||
dataset.
|
||||
This script depends on fast_align and mosesdecoder tools. You will need to
|
||||
build those before running the script.
|
||||
fast_align:
|
||||
github: http://github.com/clab/fast_align
|
||||
instructions: follow the instructions in README.md
|
||||
mosesdecoder:
|
||||
github: http://github.com/moses-smt/mosesdecoder
|
||||
instructions: http://www.statmt.org/moses/?n=Development.GetStarted
|
||||
The script produces the following files under --output_dir:
|
||||
text.joined - concatenation of lines from the source_file and the
|
||||
target_file.
|
||||
align.forward - forward pass of fast_align.
|
||||
align.backward - backward pass of fast_align.
|
||||
aligned.sym_heuristic - symmetrized alignment.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from itertools import zip_longest
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="symmetric alignment builer")
|
||||
# fmt: off
|
||||
parser.add_argument('--fast_align_dir',
|
||||
help='path to fast_align build directory')
|
||||
parser.add_argument('--mosesdecoder_dir',
|
||||
help='path to mosesdecoder root directory')
|
||||
parser.add_argument('--sym_heuristic',
|
||||
help='heuristic to use for symmetrization',
|
||||
default='grow-diag-final-and')
|
||||
parser.add_argument('--source_file',
|
||||
help='path to a file with sentences '
|
||||
'in the source language')
|
||||
parser.add_argument('--target_file',
|
||||
help='path to a file with sentences '
|
||||
'in the target language')
|
||||
parser.add_argument('--output_dir',
|
||||
help='output directory')
|
||||
# fmt: on
|
||||
args = parser.parse_args()
|
||||
|
||||
fast_align_bin = os.path.join(args.fast_align_dir, "fast_align")
|
||||
symal_bin = os.path.join(args.mosesdecoder_dir, "bin", "symal")
|
||||
sym_fast_align_bin = os.path.join(
|
||||
args.mosesdecoder_dir, "scripts", "ems", "support", "symmetrize-fast-align.perl"
|
||||
)
|
||||
|
||||
# create joined file
|
||||
joined_file = os.path.join(args.output_dir, "text.joined")
|
||||
with open(args.source_file, "r", encoding="utf-8") as src, open(
|
||||
args.target_file, "r", encoding="utf-8"
|
||||
) as tgt:
|
||||
with open(joined_file, "w", encoding="utf-8") as joined:
|
||||
for s, t in zip_longest(src, tgt):
|
||||
print("{} ||| {}".format(s.strip(), t.strip()), file=joined)
|
||||
|
||||
bwd_align_file = os.path.join(args.output_dir, "align.backward")
|
||||
|
||||
# run forward alignment
|
||||
fwd_align_file = os.path.join(args.output_dir, "align.forward")
|
||||
fwd_fast_align_cmd = "{FASTALIGN} -i {JOINED} -d -o -v > {FWD}".format(
|
||||
FASTALIGN=fast_align_bin, JOINED=joined_file, FWD=fwd_align_file
|
||||
)
|
||||
assert os.system(fwd_fast_align_cmd) == 0
|
||||
|
||||
# run backward alignment
|
||||
bwd_align_file = os.path.join(args.output_dir, "align.backward")
|
||||
bwd_fast_align_cmd = "{FASTALIGN} -i {JOINED} -d -o -v -r > {BWD}".format(
|
||||
FASTALIGN=fast_align_bin, JOINED=joined_file, BWD=bwd_align_file
|
||||
)
|
||||
assert os.system(bwd_fast_align_cmd) == 0
|
||||
|
||||
# run symmetrization
|
||||
sym_out_file = os.path.join(args.output_dir, "aligned")
|
||||
sym_cmd = "{SYMFASTALIGN} {FWD} {BWD} {SRC} {TGT} {OUT} {HEURISTIC} {SYMAL}".format(
|
||||
SYMFASTALIGN=sym_fast_align_bin,
|
||||
FWD=fwd_align_file,
|
||||
BWD=bwd_align_file,
|
||||
SRC=args.source_file,
|
||||
TGT=args.target_file,
|
||||
OUT=sym_out_file,
|
||||
HEURISTIC=args.sym_heuristic,
|
||||
SYMAL=symal_bin,
|
||||
)
|
||||
assert os.system(sym_cmd) == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python
|
||||
"""Helper script to compare two argparse.Namespace objects."""
|
||||
|
||||
from argparse import Namespace # noqa
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
ns1 = eval(input("Namespace 1: "))
|
||||
ns2 = eval(input("Namespace 2: "))
|
||||
|
||||
def keys(ns):
|
||||
ks = set()
|
||||
for k in dir(ns):
|
||||
if not k.startswith("_"):
|
||||
ks.add(k)
|
||||
return ks
|
||||
|
||||
k1 = keys(ns1)
|
||||
k2 = keys(ns2)
|
||||
|
||||
def print_keys(ks, ns1, ns2=None):
|
||||
for k in ks:
|
||||
if ns2 is None:
|
||||
print("{}\t{}".format(k, getattr(ns1, k, None)))
|
||||
else:
|
||||
print(
|
||||
"{}\t{}\t{}".format(k, getattr(ns1, k, None), getattr(ns2, k, None))
|
||||
)
|
||||
|
||||
print("Keys unique to namespace 1:")
|
||||
print_keys(k1 - k2, ns1)
|
||||
print()
|
||||
|
||||
print("Keys unique to namespace 2:")
|
||||
print_keys(k2 - k1, ns2)
|
||||
print()
|
||||
|
||||
print("Overlapping keys with different values:")
|
||||
ks = [k for k in k1 & k2 if getattr(ns1, k, "None") != getattr(ns2, k, "None")]
|
||||
print_keys(ks, ns1, ns2)
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "usage: $0 GENERATE_PY_OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GEN=$1
|
||||
|
||||
SYS=$GEN.sys
|
||||
REF=$GEN.ref
|
||||
|
||||
if [ $(tail -n 1 $GEN | grep BLEU | wc -l) -ne 1 ]; then
|
||||
echo "not done generating"
|
||||
exit
|
||||
fi
|
||||
|
||||
grep ^H $GEN | awk -F '\t' '{print $NF}' | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > $SYS
|
||||
grep ^T $GEN | cut -f2- | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > $REF
|
||||
fairseq-score --sys $SYS --ref $REF
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
"""Extracts random constraints from reference files."""
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import sys
|
||||
|
||||
from sacrebleu import extract_ngrams
|
||||
|
||||
|
||||
def get_phrase(words, index, length):
|
||||
assert index < len(words) - length + 1
|
||||
phr = " ".join(words[index : index + length])
|
||||
for i in range(index, index + length):
|
||||
words.pop(index)
|
||||
return phr
|
||||
|
||||
|
||||
def main(args):
|
||||
|
||||
if args.seed:
|
||||
random.seed(args.seed)
|
||||
|
||||
for line in sys.stdin:
|
||||
constraints = []
|
||||
|
||||
def add_constraint(constraint):
|
||||
constraints.append(constraint)
|
||||
|
||||
source = line.rstrip()
|
||||
if "\t" in line:
|
||||
source, target = line.split("\t")
|
||||
if args.add_sos:
|
||||
target = f"<s> {target}"
|
||||
if args.add_eos:
|
||||
target = f"{target} </s>"
|
||||
|
||||
if len(target.split()) >= args.len:
|
||||
words = [target]
|
||||
|
||||
num = args.number
|
||||
|
||||
choices = {}
|
||||
for i in range(num):
|
||||
if len(words) == 0:
|
||||
break
|
||||
segmentno = random.choice(range(len(words)))
|
||||
segment = words.pop(segmentno)
|
||||
tokens = segment.split()
|
||||
phrase_index = random.choice(range(len(tokens)))
|
||||
choice = " ".join(
|
||||
tokens[phrase_index : min(len(tokens), phrase_index + args.len)]
|
||||
)
|
||||
for j in range(
|
||||
phrase_index, min(len(tokens), phrase_index + args.len)
|
||||
):
|
||||
tokens.pop(phrase_index)
|
||||
if phrase_index > 0:
|
||||
words.append(" ".join(tokens[0:phrase_index]))
|
||||
if phrase_index + 1 < len(tokens):
|
||||
words.append(" ".join(tokens[phrase_index:]))
|
||||
choices[target.find(choice)] = choice
|
||||
|
||||
# mask out with spaces
|
||||
target = target.replace(choice, " " * len(choice), 1)
|
||||
|
||||
for key in sorted(choices.keys()):
|
||||
add_constraint(choices[key])
|
||||
|
||||
print(source, *constraints, sep="\t")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--number", "-n", type=int, default=1, help="number of phrases")
|
||||
parser.add_argument("--len", "-l", type=int, default=1, help="phrase length")
|
||||
parser.add_argument(
|
||||
"--add-sos", default=False, action="store_true", help="add <s> token"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--add-eos", default=False, action="store_true", help="add </s> token"
|
||||
)
|
||||
parser.add_argument("--seed", "-s", default=0, type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args)
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import sys
|
||||
|
||||
|
||||
"""Reads in a fairseq output file, and verifies that the constraints
|
||||
(C- lines) are present in the output (the first H- line). Assumes that
|
||||
constraints are listed prior to the first hypothesis.
|
||||
"""
|
||||
|
||||
constraints = []
|
||||
found = 0
|
||||
total = 0
|
||||
for line in sys.stdin:
|
||||
if line.startswith("C-"):
|
||||
constraints.append(line.rstrip().split("\t")[1])
|
||||
elif line.startswith("H-"):
|
||||
text = line.split("\t")[2]
|
||||
|
||||
for constraint in constraints:
|
||||
total += 1
|
||||
if constraint in text:
|
||||
found += 1
|
||||
else:
|
||||
print(f"No {constraint} in {text}", file=sys.stderr)
|
||||
|
||||
constraints = []
|
||||
|
||||
print(f"Found {found} / {total} = {100 * found / total:.1f}%")
|
||||
@@ -0,0 +1,34 @@
|
||||
-- Copyright (c) Facebook, Inc. and its affiliates.
|
||||
--
|
||||
-- This source code is licensed under the MIT license found in the
|
||||
-- LICENSE file in the root directory of this source tree.
|
||||
--
|
||||
-- Usage: convert_dictionary.lua <dict.th7>
|
||||
require 'fairseq'
|
||||
require 'torch'
|
||||
require 'paths'
|
||||
|
||||
if #arg < 1 then
|
||||
print('usage: convert_dictionary.lua <dict.th7>')
|
||||
os.exit(1)
|
||||
end
|
||||
if not paths.filep(arg[1]) then
|
||||
print('error: file does not exit: ' .. arg[1])
|
||||
os.exit(1)
|
||||
end
|
||||
|
||||
dict = torch.load(arg[1])
|
||||
dst = paths.basename(arg[1]):gsub('.th7', '.txt')
|
||||
assert(dst:match('.txt$'))
|
||||
|
||||
f = io.open(dst, 'w')
|
||||
for idx, symbol in ipairs(dict.index_to_symbol) do
|
||||
if idx > dict.cutoff then
|
||||
break
|
||||
end
|
||||
f:write(symbol)
|
||||
f:write(' ')
|
||||
f:write(dict.index_to_freq[idx])
|
||||
f:write('\n')
|
||||
end
|
||||
f:close()
|
||||
@@ -0,0 +1,108 @@
|
||||
-- Copyright (c) Facebook, Inc. and its affiliates.
|
||||
--
|
||||
-- This source code is licensed under the MIT license found in the
|
||||
-- LICENSE file in the root directory of this source tree.
|
||||
--
|
||||
-- Usage: convert_model.lua <model_epoch1.th7>
|
||||
require 'torch'
|
||||
local fairseq = require 'fairseq'
|
||||
|
||||
model = torch.load(arg[1])
|
||||
|
||||
function find_weight_norm(container, module)
|
||||
for _, wn in ipairs(container:listModules()) do
|
||||
if torch.type(wn) == 'nn.WeightNorm' and wn.modules[1] == module then
|
||||
return wn
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function push_state(dict, key, module)
|
||||
if torch.type(module) == 'nn.Linear' then
|
||||
local wn = find_weight_norm(model.module, module)
|
||||
assert(wn)
|
||||
dict[key .. '.weight_v'] = wn.v:float()
|
||||
dict[key .. '.weight_g'] = wn.g:float()
|
||||
elseif torch.type(module) == 'nn.TemporalConvolutionTBC' then
|
||||
local wn = find_weight_norm(model.module, module)
|
||||
assert(wn)
|
||||
local v = wn.v:float():view(wn.viewOut):transpose(2, 3)
|
||||
dict[key .. '.weight_v'] = v
|
||||
dict[key .. '.weight_g'] = wn.g:float():view(module.weight:size(3), 1, 1)
|
||||
else
|
||||
dict[key .. '.weight'] = module.weight:float()
|
||||
end
|
||||
if module.bias then
|
||||
dict[key .. '.bias'] = module.bias:float()
|
||||
end
|
||||
end
|
||||
|
||||
encoder_dict = {}
|
||||
decoder_dict = {}
|
||||
combined_dict = {}
|
||||
|
||||
function encoder_state(encoder)
|
||||
luts = encoder:findModules('nn.LookupTable')
|
||||
push_state(encoder_dict, 'embed_tokens', luts[1])
|
||||
push_state(encoder_dict, 'embed_positions', luts[2])
|
||||
|
||||
fcs = encoder:findModules('nn.Linear')
|
||||
assert(#fcs >= 2)
|
||||
local nInputPlane = fcs[1].weight:size(1)
|
||||
push_state(encoder_dict, 'fc1', table.remove(fcs, 1))
|
||||
push_state(encoder_dict, 'fc2', table.remove(fcs, #fcs))
|
||||
|
||||
for i, module in ipairs(encoder:findModules('nn.TemporalConvolutionTBC')) do
|
||||
push_state(encoder_dict, 'convolutions.' .. tostring(i - 1), module)
|
||||
if nInputPlane ~= module.weight:size(3) / 2 then
|
||||
push_state(encoder_dict, 'projections.' .. tostring(i - 1), table.remove(fcs, 1))
|
||||
end
|
||||
nInputPlane = module.weight:size(3) / 2
|
||||
end
|
||||
assert(#fcs == 0)
|
||||
end
|
||||
|
||||
function decoder_state(decoder)
|
||||
luts = decoder:findModules('nn.LookupTable')
|
||||
push_state(decoder_dict, 'embed_tokens', luts[1])
|
||||
push_state(decoder_dict, 'embed_positions', luts[2])
|
||||
|
||||
fcs = decoder:findModules('nn.Linear')
|
||||
local nInputPlane = fcs[1].weight:size(1)
|
||||
push_state(decoder_dict, 'fc1', table.remove(fcs, 1))
|
||||
push_state(decoder_dict, 'fc2', fcs[#fcs - 1])
|
||||
push_state(decoder_dict, 'fc3', fcs[#fcs])
|
||||
|
||||
table.remove(fcs, #fcs)
|
||||
table.remove(fcs, #fcs)
|
||||
|
||||
for i, module in ipairs(decoder:findModules('nn.TemporalConvolutionTBC')) do
|
||||
if nInputPlane ~= module.weight:size(3) / 2 then
|
||||
push_state(decoder_dict, 'projections.' .. tostring(i - 1), table.remove(fcs, 1))
|
||||
end
|
||||
nInputPlane = module.weight:size(3) / 2
|
||||
|
||||
local prefix = 'attention.' .. tostring(i - 1)
|
||||
push_state(decoder_dict, prefix .. '.in_projection', table.remove(fcs, 1))
|
||||
push_state(decoder_dict, prefix .. '.out_projection', table.remove(fcs, 1))
|
||||
push_state(decoder_dict, 'convolutions.' .. tostring(i - 1), module)
|
||||
end
|
||||
assert(#fcs == 0)
|
||||
end
|
||||
|
||||
|
||||
_encoder = model.module.modules[2]
|
||||
_decoder = model.module.modules[3]
|
||||
|
||||
encoder_state(_encoder)
|
||||
decoder_state(_decoder)
|
||||
|
||||
for k, v in pairs(encoder_dict) do
|
||||
combined_dict['encoder.' .. k] = v
|
||||
end
|
||||
for k, v in pairs(decoder_dict) do
|
||||
combined_dict['decoder.' .. k] = v
|
||||
end
|
||||
|
||||
|
||||
torch.save('state_dict.t7', combined_dict)
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""
|
||||
Count the number of documents and average number of lines and tokens per
|
||||
document in a large file. Documents should be separated by a single empty line.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input")
|
||||
parser.add_argument("--gzip", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
def gopen():
|
||||
if args.gzip:
|
||||
return gzip.open(args.input, "r")
|
||||
else:
|
||||
return open(args.input, "r", encoding="utf-8")
|
||||
|
||||
num_lines = []
|
||||
num_toks = []
|
||||
with gopen() as h:
|
||||
num_docs = 1
|
||||
num_lines_in_doc = 0
|
||||
num_toks_in_doc = 0
|
||||
for i, line in enumerate(h):
|
||||
if len(line.strip()) == 0: # empty line indicates new document
|
||||
num_docs += 1
|
||||
num_lines.append(num_lines_in_doc)
|
||||
num_toks.append(num_toks_in_doc)
|
||||
num_lines_in_doc = 0
|
||||
num_toks_in_doc = 0
|
||||
else:
|
||||
num_lines_in_doc += 1
|
||||
num_toks_in_doc += len(line.rstrip().split())
|
||||
if i % 1000000 == 0:
|
||||
print(i, file=sys.stderr, end="", flush=True)
|
||||
elif i % 100000 == 0:
|
||||
print(".", file=sys.stderr, end="", flush=True)
|
||||
print(file=sys.stderr, flush=True)
|
||||
|
||||
print("found {} docs".format(num_docs))
|
||||
print("average num lines per doc: {}".format(np.mean(num_lines)))
|
||||
print("average num toks per doc: {}".format(np.mean(num_toks)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import argparse
|
||||
|
||||
from fairseq.data import Dictionary, data_utils, indexed_dataset
|
||||
|
||||
|
||||
def get_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="writes text from binarized file to stdout"
|
||||
)
|
||||
# fmt: off
|
||||
parser.add_argument('--dataset-impl', help='dataset implementation',
|
||||
choices=indexed_dataset.get_available_dataset_impl())
|
||||
parser.add_argument('--dict', metavar='FP', help='dictionary containing known words', default=None)
|
||||
parser.add_argument('--input', metavar='FP', required=True, help='binarized file to read')
|
||||
# fmt: on
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = get_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
dictionary = Dictionary.load(args.dict) if args.dict is not None else None
|
||||
dataset = data_utils.load_indexed_dataset(
|
||||
args.input,
|
||||
dictionary,
|
||||
dataset_impl=args.dataset_impl,
|
||||
default="lazy",
|
||||
)
|
||||
|
||||
for tensor_line in dataset:
|
||||
if dictionary is None:
|
||||
line = " ".join([str(int(x)) for x in tensor_line])
|
||||
else:
|
||||
line = dictionary.string(tensor_line)
|
||||
|
||||
print(line)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
|
||||
pt_regexp = re.compile(r"checkpoint(\d+|_\d+_\d+|_[a-z]+)\.pt")
|
||||
pt_regexp_epoch_based = re.compile(r"checkpoint(\d+)\.pt")
|
||||
pt_regexp_update_based = re.compile(r"checkpoint_\d+_(\d+)\.pt")
|
||||
|
||||
|
||||
def parse_checkpoints(files):
|
||||
entries = []
|
||||
for f in files:
|
||||
m = pt_regexp_epoch_based.fullmatch(f)
|
||||
if m is not None:
|
||||
entries.append((int(m.group(1)), m.group(0)))
|
||||
else:
|
||||
m = pt_regexp_update_based.fullmatch(f)
|
||||
if m is not None:
|
||||
entries.append((int(m.group(1)), m.group(0)))
|
||||
return entries
|
||||
|
||||
|
||||
def last_n_checkpoints(files, n):
|
||||
entries = parse_checkpoints(files)
|
||||
return [x[1] for x in sorted(entries, reverse=True)[:n]]
|
||||
|
||||
|
||||
def every_n_checkpoints(files, n):
|
||||
entries = parse_checkpoints(files)
|
||||
return [x[1] for x in sorted(sorted(entries)[::-n])]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Recursively delete checkpoint files from `root_dir`, "
|
||||
"but preserve checkpoint_best.pt and checkpoint_last.pt"
|
||||
)
|
||||
)
|
||||
parser.add_argument("root_dirs", nargs="*")
|
||||
parser.add_argument(
|
||||
"--save-last", type=int, default=0, help="number of last checkpoints to save"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-every", type=int, default=0, help="interval of checkpoints to save"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preserve-test",
|
||||
action="store_true",
|
||||
help="preserve checkpoints in dirs that start with test_ prefix (default: delete them)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--delete-best", action="store_true", help="delete checkpoint_best.pt"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--delete-last", action="store_true", help="delete checkpoint_last.pt"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-dereference", action="store_true", help="don't dereference symlinks"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
files_to_desymlink = []
|
||||
files_to_preserve = []
|
||||
files_to_delete = []
|
||||
for root_dir in args.root_dirs:
|
||||
for root, _subdirs, files in os.walk(root_dir):
|
||||
if args.save_last > 0:
|
||||
to_save = last_n_checkpoints(files, args.save_last)
|
||||
else:
|
||||
to_save = []
|
||||
if args.save_every > 0:
|
||||
to_save += every_n_checkpoints(files, args.save_every)
|
||||
for file in files:
|
||||
if not pt_regexp.fullmatch(file):
|
||||
continue
|
||||
full_path = os.path.join(root, file)
|
||||
if (
|
||||
not os.path.basename(root).startswith("test_") or args.preserve_test
|
||||
) and (
|
||||
(file == "checkpoint_last.pt" and not args.delete_last)
|
||||
or (file == "checkpoint_best.pt" and not args.delete_best)
|
||||
or file in to_save
|
||||
):
|
||||
if os.path.islink(full_path) and not args.no_dereference:
|
||||
files_to_desymlink.append(full_path)
|
||||
else:
|
||||
files_to_preserve.append(full_path)
|
||||
else:
|
||||
files_to_delete.append(full_path)
|
||||
|
||||
if len(files_to_desymlink) == 0 and len(files_to_delete) == 0:
|
||||
print("Nothing to do.")
|
||||
sys.exit(0)
|
||||
|
||||
files_to_desymlink = sorted(files_to_desymlink)
|
||||
files_to_preserve = sorted(files_to_preserve)
|
||||
files_to_delete = sorted(files_to_delete)
|
||||
|
||||
print("Operations to perform (in order):")
|
||||
if len(files_to_desymlink) > 0:
|
||||
for file in files_to_desymlink:
|
||||
print(" - preserve (and dereference symlink): " + file)
|
||||
if len(files_to_preserve) > 0:
|
||||
for file in files_to_preserve:
|
||||
print(" - preserve: " + file)
|
||||
if len(files_to_delete) > 0:
|
||||
for file in files_to_delete:
|
||||
print(" - delete: " + file)
|
||||
while True:
|
||||
resp = input("Continue? (Y/N): ")
|
||||
if resp.strip().lower() == "y":
|
||||
break
|
||||
elif resp.strip().lower() == "n":
|
||||
sys.exit(0)
|
||||
|
||||
print("Executing...")
|
||||
if len(files_to_desymlink) > 0:
|
||||
for file in files_to_desymlink:
|
||||
realpath = os.path.realpath(file)
|
||||
print("rm " + file)
|
||||
os.remove(file)
|
||||
print("cp {} {}".format(realpath, file))
|
||||
shutil.copyfile(realpath, file)
|
||||
if len(files_to_delete) > 0:
|
||||
for file in files_to_delete:
|
||||
print("rm " + file)
|
||||
os.remove(file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ $# -ne 4 ]; then
|
||||
echo "usage: $0 TESTSET SRCLANG TGTLANG GEN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TESTSET=$1
|
||||
SRCLANG=$2
|
||||
TGTLANG=$3
|
||||
|
||||
GEN=$4
|
||||
|
||||
if ! command -v sacremoses &> /dev/null
|
||||
then
|
||||
echo "sacremoses could not be found, please install with: pip install sacremoses"
|
||||
exit
|
||||
fi
|
||||
|
||||
grep ^H $GEN \
|
||||
| sed 's/^H\-//' \
|
||||
| sort -n -k 1 \
|
||||
| cut -f 3 \
|
||||
| sacremoses detokenize \
|
||||
> $GEN.sorted.detok
|
||||
|
||||
sacrebleu --test-set $TESTSET --language-pair "${SRCLANG}-${TGTLANG}" < $GEN.sorted.detok
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""
|
||||
Split a large file into shards while respecting document boundaries. Documents
|
||||
should be separated by a single empty line.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input")
|
||||
parser.add_argument("--num-shards", type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
assert args.num_shards is not None and args.num_shards > 1
|
||||
|
||||
with open(args.input, "r", encoding="utf-8") as h:
|
||||
with contextlib.ExitStack() as stack:
|
||||
outputs = [
|
||||
stack.enter_context(
|
||||
open(args.input + ".shard" + str(i), "w", encoding="utf-8")
|
||||
)
|
||||
for i in range(args.num_shards)
|
||||
]
|
||||
|
||||
doc = []
|
||||
first_doc = [True] * args.num_shards
|
||||
|
||||
def output_doc(i):
|
||||
if not first_doc[i]:
|
||||
outputs[i].write("\n")
|
||||
first_doc[i] = False
|
||||
for line in doc:
|
||||
outputs[i].write(line)
|
||||
doc.clear()
|
||||
|
||||
num_docs = 0
|
||||
for line in h:
|
||||
if line.strip() == "": # empty line indicates new document
|
||||
output_doc(num_docs % args.num_shards)
|
||||
num_docs += 1
|
||||
else:
|
||||
doc.append(line)
|
||||
output_doc(num_docs % args.num_shards)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""
|
||||
Split a large file into a train and valid set while respecting document
|
||||
boundaries. Documents should be separated by a single empty line.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input")
|
||||
parser.add_argument("sample_output", help="train output file")
|
||||
parser.add_argument("remainder_output", help="valid output file")
|
||||
parser.add_argument("-k", type=int, help="remainder size")
|
||||
parser.add_argument(
|
||||
"--lines", action="store_true", help="split lines instead of docs"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
assert args.k is not None
|
||||
|
||||
sample = []
|
||||
remainder = []
|
||||
num_docs = [0]
|
||||
|
||||
def update_sample(doc):
|
||||
if len(sample) < args.k:
|
||||
sample.append(doc.copy())
|
||||
else:
|
||||
i = num_docs[0]
|
||||
j = random.randrange(i + 1)
|
||||
if j < args.k:
|
||||
remainder.append(sample[j])
|
||||
sample[j] = doc.copy()
|
||||
else:
|
||||
remainder.append(doc.copy())
|
||||
num_docs[0] += 1
|
||||
doc.clear()
|
||||
|
||||
with open(args.input, "r", encoding="utf-8") as h:
|
||||
doc = []
|
||||
for i, line in enumerate(h):
|
||||
if line.strip() == "": # empty line indicates new document
|
||||
update_sample(doc)
|
||||
else:
|
||||
doc.append(line)
|
||||
if args.lines:
|
||||
update_sample(doc)
|
||||
if i % 1000000 == 0:
|
||||
print(i, file=sys.stderr, end="", flush=True)
|
||||
elif i % 100000 == 0:
|
||||
print(".", file=sys.stderr, end="", flush=True)
|
||||
if len(doc) > 0:
|
||||
update_sample(doc)
|
||||
print(file=sys.stderr, flush=True)
|
||||
|
||||
assert len(sample) == args.k
|
||||
|
||||
with open(args.sample_output, "w", encoding="utf-8") as out:
|
||||
first = True
|
||||
for doc in sample:
|
||||
if not first and not args.lines:
|
||||
out.write("\n")
|
||||
first = False
|
||||
for line in doc:
|
||||
out.write(line)
|
||||
|
||||
with open(args.remainder_output, "w", encoding="utf-8") as out:
|
||||
first = True
|
||||
for doc in remainder:
|
||||
if not first and not args.lines:
|
||||
out.write("\n")
|
||||
first = False
|
||||
for line in doc:
|
||||
out.write(line)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
|
||||
import sentencepiece as spm
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model", required=True, help="sentencepiece model to use for decoding"
|
||||
)
|
||||
parser.add_argument("--input", required=True, help="input file to decode")
|
||||
parser.add_argument("--input_format", choices=["piece", "id"], default="piece")
|
||||
args = parser.parse_args()
|
||||
|
||||
sp = spm.SentencePieceProcessor()
|
||||
sp.Load(args.model)
|
||||
|
||||
if args.input_format == "piece":
|
||||
|
||||
def decode(l):
|
||||
return "".join(sp.DecodePieces(l))
|
||||
|
||||
elif args.input_format == "id":
|
||||
|
||||
def decode(l):
|
||||
return "".join(sp.DecodeIds(l))
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
def tok2int(tok):
|
||||
# remap reference-side <unk> (represented as <<unk>>) to 0
|
||||
return int(tok) if tok != "<<unk>>" else 0
|
||||
|
||||
with open(args.input, "r", encoding="utf-8") as h:
|
||||
for line in h:
|
||||
if args.input_format == "id":
|
||||
print(decode(list(map(tok2int, line.rstrip().split()))))
|
||||
elif args.input_format == "piece":
|
||||
print(decode(line.rstrip().split()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import sys
|
||||
|
||||
import sentencepiece as spm
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model", required=True, help="sentencepiece model to use for encoding"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--inputs", nargs="+", default=["-"], help="input files to filter/encode"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--outputs", nargs="+", default=["-"], help="path to save encoded outputs"
|
||||
)
|
||||
parser.add_argument("--output_format", choices=["piece", "id"], default="piece")
|
||||
parser.add_argument(
|
||||
"--min-len",
|
||||
type=int,
|
||||
metavar="N",
|
||||
help="filter sentence pairs with fewer than N tokens",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-len",
|
||||
type=int,
|
||||
metavar="N",
|
||||
help="filter sentence pairs with more than N tokens",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
assert len(args.inputs) == len(
|
||||
args.outputs
|
||||
), "number of input and output paths should match"
|
||||
|
||||
sp = spm.SentencePieceProcessor()
|
||||
sp.Load(args.model)
|
||||
|
||||
if args.output_format == "piece":
|
||||
|
||||
def encode(l):
|
||||
return sp.EncodeAsPieces(l)
|
||||
|
||||
elif args.output_format == "id":
|
||||
|
||||
def encode(l):
|
||||
return list(map(str, sp.EncodeAsIds(l)))
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
if args.min_len is not None or args.max_len is not None:
|
||||
|
||||
def valid(line):
|
||||
return (args.min_len is None or len(line) >= args.min_len) and (
|
||||
args.max_len is None or len(line) <= args.max_len
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
def valid(lines):
|
||||
return True
|
||||
|
||||
with contextlib.ExitStack() as stack:
|
||||
inputs = [
|
||||
stack.enter_context(open(input, "r", encoding="utf-8"))
|
||||
if input != "-"
|
||||
else sys.stdin
|
||||
for input in args.inputs
|
||||
]
|
||||
outputs = [
|
||||
stack.enter_context(open(output, "w", encoding="utf-8"))
|
||||
if output != "-"
|
||||
else sys.stdout
|
||||
for output in args.outputs
|
||||
]
|
||||
|
||||
stats = {
|
||||
"num_empty": 0,
|
||||
"num_filtered": 0,
|
||||
}
|
||||
|
||||
def encode_line(line):
|
||||
line = line.strip()
|
||||
if len(line) > 0:
|
||||
line = encode(line)
|
||||
if valid(line):
|
||||
return line
|
||||
else:
|
||||
stats["num_filtered"] += 1
|
||||
else:
|
||||
stats["num_empty"] += 1
|
||||
return None
|
||||
|
||||
for i, lines in enumerate(zip(*inputs), start=1):
|
||||
enc_lines = list(map(encode_line, lines))
|
||||
if not any(enc_line is None for enc_line in enc_lines):
|
||||
for enc_line, output_h in zip(enc_lines, outputs):
|
||||
print(" ".join(enc_line), file=output_h)
|
||||
if i % 10000 == 0:
|
||||
print("processed {} lines".format(i), file=sys.stderr)
|
||||
|
||||
print("skipped {} empty lines".format(stats["num_empty"]), file=sys.stderr)
|
||||
print("filtered {} lines".format(stats["num_filtered"]), file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
|
||||
import sys
|
||||
|
||||
import sentencepiece as spm
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
spm.SentencePieceTrainer.Train(" ".join(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user