wehub-resource-sync
398 lines
14 KiB
Python
398 lines
14 KiB
Python
#!/usr/bin/env python3 -u
|
|
# Copyright (c) Facebook, Inc. and its affiliates.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
"""
|
|
Generate n-best translations using a trained model.
|
|
"""
|
|
|
|
import os
|
|
import subprocess
|
|
from contextlib import redirect_stdout
|
|
|
|
from fairseq import options
|
|
from fairseq_cli import generate, preprocess
|
|
|
|
from examples.noisychannel import rerank_options, rerank_utils
|
|
|
|
|
|
def gen_and_reprocess_nbest(args):
|
|
if args.score_dict_dir is None:
|
|
args.score_dict_dir = args.data
|
|
if args.prefix_len is not None:
|
|
assert (
|
|
args.right_to_left1 is False
|
|
), "prefix length not compatible with right to left models"
|
|
assert (
|
|
args.right_to_left2 is False
|
|
), "prefix length not compatible with right to left models"
|
|
|
|
if args.nbest_list is not None:
|
|
assert args.score_model2 is None
|
|
|
|
if args.backwards1:
|
|
scorer1_src = args.target_lang
|
|
scorer1_tgt = args.source_lang
|
|
else:
|
|
scorer1_src = args.source_lang
|
|
scorer1_tgt = args.target_lang
|
|
|
|
store_data = (
|
|
os.path.join(os.path.dirname(__file__)) + "/rerank_data/" + args.data_dir_name
|
|
)
|
|
if not os.path.exists(store_data):
|
|
os.makedirs(store_data)
|
|
|
|
(
|
|
pre_gen,
|
|
left_to_right_preprocessed_dir,
|
|
right_to_left_preprocessed_dir,
|
|
backwards_preprocessed_dir,
|
|
lm_preprocessed_dir,
|
|
) = rerank_utils.get_directories(
|
|
args.data_dir_name,
|
|
args.num_rescore,
|
|
args.gen_subset,
|
|
args.gen_model_name,
|
|
args.shard_id,
|
|
args.num_shards,
|
|
args.sampling,
|
|
args.prefix_len,
|
|
args.target_prefix_frac,
|
|
args.source_prefix_frac,
|
|
)
|
|
assert not (
|
|
args.right_to_left1 and args.backwards1
|
|
), "backwards right to left not supported"
|
|
assert not (
|
|
args.right_to_left2 and args.backwards2
|
|
), "backwards right to left not supported"
|
|
assert not (
|
|
args.prefix_len is not None and args.target_prefix_frac is not None
|
|
), "target prefix frac and target prefix len incompatible"
|
|
|
|
# make directory to store generation results
|
|
if not os.path.exists(pre_gen):
|
|
os.makedirs(pre_gen)
|
|
|
|
rerank1_is_gen = (
|
|
args.gen_model == args.score_model1 and args.source_prefix_frac is None
|
|
)
|
|
rerank2_is_gen = (
|
|
args.gen_model == args.score_model2 and args.source_prefix_frac is None
|
|
)
|
|
|
|
if args.nbest_list is not None:
|
|
rerank2_is_gen = True
|
|
|
|
# make directories to store preprossed nbest list for reranking
|
|
if not os.path.exists(left_to_right_preprocessed_dir):
|
|
os.makedirs(left_to_right_preprocessed_dir)
|
|
if not os.path.exists(right_to_left_preprocessed_dir):
|
|
os.makedirs(right_to_left_preprocessed_dir)
|
|
if not os.path.exists(lm_preprocessed_dir):
|
|
os.makedirs(lm_preprocessed_dir)
|
|
if not os.path.exists(backwards_preprocessed_dir):
|
|
os.makedirs(backwards_preprocessed_dir)
|
|
|
|
score1_file = rerank_utils.rescore_file_name(
|
|
pre_gen,
|
|
args.prefix_len,
|
|
args.model1_name,
|
|
target_prefix_frac=args.target_prefix_frac,
|
|
source_prefix_frac=args.source_prefix_frac,
|
|
backwards=args.backwards1,
|
|
)
|
|
if args.score_model2 is not None:
|
|
score2_file = rerank_utils.rescore_file_name(
|
|
pre_gen,
|
|
args.prefix_len,
|
|
args.model2_name,
|
|
target_prefix_frac=args.target_prefix_frac,
|
|
source_prefix_frac=args.source_prefix_frac,
|
|
backwards=args.backwards2,
|
|
)
|
|
|
|
predictions_bpe_file = pre_gen + "/generate_output_bpe.txt"
|
|
|
|
using_nbest = args.nbest_list is not None
|
|
|
|
if using_nbest:
|
|
print("Using predefined n-best list from interactive.py")
|
|
predictions_bpe_file = args.nbest_list
|
|
|
|
else:
|
|
if not os.path.isfile(predictions_bpe_file):
|
|
print("STEP 1: generate predictions using the p(T|S) model with bpe")
|
|
print(args.data)
|
|
param1 = [
|
|
args.data,
|
|
"--path",
|
|
args.gen_model,
|
|
"--shard-id",
|
|
str(args.shard_id),
|
|
"--num-shards",
|
|
str(args.num_shards),
|
|
"--nbest",
|
|
str(args.num_rescore),
|
|
"--batch-size",
|
|
str(args.batch_size),
|
|
"--beam",
|
|
str(args.num_rescore),
|
|
"--batch-size",
|
|
str(args.num_rescore),
|
|
"--gen-subset",
|
|
args.gen_subset,
|
|
"--source-lang",
|
|
args.source_lang,
|
|
"--target-lang",
|
|
args.target_lang,
|
|
]
|
|
if args.sampling:
|
|
param1 += ["--sampling"]
|
|
|
|
gen_parser = options.get_generation_parser()
|
|
input_args = options.parse_args_and_arch(gen_parser, param1)
|
|
|
|
print(input_args)
|
|
with open(predictions_bpe_file, "w") as f:
|
|
with redirect_stdout(f):
|
|
generate.main(input_args)
|
|
|
|
gen_output = rerank_utils.BitextOutputFromGen(
|
|
predictions_bpe_file,
|
|
bpe_symbol=args.post_process,
|
|
nbest=using_nbest,
|
|
prefix_len=args.prefix_len,
|
|
target_prefix_frac=args.target_prefix_frac,
|
|
)
|
|
|
|
if args.diff_bpe:
|
|
rerank_utils.write_reprocessed(
|
|
gen_output.no_bpe_source,
|
|
gen_output.no_bpe_hypo,
|
|
gen_output.no_bpe_target,
|
|
pre_gen + "/source_gen_bpe." + args.source_lang,
|
|
pre_gen + "/target_gen_bpe." + args.target_lang,
|
|
pre_gen + "/reference_gen_bpe." + args.target_lang,
|
|
)
|
|
bitext_bpe = args.rescore_bpe_code
|
|
bpe_src_param = [
|
|
"-c",
|
|
bitext_bpe,
|
|
"--input",
|
|
pre_gen + "/source_gen_bpe." + args.source_lang,
|
|
"--output",
|
|
pre_gen + "/rescore_data." + args.source_lang,
|
|
]
|
|
bpe_tgt_param = [
|
|
"-c",
|
|
bitext_bpe,
|
|
"--input",
|
|
pre_gen + "/target_gen_bpe." + args.target_lang,
|
|
"--output",
|
|
pre_gen + "/rescore_data." + args.target_lang,
|
|
]
|
|
|
|
subprocess.call(
|
|
[
|
|
"python",
|
|
os.path.join(
|
|
os.path.dirname(__file__), "subword-nmt/subword_nmt/apply_bpe.py"
|
|
),
|
|
]
|
|
+ bpe_src_param,
|
|
shell=False,
|
|
)
|
|
|
|
subprocess.call(
|
|
[
|
|
"python",
|
|
os.path.join(
|
|
os.path.dirname(__file__), "subword-nmt/subword_nmt/apply_bpe.py"
|
|
),
|
|
]
|
|
+ bpe_tgt_param,
|
|
shell=False,
|
|
)
|
|
|
|
if (not os.path.isfile(score1_file) and not rerank1_is_gen) or (
|
|
args.score_model2 is not None
|
|
and not os.path.isfile(score2_file)
|
|
and not rerank2_is_gen
|
|
):
|
|
print(
|
|
"STEP 2: process the output of generate.py so we have clean text files with the translations"
|
|
)
|
|
|
|
rescore_file = "/rescore_data"
|
|
if args.prefix_len is not None:
|
|
prefix_len_rescore_file = rescore_file + "prefix" + str(args.prefix_len)
|
|
if args.target_prefix_frac is not None:
|
|
target_prefix_frac_rescore_file = (
|
|
rescore_file + "target_prefix_frac" + str(args.target_prefix_frac)
|
|
)
|
|
if args.source_prefix_frac is not None:
|
|
source_prefix_frac_rescore_file = (
|
|
rescore_file + "source_prefix_frac" + str(args.source_prefix_frac)
|
|
)
|
|
|
|
if not args.right_to_left1 or not args.right_to_left2:
|
|
if not args.diff_bpe:
|
|
rerank_utils.write_reprocessed(
|
|
gen_output.source,
|
|
gen_output.hypo,
|
|
gen_output.target,
|
|
pre_gen + rescore_file + "." + args.source_lang,
|
|
pre_gen + rescore_file + "." + args.target_lang,
|
|
pre_gen + "/reference_file",
|
|
bpe_symbol=args.post_process,
|
|
)
|
|
if args.prefix_len is not None:
|
|
bw_rescore_file = prefix_len_rescore_file
|
|
rerank_utils.write_reprocessed(
|
|
gen_output.source,
|
|
gen_output.hypo,
|
|
gen_output.target,
|
|
pre_gen + prefix_len_rescore_file + "." + args.source_lang,
|
|
pre_gen + prefix_len_rescore_file + "." + args.target_lang,
|
|
pre_gen + "/reference_file",
|
|
prefix_len=args.prefix_len,
|
|
bpe_symbol=args.post_process,
|
|
)
|
|
elif args.target_prefix_frac is not None:
|
|
bw_rescore_file = target_prefix_frac_rescore_file
|
|
rerank_utils.write_reprocessed(
|
|
gen_output.source,
|
|
gen_output.hypo,
|
|
gen_output.target,
|
|
pre_gen
|
|
+ target_prefix_frac_rescore_file
|
|
+ "."
|
|
+ args.source_lang,
|
|
pre_gen
|
|
+ target_prefix_frac_rescore_file
|
|
+ "."
|
|
+ args.target_lang,
|
|
pre_gen + "/reference_file",
|
|
bpe_symbol=args.post_process,
|
|
target_prefix_frac=args.target_prefix_frac,
|
|
)
|
|
else:
|
|
bw_rescore_file = rescore_file
|
|
|
|
if args.source_prefix_frac is not None:
|
|
fw_rescore_file = source_prefix_frac_rescore_file
|
|
rerank_utils.write_reprocessed(
|
|
gen_output.source,
|
|
gen_output.hypo,
|
|
gen_output.target,
|
|
pre_gen
|
|
+ source_prefix_frac_rescore_file
|
|
+ "."
|
|
+ args.source_lang,
|
|
pre_gen
|
|
+ source_prefix_frac_rescore_file
|
|
+ "."
|
|
+ args.target_lang,
|
|
pre_gen + "/reference_file",
|
|
bpe_symbol=args.post_process,
|
|
source_prefix_frac=args.source_prefix_frac,
|
|
)
|
|
else:
|
|
fw_rescore_file = rescore_file
|
|
|
|
if args.right_to_left1 or args.right_to_left2:
|
|
rerank_utils.write_reprocessed(
|
|
gen_output.source,
|
|
gen_output.hypo,
|
|
gen_output.target,
|
|
pre_gen + "/right_to_left_rescore_data." + args.source_lang,
|
|
pre_gen + "/right_to_left_rescore_data." + args.target_lang,
|
|
pre_gen + "/right_to_left_reference_file",
|
|
right_to_left=True,
|
|
bpe_symbol=args.post_process,
|
|
)
|
|
|
|
print("STEP 3: binarize the translations")
|
|
if (
|
|
not args.right_to_left1
|
|
or args.score_model2 is not None
|
|
and not args.right_to_left2
|
|
or not rerank1_is_gen
|
|
):
|
|
|
|
if args.backwards1 or args.backwards2:
|
|
if args.backwards_score_dict_dir is not None:
|
|
bw_dict = args.backwards_score_dict_dir
|
|
else:
|
|
bw_dict = args.score_dict_dir
|
|
bw_preprocess_param = [
|
|
"--source-lang",
|
|
scorer1_src,
|
|
"--target-lang",
|
|
scorer1_tgt,
|
|
"--trainpref",
|
|
pre_gen + bw_rescore_file,
|
|
"--srcdict",
|
|
bw_dict + "/dict." + scorer1_src + ".txt",
|
|
"--tgtdict",
|
|
bw_dict + "/dict." + scorer1_tgt + ".txt",
|
|
"--destdir",
|
|
backwards_preprocessed_dir,
|
|
]
|
|
preprocess_parser = options.get_preprocessing_parser()
|
|
input_args = preprocess_parser.parse_args(bw_preprocess_param)
|
|
preprocess.main(input_args)
|
|
|
|
preprocess_param = [
|
|
"--source-lang",
|
|
scorer1_src,
|
|
"--target-lang",
|
|
scorer1_tgt,
|
|
"--trainpref",
|
|
pre_gen + fw_rescore_file,
|
|
"--srcdict",
|
|
args.score_dict_dir + "/dict." + scorer1_src + ".txt",
|
|
"--tgtdict",
|
|
args.score_dict_dir + "/dict." + scorer1_tgt + ".txt",
|
|
"--destdir",
|
|
left_to_right_preprocessed_dir,
|
|
]
|
|
preprocess_parser = options.get_preprocessing_parser()
|
|
input_args = preprocess_parser.parse_args(preprocess_param)
|
|
preprocess.main(input_args)
|
|
|
|
if args.right_to_left1 or args.right_to_left2:
|
|
preprocess_param = [
|
|
"--source-lang",
|
|
scorer1_src,
|
|
"--target-lang",
|
|
scorer1_tgt,
|
|
"--trainpref",
|
|
pre_gen + "/right_to_left_rescore_data",
|
|
"--srcdict",
|
|
args.score_dict_dir + "/dict." + scorer1_src + ".txt",
|
|
"--tgtdict",
|
|
args.score_dict_dir + "/dict." + scorer1_tgt + ".txt",
|
|
"--destdir",
|
|
right_to_left_preprocessed_dir,
|
|
]
|
|
preprocess_parser = options.get_preprocessing_parser()
|
|
input_args = preprocess_parser.parse_args(preprocess_param)
|
|
preprocess.main(input_args)
|
|
|
|
return gen_output
|
|
|
|
|
|
def cli_main():
|
|
parser = rerank_options.get_reranking_parser()
|
|
args = options.parse_args_and_arch(parser)
|
|
gen_and_reprocess_nbest(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli_main()
|