wehub-resource-sync
53 lines
1.8 KiB
Python
53 lines
1.8 KiB
Python
# Copyright (c) Facebook, Inc. and its affiliates.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
|
|
|
|
import os
|
|
import glob
|
|
import argparse
|
|
from utils.dedup import deup
|
|
|
|
import sys
|
|
WORKDIR_ROOT = os.environ.get('WORKDIR_ROOT', None)
|
|
|
|
if WORKDIR_ROOT is None or not WORKDIR_ROOT.strip():
|
|
print('please specify your working directory root in OS environment variable WORKDIR_ROOT. Exitting..."')
|
|
sys.exit(-1)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--from-folder", type=str, required=True,
|
|
help="the data folder to be dedup")
|
|
parser.add_argument("--to-folder", type=str, required=True,
|
|
help="the data folder to save deduped data")
|
|
parser.add_argument('--directions', type=str, default=None, required=False)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.directions is None:
|
|
raw_files = glob.glob(f'{args.from_folder}/train*')
|
|
|
|
directions = [os.path.split(file_path)[-1].split('.')[1] for file_path in raw_files]
|
|
else:
|
|
directions = args.directions.split(',')
|
|
directions = sorted(set(directions))
|
|
|
|
for direction in directions:
|
|
src, tgt = direction.split('-')
|
|
src_file = f'{args.from_folder}/train.{src}-{tgt}.{src}'
|
|
tgt_file = f'{args.from_folder}/train.{src}-{tgt}.{tgt}'
|
|
src_file_out = f'{args.to_folder}/train.{src}-{tgt}.{src}'
|
|
tgt_file_out = f'{args.to_folder}/train.{src}-{tgt}.{tgt}'
|
|
assert src_file != src_file_out
|
|
assert tgt_file != tgt_file_out
|
|
print(f'deduping {src_file}, {tgt_file}')
|
|
deup(src_file, tgt_file, src_file_out, tgt_file_out)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|