chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:24:13 +08:00
commit 1037506f2e
6050 changed files with 1731598 additions and 0 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+27
View File
@@ -0,0 +1,27 @@
"""
#!/usr/bin/env python3 -u
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the LICENSE file in
# the root directory of this source tree. An additional grant of patent rights
# can be found in the PATENTS file in the same directory.
"""
mkdir data
# WMT16 EN-RO
cd data
mkdir wmt16.en-ro
cd wmt16.en-ro
gdown https://drive.google.com/uc?id=1YrAwCEuktG-iDVxtEW-FE72uFTLc5QMl
tar -zxvf wmt16.tar.gz
mv wmt16/en-ro/train/corpus.bpe.en train.en
mv wmt16/en-ro/train/corpus.bpe.ro train.ro
mv wmt16/en-ro/dev/dev.bpe.en valid.en
mv wmt16/en-ro/dev/dev.bpe.ro valid.ro
mv wmt16/en-ro/test/test.bpe.en test.en
mv wmt16/en-ro/test/test.bpe.ro test.ro
rm wmt16.tar.gz
rm -r wmt16