chore: import upstream snapshot with attribution

xtune/scripts/cross-lingual-transfer/train_mlqa.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+
+cp -r $DATA_DIR/squad/ $DATA_DIR/mlqa/squad1.1/
+
+TASK='mlqa'
+
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/SQuAD/translate-train/
+MODEL_PATH=$DATA_DIR/$MODEL
+
+EPOCH=4
+MAXL=384
+LANGS="en,es,de,ar,hi,vi,zh"
+BSR=0.3
+SA=0.3
+SNBS=-1
+CSR=0.3
+R1_LAMBDA=5.0
+R2_LAMBDA=5.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=4
+  GRAD_ACC=8
+  LR=1.5e-5
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=3e-5
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages es,de,ar,hi,vi,zh \
+        --noised_max_seq_length $MAXL
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-SS-R2_Lambda${R2_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method ss \
+        --max_steps 24000 \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_MODEL_PATH
+fi
+
+

xtune/scripts/cross-lingual-transfer/train_panx.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='panx'
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAX_LENGTH=128
+LANGS="ar,he,vi,id,jv,ms,tl,eu,ml,ta,te,af,nl,en,de,el,bn,hi,mr,ur,fa,fr,it,pt,es,bg,ru,ja,ka,ko,th,sw,yo,my,zh,kk,tr,et,fi,hu"
+EVALUATE_STEPS=1000
+BSR=0.3
+SA=0.3
+SNBS=-1
+R1_LAMBDA=5.0
+R2_LAMBDA=5.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=7e-6
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=1e-5
+fi
+
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/translate_train.panx.txt
+
+DATA_DIR=$DATA_DIR/$TASK/${TASK}_processed_maxlen${MAX_LENGTH}/
+
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-SS-R2_Lambda${R2_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method ss \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH \
+        --use_hard_labels
+fi

xtune/scripts/cross-lingual-transfer/train_pawsx.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='pawsx'
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/PAWSX/
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAXL=256
+LANGS="de,en,es,fr,ja,ko,zh"
+EVALUATE_STEPS=1000
+CSR=0.5
+R1_LAMBDA=5.0
+R2_LAMBDA=2.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=16
+  GRAD_ACC=2
+  LR=1e-5
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=1e-5
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages de,es,fr,ja,ko,zh
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_Lambda${R1_LAMBDA}-Aug1.0-CS-R2_Lambda${R2_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages de,es,fr,ja,ko,zh \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method cs \
+        --r2_lambda $R2_LAMBDA
+fi

xtune/scripts/cross-lingual-transfer/train_tydiqa.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='tydiqa'
+MODEL_PATH=$DATA_DIR/$MODEL
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/TyDiQA-GoldP/translate-train/
+
+
+MAXL=384
+LANGS="en,ar,bn,fi,id,ko,ru,sw,te"
+BSR=0.3
+SA=0.3
+SNBS=-1
+R1_LAMBDA=5.0
+R2_LAMBDA=5.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=4
+  GRAD_ACC=8
+  LR=1.5e-5
+  EPOCH=10
+  MAX_STEPS=2500
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=3e-5
+  EPOCH=20
+  MAX_STEPS=5000
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-SS-R2_Lambda${R2_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method ss \
+        --max_steps $MAX_STEPS \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH
+fi
+
+

xtune/scripts/cross-lingual-transfer/train_udpos.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='udpos'
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAX_LENGTH=128
+LANGS="af,ar,bg,de,el,en,es,et,eu,fa,fi,fr,he,hi,hu,id,it,ja,kk,ko,mr,nl,pt,ru,ta,te,th,tl,tr,ur,vi,yo,zh"
+EVALUATE_STEPS=500
+BSR=0.5
+SA=0.3
+SNBS=-1
+R1_LAMBDA=5.0
+R2_LAMBDA=0.3
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=5e-6
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=2e-5
+fi
+
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/translate_train.udpos.txt
+
+DATA_DIR=$DATA_DIR/$TASK/${TASK}_processed_maxlen${MAX_LENGTH}/
+
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --use_pooling_strategy \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-SS-R2_Lambda${R2_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --use_pooling_strategy \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method ss \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH
+fi

xtune/scripts/cross-lingual-transfer/train_xnli.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='xnli'
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/XNLI/
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAXL=256
+LANGS="ar,bg,de,el,en,es,fr,hi,ru,sw,th,tr,ur,vi,zh"
+EVALUATE_STEPS=5000
+CSR=0.3
+R1_LAMBDA=5.0
+R2_LAMBDA=5.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=16
+  GRAD_ACC=2
+  LR=5e-6
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=7e-6
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages ar,bg,de,el,es,fr,hi,ru,sw,th,tr,ur,vi,zh
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_Lambda${R1_LAMBDA}-Aug1.0-CS-R2_Lambda${R2_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages ar,bg,de,el,es,fr,hi,ru,sw,th,tr,ur,vi,zh \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method cs \
+        --r2_lambda $R2_LAMBDA
+fi

xtune/scripts/cross-lingual-transfer/train_xquad.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+cp -r $DATA_DIR/squad/ $DATA_DIR/xquad/squad1.1/
+
+TASK='xquad'
+MODEL_PATH=$DATA_DIR/$MODEL
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/SQuAD/translate-train/
+
+EPOCH=4
+MAXL=384
+LANGS="ar,de,el,en,es,hi,ru,th,tr,vi,zh"
+BSR=0.3
+SA=0.3
+SNBS=-1
+CSR=0.3
+R1_LAMBDA=5.0
+R2_LAMBDA=5.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=4
+  GRAD_ACC=8
+  LR=1.5e-5
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=3e-5
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages ar,de,el,es,hi,ru,th,tr,vi,zh \
+        --noised_max_seq_length $MAXL
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-SS-R2_Lambda${R2_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method ss \
+        --max_steps 24000 \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH
+fi
+
+

xtune/scripts/download_data.sh

@@ -0,0 +1,200 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+DIR=$REPO/download/
+mkdir -p $DIR
+
+# download XNLI dataset
+function download_xnli {
+    OUTPATH=$DIR/xnli-tmp/
+    if [ ! -d $OUTPATH/XNLI-MT-1.0 ]; then
+      if [ ! -f $OUTPATH/XNLI-MT-1.0.zip ]; then
+        wget -c https://dl.fbaipublicfiles.com/XNLI/XNLI-MT-1.0.zip -P $OUTPATH -q --show-progress
+      fi
+      unzip -qq $OUTPATH/XNLI-MT-1.0.zip -d $OUTPATH
+    fi
+    if [ ! -d $OUTPATH/XNLI-1.0 ]; then
+      if [ ! -f $OUTPATH/XNLI-1.0.zip ]; then
+        wget -c https://dl.fbaipublicfiles.com/XNLI/XNLI-1.0.zip -P $OUTPATH -q --show-progress
+      fi
+      unzip -qq $OUTPATH/XNLI-1.0.zip -d $OUTPATH
+    fi
+    python $REPO/utils_preprocess.py \
+      --data_dir $OUTPATH \
+      --output_dir $DIR/xnli/ \
+      --task xnli
+    rm -rf $OUTPATH
+    echo "Successfully downloaded data at $DIR/xnli" >> $DIR/download.log
+}
+
+# download PAWS-X dataset
+function download_pawsx {
+    cd $DIR
+    wget https://storage.googleapis.com/paws/pawsx/x-final.tar.gz -q --show-progress
+    tar xzf x-final.tar.gz -C $DIR/
+    python $REPO/utils_preprocess.py \
+      --data_dir $DIR/x-final \
+      --output_dir $DIR/pawsx/ \
+      --task pawsx
+    rm -rf x-final x-final.tar.gz
+    echo "Successfully downloaded data at $DIR/pawsx" >> $DIR/download.log
+}
+
+# download UD-POS dataset
+function download_udpos {
+    base_dir=$DIR/udpos-tmp
+    out_dir=$base_dir/conll/
+    mkdir -p $out_dir
+    cd $base_dir
+    curl -s --remote-name-all https://lindat.mff.cuni.cz/repository/xmlui/bitstream/handle/11234/1-3105/ud-treebanks-v2.5.tgz
+    tar -xzf $base_dir/ud-treebanks-v2.5.tgz
+
+    langs=(af ar bg de el en es et eu fa fi fr he hi hu id it ja kk ko mr nl pt ru ta te th tl tr ur vi yo zh)
+    for x in $base_dir/ud-treebanks-v2.5/*/*.conllu; do
+        file="$(basename $x)"
+        IFS='_' read -r -a array <<< "$file"
+        lang=${array[0]}
+        if [[ " ${langs[@]} " =~ " ${lang} " ]]; then
+            lang_dir=$out_dir/$lang/
+            mkdir -p $lang_dir
+            y=$lang_dir/${file/conllu/conll}
+            if [ ! -f "$y" ]; then
+                echo "python $REPO/src/ud-conversion-tools/conllu_to_conll.py $x $y --lang $lang --replace_subtokens_with_fused_forms --print_fused_forms"
+                python $REPO/src/ud-conversion-tools/conllu_to_conll.py $x $y --lang $lang --replace_subtokens_with_fused_forms --print_fused_forms
+            else
+                echo "${y} exists"
+            fi
+        fi
+    done
+
+    python $REPO/utils_preprocess.py --data_dir $out_dir/ --output_dir $DIR/udpos/ --task  udpos
+    rm -rf $out_dir ud-treebanks-v2.tgz $DIR/udpos-tmp
+    echo "Successfully downloaded data at $DIR/udpos" >> $DIR/download.log
+}
+
+function download_panx {
+    echo "Download panx NER dataset"
+    if [ -f $DIR/AmazonPhotos.zip ]; then
+        base_dir=$DIR/panx_dataset/
+        unzip -qq -j $DIR/AmazonPhotos.zip -d $base_dir
+        cd $base_dir
+        langs=(ar he vi id jv ms tl eu ml ta te af nl en de el bn hi mr ur fa fr it pt es bg ru ja ka ko th sw yo my zh kk tr et fi hu)
+        for lg in ${langs[@]}; do
+            tar xzf $base_dir/${lg}.tar.gz
+            for f in dev test train; do mv $base_dir/$f $base_dir/${lg}-${f}; done
+        done
+        cd ..
+        python $REPO/utils_preprocess.py \
+            --data_dir $base_dir \
+            --output_dir $DIR/panx \
+            --task panx
+        rm -rf $base_dir
+        echo "Successfully downloaded data at $DIR/panx" >> $DIR/download.log
+    else
+        echo "Please download the AmazonPhotos.zip file on Amazon Cloud Drive mannually and save it to $DIR/AmazonPhotos.zip"
+        echo "https://www.amazon.com/clouddrive/share/d3KGCRCIYwhKJF0H3eWA26hjg2ZCRhjpEQtDL70FSBN"
+    fi
+}
+
+function download_tatoeba {
+    base_dir=$DIR/tatoeba-tmp/
+    wget https://github.com/facebookresearch/LASER/archive/master.zip
+    unzip -qq -o master.zip -d $base_dir/
+    mv $base_dir/LASER-master/data/tatoeba/v1/* $base_dir/
+    python $REPO/utils_preprocess.py \
+      --data_dir $base_dir \
+      --output_dir $DIR/tatoeba \
+      --task tatoeba
+    rm -rf $base_dir master.zip
+    echo "Successfully downloaded data at $DIR/tatoeba" >> $DIR/download.log
+}
+
+function download_bucc18 {
+    base_dir=$DIR/bucc2018/
+    cd $DIR
+    for lg in zh ru de fr; do
+        wget https://comparable.limsi.fr/bucc2018/bucc2018-${lg}-en.training-gold.tar.bz2 -q --show-progress
+        tar -xjf bucc2018-${lg}-en.training-gold.tar.bz2
+        wget https://comparable.limsi.fr/bucc2018/bucc2018-${lg}-en.sample-gold.tar.bz2 -q --show-progress
+        tar -xjf bucc2018-${lg}-en.sample-gold.tar.bz2
+    done
+    mv $base_dir/*/* $base_dir/
+    for f in $base_dir/*training*; do mv $f ${f/training/test}; done
+    for f in $base_dir/*sample*; do mv $f ${f/sample/dev}; done
+    rm -rf $base_dir/*test.gold $DIR/bucc2018*tar.bz2 $base_dir/{zh,ru,de,fr}-en/
+    echo "Successfully downloaded data at $DIR/bucc2018" >> $DIR/download.log
+}
+
+function download_squad {
+    echo "download squad"
+    base_dir=$DIR/squad/
+    mkdir -p $base_dir && cd $base_dir
+    wget https://raw.githubusercontent.com/rajpurkar/SQuAD-explorer/master/dataset/train-v1.1.json -q --show-progress
+    wget https://raw.githubusercontent.com/rajpurkar/SQuAD-explorer/master/dataset/dev-v1.1.json -q --show-progress
+    echo "Successfully downloaded data at $DIR/squad"  >> $DIR/download.log
+}
+
+function download_xquad {
+    echo "download xquad"
+    base_dir=$DIR/xquad/
+    mkdir -p $base_dir && cd $base_dir
+    for lang in ar de el en es hi ru th tr vi zh; do
+      wget https://raw.githubusercontent.com/deepmind/xquad/master/xquad.${lang}.json -q --show-progress
+    done
+    python $REPO/utils_preprocess.py --data_dir $base_dir --output_dir $base_dir --task xquad
+    echo "Successfully downloaded data at $DIR/xquad" >> $DIR/download.log
+}
+
+function download_mlqa {
+    echo "download mlqa"
+    base_dir=$DIR/mlqa/
+    mkdir -p $base_dir && cd $base_dir
+    zip_file=MLQA_V1.zip
+    wget https://dl.fbaipublicfiles.com/MLQA/${zip_file} -q --show-progress
+    unzip -qq ${zip_file}
+    rm ${zip_file}
+    python $REPO/utils_preprocess.py --data_dir $base_dir/MLQA_V1/test --output_dir $base_dir --task mlqa
+    echo "Successfully downloaded data at $DIR/mlqa" >> $DIR/download.log
+}
+
+function download_tydiqa {
+    echo "download tydiqa-goldp"
+    base_dir=$DIR/tydiqa/
+    mkdir -p $base_dir && cd $base_dir
+    tydiqa_train_file=tydiqa-goldp-v1.1-train.json
+    tydiqa_dev_file=tydiqa-goldp-v1.1-dev.tgz
+    wget https://storage.googleapis.com/tydiqa/v1.1/${tydiqa_train_file} -q --show-progress
+    wget https://storage.googleapis.com/tydiqa/v1.1/${tydiqa_dev_file} -q --show-progress
+    tar -xf ${tydiqa_dev_file}
+    rm ${tydiqa_dev_file}
+    out_dir=$base_dir/tydiqa-goldp-v1.1-train
+    python $REPO/utils_preprocess.py --data_dir $base_dir --output_dir $out_dir --task tydiqa
+    mv $base_dir/$tydiqa_train_file $out_dir/
+    echo "Successfully downloaded data at $DIR/tydiqa" >> $DIR/download.log
+}
+
+download_xnli
+download_pawsx
+download_tatoeba
+download_bucc18
+download_squad
+download_xquad
+download_mlqa
+download_tydiqa
+download_udpos
+download_panx
+
+cp -r $DIR/squad/ $DIR/xquad/squad1.1/

xtune/scripts/download_model.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+DIR=$REPO/download/
+mkdir -p $DIR
+
+# download xlm-roberta-base
+function download_xlm-roberta-base {
+    mkdir -p $DIR/xlm-roberta-base/
+    cd $DIR/xlm-roberta-base/
+    wget https://huggingface.co/xlm-roberta-base/resolve/main/pytorch_model.bin -q --show-progress
+    wget https://huggingface.co/xlm-roberta-base/resolve/main/config.json -q --show-progress
+    wget https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model -q --show-progress
+    wget https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json -q --show-progress
+    echo "Successfully downloaded xlm-roberta-base at $DIR/xlm-roberta-base" >> $DIR/download_model.log
+}
+
+# download xlm-roberta-large
+function download_xlm-roberta-large {
+    mkdir -p $DIR/xlm-roberta-large/
+    cd $DIR/xlm-roberta-large/
+    wget https://huggingface.co/xlm-roberta-large/resolve/main/pytorch_model.bin -q --show-progress
+    wget https://huggingface.co/xlm-roberta-large/resolve/main/config.json -q --show-progress
+    wget https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model -q --show-progress
+    wget https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json -q --show-progress
+    echo "Successfully downloaded xlm-roberta-large at $DIR/xlm-roberta-large" >> $DIR/download_model.log
+}
+
+download_xlm-roberta-base
+download_xlm-roberta-large

xtune/scripts/preprocess_panx.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-bert-base-multilingual-cased}
+DATA_DIR=${2:-"$REPO/download/"}
+
+TASK='panx'
+MAXL=128
+LANGS="ar,he,vi,id,jv,ms,tl,eu,ml,ta,te,af,nl,en,de,el,bn,hi,mr,ur,fa,fr,it,pt,es,bg,ru,ja,ka,ko,th,sw,yo,my,zh,kk,tr,et,fi,hu"
+LC=""
+if [ $MODEL == "bert-base-multilingual-cased" ]; then
+  MODEL_TYPE="bert"
+elif [ $MODEL == "xlm-mlm-100-1280" ] || [ $MODEL == "xlm-mlm-tlm-xnli15-1024" ]; then
+  MODEL_TYPE="xlm"
+  LC=" --do_lower_case"
+elif [ $MODEL == "xlm-roberta-large" ] || [ $MODEL == "xlm-roberta-base" ]; then
+  MODEL_TYPE="xlmr"
+fi
+SAVE_DIR="$DATA_DIR/$TASK/${TASK}_processed_maxlen${MAXL}"
+mkdir -p $SAVE_DIR
+python3 $REPO/utils_preprocess.py \
+  --data_dir $DATA_DIR/$TASK/ \
+  --task panx_tokenize \
+  --model_name_or_path $MODEL \
+  --model_type $MODEL_TYPE \
+  --max_len $MAXL \
+  --output_dir $SAVE_DIR \
+  --languages $LANGS $LC >> $SAVE_DIR/preprocess.log
+if [ ! -f $SAVE_DIR/labels.txt ]; then
+  cat $SAVE_DIR/*/*.${MODEL} | cut -f 2 | grep -v "^$" | sort | uniq > $SAVE_DIR/labels.txt
+fi

xtune/scripts/preprocess_udpos.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-bert-base-multilingual-cased}
+DATA_DIR=${2:-"$REPO/download/"}
+
+TASK='udpos'
+MAXL=128
+LANGS='af,ar,bg,de,el,en,es,et,eu,fa,fi,fr,he,hi,hu,id,it,ja,kk,ko,mr,nl,pt,ru,ta,te,th,tl,tr,ur,vi,yo,zh'
+LC=""
+if [ $MODEL == "bert-base-multilingual-cased" ]; then
+  MODEL_TYPE="bert"
+elif [ $MODEL == "xlm-mlm-100-1280" ] || [ $MODEL == "xlm-mlm-tlm-xnli15-1024" ]; then
+  MODEL_TYPE="xlm"
+  LC=" --do_lower_case"
+elif [ $MODEL == "xlm-roberta-large" ] || [ $MODEL == "xlm-roberta-base" ]; then
+  MODEL_TYPE="xlmr"
+fi
+
+SAVE_DIR="$DATA_DIR/${TASK}/udpos_processed_maxlen${MAXL}"
+mkdir -p $SAVE_DIR
+python3 $REPO/utils_preprocess.py \
+  --data_dir $DATA_DIR/${TASK}/ \
+  --task udpos_tokenize \
+  --model_name_or_path $MODEL \
+  --model_type $MODEL_TYPE \
+  --max_len $MAXL \
+  --output_dir $SAVE_DIR \
+  --languages $LANGS $LC >> $SAVE_DIR/process.log
+if [ ! -f $SAVE_DIR/labels.txt ]; then
+  echo "create label"
+  cat $SAVE_DIR/*/*.${MODEL} | cut -f 2 | grep -v "^$" | sort | uniq > $SAVE_DIR/labels.txt
+fi

xtune/scripts/train.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+SETTING=${1:-cross-lingual-transfer}
+TASK=${2:-xnli}
+MODEL=${3:-"xlm-roberta-base"}
+STAGE=${4:-1}
+GPU=${5:-0}
+DATA_DIR=${6:-"$REPO/download/"}
+OUT_DIR=${7:-"$REPO/outputs/"}
+SEED=${8:-1}
+
+echo "Fine-tuning $MODEL on $TASK using GPU $GPU in STAGE $STAGE with SETTING $SETTING"
+echo "Load data from $DATA_DIR, and save models to $OUT_DIR"
+
+if [ $TASK == "udpos" ]; then
+  bash $REPO/scripts/preprocess_udpos.sh $MODEL $DATA_DIR
+elif [ $TASK == "panx" ]; then
+  bash $REPO/scripts/preprocess_panx.sh $MODEL $DATA_DIR
+fi
+
+bash $REPO/scripts/$SETTING/train_${TASK}.sh $MODEL $STAGE $GPU $DATA_DIR $OUT_DIR $SEED

xtune/scripts/translate-train-all/train_mlqa.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+
+cp -r $DATA_DIR/squad/ $DATA_DIR/mlqa/squad1.1/
+
+TASK='mlqa'
+
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/SQuAD/translate-train/
+MODEL_PATH=$DATA_DIR/$MODEL
+
+EPOCH=4
+MAXL=384
+LANGS="en,es,de,ar,hi,vi,zh"
+BSR=0.3
+SA=0.3
+SNBS=-1
+CSR=0.3
+R1_LAMBDA=5.0
+R2_LAMBDA=0.5
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=4
+  GRAD_ACC=8
+  LR=1.5e-5
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=3e-5
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages es,de,ar,hi,vi,zh \
+        --noised_max_seq_length $MAXL
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-MT-R2_Lambda${R2_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method mt \
+        --translation_path $TRANSLATION_PATH \
+        --max_steps 24000 \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH
+fi
+
+

xtune/scripts/translate-train-all/train_panx.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='panx'
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAX_LENGTH=128
+LANGS="ar,he,vi,id,jv,ms,tl,eu,ml,ta,te,af,nl,en,de,el,bn,hi,mr,ur,fa,fr,it,pt,es,bg,ru,ja,ka,ko,th,sw,yo,my,zh,kk,tr,et,fi,hu"
+EVALUATE_STEPS=1000
+BSR=0.3
+SA=0.3
+SNBS=-1
+R1_LAMBDA=5.0
+R2_LAMBDA=1.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=7e-6
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=1e-5
+fi
+
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/translate_train.panx.txt
+
+DATA_DIR=$DATA_DIR/$TASK/${TASK}_processed_maxlen${MAX_LENGTH}/
+
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-MT-R2_Lambda${R2_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method mt \
+        --translation_path $TRANSLATION_PATH \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH \
+        --use_hard_labels
+fi

xtune/scripts/translate-train-all/train_pawsx.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='pawsx'
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/PAWSX/
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAXL=256
+LANGS="de,en,es,fr,ja,ko,zh"
+EVALUATE_STEPS=1000
+R1_LAMBDA=5.0
+R2_LAMBDA=1.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=16
+  GRAD_ACC=2
+  LR=1e-5
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=1e-5
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-Translate-R1_LAMBDA${R1_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --enable_translate_data \
+        --translation_path $TRANSLATION_PATH
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-Translate-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-Translate-R1_Lambda${R1_LAMBDA}-Aug1.0-MT-R2_Lambda${R2_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --enable_translate_data \
+        --translation_path $TRANSLATION_PATH \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method mt \
+        --r2_lambda $R2_LAMBDA
+fi
+
+

xtune/scripts/translate-train-all/train_tydiqa.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='tydiqa'
+MODEL_PATH=$DATA_DIR/$MODEL
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/TyDiQA-GoldP/translate-train/
+
+
+MAXL=384
+LANGS="en,ar,bn,fi,id,ko,ru,sw,te"
+BSR=0.3
+SA=0.3
+SNBS=-1
+R1_LAMBDA=5.0
+R2_LAMBDA=0.3
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=4
+  GRAD_ACC=8
+  LR=1.5e-5
+  EPOCH=10
+  MAX_STEPS=2500
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=3e-5
+  EPOCH=20
+  MAX_STEPS=5000
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-MT-R2_Lambda${R2_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method mt \
+        --translation_path $TRANSLATION_PATH \
+        --max_steps $MAX_STEPS \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH
+fi
+
+

xtune/scripts/translate-train-all/train_udpos.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='udpos'
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAX_LENGTH=128
+LANGS="af,ar,bg,de,el,en,es,et,eu,fa,fi,fr,he,hi,hu,id,it,ja,kk,ko,mr,nl,pt,ru,ta,te,th,tl,tr,ur,vi,yo,zh"
+EVALUATE_STEPS=500
+BSR=0.5
+SA=0.3
+SNBS=-1
+R1_LAMBDA=5.0
+R2_LAMBDA=0.3
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=5e-6
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=2e-5
+fi
+
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/translate_train.udpos.txt
+
+DATA_DIR=$DATA_DIR/$TASK/${TASK}_processed_maxlen${MAX_LENGTH}/
+
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --use_pooling_strategy \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-MT-R2_Lambda${R2_LAMBDA}/"
+  python src/run_tag.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --do_predict \
+        --do_predict_dev \
+        --predict_langs $LANGS \
+        --train_langs en \
+        --data_dir $DATA_DIR \
+        --labels $DATA_DIR/labels.txt \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAX_LENGTH \
+        --noised_max_seq_length $MAX_LENGTH \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps $EVALUATE_STEPS \
+        --seed $SEED \
+        --warmup_steps -1 \
+        --save_only_best_checkpoint \
+        --eval_all_checkpoints \
+        --eval_patience -1 \
+        --fp16 --fp16_opt_level O2 \
+        --hidden_dropout_prob 0.1 \
+        --original_loss \
+        --use_pooling_strategy \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --use_token_label_probs \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method mt \
+        --translation_path $TRANSLATION_PATH \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH
+fi

xtune/scripts/translate-train-all/train_xnli.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+TASK='xnli'
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/XNLI/
+MODEL_PATH=$DATA_DIR/$MODEL
+EPOCH=10
+MAXL=256
+LANGS="ar,bg,de,el,en,es,fr,hi,ru,sw,th,tr,ur,vi,zh"
+EVALUATE_STEPS=5000
+R1_LAMBDA=5.0
+R2_LAMBDA=1.0
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=16
+  GRAD_ACC=2
+  LR=5e-6
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=7e-6
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-Translate-R1_LAMBDA${R1_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --enable_translate_data \
+        --translation_path $TRANSLATION_PATH
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-Translate-R1_LAMBDA${R1_LAMBDA}/checkpoint-best"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-Translate-R1_Lambda${R1_LAMBDA}-Aug1.0-MT-R2_Lambda${R2_LAMBDA}/"
+  mkdir -p $OUTPUT_DIR
+  python ./src/run_cls.py --model_type xlmr \
+        --model_name_or_path $MODEL_PATH \
+        --language $LANGS \
+        --train_language en \
+        --do_train \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 64 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --max_seq_length $MAXL \
+        --output_dir $OUTPUT_DIR \
+        --task_name $TASK \
+        --save_steps -1 \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --evaluate_steps $EVALUATE_STEPS \
+        --logging_steps 50 \
+        --logging_steps_in_sample -1 \
+        --logging_each_epoch \
+        --gpu_id 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --enable_translate_data \
+        --translation_path $TRANSLATION_PATH \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method mt \
+        --r2_lambda $R2_LAMBDA
+fi

xtune/scripts/translate-train-all/train_xquad.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+# Copyright 2020 Google and DeepMind.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPO=$PWD
+MODEL=${1:-"xlm-roberta-base"}
+STAGE=${2:-1}
+GPU=${3:-0}
+DATA_DIR=${4:-"$REPO/download/"}
+OUT_DIR=${5:-"$REPO/outputs/"}
+SEED=${6:-1}
+
+export CUDA_VISIBLE_DEVICES=$GPU
+
+cp -r $DATA_DIR/squad/ $DATA_DIR/xquad/squad1.1/
+
+TASK='xquad'
+MODEL_PATH=$DATA_DIR/$MODEL
+TRANSLATION_PATH=$DATA_DIR/xtreme_translations/SQuAD/translate-train/
+
+EPOCH=4
+MAXL=384
+LANGS="ar,de,el,en,es,hi,ru,th,tr,vi,zh"
+BSR=0.3
+SA=0.3
+SNBS=-1
+CSR=0.3
+R1_LAMBDA=5.0
+R2_LAMBDA=0.1
+if [ $MODEL == "xlm-roberta-large" ]; then
+  BATCH_SIZE=4
+  GRAD_ACC=8
+  LR=1.5e-5
+else
+  BATCH_SIZE=32
+  GRAD_ACC=1
+  LR=3e-5
+fi
+
+if [ $STAGE == 1 ]; then
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_code_switch \
+        --code_switch_ratio $CSR \
+        --dict_dir $DATA_DIR/dicts \
+        --dict_languages ar,de,el,es,hi,ru,th,tr,vi,zh \
+        --noised_max_seq_length $MAXL
+elif [ $STAGE == 2 ]; then
+  FIRST_STAGE_MODEL_PATH="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-CS-csr${CSR}-R1_LAMBDA${R1_LAMBDA}/"
+  OUTPUT_DIR="${OUT_DIR}/${TASK}/${MODEL}-LR${LR}-epoch${EPOCH}-MaxLen${MAXL}-SS-bsr${BSR}-sa${SA}-snbs${SNBS}-R1_Lambda${R1_LAMBDA}-Aug1.0-MT-R2_Lambda${R2_LAMBDA}/"
+  python ./src/run_qa.py --model_type xlmr \
+        --task_name $TASK \
+        --model_name_or_path $MODEL_PATH \
+        --do_train \
+        --do_eval \
+        --language $LANGS \
+        --train_language en \
+        --data_dir $DATA_DIR/$TASK/ \
+        --per_gpu_train_batch_size $BATCH_SIZE \
+        --gradient_accumulation_steps $GRAD_ACC \
+        --per_gpu_eval_batch_size 128 \
+        --learning_rate $LR \
+        --num_train_epochs $EPOCH \
+        --save_steps 0 \
+        --logging_each_epoch \
+        --max_seq_length $MAXL \
+        --doc_stride 128 \
+        --output_dir $OUTPUT_DIR \
+        --overwrite_output_dir \
+        --evaluate_during_training \
+        --logging_steps 50 \
+        --evaluate_steps 0 \
+        --seed $SEED \
+        --fp16 --fp16_opt_level O2 \
+        --warmup_steps -1 \
+        --enable_r1_loss \
+        --r1_lambda $R1_LAMBDA \
+        --original_loss \
+        --overall_ratio 1.0 \
+        --keep_boundary_unchanged \
+        --enable_bpe_sampling \
+        --bpe_sampling_ratio $BSR \
+        --sampling_alpha $SA \
+        --sampling_nbest_size $SNBS \
+        --noised_max_seq_length $MAXL \
+        --enable_data_augmentation \
+        --augment_ratio 1.0 \
+        --augment_method mt \
+        --translation_path $TRANSLATION_PATH \
+        --max_steps 24000 \
+        --r2_lambda $R2_LAMBDA \
+        --first_stage_model_path $FIRST_STAGE_MODEL_PATH
+fi
+
+