wehub-resource-sync
429 lines
19 KiB
Bash
429 lines
19 KiB
Bash
#!/bin/bash
|
|
|
|
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
source test_tipc/common_func.sh
|
|
|
|
FILENAME=$1
|
|
|
|
# MODE be one of ['lite_train_lite_infer' 'lite_train_whole_infer' 'whole_train_whole_infer',
|
|
# 'whole_infer', 'klquant_whole_infer',
|
|
# 'cpp_infer', 'serving_infer']
|
|
# PaddleNLP supports 'lite_train_lite_infer', 'lite_train_whole_infer', 'whole_train_whole_infer' and
|
|
# 'whole_infer' mode now.
|
|
|
|
MODE=$2
|
|
|
|
dataline=$(cat ${FILENAME})
|
|
|
|
# parser params
|
|
IFS=$'\n'
|
|
lines=(${dataline})
|
|
|
|
# The training params
|
|
model_name=$(func_parser_value "${lines[1]}")
|
|
|
|
trainer_list=$(func_parser_value "${lines[14]}")
|
|
|
|
if [ ${MODE} = "lite_train_lite_infer" ];then
|
|
if [ ${model_name} == "bigru_crf" ]; then
|
|
rm -rf ./data/lexical_analysis_dataset_tiny ./data/lexical_analysis_dataset_tiny.tar.gz
|
|
wget -nc -P ./data/ https://bj.bcebos.com/paddlenlp/datasets/lexical_analysis_dataset_tiny.tar.gz --no-check-certificate
|
|
cd ./data/ && tar xfz lexical_analysis_dataset_tiny.tar.gz && cd ..
|
|
fi
|
|
|
|
if [ ${model_name} == "ernie_information_extraction" ]; then
|
|
python ../slm/examples/information_extraction/waybill_ie/download.py --data_dir ./waybill_ie
|
|
fi
|
|
|
|
if [[ ${model_name} =~ transformer* ]]; then
|
|
cd ../slm/examples/machine_translation/transformer/
|
|
|
|
# The whole procedure of lite_train_infer should be less than 15min.
|
|
# Hence, set maximum output length is 16.
|
|
sed -i "s/^max_out_len.*/max_out_len: 16/g" configs/transformer.base.yaml
|
|
sed -i "s/^max_out_len.*/max_out_len: 16/g" configs/transformer.big.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: 128/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: False/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: False/g" configs/transformer.base.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: 128/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: False/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: False/g" configs/transformer.big.yaml
|
|
|
|
# Data set prepared.
|
|
if [ ! -f WMT14.en-de.partial.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/datasets/WMT14.en-de.partial.tar.gz
|
|
tar -zxf WMT14.en-de.partial.tar.gz
|
|
fi
|
|
# Set soft link.
|
|
if [ -f train.en ]; then
|
|
rm -f train.en
|
|
fi
|
|
if [ -f train.de ]; then
|
|
rm -f train.de
|
|
fi
|
|
if [ -f dev.en ]; then
|
|
rm -f dev.en
|
|
fi
|
|
if [ -f dev.de ]; then
|
|
rm -f dev.de
|
|
fi
|
|
if [ -f test.en ]; then
|
|
rm -f test.en
|
|
fi
|
|
if [ -f test.de ]; then
|
|
rm -f test.de
|
|
fi
|
|
rm -f vocab_all.bpe.33712
|
|
rm -f vocab_all.bpe.33708
|
|
# Vocab
|
|
cp -f WMT14.en-de.partial/wmt14_ende_data_bpe/vocab_all.bpe.33712 ./
|
|
cp -f WMT14.en-de.partial/wmt14_ende_data_bpe/vocab_all.bpe.33708 ./
|
|
# Train
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/train.tok.clean.bpe.en train.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/train.tok.clean.bpe.de train.de
|
|
# Dev
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/dev.tok.bpe.en dev.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/dev.tok.bpe.de dev.de
|
|
#Test
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/test.tok.bpe.en test.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/test.tok.bpe.de test.de
|
|
cd -
|
|
fi
|
|
elif [ ${MODE} = "whole_train_whole_infer" ];then
|
|
if [ ${model_name} == "bigru_crf" ]; then
|
|
rm -rf ./data/lexical_analysis_dataset_tiny ./data/lexical_analysis_dataset_tiny.tar.gz
|
|
wget -nc -P ./data/ https://bj.bcebos.com/paddlenlp/datasets/lexical_analysis_dataset_tiny.tar.gz --no-check-certificate
|
|
cd ./data/ && tar xfz lexical_analysis_dataset_tiny.tar.gz && cd ..
|
|
fi
|
|
|
|
if [ ${model_name} == "ernie_information_extraction" ]; then
|
|
python ../slm/examples/information_extraction/waybill_ie/download.py --data_dir ./waybill_ie
|
|
fi
|
|
|
|
if [[ ${model_name} =~ transformer* ]]; then
|
|
cd ../slm/examples/machine_translation/transformer/
|
|
sed -i "s/^max_out_len.*/max_out_len: 256/g" configs/transformer.base.yaml
|
|
sed -i "s/^max_out_len.*/max_out_len: 1024/g" configs/transformer.big.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: None/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: True/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: True/g" configs/transformer.base.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: None/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: True/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: True/g" configs/transformer.big.yaml
|
|
|
|
# Whole data set prepared.
|
|
if [ ! -f WMT14.en-de.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/datasets/WMT14.en-de.tar.gz
|
|
tar -zxf WMT14.en-de.tar.gz
|
|
fi
|
|
# Set soft link.
|
|
if [ -f train.en ]; then
|
|
rm -f train.en
|
|
fi
|
|
if [ -f train.de ]; then
|
|
rm -f train.de
|
|
fi
|
|
if [ -f dev.en ]; then
|
|
rm -f dev.en
|
|
fi
|
|
if [ -f dev.de ]; then
|
|
rm -f dev.de
|
|
fi
|
|
if [ -f test.en ]; then
|
|
rm -f test.en
|
|
fi
|
|
if [ -f test.de ]; then
|
|
rm -f test.de
|
|
fi
|
|
rm -f vocab_all.bpe.33712
|
|
rm -f vocab_all.bpe.33708
|
|
# Vocab
|
|
cp -f WMT14.en-de/wmt14_ende_data_bpe/vocab_all.bpe.33712 ./
|
|
cp -f WMT14.en-de/wmt14_ende_data_bpe/vocab_all.bpe.33708 ./
|
|
# Train with whole data.
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/train.tok.clean.bpe.33708.en train.en
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/train.tok.clean.bpe.33708.de train.de
|
|
# Dev with whole data.
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2013.tok.bpe.33708.en dev.en
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2013.tok.bpe.33708.de dev.de
|
|
# Test with whole data.
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.en test.en
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.de test.de
|
|
cd -
|
|
fi
|
|
elif [ ${MODE} = "lite_train_whole_infer" ];then
|
|
if [ ${model_name} == "bigru_crf" ]; then
|
|
rm -rf ./data/lexical_analysis_dataset_tiny ./data/lexical_analysis_dataset_tiny.tar.gz
|
|
wget -nc -P ./data/ https://bj.bcebos.com/paddlenlp/datasets/lexical_analysis_dataset_tiny.tar.gz --no-check-certificate
|
|
cd ./data/ && tar xfz lexical_analysis_dataset_tiny.tar.gz && cd ..
|
|
fi
|
|
|
|
if [ ${model_name} == "ernie_information_extraction" ]; then
|
|
python ../slm/examples/information_extraction/waybill_ie/download.py --data_dir ./waybill_ie
|
|
fi
|
|
|
|
if [[ ${model_name} =~ transformer* ]]; then
|
|
cd ../slm/examples/machine_translation/transformer/
|
|
sed -i "s/^max_out_len.*/max_out_len: 256/g" configs/transformer.base.yaml
|
|
sed -i "s/^max_out_len.*/max_out_len: 1024/g" configs/transformer.big.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: None/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: True/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: True/g" configs/transformer.base.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: None/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: True/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: True/g" configs/transformer.big.yaml
|
|
|
|
# Trained transformer base model checkpoint.
|
|
# For infer.
|
|
if [ ! -f transformer-base-wmt_ende_bpe.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/models/transformers/transformer/transformer-base-wmt_ende_bpe.tar.gz
|
|
tar -zxf transformer-base-wmt_ende_bpe.tar.gz
|
|
mv base_trained_models/ trained_models/
|
|
fi
|
|
# For train.
|
|
if [ ! -f WMT14.en-de.partial.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/datasets/WMT14.en-de.partial.tar.gz
|
|
tar -zxf WMT14.en-de.partial.tar.gz
|
|
fi
|
|
# Whole data set prepared.
|
|
if [ ! -f WMT14.en-de.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/datasets/WMT14.en-de.tar.gz
|
|
tar -zxf WMT14.en-de.tar.gz
|
|
fi
|
|
# Set soft link.
|
|
if [ -f train.en ]; then
|
|
rm -f train.en
|
|
fi
|
|
if [ -f train.de ]; then
|
|
rm -f train.de
|
|
fi
|
|
if [ -f dev.en ]; then
|
|
rm -f dev.en
|
|
fi
|
|
if [ -f dev.de ]; then
|
|
rm -f dev.de
|
|
fi
|
|
if [ -f test.en ]; then
|
|
rm -f test.en
|
|
fi
|
|
if [ -f test.de ]; then
|
|
rm -f test.de
|
|
fi
|
|
rm -f vocab_all.bpe.33712
|
|
rm -f vocab_all.bpe.33708
|
|
# Vocab
|
|
cp -f WMT14.en-de.partial/wmt14_ende_data_bpe/vocab_all.bpe.33712 ./
|
|
cp -f WMT14.en-de.partial/wmt14_ende_data_bpe/vocab_all.bpe.33708 ./
|
|
# Train with partial data.
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/train.tok.clean.bpe.en train.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/train.tok.clean.bpe.de train.de
|
|
# Dev with partial data.
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/dev.tok.bpe.en dev.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/dev.tok.bpe.de dev.de
|
|
# Test with whole data.
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.en test.en
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.de test.de
|
|
cd -
|
|
fi
|
|
elif [ ${MODE} = "whole_infer" ];then
|
|
if [ ${model_name} == "bigru_crf" ]; then
|
|
rm -rf ./data/lexical_analysis_dataset_tiny ./data/lexical_analysis_dataset_tiny.tar.gz
|
|
wget -nc -P ./data/ https://bj.bcebos.com/paddlenlp/datasets/lexical_analysis_dataset_tiny.tar.gz --no-check-certificate
|
|
cd ./data/ && tar xfz lexical_analysis_dataset_tiny.tar.gz && cd ..
|
|
# Download static model
|
|
rm -rf ./test_tipc/bigru_crf/infer_model
|
|
wget -nc -P ./test_tipc/bigru_crf/ https://bj.bcebos.com/paddlenlp/models/bigru_crf_infer_model.tgz --no-check-certificate
|
|
cd ./test_tipc/bigru_crf && tar xfz bigru_crf_infer_model.tgz && cd ../..
|
|
fi
|
|
|
|
if [ ${model_name} == "ernie_information_extraction" ]; then
|
|
python ../slm/examples/information_extraction/waybill_ie/download.py --data_dir ./waybill_ie
|
|
fi
|
|
|
|
if [[ ${model_name} =~ transformer* ]]; then
|
|
cd ../slm/examples/machine_translation/transformer/
|
|
sed -i "s/^max_out_len.*/max_out_len: 256/g" configs/transformer.base.yaml
|
|
sed -i "s/^max_out_len.*/max_out_len: 1024/g" configs/transformer.big.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: None/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: True/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: True/g" configs/transformer.base.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: None/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: True/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: True/g" configs/transformer.big.yaml
|
|
|
|
# Trained transformer base model checkpoint.
|
|
if [ ! -f transformer-base-wmt_ende_bpe.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/models/transformers/transformer/transformer-base-wmt_ende_bpe.tar.gz
|
|
tar -zxf transformer-base-wmt_ende_bpe.tar.gz
|
|
mv base_trained_models/ trained_models/
|
|
fi
|
|
# Whole data set prepared.
|
|
if [ ! -f WMT14.en-de.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/datasets/WMT14.en-de.tar.gz
|
|
tar -zxf WMT14.en-de.tar.gz
|
|
fi
|
|
# Set soft link.
|
|
if [ -f test.en ]; then
|
|
rm -f test.en
|
|
fi
|
|
if [ -f test.de ]; then
|
|
rm -f test.de
|
|
fi
|
|
rm -f vocab_all.bpe.33712
|
|
rm -f vocab_all.bpe.33708
|
|
# Vocab
|
|
cp -f WMT14.en-de/wmt14_ende_data_bpe/vocab_all.bpe.33712 ./
|
|
cp -f WMT14.en-de/wmt14_ende_data_bpe/vocab_all.bpe.33708 ./
|
|
# Test with whole data.
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.en test.en
|
|
ln -s WMT14.en-de/wmt14_ende_data_bpe/newstest2014.tok.bpe.33708.de test.de
|
|
cd -
|
|
fi
|
|
elif [ ${MODE} = "benchmark_train" ];then
|
|
if [ ${model_name} == "bigru_crf" ]; then
|
|
rm -rf ./data/lexical_analysis_dataset_tiny ./data/lexical_analysis_dataset_tiny.tar.gz
|
|
python ${BENCHMARK_ROOT}/paddlecloud/file_upload_download.py --remote-path frame_benchmark/paddle/PaddleNLP/lexical_analysis_dataset_tiny/ --local-path ./data/ --mode download
|
|
cd ./data/ && tar xfz lexical_analysis_dataset_tiny.tar.gz && cd ..
|
|
fi
|
|
|
|
if [[ ${model_name} =~ bert* ]]; then
|
|
rm -rf ./data/wikicorpus_en_seqlen128/ wikicorpus_en_seqlen128.tar wikicorpus_en_seqlen512 hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/ hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5.tar
|
|
wget -nc -P ./data/ https://bj.bcebos.com/paddlenlp/datasets/benchmark_wikicorpus_en_seqlen128.tar --no-check-certificate
|
|
wget -nc -P ./data/ https://bj.bcebos.com/paddlenlp/datasets/benchmark_hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5.tar --no-check-certificate
|
|
|
|
cd ./data/
|
|
tar -xf benchmark_wikicorpus_en_seqlen128.tar
|
|
tar -xf benchmark_hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5.tar
|
|
|
|
ln -s hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/wikicorpus_en_seqlen512/ wikicorpus_en_seqlen512
|
|
|
|
cd ..
|
|
|
|
python -m pip install h5py -i https://pip.baidu-int.com/simple
|
|
fi
|
|
|
|
if [[ ${model_name} == "gpt2" ]]; then
|
|
python -m pip install fast_dataindex -i https://pip.baidu-int.com/simple
|
|
mkdir -p data && cd data
|
|
wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_ids.npy -o .tmp
|
|
wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_idx.npz -o .tmp
|
|
cd -
|
|
fi
|
|
|
|
if [[ ${model_name} == "gpt3" ]]; then
|
|
python -m pip install fast_dataindex -i https://pip.baidu-int.com/simple
|
|
mkdir -p data && cd data
|
|
wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_ids.npy -o .tmp
|
|
wget https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_idx.npz -o .tmp
|
|
cd -
|
|
fi
|
|
|
|
if [[ ${model_name} =~ transformer* ]]; then
|
|
cd ../slm/examples/machine_translation/transformer/
|
|
|
|
git checkout .
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: 128/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: False/g" configs/transformer.base.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: False/g" configs/transformer.base.yaml
|
|
|
|
sed -i "s/^random_seed:.*/random_seed: 128/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle_batch:.*/shuffle_batch: False/g" configs/transformer.big.yaml
|
|
sed -i "s/^shuffle:.*/shuffle: False/g" configs/transformer.big.yaml
|
|
|
|
# Data set prepared.
|
|
if [ ! -f WMT14.en-de.partial.tar.gz ]; then
|
|
wget https://bj.bcebos.com/paddlenlp/datasets/WMT14.en-de.partial.tar.gz
|
|
tar -zxf WMT14.en-de.partial.tar.gz
|
|
fi
|
|
# Set soft link.
|
|
if [ -f train.en ]; then
|
|
rm -f train.en
|
|
fi
|
|
if [ -f train.de ]; then
|
|
rm -f train.de
|
|
fi
|
|
if [ -f dev.en ]; then
|
|
rm -f dev.en
|
|
fi
|
|
if [ -f dev.de ]; then
|
|
rm -f dev.de
|
|
fi
|
|
if [ -f test.en ]; then
|
|
rm -f test.en
|
|
fi
|
|
if [ -f test.de ]; then
|
|
rm -f test.de
|
|
fi
|
|
rm -f vocab_all.bpe.33712
|
|
rm -f vocab_all.bpe.33708
|
|
# Vocab
|
|
cp -f WMT14.en-de.partial/wmt14_ende_data_bpe/vocab_all.bpe.33712 ./
|
|
cp -f WMT14.en-de.partial/wmt14_ende_data_bpe/vocab_all.bpe.33708 ./
|
|
# Train
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/train.tok.clean.bpe.en train.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/train.tok.clean.bpe.de train.de
|
|
# Dev
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/dev.tok.bpe.en dev.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/dev.tok.bpe.de dev.de
|
|
#Test
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/test.tok.bpe.en test.en
|
|
ln -s WMT14.en-de.partial/wmt14_ende_data_bpe/test.tok.bpe.de test.de
|
|
cd -
|
|
fi
|
|
|
|
if [[ ${model_name} =~ "latent_diffusion_model" ]]; then
|
|
rm -rf laion400m_demo_data.tar.gz
|
|
wget https://paddlenlp.bj.bcebos.com/models/community/junnyu/develop/laion400m_demo_data.tar.gz
|
|
rm -rf data
|
|
tar -zxvf laion400m_demo_data.tar.gz
|
|
fi
|
|
|
|
if [[ ${model_name} =~ "stable_diffusion_model" ]]; then
|
|
rm -rf laion400m_demo_data.tar.gz
|
|
wget https://paddlenlp.bj.bcebos.com/models/community/junnyu/develop/laion400m_demo_data.tar.gz
|
|
rm -rf data
|
|
tar -zxvf laion400m_demo_data.tar.gz
|
|
fi
|
|
|
|
if [[ ${model_name} =~ "llama" ]]; then
|
|
rm -rf llama_sft_demo_data.tar.gz
|
|
wget https://paddlenlp.bj.bcebos.com/models/community/facebook/llama_sft_demo_data.tar.gz
|
|
tar -xvf llama_sft_demo_data.tar.gz
|
|
fi
|
|
|
|
export PYTHONPATH=$(dirname "$PWD"):$PYTHONPATH
|
|
python -m pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|
python -m pip install einops -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|
python -m pip install setuptools_scm
|
|
python -m pip install Cython
|
|
python -m pip install -r ../requirements.txt #-i https://pypi.tuna.tsinghua.edu.cn/simple
|
|
python -m pip install pybind11 regex sentencepiece tqdm visualdl attrdict easydict pyyaml rouge -i https://pip.baidu-int.com/simple
|
|
|
|
python -m pip install -e ../
|
|
# python -m pip install paddlenlp # PDC 镜像中安装失败
|
|
python -m pip list
|
|
cd -
|
|
fi
|