wehub-resource-sync
106 lines
3.4 KiB
Bash
106 lines
3.4 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
export Testcase=$3
|
|
export cudaid2=$2
|
|
export cudaid1=$1
|
|
export nlp_dir=${PWD}
|
|
mkdir ${nlp_dir}/logs
|
|
mkdir ${nlp_dir}/model_logs
|
|
mkdir ${nlp_dir}/unittest_logs
|
|
export log_path=${nlp_dir}/logs
|
|
####################################
|
|
# for paddlenlp env
|
|
python -c 'import sys; print(sys.version_info[:])'
|
|
python -c 'import nltk; nltk.download("punkt")'
|
|
set -x
|
|
python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)";
|
|
nlp1_build (){
|
|
echo -e "\033[35m ---- only install paddlenlp \033[0m"
|
|
python -m pip install paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
|
|
}
|
|
nlp2_build (){
|
|
echo -e "\033[35m ---- build and install paddlenlp \033[0m"
|
|
rm -rf build/
|
|
rm -rf paddlenlp.egg-info/
|
|
rm -rf dist/
|
|
|
|
python -m pip install -r requirements.txt
|
|
python setup.py bdist_wheel
|
|
python -m pip install -U dist/paddlenlp****.whl
|
|
}
|
|
nlp2_build
|
|
python -c 'from visualdl import LogWriter'
|
|
pip list
|
|
set +x
|
|
####################################
|
|
# run p0case
|
|
export P0case_list=()
|
|
export P0case_time=0
|
|
export all_P0case_time=0
|
|
declare -A all_P0case_dic
|
|
get_diff_TO_P0case(){
|
|
if [[ ${Testcase} =~ "all" ]];then
|
|
P0case_list=(msra_ner glue bert skep bigbird gpt ernie-1.0 xlnet ofa squad tinybert lexical_analysis seq2seq \
|
|
word_embedding ernie-ctm distilbert stacl transformer simbert pointer_summarizer question_matching ernie-csc \
|
|
nptag clue taskflow transformers fast_generation ernie-3.0 fast_transformer fast_gpt llama)
|
|
elif [[ ${Testcase} =~ "p0" ]];then
|
|
P0case_list=(glue bert skep gpt ernie-1.0 transformer clue)
|
|
else
|
|
P0case_list=${Testcase}
|
|
fi
|
|
}
|
|
get_diff_TO_P0case
|
|
echo -e "\033[35m =======CI Check P0case========= \033[0m"
|
|
echo -e "\033[35m ---- P0case_list length: ${#P0case_list[*]}, cases: ${P0case_list[*]} \033[0m"
|
|
set +e
|
|
echo -e "\033[35m ---- start run P0case \033[0m"
|
|
case_num=1
|
|
for p0case in ${P0case_list[*]};do
|
|
echo -e "\033[35m ---- running P0case $case_num/${#P0case_list[*]}: ${p0case} \033[0m"
|
|
bash ${nlp_dir}/scripts/regression/ci_case.sh ${p0case} ${cudaid1} ${cudaid2}
|
|
let case_num++
|
|
done
|
|
echo -e "\033[35m ---- end run P0case \033[0m"
|
|
cd ${nlp_dir}
|
|
upload(){
|
|
if [ -f '/ssd1/paddlenlp/bos/upload.py' ];then
|
|
cp -r /ssd1/paddlenlp/bos/* ./
|
|
tar -zcvf model_logs.tar model_logs/
|
|
mkdir upload && mv model_logs.tar upload
|
|
python upload.py upload 'paddle-qa/paddlenlp'
|
|
else
|
|
echo 'No upload script found'
|
|
fi
|
|
}
|
|
upload
|
|
cd model_logs/
|
|
FF=`ls *_FAIL*|wc -l`
|
|
if [ "${FF}" -gt "0" ];then
|
|
P0case_EXCODE=1
|
|
else
|
|
P0case_EXCODE=0
|
|
fi
|
|
if [ $P0case_EXCODE -ne 0 ] ; then
|
|
cd model_logs/
|
|
FF=`ls *_FAIL*|wc -l`
|
|
echo -e "\033[31m ---- P0case failed number: ${FF} \033[0m"
|
|
ls *_FAIL*
|
|
exit $P0case_EXCODE
|
|
else
|
|
echo -e "\033[32m ---- P0case Success \033[0m"
|
|
fi
|