wehub-resource-sync
111 lines
5.0 KiB
Bash
111 lines
5.0 KiB
Bash
#!/usr/bin/env bash
|
||
|
||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
# Test training benchmark for a model.
|
||
# Usage：bash benchmark/run_benchmark.sh ${model_name_or_path} ${per_device_train_batch_size} ${tensor_parallel_degree} ${pipeline_parallel_degree} ${virtual_pp_degree} ${sequence_parallel} ${sharding_parallel_degree} ${sharding} ${recompute} ${run_mode} ${device_num}
|
||
function _set_params(){
|
||
model_name_or_path=${model_name_or_path:-"llama"}
|
||
run_mode=${run_mode:-"DP1-mbs1"}
|
||
device_num=${device_num:-"N1C1"}
|
||
batch_size=${batch_size:-2}
|
||
model_item=${model_item:-"llama-7b"}
|
||
base_batch_size=${batch_size}
|
||
dtype=${dtype:-"fp16"}
|
||
benchmark=${benchmark:-0}
|
||
|
||
profiling=${PROFILING:-"false"} # (必选) Profiling 开关，默认关闭，通过全局变量传递
|
||
model_repo="PaddleNLP" # (必选) 模型套件的名字
|
||
speed_unit="tokens/s" # (必选)速度指标单位
|
||
skip_steps=0 # (必选)解析日志，跳过模型前几个性能不稳定的step
|
||
keyword="IPS:" # (必选)解析日志，筛选出性能数据所在行的关键字
|
||
|
||
convergence_key="precision:" # (可选)解析日志，筛选出收敛数据所在行的关键字 如：convergence_key="loss:"
|
||
|
||
fp_item=${dtype}
|
||
# 以下为通用执行命令，无特殊可不用修改
|
||
model_name=${model_item}_bs${batch_size}_${fp_item}_${run_mode} # (必填) 且格式不要改动,与竞品名称对齐
|
||
device=${CUDA_VISIBLE_DEVICES//,/ }
|
||
arr=(${device})
|
||
num_gpu_devices=${#arr[*]}
|
||
run_log_path=${TRAIN_LOG_DIR:-$(pwd)} # （必填） TRAIN_LOG_DIR benchmark框架设置该参数为全局变量
|
||
profiling_log_path=${PROFILING_LOG_DIR:-$(pwd)} # （必填） PROFILING_LOG_DIR benchmark框架设置该参数为全局变量
|
||
speed_log_path=${LOG_PATH_INDEX_DIR:-$(pwd)}
|
||
train_log_file=${run_log_path}/${model_repo}_${model_name}_${device_num}_log
|
||
mkdir -p $(dirname ${train_log_file})
|
||
|
||
profiling_log_file=${profiling_log_path}/${model_repo}_${model_name}_${device_num}_profiling
|
||
mkdir -p $(dirname ${profiling_log_file})
|
||
|
||
speed_log_file=${speed_log_path}/${model_repo}_${model_name}_${device_num}_speed
|
||
mkdir -p $(dirname ${speed_log_file})
|
||
|
||
OUTPUT_PATH=${run_log_path}/output
|
||
|
||
log_file=${train_log_file}
|
||
is_large_model=True
|
||
}
|
||
|
||
function _train(){
|
||
batch_size=${per_device_train_batch_size} # 如果模型跑多卡单进程时,请在_train函数中计算出多卡需要的bs
|
||
|
||
if [ -d $OUTPUT_PATH ]; then
|
||
rm -rf $OUTPUT_PATH
|
||
fi
|
||
mkdir $OUTPUT_PATH
|
||
|
||
echo "current CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}, model_name=${model_name}, device_num=${device_num}, is profiling=${profiling}"
|
||
|
||
use_pure_fp16=False
|
||
export MODEL_NAME=${model_name_or_path}
|
||
# 以下为通用执行命令，无特殊可不用修改
|
||
case ${device_num} in
|
||
N1C1) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
|
||
train_cmd="python -m pytest -s -v test_tipc/llm/test_predictor.py"
|
||
workerlog_id=0
|
||
;;
|
||
*) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
|
||
train_cmd="${} python -m paddle.distributed.launch --log_dir=./mylog --gpus=0,1,2,3,4,5,6,7 ${PADDLE_RANK_OPTION}\
|
||
run_pretrain.py ${train_cmd}"
|
||
workerlog_id=0
|
||
;;
|
||
esac
|
||
echo "train_cmd: ${train_cmd} log_file: ${log_file}"
|
||
python -c "import paddlenlp"
|
||
if [[ ${model_name} =~ "CE" ]];then # CE精度-不限制执行时间
|
||
${train_cmd} > ${log_file} 2>&1
|
||
else
|
||
timeout 30m ${train_cmd} > ${log_file} 2>&1
|
||
# echo ${train_cmd}
|
||
fi
|
||
if [ $? -ne 0 ];then
|
||
echo -e "${model_name}, FAIL"
|
||
else
|
||
echo -e "${model_name}, SUCCESS"
|
||
fi
|
||
#kill -9 `ps -ef|grep 'python'|awk '{print $2}'`
|
||
if [ ${device_num} != "N1C1" -a -d mylog ]; then
|
||
rm ${log_file}
|
||
cp mylog/workerlog.${workerlog_id} ${log_file}
|
||
fi
|
||
}
|
||
|
||
export PYTHONPATH=$(dirname "$PWD"):$(dirname "$PWD")/llm:$PYTHONPATH
|
||
|
||
source ${BENCHMARK_ROOT}/scripts/run_model.sh # 在该脚本中会对符合benchmark规范的log使用analysis.py 脚本进行性能数据解析;如果不联调只想要产出训练log可以注掉本行,提交时需打开
|
||
_set_params $@
|
||
# _train # 如果只产出训练log,不解析,可取消注释
|
||
_run # 该函数在run_model.sh中,执行时会调用_train; 如果不联调只产出训练log可以注掉本行,提交时需打开
|