wehub-resource-sync
89 lines
3.4 KiB
Bash
Executable File
89 lines
3.4 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
if [[ ! -d logs ]]
|
|
then
|
|
mkdir logs
|
|
fi
|
|
|
|
validate_file() {
|
|
file=$1
|
|
file_name=$2
|
|
|
|
if [[ -f $file ]]; then
|
|
echo "Using ${file_name}: ${file}"
|
|
else
|
|
echo "${file} not found"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
validate_folder() {
|
|
dir=$1
|
|
dir_name=$2
|
|
|
|
if [[ -d ${dir} ]]; then
|
|
echo "Using ${dir_name}: ${dir}"
|
|
else
|
|
echo "${dir} folder not found"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Validate path to BingBertSquad script
|
|
if [ -z "${BingBertSquad_DIR+x}" ]; then
|
|
export BingBertSquad_DIR=../../../DeepSpeedExamples/training/BingBertSquad
|
|
echo "BingBertSquad_DIR environment variable not set; trying default: ${BingBertSquad_DIR}"
|
|
fi
|
|
validate_folder ${BingBertSquad_DIR} "BingBertSquad_DIR"
|
|
|
|
fp16_config_json=deepspeed_bsz24_fp16_config.json
|
|
validate_file ${fp16_config_json} "fp16_config_json"
|
|
fp32_config_json=deepspeed_bsz24_fp32_config.json
|
|
validate_file ${fp32_config_json} "fp32_config_json"
|
|
|
|
start_time=`date +"%D %T"`
|
|
echo "---------------begin @ ${start_time}--------------"
|
|
# Note: you may play around with commented parts below (num_gpus and nohup command) for simultaneous runs; just make sure your hardware allocation can support it
|
|
for num_gpus in 8 1 # 4 2
|
|
do
|
|
#run_cmd="nohup bash run_BingBertSquad.sh -g ${num_gpus} -d --deepspeed_config ${fp16_config_json} --fp16 > logs/deepspeed_fp16_${num_gpus}_`date +"%Y%m%d%H%M%S"`.out 2> logs/deepspeed_fp16_${num_gpus}_`date +"%Y%m%d%H%M%S"`.err &"
|
|
run_cmd="bash run_BingBertSquad.sh -g ${num_gpus} -d --deepspeed_config ${fp16_config_json} --fp16"
|
|
start_time=`date +"%D %T"`
|
|
echo "---------------begin @ ${start_time}--------------"
|
|
echo ${run_cmd}
|
|
eval ${run_cmd}
|
|
end_time=`date +"%D %T"`
|
|
echo "---------------finish @ ${end_time} --------------"
|
|
|
|
#run_cmd="nohup bash run_BingBertSquad.sh -g ${num_gpus} -d --deepspeed_config ${fp32_config_json} > logs/deepspeed_fp32_${num_gpus}_`date +"%Y%m%d%H%M%S"`.out 2> logs/deepspeed_fp32_${num_gpus}_`date +"%Y%m%d%H%M%S"`.err &"
|
|
run_cmd="bash run_BingBertSquad.sh -g ${num_gpus} -d --deepspeed_config ${fp32_config_json}"
|
|
start_time=`date +"%D %T"`
|
|
echo "---------------begin @ ${start_time}--------------"
|
|
echo ${run_cmd}
|
|
eval ${run_cmd}
|
|
end_time=`date +"%D %T"`
|
|
echo "---------------finish @ ${end_time} --------------"
|
|
|
|
#run_cmd="nohup bash run_BingBertSquad.sh -g ${num_gpus} --fp16 > logs/baseline_fp16_${num_gpus}_`date +"%Y%m%d%H%M%S"`.out 2> logs/baseline_fp16_${num_gpus}_`date +"%Y%m%d%H%M%S"`.err &"
|
|
run_cmd="bash run_BingBertSquad.sh -g ${num_gpus} --fp16"
|
|
start_time=`date +"%D %T"`
|
|
echo "---------------begin @ ${start_time}--------------"
|
|
echo ${run_cmd}
|
|
eval ${run_cmd}
|
|
end_time=`date +"%D %T"`
|
|
echo "---------------finish @ ${end_time} --------------"
|
|
|
|
#run_cmd="nohup bash run_BingBertSquad.sh -g ${num_gpus} > logs/baseline_fp32_${num_gpus}_`date +"%Y%m%d%H%M%S"`.out 2> logs/baseline_fp32_${num_gpus}_`date +"%Y%m%d%H%M%S"`.err &"
|
|
run_cmd="bash run_BingBertSquad.sh -g ${num_gpus}"
|
|
start_time=`date +"%D %T"`
|
|
echo "---------------begin @ ${start_time}--------------"
|
|
echo ${run_cmd}
|
|
eval ${run_cmd}
|
|
end_time=`date +"%D %T"`
|
|
echo "---------------finish @ ${end_time} --------------"
|
|
done
|
|
end_time=`date +"%D %T"`
|
|
echo "---------------finish @ ${end_time} --------------"
|
|
|
|
set +x
|