wehub-resource-sync
59 lines
2.5 KiB
Bash
59 lines
2.5 KiB
Bash
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
unset GREP_OPTIONS
|
|
rm ./run_detail.log
|
|
rm ./UT_resource.log
|
|
rm ./UT_resource_sort.log
|
|
rm ./while_list.log
|
|
|
|
export LD_LIBRARY_PATH="$PWD/python/paddle/libs;$LD_LIBRARY_PATH"
|
|
export CUDA_VISIBLE_DEVICES=0,1
|
|
|
|
test_cases=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d')
|
|
use_memory_base=$(nvidia-smi -q -i 0 | grep "Used" | head -1 | grep -o "[0-9]*")
|
|
for unittest in $test_cases
|
|
do
|
|
use_memory=0
|
|
gpu_utilization=0
|
|
memory_utilization=0
|
|
ctest -R "^${unittest}$" --repeat-until-fail 5 -j 1 &
|
|
PID=$!
|
|
echo -e "******************************************************"
|
|
echo -e "[$unittest]: PID:$PID \n"
|
|
while [[ $(ps aux | awk '{print $2}' | grep "^$PID$" | grep -v "grep" | wc -l) -ge 1 ]]
|
|
do
|
|
use_memory_current=$(nvidia-smi -q -i 0 | grep "Used" | head -1 | grep -o "[0-9]*")
|
|
if [[ $use_memory_current -gt $use_memory ]];then
|
|
use_memory=$use_memory_current
|
|
fi
|
|
memory_utilization_current=$(nvidia-smi -q -i 0 | grep "Memory" | sed -n '3p' | grep -o "[0-9]*")
|
|
if [[ $memory_utilization_current -gt $memory_utilization ]];then
|
|
memory_utilization=$memory_utilization_current
|
|
fi
|
|
|
|
gpu_utilization_current=$(nvidia-smi -q -i 0 | grep "Gpu" | grep -o "[0-9]*")
|
|
if [[ $gpu_utilization_current -gt $gpu_utilization ]];then
|
|
gpu_utilization=$gpu_utilization_current
|
|
fi
|
|
done
|
|
use_memory=`expr $use_memory - $use_memory_base`
|
|
echo -e " use_memory:$use_memory \n memory_utilization:$memory_utilization \n gpu_utilization:$gpu_utilization\n"
|
|
echo -e "[$unittest]: \n use_memory:$use_memory \n memory_utilization:$memory_utilization \n gpu_utilization:$gpu_utilization\n" >> run_detail.log
|
|
echo -e "$unittest:$use_memory:$memory_utilization:$gpu_utilization" >> UT_resource.log
|
|
done
|
|
|
|
sort -r -n -k 2 -t : UT_resource.log > UT_resource_sort.log
|
|
cat UT_resource_sort.log | awk -F ':' '{print $1}' > while_list.log
|