chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,393 @@
|
||||
name: Night-Coverage
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 18 * * *"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
name: Coverage build docker
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
outputs:
|
||||
docker_coverage_image: ${{ steps.build-docker-images.outputs.docker_coverage_image }}
|
||||
runs-on:
|
||||
group: HK-Clone
|
||||
steps:
|
||||
- name: build-docker-images
|
||||
id: build-docker-images
|
||||
run: |
|
||||
set -x
|
||||
cd ${{ github.workspace }}
|
||||
pwd
|
||||
git clone --depth=1000 https://github.com/PaddlePaddle/Paddle.git
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
|
||||
cd Paddle/tools/dockerfile
|
||||
bash ci_dockerfile.sh
|
||||
md5_value=`md5sum Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage |awk '{print $1}'`
|
||||
echo "docker_coverage_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/ci/paddle:${md5_value}" >> $GITHUB_OUTPUT
|
||||
|
||||
# clean workspace
|
||||
cd ${{ github.workspace }}
|
||||
rm -rf * .[^.]*
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [build-docker]
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_PIP_CUDA_LIBRARIES: "OFF"
|
||||
WITH_FLAGCX: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
ON_INFER: "ON"
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
UT_RUN_TYPE_SETTING: WITHOUT_HYBRID
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_PIP_CUDA_LIBRARIES \
|
||||
-e WITH_FLAGCX \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ON_INFER \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e UT_RUN_TYPE_SETTING \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
echo "Clone Paddle"
|
||||
git clone --depth=1000 https://github.com/PaddlePaddle/Paddle.git .
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
echo "Extracting Paddle"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git checkout test
|
||||
git submodule update --init --recursive
|
||||
echo "Pull upstream $BRANCH"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash $ci_scripts/coverage_build.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
Build_Size=$(du -h --max-depth=0 ${work_dir}/build |awk '"'"'{print $1}'"'"')
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
echo "Build_Size=${Build_Size}" > ${work_dir}/dist/coverage_build_size
|
||||
find ./ -type f -size +200M | xargs du -lh
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf paddle_inference_install_dir
|
||||
rm -rf paddle_inference_c_install_dir
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle/dist
|
||||
coverage_tag=$(date +%Y-%m-%d)
|
||||
mkdir -p ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
echo "Uploading coverage build size"
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/night/coverage/${coverage_tag}
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/night/coverage
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/night/coverage/${coverage_tag}
|
||||
cd /
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
cp Paddle.tar.gz ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, build-docker]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
WITH_ALL_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e WITH_ALL_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
coverage_tag=$(date +%Y-%m-%d)
|
||||
cp ${CFS_DIR}/coverage_night/${coverage_tag}/Paddle.tar.gz .
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/coverage_test.sh
|
||||
TEST_EXIT_CODE=$?
|
||||
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> ${{ github.env }}
|
||||
if [[ "$TEST_EXIT_CODE" -ne 0 && "$TEST_EXIT_CODE" -ne 9 ]]; then
|
||||
exit $TEST_EXIT_CODE
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Generate coverage information
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
commit_info=$(git log --format=fuller |head -1|awk "{print \$2}")
|
||||
touch ${PADDLE_ROOT}/night_coverage.txt
|
||||
echo "commit:${commit_info}" >>${PADDLE_ROOT}/night_coverage.txt
|
||||
unset GREP_OPTIONS
|
||||
export WITH_ALL_COVERAGE=ON
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/utils.sh; check_coverage
|
||||
coverage_tag=$(date +"%m-%d")
|
||||
mkdir -p ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
cp build/coverage_files/* ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
echo "Uploading night_coverage.txt"
|
||||
coverage_time=$(date +%Y-%m-%d)
|
||||
python ${{ env.bos_file }} night_coverage.txt paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded night_coverage.txt"
|
||||
echo "Uploading coverage-full.info"
|
||||
python ${{ env.bos_file }} build/coverage-full.info paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded coverage-full.info"
|
||||
echo "Uploading python-coverage-full.info"
|
||||
python ${{ env.bos_file }} build/python-coverage-full.info paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded python-coverage-full.info"
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
Reference in New Issue
Block a user