chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,322 @@
|
||||
name: PR-CI-Inference
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_inference_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
docker_image: ${{ inputs.docker_inference_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "inference"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_build
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: "0.15"
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
|
||||
WITH_PYTHON: "OFF"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_NVCC_LAZY: "OFF"
|
||||
PYTHON_ABI: cp310-cp310
|
||||
CACHE_DIR: /root/.cache/inference
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: "0.8"
|
||||
CUDA_ARCH_NAME: Pascal
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/home/data/cfs/.ccache/inference:/root/.ccache/inference" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:/action-runner" \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_PYTHON \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e WITH_NVCC_LAZY \
|
||||
-e PYTHON_ABI \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e PY_VERSION \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-build/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git config --global --add safe.directory "*"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
source /action-runner/proxy
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Run build
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
source /action-runner/proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/inference_build.sh
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload build.tar.gz
|
||||
env:
|
||||
home_path: /action-runner
|
||||
bos_file: /action-runner/bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source /action-runner/proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
rm -rf `find . -name "*.o"`
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --tries=5 --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
source /action-runner/unproxy
|
||||
cd /
|
||||
tar --use-compress-program="pigz -1" -cpPf build.tar.gz paddle
|
||||
echo "Uploading build.tar.gz to cfs"
|
||||
mkdir -p ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}
|
||||
cp build.tar.gz ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID} && echo success
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
# docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: build
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_test
|
||||
runs-on:
|
||||
group: Inference-P4
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: "0.15"
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
|
||||
WITH_PYTHON: "OFF"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_NVCC_LAZY: "OFF"
|
||||
PYTHON_ABI: cp310-cp310
|
||||
CACHE_DIR: /root/.cache/inference
|
||||
CCACHE_DIR: /root/.ccache/inference
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: "0.8"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:/action-runner" \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_PYTHON \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e WITH_NVCC_LAZY \
|
||||
-e PYTHON_ABI \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e PY_VERSION \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
tar --use-compress-program="pigz -1" -xpf build.tar.gz --strip-components=1
|
||||
rm build.tar.gz
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
echo "export PATH=${PATH}" >> ~/.bashrc
|
||||
echo "::group::Install dependencies"
|
||||
source /action-runner/proxy
|
||||
pip install -r "python/unittest_py/requirements.txt"
|
||||
echo "::endgroup::"
|
||||
'
|
||||
|
||||
- name: Test fluid library for inference
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
test_fluid_lib
|
||||
'
|
||||
|
||||
- name: Test go inference api
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
test_go_inference_api
|
||||
'
|
||||
|
||||
- name: Check approvals of unittest
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
check_approvals_of_unittest 3
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
# docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
Reference in New Issue
Block a user