wehub-resource-sync
737 lines
31 KiB
YAML
737 lines
31 KiB
YAML
name: CI-H
|
|
|
|
on:
|
|
pull_request:
|
|
types: [opened, synchronize]
|
|
branches: [develop, release/**]
|
|
|
|
permissions: read-all
|
|
|
|
concurrency:
|
|
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
PR_ID: ${{ github.event.pull_request.number }}
|
|
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
|
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
|
ci_scripts: /paddle/ci
|
|
BRANCH: ${{ github.base_ref }}
|
|
work_dir: /paddle
|
|
PADDLE_ROOT: /paddle
|
|
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
|
CI_name: h-coverage
|
|
CFS_DIR: /home/data/cfs
|
|
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
|
fleet_branch: "release/0.2"
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
check-skill:
|
|
name: Check file paths
|
|
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
|
runs-on:
|
|
group: APPROVAL
|
|
continue-on-error: true
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
outputs:
|
|
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
|
steps:
|
|
- name: Cleanup
|
|
run: |
|
|
rm -rf * .[^.]*
|
|
- name: Checkout
|
|
uses: actions/checkout@v6
|
|
- name: Run skip check
|
|
id: skip-skill
|
|
uses: ./.github/actions/skip-skill
|
|
with:
|
|
ignore-paths: '.agents/skills/**'
|
|
- name: Print outputs
|
|
if: always()
|
|
run: |
|
|
echo "=== check-skill outputs ==="
|
|
echo "steps.skip-skill.outputs.should-skip: '${{ steps.skip-skill.outputs.should-skip }}'"
|
|
echo "============================"
|
|
|
|
clone:
|
|
name: Coverage clone
|
|
needs: [check-skill]
|
|
uses: ./.github/workflows/_Clone-linux.yml
|
|
with:
|
|
workflow-name: "coverage"
|
|
clone_dir: h-ci
|
|
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
|
|
|
check-bypass:
|
|
name: Check bypass
|
|
needs: [clone]
|
|
if: ${{ needs.clone.outputs.can-skip != 'true' }}
|
|
uses: ./.github/workflows/check-bypass.yml
|
|
with:
|
|
workflow-name: h-ci
|
|
secrets:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
build:
|
|
name: Coverage build
|
|
needs: [clone, check-bypass]
|
|
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && needs.clone.outputs.can-skip != 'true' }}
|
|
runs-on:
|
|
group: GZ_BD-CPU
|
|
timeout-minutes: 120
|
|
outputs:
|
|
can-skip: ${{ needs.check-bypass.outputs.can-skip == 'true' || needs.clone.outputs.can-skip == 'true' || 'false' }}
|
|
steps:
|
|
- name: Check docker image and run container
|
|
env:
|
|
CACHE_DIR: "/root/.cache/coverage"
|
|
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
|
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
|
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
|
CTEST_PARALLEL_LEVEL: 2
|
|
WITH_GPU: "ON"
|
|
CUDA_ARCH_NAME: Hopper
|
|
WITH_AVX: "ON"
|
|
PADDLE_VERSION: 0.0.0
|
|
CUDA_VISIBLE_DEVICES: 0,1
|
|
WITH_DISTRIBUTE: "ON"
|
|
LITE_GIT_TAG: develop
|
|
WITH_UNITY_BUILD: "ON"
|
|
WITH_FA_BUILD_WITH_CACHE: "ON"
|
|
PY_VERSION: "3.12"
|
|
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
|
CCACHE_MAXSIZE: 50G
|
|
CCACHE_LIMIT_MULTIPLE: 0.8
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
|
echo "container_name=${container_name}" >> ${{ github.env }}
|
|
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
|
docker run -d -t --name ${container_name} \
|
|
-v "/home/data/cfs:/home/data/cfs" \
|
|
-v "/home/data/cfs/.cache:/root/.cache" \
|
|
-v "/home/data/shared:/home/data/shared" \
|
|
-v "/dev/shm:/dev/shm" \
|
|
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
|
-v ${{ github.workspace }}:/paddle \
|
|
-e CI_name \
|
|
-e BRANCH \
|
|
-e PR_ID \
|
|
-e COMMIT_ID \
|
|
-e work_dir \
|
|
-e PADDLE_ROOT \
|
|
-e GIT_PR_ID \
|
|
-e CACHE_DIR \
|
|
-e CCACHE_DIR \
|
|
-e CCACHE_SECONDARY_STORAGE \
|
|
-e ci_scripts \
|
|
-e FLAGS_fraction_of_gpu_memory_to_use \
|
|
-e CTEST_PARALLEL_LEVEL \
|
|
-e WITH_GPU \
|
|
-e CUDA_ARCH_NAME \
|
|
-e WITH_AVX \
|
|
-e PADDLE_VERSION \
|
|
-e WITH_DISTRIBUTE \
|
|
-e LITE_GIT_TAG \
|
|
-e WITH_UNITY_BUILD \
|
|
-e WITH_FA_BUILD_WITH_CACHE \
|
|
-e PY_VERSION \
|
|
-e INFERENCE_DEMO_INSTALL_DIR \
|
|
-e CCACHE_MAXSIZE \
|
|
-e CCACHE_LIMIT_MULTIPLE \
|
|
-e GITHUB_TOKEN \
|
|
-e GITHUB_API_TOKEN \
|
|
-e CFS_DIR \
|
|
-e no_proxy \
|
|
-w /paddle --network host ${docker_image}
|
|
|
|
- name: Download paddle.tar.gz and update test branch
|
|
timeout-minutes: 30
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
rm -rf * .[^.]*
|
|
set -e
|
|
echo "Downloading Paddle.tar.gz"
|
|
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-ci/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
|
echo "Extracting Paddle.tar.gz"
|
|
tar -xf Paddle.tar.gz --strip-components=1
|
|
rm Paddle.tar.gz
|
|
git config --global --add safe.directory "*"
|
|
git remote -v
|
|
set +e
|
|
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
|
set -e
|
|
git config pull.rebase false
|
|
git checkout test
|
|
echo "Pull upstream $BRANCH"
|
|
source ${{ github.workspace }}/../../../proxy
|
|
bash ci/git_pull.sh $BRANCH
|
|
git submodule update
|
|
'
|
|
|
|
- name: Build
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
flashattn_version=$(git submodule status|grep flashattn|awk "{print \$1}"|sed "s#-##g")
|
|
echo flashattn_version:$flashattn_version
|
|
url="https://xly-devops.bj.bcebos.com/gpups/flash-attention/cu90/flashattn_libs_${flashattn_version}.tar"
|
|
echo url:$url
|
|
url_return=`curl -s -o /dev/null -w "%{http_code}" $url`
|
|
if [ "$url_return" != "200" ];then
|
|
echo "flashattn cache not found, please contact umiswing"
|
|
exit 7
|
|
fi
|
|
mkdir -p ${CFS_DIR}/.cache/coverage
|
|
mkdir -p ${CFS_DIR}/.ccache/coverage
|
|
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
|
|
source ${{ github.workspace }}/../../../proxy
|
|
bash ${ci_scripts}/cmake-predownload.sh
|
|
bash ${ci_scripts}/cmake_preload_data_file.sh
|
|
pip install -r python/requirements.txt
|
|
mkdir -p build && cd build
|
|
ccache -z
|
|
cmake .. -DPY_VERSION=3.12 -DWITH_GPU=ON -DWITH_DISTRIBUTE=ON -DWITH_TESTING=ON -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN="90" -DFA_JOB_POOLS_COMPILE=1 -DWITH_CUDNN_FRONTEND=ON -DON_INFER=OFF -DWITH_NVSHMEM=ON
|
|
make -j20
|
|
EXIT_CODE=$?
|
|
ccache -s
|
|
exit $EXIT_CODE
|
|
'
|
|
|
|
- name: Clean up env
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
source ~/.bashrc
|
|
source ${ci_scripts}/utils.sh; clean_build_files
|
|
: "${PY_VERSION:?PY_VERSION is required}"
|
|
rm -rf $(find . -name "*.a")
|
|
rm -rf $(find . -name "*.o")
|
|
rm -rf lib.linux-x86_64-${PY_VERSION}
|
|
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
|
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
|
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
|
cd /
|
|
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
|
'
|
|
|
|
- name: Upload coverage product
|
|
env:
|
|
home_path: ${{ github.workspace }}/..
|
|
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
|
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
echo "::group::Install bce-python-sdk"
|
|
python -m pip install bce-python-sdk==0.8.74
|
|
echo "::endgroup::"
|
|
export AK=paddle
|
|
export SK=paddle
|
|
if [ ! -f "${{ env.bos_file }}" ]; then
|
|
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
|
mkdir ${{ env.home_path }}/bos_retry
|
|
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
|
fi
|
|
cd /paddle
|
|
mv /Paddle.tar.gz .
|
|
cp ./build/python/dist/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl .
|
|
echo "Uploading Paddle.tar.gz"
|
|
python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
|
echo "Uploading coverage wheel"
|
|
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
|
echo "End Upload"
|
|
'
|
|
|
|
- name: Terminate and delete the container
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
|
docker stop ${{ env.container_name }}
|
|
docker rm ${{ env.container_name }}
|
|
|
|
test:
|
|
name: Coverage test
|
|
needs: [build, check-bypass, check-skill]
|
|
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
|
runs-on:
|
|
group: H-Coverage
|
|
timeout-minutes: 120
|
|
steps:
|
|
- name: Determine the runner
|
|
run: |
|
|
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
|
echo $runner_name
|
|
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
|
source utils.sh
|
|
determine_gpu_runner ${runner_name}
|
|
|
|
- name: Check docker image and run container
|
|
env:
|
|
CACHE_DIR: "/root/.cache/coverage"
|
|
CCACHE_DIR: "/root/.ccache/coverage"
|
|
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
|
CTEST_PARALLEL_LEVEL: 2
|
|
WITH_GPU: "ON"
|
|
CUDA_ARCH_NAME: Hopper
|
|
WITH_AVX: "ON"
|
|
COVERALLS_UPLOAD: "ON"
|
|
PADDLE_VERSION: 0.0.0
|
|
WITH_DISTRIBUTE: "ON"
|
|
WITH_UNITY_BUILD: "ON"
|
|
PY_VERSION: "3.12"
|
|
WITH_SHARED_PHI: "ON"
|
|
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
|
WITH_CINN: "ON"
|
|
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
|
CCACHE_MAXSIZE: 200G
|
|
CCACHE_LIMIT_MULTIPLE: 0.8
|
|
FLAGS_PIR_OPTEST: "TRUE"
|
|
ON_INFER: "ON"
|
|
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
|
echo "container_name=${container_name}" >> ${{ github.env }}
|
|
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
|
docker run -d -t --gpus "\"device=${GPU_DEVICES}\"" --name ${container_name} \
|
|
-v "/home/data/cfs:/home/data/cfs" \
|
|
-v "/home/data/cfs/.cache:/root/.cache" \
|
|
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
|
-v "/dev/shm:/dev/shm" \
|
|
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
|
-v ${{ github.workspace }}:/paddle \
|
|
-e CI_name \
|
|
-e BRANCH \
|
|
-e PR_ID \
|
|
-e COMMIT_ID \
|
|
-e work_dir \
|
|
-e PADDLE_ROOT \
|
|
-e GIT_PR_ID \
|
|
-e CACHE_DIR \
|
|
-e CCACHE_DIR \
|
|
-e ci_scripts \
|
|
-e FLAGS_fraction_of_gpu_memory_to_use \
|
|
-e CTEST_PARALLEL_LEVEL \
|
|
-e WITH_GPU \
|
|
-e CUDA_ARCH_NAME \
|
|
-e WITH_AVX \
|
|
-e WITH_COVERAGE \
|
|
-e COVERALLS_UPLOAD \
|
|
-e PADDLE_VERSION \
|
|
-e WITH_DISTRIBUTE \
|
|
-e WITH_UNITY_BUILD \
|
|
-e PY_VERSION \
|
|
-e WITH_SHARED_PHI \
|
|
-e WITH_CINN \
|
|
-e INFERENCE_DEMO_INSTALL_DIR \
|
|
-e CCACHE_MAXSIZE \
|
|
-e CCACHE_LIMIT_MULTIPLE \
|
|
-e FLAGS_PIR_OPTEST \
|
|
-e ON_INFER \
|
|
-e COVERAGE_FILE \
|
|
-e GITHUB_TOKEN \
|
|
-e GITHUB_API_TOKEN \
|
|
-e CFS_DIR \
|
|
-e no_proxy \
|
|
-w /paddle --network host ${docker_image}
|
|
|
|
- name: Download paddle.tar.gz and update test branch
|
|
timeout-minutes: 30
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
rm -rf * .[^.]*
|
|
set -e
|
|
echo "Downloading Paddle.tar.gz from cfs"
|
|
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
|
echo "Extracting Paddle.tar.gz"
|
|
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
|
rm Paddle.tar.gz
|
|
'
|
|
|
|
- name: Test
|
|
id: unit_test
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
source ${{ github.workspace }}/../../../proxy
|
|
pip install build//python/dist/*.whl --no-deps
|
|
pip install -r python/unittest_py/requirements.txt
|
|
bash $ci_scripts/h-test.sh
|
|
'
|
|
|
|
- name: FA Test
|
|
if: (success() || failure()) && steps.unit_test.conclusion != 'skipped'
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
source ${{ github.workspace }}/../../../proxy
|
|
python -m pip install pytest einops
|
|
cd test/test_flashmask_ci
|
|
bash run.sh
|
|
'
|
|
|
|
- name: Terminate and delete the container
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
rm Paddle.tar.gz
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
|
docker stop ${{ env.container_name }}
|
|
docker rm ${{ env.container_name }}
|
|
|
|
fleet-build:
|
|
name: Build Fleet whl
|
|
needs: [build, check-bypass, check-skill]
|
|
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
|
runs-on:
|
|
group: GZ_BD-CPU
|
|
timeout-minutes: 60
|
|
env:
|
|
TASK: fleet-ci-paddle-build-whl-${{ github.event.pull_request.number }}
|
|
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
|
PY_VERSION: "3.12"
|
|
steps:
|
|
- name: Check docker image and run container
|
|
env:
|
|
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
|
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
|
CCACHE_MAXSIZE: 50G
|
|
CCACHE_LIMIT_MULTIPLE: 0.8
|
|
CCACHE_STATSLOG: /paddle/build/.stats.log
|
|
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
|
run: |
|
|
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
|
echo "container_name=${container_name}" >> ${{ github.env }}
|
|
docker pull $docker_image
|
|
docker run -d -t --name ${container_name} \
|
|
-v "/dev/shm:/dev/shm" \
|
|
-v "/home/data/cfs:/home/data/cfs" \
|
|
-v "/home/data/shared:/home/data/shared" \
|
|
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
|
-v ${{ github.workspace }}:/paddle \
|
|
-e BRANCH \
|
|
-e PR_ID \
|
|
-e COMMIT_ID \
|
|
-e PADDLE_ROOT \
|
|
-e ci_scripts \
|
|
-e no_proxy \
|
|
-e fleet_branch \
|
|
-e CI_name \
|
|
-e CCACHE_DIR \
|
|
-e CCACHE_SECONDARY_STORAGE \
|
|
-e CCACHE_MAXSIZE \
|
|
-e CCACHE_LIMIT_MULTIPLE \
|
|
-e CCACHE_STATSLOG \
|
|
-e CCACHE_SLOPPINESS \
|
|
-e PY_VERSION \
|
|
-w /paddle --network host ${docker_image}
|
|
|
|
- name: uv build whl
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
|
rm -rf * .[^.]*
|
|
set -x
|
|
source ${{ github.workspace }}/../../../proxy
|
|
apt-get update
|
|
apt-get install -y libnvidia-compute-535
|
|
python -m pip install --upgrade pip wheel setuptools
|
|
git clone https://github.com/PaddlePaddle/PaddleFleet.git .
|
|
git config --global --add safe.directory /paddle
|
|
git config user.name "PaddleCI"
|
|
git config user.email "paddle_ci@example.com"
|
|
git config pull.rebase false
|
|
if [ "${BRANCH}" != "develop" ]; then
|
|
git checkout $fleet_branch
|
|
echo "Checked out fleet branch: $fleet_branch"
|
|
else
|
|
git checkout develop
|
|
echo "Checked out fleet branch: develop"
|
|
fi
|
|
mkdir -p /home/.cache/pip
|
|
pip cache dir
|
|
echo "Install uv"
|
|
export UV_HTTP_TIMEOUT=300
|
|
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
|
|
echo "uv build"
|
|
git submodule update --init --recursive
|
|
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:$LD_LIBRARY_PATH
|
|
export IS_NVIDIA=True
|
|
echo "Downloading Paddle.tar.gz from cfs"
|
|
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
|
pip uninstall paddlepaddle-gpu -y
|
|
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
|
pip install paddle-nvidia-nvshmem-cu12 --extra-index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
|
uv build --wheel --no-build-isolation -v --python $(which python)
|
|
'
|
|
|
|
- name: download ops whl from latest
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
|
set -x
|
|
OPS_DST=paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl
|
|
wget -q --no-proxy "https://paddle-whl.bj.bcebos.com/nightly/cu129/paddlefleet-ops/${OPS_DST}" -O dist/${OPS_DST}
|
|
'
|
|
|
|
- name: upload whl to BOS
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
|
export AK=paddle
|
|
export SK=paddle
|
|
cd /
|
|
if [ ! -f "bos/BosClient.py" ]; then
|
|
wget -q --no-proxy -O bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
|
mkdir bos
|
|
tar xf bos_new.tar.gz -C bos
|
|
fi
|
|
target_path="paddle-github-action/PR/paddlefleet/${{ env.PR_ID }}/${{ env.COMMIT_ID }}"
|
|
tar zcf paddldfleet.tar.gz paddle
|
|
python /bos/BosClient.py paddldfleet.tar.gz ${target_path}
|
|
'
|
|
- name: Terminate and delete the container
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c 'bash ci/clean_uv_cache.sh; rm -rf * .[^.]*'
|
|
docker rm -f ${{ env.container_name }}
|
|
|
|
fleet_single_card_test:
|
|
name: Fleet Unit test (single card)
|
|
needs: [build, fleet-build]
|
|
if: needs.build.outputs.can-skip != 'true'
|
|
runs-on:
|
|
group: Fleet-H-single-card
|
|
timeout-minutes: 90
|
|
env:
|
|
PIP_CACHE_DIR: /root/.cache/pip
|
|
CACHE_DIR: /root/.cache
|
|
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-single-card-test
|
|
PY_VERSION: "3.12"
|
|
steps:
|
|
- name: Determine the runner
|
|
run: |
|
|
gpu_id=$(( $(echo $PWD | awk -F'/' '{print $3}' | awk -F'-' '{print $2}') - 1 ))
|
|
echo GPU_DEVICES="$gpu_id" >> $GITHUB_ENV
|
|
|
|
- name: Check docker image and run container
|
|
env:
|
|
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
|
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
|
run: |
|
|
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
|
echo "container_name=${container_name}" >> ${{ github.env }}
|
|
docker pull $docker_image
|
|
docker run -d -t --name ${container_name} --gpus "\"device=${GPU_DEVICES}\"" --shm-size=32G \
|
|
-v "/dev/shm:/dev/shm" \
|
|
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
|
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
|
-v /ssd1/paddle-1/action_cache:/root/.cache \
|
|
-v ${{ github.workspace }}:/paddle \
|
|
-e BRANCH \
|
|
-e PR_ID \
|
|
-e COMMIT_ID \
|
|
-e PADDLE_ROOT \
|
|
-e ci_scripts \
|
|
-e CACHE_DIR \
|
|
-e no_proxy \
|
|
-e CI_name \
|
|
-e PIP_CACHE_DIR \
|
|
-e PY_VERSION \
|
|
-e work_dir \
|
|
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
|
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
|
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
|
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
|
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
|
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
|
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
|
-w /paddle --network host ${docker_image}
|
|
|
|
- name: Install PaddleFleet
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
|
rm -rf * .[^.]*
|
|
source /root/proxy
|
|
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
|
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
|
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
|
tar -xf paddldfleet.tar.gz --strip-components=1
|
|
git config --global --add safe.directory /paddle
|
|
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
|
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
|
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
|
chmod +x /usr/local/bin/yq
|
|
'
|
|
|
|
- name: Download paddle.tar.gz and install paddle whl
|
|
timeout-minutes: 30
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
set -e
|
|
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
|
mkdir -p /PaddlePaddle
|
|
cd /PaddlePaddle
|
|
echo "Downloading Paddle.tar.gz from cfs"
|
|
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
|
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
|
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
|
export UV_HTTP_TIMEOUT=300
|
|
pip uninstall paddlepaddle-gpu -y
|
|
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
|
echo "paddlefleet commit:"
|
|
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
|
'
|
|
|
|
- name: Single card test
|
|
timeout-minutes: 60
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -xce '
|
|
pwd
|
|
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
|
if [ "${BRANCH}" != "develop" ]; then
|
|
git checkout $fleet_branch
|
|
echo "Checked out fleet branch: $fleet_branch"
|
|
else
|
|
git checkout develop
|
|
echo "Checked out fleet branch: develop"
|
|
fi
|
|
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
|
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
|
export UV_HTTP_TIMEOUT=300
|
|
python -c "import paddle; print(paddle.version.commit)"
|
|
bash ci/single_card_test.sh
|
|
single_card_exit_code=$?
|
|
if [[ "$single_card_exit_code" != "0" ]]; then
|
|
echo -e "::error:: \033[31mSingle card test failed.\033[0m"
|
|
exit 1
|
|
else
|
|
echo -e "\033[32mSingle card test succeeded.\033[0m"
|
|
fi
|
|
'
|
|
|
|
- name: Terminate and delete the container
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
|
docker rm -f ${{ env.container_name }}
|
|
|
|
fleet-multi-card_test:
|
|
name: Fleet Unit test (multi-card)
|
|
needs: [build, fleet-build, check-bypass, check-skill]
|
|
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
|
runs-on:
|
|
group: Fleet-H-multi-card
|
|
timeout-minutes: 60
|
|
env:
|
|
PIP_CACHE_DIR: /root/.cache/pip
|
|
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-multi-card_test
|
|
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
|
PY_VERSION: "3.12"
|
|
steps:
|
|
- name: Check docker image and run container
|
|
run: |
|
|
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
|
echo "container_name=${container_name}" >> ${{ github.env }}
|
|
docker pull $docker_image
|
|
docker run -d -t --gpus all --name ${container_name} \
|
|
-v "/dev/shm:/dev/shm" \
|
|
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
|
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
|
-v ${{ github.workspace }}/../../../.cache:/root/.cache \
|
|
-v ${{ github.workspace }}:/paddle \
|
|
-e BRANCH \
|
|
-e PR_ID \
|
|
-e COMMIT_ID \
|
|
-e PADDLE_ROOT \
|
|
-e ci_scripts \
|
|
-e CACHE_DIR \
|
|
-e fleet_branch \
|
|
-e no_proxy \
|
|
-e CI_name \
|
|
-e PIP_CACHE_DIR \
|
|
-e PY_VERSION \
|
|
-e work_dir \
|
|
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
|
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
|
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
|
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
|
-e GITHUB_EVENT_NAME="${{ github.event_name }}" \
|
|
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
|
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
|
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
|
-w /paddle --network host ${docker_image}
|
|
|
|
- name: Install PaddleFleet
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
|
rm -rf * .[^.]*
|
|
source /root/proxy
|
|
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
|
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
|
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
|
tar -xf paddldfleet.tar.gz --strip-components=1
|
|
git config --global --add safe.directory /paddle
|
|
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
|
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
|
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
|
chmod +x /usr/local/bin/yq
|
|
'
|
|
|
|
- name: Download paddle.tar.gz and install paddle whl
|
|
timeout-minutes: 30
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
set -e
|
|
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
|
mkdir -p /PaddlePaddle
|
|
cd /PaddlePaddle
|
|
echo "Downloading Paddle.tar.gz from cfs"
|
|
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
|
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
|
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
|
export UV_HTTP_TIMEOUT=300
|
|
pip uninstall paddlepaddle-gpu -y
|
|
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
|
echo "paddlefleet commit:"
|
|
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
|
'
|
|
|
|
- name: Multi-card test
|
|
timeout-minutes: 60
|
|
run: |
|
|
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
|
export PYTHONPATH=$(pwd)
|
|
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
|
if [ "${BRANCH}" != "develop" ]; then
|
|
git checkout $fleet_branch
|
|
echo "Checked out fleet branch: $fleet_branch"
|
|
else
|
|
git checkout develop
|
|
echo "Checked out fleet branch: develop"
|
|
fi
|
|
python -c "import paddle; print(paddle.version.commit)"
|
|
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
|
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
|
export UV_HTTP_TIMEOUT=300
|
|
bash ci/multi-card_test.sh
|
|
multi_card_exit_code=$?
|
|
if [[ "$multi_card_exit_code" != "0" ]]; then
|
|
echo -e "::error:: \033[31mMulti card test failed.\033[0m"
|
|
exit 1
|
|
else
|
|
echo -e "\033[32mMulti card test succeeded.\033[0m"
|
|
fi
|
|
'
|
|
|
|
- name: Terminate and delete the container
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
|
docker rm -f ${{ env.container_name }}
|