chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,736 @@
|
||||
name: CI-H
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: h-coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
fleet_branch: "release/0.2"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
- name: Print outputs
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== check-skill outputs ==="
|
||||
echo "steps.skip-skill.outputs.should-skip: '${{ steps.skip-skill.outputs.should-skip }}'"
|
||||
echo "============================"
|
||||
|
||||
clone:
|
||||
name: Coverage clone
|
||||
needs: [check-skill]
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
workflow-name: "coverage"
|
||||
clone_dir: h-ci
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
needs: [clone]
|
||||
if: ${{ needs.clone.outputs.can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: h-ci
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [clone, check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && needs.clone.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip == 'true' || needs.clone.outputs.can-skip == 'true' || 'false' }}
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Hopper
|
||||
WITH_AVX: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_FA_BUILD_WITH_CACHE: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e WITH_FA_BUILD_WITH_CACHE \
|
||||
-e PY_VERSION \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-ci/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global --add safe.directory "*"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream $BRANCH"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
flashattn_version=$(git submodule status|grep flashattn|awk "{print \$1}"|sed "s#-##g")
|
||||
echo flashattn_version:$flashattn_version
|
||||
url="https://xly-devops.bj.bcebos.com/gpups/flash-attention/cu90/flashattn_libs_${flashattn_version}.tar"
|
||||
echo url:$url
|
||||
url_return=`curl -s -o /dev/null -w "%{http_code}" $url`
|
||||
if [ "$url_return" != "200" ];then
|
||||
echo "flashattn cache not found, please contact umiswing"
|
||||
exit 7
|
||||
fi
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/cmake_preload_data_file.sh
|
||||
pip install -r python/requirements.txt
|
||||
mkdir -p build && cd build
|
||||
ccache -z
|
||||
cmake .. -DPY_VERSION=3.12 -DWITH_GPU=ON -DWITH_DISTRIBUTE=ON -DWITH_TESTING=ON -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN="90" -DFA_JOB_POOLS_COMPILE=1 -DWITH_CUDNN_FRONTEND=ON -DON_INFER=OFF -DWITH_NVSHMEM=ON
|
||||
make -j20
|
||||
EXIT_CODE=$?
|
||||
ccache -s
|
||||
exit $EXIT_CODE
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
: "${PY_VERSION:?PY_VERSION is required}"
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle
|
||||
mv /Paddle.tar.gz .
|
||||
cp ./build/python/dist/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl .
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "End Upload"
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: H-Coverage
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
||||
source utils.sh
|
||||
determine_gpu_runner ${runner_name}
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Hopper
|
||||
WITH_AVX: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
||||
docker run -d -t --gpus "\"device=${GPU_DEVICES}\"" --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
id: unit_test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
pip install build//python/dist/*.whl --no-deps
|
||||
pip install -r python/unittest_py/requirements.txt
|
||||
bash $ci_scripts/h-test.sh
|
||||
'
|
||||
|
||||
- name: FA Test
|
||||
if: (success() || failure()) && steps.unit_test.conclusion != 'skipped'
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install pytest einops
|
||||
cd test/test_flashmask_ci
|
||||
bash run.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
fleet-build:
|
||||
name: Build Fleet whl
|
||||
needs: [build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TASK: fleet-ci-paddle-build-whl-${{ github.event.pull_request.number }}
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_STATSLOG: /paddle/build/.stats.log
|
||||
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e fleet_branch \
|
||||
-e CI_name \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e CCACHE_STATSLOG \
|
||||
-e CCACHE_SLOPPINESS \
|
||||
-e PY_VERSION \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: uv build whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
set -x
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
apt-get update
|
||||
apt-get install -y libnvidia-compute-535
|
||||
python -m pip install --upgrade pip wheel setuptools
|
||||
git clone https://github.com/PaddlePaddle/PaddleFleet.git .
|
||||
git config --global --add safe.directory /paddle
|
||||
git config user.name "PaddleCI"
|
||||
git config user.email "paddle_ci@example.com"
|
||||
git config pull.rebase false
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
mkdir -p /home/.cache/pip
|
||||
pip cache dir
|
||||
echo "Install uv"
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
|
||||
echo "uv build"
|
||||
git submodule update --init --recursive
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:$LD_LIBRARY_PATH
|
||||
export IS_NVIDIA=True
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
pip install paddle-nvidia-nvshmem-cu12 --extra-index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
uv build --wheel --no-build-isolation -v --python $(which python)
|
||||
'
|
||||
|
||||
- name: download ops whl from latest
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
set -x
|
||||
OPS_DST=paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
wget -q --no-proxy "https://paddle-whl.bj.bcebos.com/nightly/cu129/paddlefleet-ops/${OPS_DST}" -O dist/${OPS_DST}
|
||||
'
|
||||
|
||||
- name: upload whl to BOS
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
cd /
|
||||
if [ ! -f "bos/BosClient.py" ]; then
|
||||
wget -q --no-proxy -O bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir bos
|
||||
tar xf bos_new.tar.gz -C bos
|
||||
fi
|
||||
target_path="paddle-github-action/PR/paddlefleet/${{ env.PR_ID }}/${{ env.COMMIT_ID }}"
|
||||
tar zcf paddldfleet.tar.gz paddle
|
||||
python /bos/BosClient.py paddldfleet.tar.gz ${target_path}
|
||||
'
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'bash ci/clean_uv_cache.sh; rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
fleet_single_card_test:
|
||||
name: Fleet Unit test (single card)
|
||||
needs: [build, fleet-build]
|
||||
if: needs.build.outputs.can-skip != 'true'
|
||||
runs-on:
|
||||
group: Fleet-H-single-card
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
PIP_CACHE_DIR: /root/.cache/pip
|
||||
CACHE_DIR: /root/.cache
|
||||
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-single-card-test
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
gpu_id=$(( $(echo $PWD | awk -F'/' '{print $3}' | awk -F'-' '{print $2}') - 1 ))
|
||||
echo GPU_DEVICES="$gpu_id" >> $GITHUB_ENV
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --name ${container_name} --gpus "\"device=${GPU_DEVICES}\"" --shm-size=32G \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
||||
-v /ssd1/paddle-1/action_cache:/root/.cache \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CACHE_DIR \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e PIP_CACHE_DIR \
|
||||
-e PY_VERSION \
|
||||
-e work_dir \
|
||||
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
||||
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
||||
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
||||
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
||||
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
||||
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
||||
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Install PaddleFleet
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
source /root/proxy
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
||||
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
||||
tar -xf paddldfleet.tar.gz --strip-components=1
|
||||
git config --global --add safe.directory /paddle
|
||||
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
||||
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
||||
chmod +x /usr/local/bin/yq
|
||||
'
|
||||
|
||||
- name: Download paddle.tar.gz and install paddle whl
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
mkdir -p /PaddlePaddle
|
||||
cd /PaddlePaddle
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
echo "paddlefleet commit:"
|
||||
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
||||
'
|
||||
|
||||
- name: Single card test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -xce '
|
||||
pwd
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
bash ci/single_card_test.sh
|
||||
single_card_exit_code=$?
|
||||
if [[ "$single_card_exit_code" != "0" ]]; then
|
||||
echo -e "::error:: \033[31mSingle card test failed.\033[0m"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\033[32mSingle card test succeeded.\033[0m"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
fleet-multi-card_test:
|
||||
name: Fleet Unit test (multi-card)
|
||||
needs: [build, fleet-build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Fleet-H-multi-card
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
PIP_CACHE_DIR: /root/.cache/pip
|
||||
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-multi-card_test
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
||||
-v ${{ github.workspace }}/../../../.cache:/root/.cache \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CACHE_DIR \
|
||||
-e fleet_branch \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e PIP_CACHE_DIR \
|
||||
-e PY_VERSION \
|
||||
-e work_dir \
|
||||
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
||||
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
||||
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
||||
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
||||
-e GITHUB_EVENT_NAME="${{ github.event_name }}" \
|
||||
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
||||
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
||||
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Install PaddleFleet
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
source /root/proxy
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
||||
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
||||
tar -xf paddldfleet.tar.gz --strip-components=1
|
||||
git config --global --add safe.directory /paddle
|
||||
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
||||
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
||||
chmod +x /usr/local/bin/yq
|
||||
'
|
||||
|
||||
- name: Download paddle.tar.gz and install paddle whl
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
mkdir -p /PaddlePaddle
|
||||
cd /PaddlePaddle
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
echo "paddlefleet commit:"
|
||||
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
||||
'
|
||||
|
||||
- name: Multi-card test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
export PYTHONPATH=$(pwd)
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
bash ci/multi-card_test.sh
|
||||
multi_card_exit_code=$?
|
||||
if [[ "$multi_card_exit_code" != "0" ]]; then
|
||||
echo -e "::error:: \033[31mMulti card test failed.\033[0m"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\033[32mMulti card test succeeded.\033[0m"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
Reference in New Issue
Block a user