chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,197 @@
|
||||
name: Linux-NPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
docker_npu_image:
|
||||
type: string
|
||||
required: true
|
||||
|
||||
env:
|
||||
docker_image: ${{ inputs.docker_npu_image }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
ci_scripts_runner: ${{ github.workspace }}/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-npu
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: npu
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "npu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: NPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker run -i --rm \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e BRANCH \
|
||||
-w /paddle $docker_image /bin/bash -c 'rm -rf * .[^.]*'
|
||||
source ~/.bashrc
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
||||
rm Paddle.tar.gz
|
||||
cd Paddle
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
source ${{ github.workspace }}/Paddle/ci/utils.sh
|
||||
determine_npu_runner ${runner_name}
|
||||
echo no_proxy="localhost,127.0.0.1,localaddress,.bj.bcebos.com,.localdomain.com,.cdn.bcebos.com,.baidu.com,.bcebos.com" >> ${{ github.env }}
|
||||
|
||||
- name: Clone PaddleCustomDevice repository
|
||||
run: |
|
||||
source ~/.bashrc
|
||||
git clone --depth=1000 -b develop https://github.com/PaddlePaddle/PaddleCustomDevice.git
|
||||
cd PaddleCustomDevice
|
||||
cp -r ../Paddle .
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_COVERAGE: "OFF"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PYTHON_VERSION: "3.10"
|
||||
no_proxy: bcebos.com
|
||||
USE_910B: 1
|
||||
FLAGS_set_to_1d: "False"
|
||||
NVIDIA_TF32_OVERRIDE: 0
|
||||
paddle_submodule: "ON"
|
||||
MC2: 1
|
||||
HCCL_OP_BASE_FFTS_MODE_ENABLE: "TRUE"
|
||||
PADDLE_XCCL_BACKEND: npu
|
||||
HCCL_SOCKET_IFNAME: =xgbe0
|
||||
FLAGS_eager_communication_connection: 1
|
||||
FLAGS_use_stride_kernel: 0
|
||||
FLAGS_allocator_strategy: naive_best_fit
|
||||
FLAGS_npu_storage_format: 0
|
||||
TEST_IMPORTANT: "ON"
|
||||
PADDLE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
echo ${ASCEND_RT_VISIBLE_DEVICES}
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --privileged --pids-limit 409600 --shm-size=128G -d -t \
|
||||
--cap-add=SYS_PTRACE --security-opt seccomp=unconfined --name ${container_name} \
|
||||
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/dcmi:/usr/local/dcmi \
|
||||
-v /ssd2/workspace/npu-dev/.cache:/root/.cache \
|
||||
-v /ssd2/workspace/npu-dev/.ccache:/root/.ccache \
|
||||
-v $home_dir/actions-runner:$home_dir/actions-runner \
|
||||
-v ${{ github.workspace }}/PaddleCustomDevice:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e PADDLE_VERSION \
|
||||
-e ASCEND_RT_VISIBLE_DEVICES \
|
||||
-e WITH_COVERAGE \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e AGILE_PULL_ID \
|
||||
-e AGILE_REVISION \
|
||||
-e GIT_PR_ID \
|
||||
-e PYTHON_VERSION \
|
||||
-e no_proxy \
|
||||
-e USE_910B \
|
||||
-e FLAGS_set_to_1d \
|
||||
-e NVIDIA_TF32_OVERRIDE \
|
||||
-e paddle_submodule \
|
||||
-e MC2 \
|
||||
-e HCCL_OP_BASE_FFTS_MODE_ENABLE \
|
||||
-e PADDLE_XCCL_BACKEND \
|
||||
-e HCCL_SOCKET_IFNAME \
|
||||
-e FLAGS_eager_communication_connection \
|
||||
-e FLAGS_use_stride_kernel \
|
||||
-e FLAGS_allocator_strategy \
|
||||
-e FLAGS_npu_storage_format \
|
||||
-e TEST_IMPORTANT \
|
||||
-e PADDLE_BRANCH \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Install Paddle-CPU
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -x
|
||||
source ~/.bashrc
|
||||
wget -q https://sys-p0.bj.bcebos.com/libstdc%2B%2B6_13.1.0-8ubuntu1_20.04.2_amd64.deb
|
||||
dpkg -i *deb
|
||||
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/cpu_whl/${PR_ID}/${COMMIT_ID}/paddlepaddle-0.0.0-cp310-cp310-linux_x86_64.whl --no-check-certificate
|
||||
PATH=/usr/local/bin:${PATH}
|
||||
echo "export PATH=$PATH" >> ~/.bashrc
|
||||
ln -sf $(which python3.10) /usr/local/bin/python
|
||||
ln -sf $(which pip3.10) /usr/local/bin/pip
|
||||
ln -sf $(which python3.10) /usr/bin/python
|
||||
ln -sf $(which pip3.10) /usr/bin/pip
|
||||
echo "::group::Install Paddle"
|
||||
pip install paddlepaddle-*.whl && rm -rf paddlepaddle-*.whl
|
||||
python -c "import paddle; print(paddle.__version__)"
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
echo "::endgroup::"
|
||||
'
|
||||
|
||||
- name: Build and test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
set -x
|
||||
echo "::group::Install dependencies"
|
||||
python -m pip install PyGithub
|
||||
python -m pip install wheel
|
||||
echo "::endgroup::"
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
pip install -U numpy==1.26.4
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
git config --global --add safe.directory ${work_dir}/Paddle
|
||||
cd Paddle
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
cd ..
|
||||
bash backends/npu/tools/pr_ci_npu.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
Reference in New Issue
Block a user