chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,87 @@
|
||||
name: Api-benchmark-baseline
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
PR_ID:
|
||||
required: false
|
||||
type: string
|
||||
COMMIT_ID:
|
||||
required: false
|
||||
type: string
|
||||
job-name:
|
||||
required: true
|
||||
default: 'api-benchmark'
|
||||
type: choice
|
||||
options:
|
||||
- api-benchmark
|
||||
- others
|
||||
schedule:
|
||||
- cron: '0 21 * * *'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Api benchmark clone
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
is_pr: 'false'
|
||||
|
||||
build-docker:
|
||||
name: Api benchmark build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
build:
|
||||
name: Api benchmark build
|
||||
if: github.event_name == 'schedule' && github.event.schedule == '0 21 * * *'
|
||||
needs: [clone, build-docker]
|
||||
uses: ./.github/workflows/_Linux-build.yml
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
is_pr: 'false'
|
||||
|
||||
api-benchmark-baseline-schedule:
|
||||
name: Api benchmark baseline with schedule
|
||||
strategy:
|
||||
matrix:
|
||||
run-labels: [api-bm-20, api-bm-27]
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
baseline: 'true'
|
||||
run-labels: ${{ matrix.run-labels }}
|
||||
|
||||
api-benchmark-baseline-pr-20:
|
||||
name: Api benchmark baseline with PR on 20
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.job-name == 'api-benchmark'
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
baseline: 'true'
|
||||
MANUALLY_PR_ID: ${{ inputs.PR_ID }}
|
||||
MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
|
||||
run-labels: api-bm-20
|
||||
|
||||
api-benchmark-baseline-pr-27:
|
||||
name: Api benchmark baseline with PR on 27
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.job-name == 'api-benchmark'
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
baseline: 'true'
|
||||
MANUALLY_PR_ID: ${{ inputs.PR_ID }}
|
||||
MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
|
||||
run-labels: api-bm-27
|
||||
@@ -0,0 +1,53 @@
|
||||
name: Approval
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
|
||||
env:
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
token: ${{ vars.ACTION_GITHUB_TOKEN }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: 'approval'
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
check-approvers:
|
||||
name: Check approval
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 200
|
||||
|
||||
- name: Update paddle
|
||||
run: |
|
||||
git switch -c test
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git fetch upstream $BRANCH
|
||||
git checkout $BRANCH
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git merge test
|
||||
|
||||
- name: Display Required Approvers
|
||||
run: |
|
||||
bash ci/check_approval.sh
|
||||
@@ -0,0 +1,154 @@
|
||||
name: CI-Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**, fleety_*]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip || 'false' }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Clone-linux
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
workflow-name: 'CI-build'
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
build-docker:
|
||||
name: build docker images
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
inference:
|
||||
name: PR-CI-Inference
|
||||
uses: ./.github/workflows/_Inference.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_inference_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
build:
|
||||
name: Linux-build
|
||||
uses: ./.github/workflows/_Linux-build.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
static-check:
|
||||
name: Static-Check
|
||||
uses: ./.github/workflows/_Static-Check.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
ce-framework:
|
||||
name: CE-Framework
|
||||
uses: ./.github/workflows/_CE-Framework.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
ce-cinn-framework:
|
||||
name: CE-CINN-Framework
|
||||
uses: ./.github/workflows/_CE-CINN-Framework.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
api-benchmark:
|
||||
name: Api-Benchmark
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
# auto-parallel:
|
||||
# name: Auto-Parallel
|
||||
# uses: ./.github/workflows/_Auto-Parallel.yml
|
||||
# needs: [clone, build-docker, build]
|
||||
# with:
|
||||
# can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
# docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
model-benchmark:
|
||||
name: Model-Benchmark
|
||||
uses: ./.github/workflows/_Model-Benchmark.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
doc-preview:
|
||||
name: Doc-Preview
|
||||
uses: ./.github/workflows/_Doc-Preview.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_doc_image: ${{ needs.build-docker.outputs.docker_doc_image }}
|
||||
|
||||
slice:
|
||||
name: Slice
|
||||
uses: ./.github/workflows/_Slice.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
slice-check: ${{ needs.clone.outputs.slice-check }}
|
||||
|
||||
deepmd-test:
|
||||
name: DeepMD-Kit-Test
|
||||
uses: ./.github/workflows/_Deepmd-pd-test.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
@@ -0,0 +1,66 @@
|
||||
name: CI-Windows
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**, fleety_*]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Clone-windows
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-windows.yml
|
||||
with:
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
win-openblas:
|
||||
name: Windows-OPENBLAS
|
||||
uses: ./.github/workflows/_Windows-OPENBLAS.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
win-gpu:
|
||||
name: Windows-GPU
|
||||
uses: ./.github/workflows/_Windows-GPU.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
win-inference:
|
||||
name: Windows-Inference
|
||||
uses: ./.github/workflows/_Windows-Inference.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
@@ -0,0 +1,129 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**, fleety_*]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Clone-linux
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
workflow-name: 'CI'
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
build-docker:
|
||||
name: build docker images
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
|
||||
sot:
|
||||
name: PR-CI-SOT
|
||||
uses: ./.github/workflows/_SOT.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_sot_image: ${{ needs.build-docker.outputs.docker_sot_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
mac:
|
||||
name: Mac-CPU
|
||||
uses: ./.github/workflows/_Mac.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
xpu:
|
||||
name: Linux-XPU
|
||||
uses: ./.github/workflows/_Linux-XPU.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_xpu_image: ${{ needs.build-docker.outputs.docker_xpu_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
dcu:
|
||||
name: Linux-DCU
|
||||
uses: ./.github/workflows/_Linux-DCU.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_dcu_image: ${{ needs.build-docker.outputs.docker_dcu_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
cpu:
|
||||
name: Linux-CPU
|
||||
uses: ./.github/workflows/_Linux-CPU.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_cpu_image: ${{ needs.build-docker.outputs.docker_cpu_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
npu:
|
||||
name: Linux-NPU
|
||||
uses: ./.github/workflows/_Linux-NPU.yml
|
||||
needs: [clone, cpu, build-docker]
|
||||
with:
|
||||
can-skip: ${{ needs.cpu.outputs.can-skip == 'true' || needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_npu_image: ${{ needs.build-docker.outputs.docker_npu_image }}
|
||||
|
||||
# ixuca:
|
||||
# name: Linux-IXUCA
|
||||
# uses: ./.github/workflows/_Linux-IXUCA.yml
|
||||
# needs: [clone]
|
||||
# with:
|
||||
# can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
distribute:
|
||||
name: Distribute-stable-build
|
||||
uses: ./.github/workflows/_Distribute-stable.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_distribute_image: ${{ needs.build-docker.outputs.docker_distribute_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
# distribute-test:
|
||||
# name: Distribute-stable-test
|
||||
# uses: ./.github/workflows/_Distribute-stable-Test.yml
|
||||
# needs: [clone, build-docker, distribute]
|
||||
# with:
|
||||
# docker_distribute_image: ${{ needs.build-docker.outputs.docker_distribute_image }}
|
||||
# clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
# distribute-formers:
|
||||
# name: Distribute-stable-formers
|
||||
# uses: ./.github/workflows/_Distribute-stable-Formers.yml
|
||||
# needs: [clone, build-docker, distribute]
|
||||
# with:
|
||||
# docker_distribute_image: ${{ needs.build-docker.outputs.docker_distribute_image }}
|
||||
# clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
@@ -0,0 +1,62 @@
|
||||
name: Check PR Template
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [develop, release/**]
|
||||
types: [opened, synchronize, reopened, edited]
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: 'template'
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
check:
|
||||
name: Check
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Template
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Clone paddle
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Check PR Template
|
||||
env:
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
AGILE_COMPILE_BRANCH: ${{ github.base_ref }}
|
||||
AGILE_CHECKIN_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
method: check_pr
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
run: |
|
||||
set +e
|
||||
python3 tools/CheckPRTemplate.py
|
||||
EXCODE=$?
|
||||
set -e
|
||||
echo "EXCODE: $EXCODE"
|
||||
echo "ipipe_log_param_EXCODE: $EXCODE"
|
||||
if [[ "$EXCODE" != "0" ]];then
|
||||
echo -e "######################################################"
|
||||
echo -e "If you encounter a situation where the PR template does not match the error message, please use the following link to update your PR: [ https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/.github/PULL_REQUEST_TEMPLATE.md ]"
|
||||
echo -e "## Reference Documentation: ##"
|
||||
echo -e "[ https://github.com/PaddlePaddle/Paddle/wiki/PULL-REQUEST-TEMPLATE--REFERENCE ]"
|
||||
echo -e "[ https://github.com/PaddlePaddle/Paddle/wiki/paddle_ci_manual ]"
|
||||
echo -e "######################################################"
|
||||
exit $EXCODE
|
||||
fi
|
||||
export method=$method
|
||||
echo $method
|
||||
set +x
|
||||
source ~/.icafe
|
||||
wget -q --no-check-certificate https://paddle-qa.bj.bcebos.com/baidu/cloud/modify_icafe.py
|
||||
set -x
|
||||
python3 modify_icafe.py
|
||||
@@ -0,0 +1,62 @@
|
||||
name: Codestyle-Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [develop, release/**]
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "codestyle"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
pre-commit:
|
||||
name: Pre Commit
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
BRANCH: develop
|
||||
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout base repo
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
fetch-depth: 1000
|
||||
|
||||
- name: Merge PR to test branch
|
||||
run: |
|
||||
git fetch origin pull/${PR_ID}/merge
|
||||
git checkout -b test FETCH_HEAD
|
||||
|
||||
- name: Setup python3.12
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install pre-commit==2.17.0
|
||||
pip install ast-grep-cli==0.44.0 # This version should be consistent with the one in .pre-commit-config.yaml
|
||||
|
||||
- name: Run ast-grep unit tests
|
||||
run: |
|
||||
ast-grep test
|
||||
|
||||
- name: Check pre-commit
|
||||
run: |
|
||||
set +e
|
||||
bash -x tools/codestyle/pre_commit.sh;EXCODE=$?
|
||||
exit $EXCODE
|
||||
@@ -0,0 +1,426 @@
|
||||
name: Coverage
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip || 'false' }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Coverage clone
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
workflow-name: "coverage"
|
||||
clone_dir: Paddle-coverage
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
build-docker:
|
||||
name: Coverage build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-coverage
|
||||
task: coverage
|
||||
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
needs: [clone]
|
||||
if: needs.clone.outputs.can-skip != 'true'
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: coverage
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [clone, build-docker, check-bypass]
|
||||
if: ${{ needs.clone.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip }}
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_PIP_CUDA_LIBRARIES: "OFF"
|
||||
WITH_FLAGCX: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
ON_INFER: "ON"
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
UT_RUN_TYPE_SETTING: WITHOUT_HYBRID
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_PIP_CUDA_LIBRARIES \
|
||||
-e WITH_FLAGCX \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ON_INFER \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e UT_RUN_TYPE_SETTING \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream $BRANCH"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash $ci_scripts/coverage_build.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Check added unittests
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/check_added_ut.sh
|
||||
'
|
||||
|
||||
- name: Check coverage build size requires approval
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/coverage_build_size_approval.sh
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
Build_Size=$(du -h --max-depth=0 ${work_dir}/build |awk '"'"'{print $1}'"'"')
|
||||
echo "Build_Size=${Build_Size}" > ${work_dir}/dist/coverage_build_size
|
||||
find ./ -type f -size +200M | xargs du -lh
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf paddle_inference_install_dir
|
||||
rm -rf paddle_inference_c_install_dir
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle/dist
|
||||
mkdir -p ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
|
||||
echo "Uploading coverage build size"
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
cd /
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
cp Paddle.tar.gz ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, build-docker]
|
||||
if: needs.build.outputs.can-skip != 'true'
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz .
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/coverage_test.sh
|
||||
TEST_EXIT_CODE=$?
|
||||
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> ${{ github.env }}
|
||||
if [[ "$TEST_EXIT_CODE" -ne 0 && "$TEST_EXIT_CODE" -ne 9 ]]; then
|
||||
exit $TEST_EXIT_CODE
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Generate coverage information
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
unset GREP_OPTIONS
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/utils.sh; check_coverage
|
||||
mkdir -p /home/data/cfs/coverage/${PR_ID}/${COMMIT_ID}
|
||||
cp build/coverage_files/* /home/data/cfs/coverage/${PR_ID}/${COMMIT_ID}
|
||||
'
|
||||
|
||||
- name: Upload coverage report
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
directory: build/coverage_files
|
||||
verbose: true
|
||||
|
||||
- name: Determine whether the coverage rate reaches 90%
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
unset GREP_OPTIONS
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/utils.sh
|
||||
bash $ci_scripts/coverage_judge.sh
|
||||
COVERAGE_EXIT_CODE=$?
|
||||
echo $COVERAGE_EXIT_CODE
|
||||
if [ "$COVERAGE_EXIT_CODE" -ne 0 ]; then
|
||||
echo "Coverage check failed, unit tests have all passed, please do not rerun, check whether the newly added code lines are fully covered by unit tests. If you have any questions, please contact XieYunshen."
|
||||
exit $COVERAGE_EXIT_CODE
|
||||
else
|
||||
echo "Coverage passed"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,736 @@
|
||||
name: CI-H
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: h-coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
fleet_branch: "release/0.2"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
- name: Print outputs
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== check-skill outputs ==="
|
||||
echo "steps.skip-skill.outputs.should-skip: '${{ steps.skip-skill.outputs.should-skip }}'"
|
||||
echo "============================"
|
||||
|
||||
clone:
|
||||
name: Coverage clone
|
||||
needs: [check-skill]
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
workflow-name: "coverage"
|
||||
clone_dir: h-ci
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
needs: [clone]
|
||||
if: ${{ needs.clone.outputs.can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: h-ci
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [clone, check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && needs.clone.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip == 'true' || needs.clone.outputs.can-skip == 'true' || 'false' }}
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Hopper
|
||||
WITH_AVX: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_FA_BUILD_WITH_CACHE: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e WITH_FA_BUILD_WITH_CACHE \
|
||||
-e PY_VERSION \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-ci/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global --add safe.directory "*"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream $BRANCH"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
flashattn_version=$(git submodule status|grep flashattn|awk "{print \$1}"|sed "s#-##g")
|
||||
echo flashattn_version:$flashattn_version
|
||||
url="https://xly-devops.bj.bcebos.com/gpups/flash-attention/cu90/flashattn_libs_${flashattn_version}.tar"
|
||||
echo url:$url
|
||||
url_return=`curl -s -o /dev/null -w "%{http_code}" $url`
|
||||
if [ "$url_return" != "200" ];then
|
||||
echo "flashattn cache not found, please contact umiswing"
|
||||
exit 7
|
||||
fi
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/cmake_preload_data_file.sh
|
||||
pip install -r python/requirements.txt
|
||||
mkdir -p build && cd build
|
||||
ccache -z
|
||||
cmake .. -DPY_VERSION=3.12 -DWITH_GPU=ON -DWITH_DISTRIBUTE=ON -DWITH_TESTING=ON -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN="90" -DFA_JOB_POOLS_COMPILE=1 -DWITH_CUDNN_FRONTEND=ON -DON_INFER=OFF -DWITH_NVSHMEM=ON
|
||||
make -j20
|
||||
EXIT_CODE=$?
|
||||
ccache -s
|
||||
exit $EXIT_CODE
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
: "${PY_VERSION:?PY_VERSION is required}"
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle
|
||||
mv /Paddle.tar.gz .
|
||||
cp ./build/python/dist/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl .
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "End Upload"
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: H-Coverage
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
||||
source utils.sh
|
||||
determine_gpu_runner ${runner_name}
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Hopper
|
||||
WITH_AVX: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
||||
docker run -d -t --gpus "\"device=${GPU_DEVICES}\"" --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
id: unit_test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
pip install build//python/dist/*.whl --no-deps
|
||||
pip install -r python/unittest_py/requirements.txt
|
||||
bash $ci_scripts/h-test.sh
|
||||
'
|
||||
|
||||
- name: FA Test
|
||||
if: (success() || failure()) && steps.unit_test.conclusion != 'skipped'
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install pytest einops
|
||||
cd test/test_flashmask_ci
|
||||
bash run.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
fleet-build:
|
||||
name: Build Fleet whl
|
||||
needs: [build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TASK: fleet-ci-paddle-build-whl-${{ github.event.pull_request.number }}
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_STATSLOG: /paddle/build/.stats.log
|
||||
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e fleet_branch \
|
||||
-e CI_name \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e CCACHE_STATSLOG \
|
||||
-e CCACHE_SLOPPINESS \
|
||||
-e PY_VERSION \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: uv build whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
set -x
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
apt-get update
|
||||
apt-get install -y libnvidia-compute-535
|
||||
python -m pip install --upgrade pip wheel setuptools
|
||||
git clone https://github.com/PaddlePaddle/PaddleFleet.git .
|
||||
git config --global --add safe.directory /paddle
|
||||
git config user.name "PaddleCI"
|
||||
git config user.email "paddle_ci@example.com"
|
||||
git config pull.rebase false
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
mkdir -p /home/.cache/pip
|
||||
pip cache dir
|
||||
echo "Install uv"
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
|
||||
echo "uv build"
|
||||
git submodule update --init --recursive
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:$LD_LIBRARY_PATH
|
||||
export IS_NVIDIA=True
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
pip install paddle-nvidia-nvshmem-cu12 --extra-index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
uv build --wheel --no-build-isolation -v --python $(which python)
|
||||
'
|
||||
|
||||
- name: download ops whl from latest
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
set -x
|
||||
OPS_DST=paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
wget -q --no-proxy "https://paddle-whl.bj.bcebos.com/nightly/cu129/paddlefleet-ops/${OPS_DST}" -O dist/${OPS_DST}
|
||||
'
|
||||
|
||||
- name: upload whl to BOS
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
cd /
|
||||
if [ ! -f "bos/BosClient.py" ]; then
|
||||
wget -q --no-proxy -O bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir bos
|
||||
tar xf bos_new.tar.gz -C bos
|
||||
fi
|
||||
target_path="paddle-github-action/PR/paddlefleet/${{ env.PR_ID }}/${{ env.COMMIT_ID }}"
|
||||
tar zcf paddldfleet.tar.gz paddle
|
||||
python /bos/BosClient.py paddldfleet.tar.gz ${target_path}
|
||||
'
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'bash ci/clean_uv_cache.sh; rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
fleet_single_card_test:
|
||||
name: Fleet Unit test (single card)
|
||||
needs: [build, fleet-build]
|
||||
if: needs.build.outputs.can-skip != 'true'
|
||||
runs-on:
|
||||
group: Fleet-H-single-card
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
PIP_CACHE_DIR: /root/.cache/pip
|
||||
CACHE_DIR: /root/.cache
|
||||
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-single-card-test
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
gpu_id=$(( $(echo $PWD | awk -F'/' '{print $3}' | awk -F'-' '{print $2}') - 1 ))
|
||||
echo GPU_DEVICES="$gpu_id" >> $GITHUB_ENV
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --name ${container_name} --gpus "\"device=${GPU_DEVICES}\"" --shm-size=32G \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
||||
-v /ssd1/paddle-1/action_cache:/root/.cache \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CACHE_DIR \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e PIP_CACHE_DIR \
|
||||
-e PY_VERSION \
|
||||
-e work_dir \
|
||||
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
||||
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
||||
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
||||
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
||||
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
||||
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
||||
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Install PaddleFleet
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
source /root/proxy
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
||||
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
||||
tar -xf paddldfleet.tar.gz --strip-components=1
|
||||
git config --global --add safe.directory /paddle
|
||||
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
||||
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
||||
chmod +x /usr/local/bin/yq
|
||||
'
|
||||
|
||||
- name: Download paddle.tar.gz and install paddle whl
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
mkdir -p /PaddlePaddle
|
||||
cd /PaddlePaddle
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
echo "paddlefleet commit:"
|
||||
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
||||
'
|
||||
|
||||
- name: Single card test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -xce '
|
||||
pwd
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
bash ci/single_card_test.sh
|
||||
single_card_exit_code=$?
|
||||
if [[ "$single_card_exit_code" != "0" ]]; then
|
||||
echo -e "::error:: \033[31mSingle card test failed.\033[0m"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\033[32mSingle card test succeeded.\033[0m"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
fleet-multi-card_test:
|
||||
name: Fleet Unit test (multi-card)
|
||||
needs: [build, fleet-build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Fleet-H-multi-card
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
PIP_CACHE_DIR: /root/.cache/pip
|
||||
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-multi-card_test
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
||||
-v ${{ github.workspace }}/../../../.cache:/root/.cache \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CACHE_DIR \
|
||||
-e fleet_branch \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e PIP_CACHE_DIR \
|
||||
-e PY_VERSION \
|
||||
-e work_dir \
|
||||
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
||||
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
||||
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
||||
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
||||
-e GITHUB_EVENT_NAME="${{ github.event_name }}" \
|
||||
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
||||
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
||||
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Install PaddleFleet
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
source /root/proxy
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
||||
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
||||
tar -xf paddldfleet.tar.gz --strip-components=1
|
||||
git config --global --add safe.directory /paddle
|
||||
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
||||
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
||||
chmod +x /usr/local/bin/yq
|
||||
'
|
||||
|
||||
- name: Download paddle.tar.gz and install paddle whl
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
mkdir -p /PaddlePaddle
|
||||
cd /PaddlePaddle
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
echo "paddlefleet commit:"
|
||||
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
||||
'
|
||||
|
||||
- name: Multi-card test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
export PYTHONPATH=$(pwd)
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
bash ci/multi-card_test.sh
|
||||
multi_card_exit_code=$?
|
||||
if [[ "$multi_card_exit_code" != "0" ]]; then
|
||||
echo -e "::error:: \033[31mMulti card test failed.\033[0m"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\033[32mMulti card test succeeded.\033[0m"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,393 @@
|
||||
name: Night-Coverage
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 18 * * *"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
name: Coverage build docker
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
outputs:
|
||||
docker_coverage_image: ${{ steps.build-docker-images.outputs.docker_coverage_image }}
|
||||
runs-on:
|
||||
group: HK-Clone
|
||||
steps:
|
||||
- name: build-docker-images
|
||||
id: build-docker-images
|
||||
run: |
|
||||
set -x
|
||||
cd ${{ github.workspace }}
|
||||
pwd
|
||||
git clone --depth=1000 https://github.com/PaddlePaddle/Paddle.git
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
|
||||
cd Paddle/tools/dockerfile
|
||||
bash ci_dockerfile.sh
|
||||
md5_value=`md5sum Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage |awk '{print $1}'`
|
||||
echo "docker_coverage_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/ci/paddle:${md5_value}" >> $GITHUB_OUTPUT
|
||||
|
||||
# clean workspace
|
||||
cd ${{ github.workspace }}
|
||||
rm -rf * .[^.]*
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [build-docker]
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_PIP_CUDA_LIBRARIES: "OFF"
|
||||
WITH_FLAGCX: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
ON_INFER: "ON"
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
UT_RUN_TYPE_SETTING: WITHOUT_HYBRID
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_PIP_CUDA_LIBRARIES \
|
||||
-e WITH_FLAGCX \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ON_INFER \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e UT_RUN_TYPE_SETTING \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
echo "Clone Paddle"
|
||||
git clone --depth=1000 https://github.com/PaddlePaddle/Paddle.git .
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
echo "Extracting Paddle"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git checkout test
|
||||
git submodule update --init --recursive
|
||||
echo "Pull upstream $BRANCH"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash $ci_scripts/coverage_build.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
Build_Size=$(du -h --max-depth=0 ${work_dir}/build |awk '"'"'{print $1}'"'"')
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
echo "Build_Size=${Build_Size}" > ${work_dir}/dist/coverage_build_size
|
||||
find ./ -type f -size +200M | xargs du -lh
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf paddle_inference_install_dir
|
||||
rm -rf paddle_inference_c_install_dir
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle/dist
|
||||
coverage_tag=$(date +%Y-%m-%d)
|
||||
mkdir -p ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
echo "Uploading coverage build size"
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/night/coverage/${coverage_tag}
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/night/coverage
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/night/coverage/${coverage_tag}
|
||||
cd /
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
cp Paddle.tar.gz ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, build-docker]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
WITH_ALL_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e WITH_ALL_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
coverage_tag=$(date +%Y-%m-%d)
|
||||
cp ${CFS_DIR}/coverage_night/${coverage_tag}/Paddle.tar.gz .
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/coverage_test.sh
|
||||
TEST_EXIT_CODE=$?
|
||||
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> ${{ github.env }}
|
||||
if [[ "$TEST_EXIT_CODE" -ne 0 && "$TEST_EXIT_CODE" -ne 9 ]]; then
|
||||
exit $TEST_EXIT_CODE
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Generate coverage information
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
commit_info=$(git log --format=fuller |head -1|awk "{print \$2}")
|
||||
touch ${PADDLE_ROOT}/night_coverage.txt
|
||||
echo "commit:${commit_info}" >>${PADDLE_ROOT}/night_coverage.txt
|
||||
unset GREP_OPTIONS
|
||||
export WITH_ALL_COVERAGE=ON
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/utils.sh; check_coverage
|
||||
coverage_tag=$(date +"%m-%d")
|
||||
mkdir -p ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
cp build/coverage_files/* ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
echo "Uploading night_coverage.txt"
|
||||
coverage_time=$(date +%Y-%m-%d)
|
||||
python ${{ env.bos_file }} night_coverage.txt paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded night_coverage.txt"
|
||||
echo "Uploading coverage-full.info"
|
||||
python ${{ env.bos_file }} build/coverage-full.info paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded coverage-full.info"
|
||||
echo "Uploading python-coverage-full.info"
|
||||
python ${{ env.bos_file }} build/python-coverage-full.info paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded python-coverage-full.info"
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,54 @@
|
||||
name: Slice-baseline-paddle
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
PR_ID:
|
||||
required: false
|
||||
type: string
|
||||
COMMIT_ID:
|
||||
required: false
|
||||
type: string
|
||||
# schedule:
|
||||
# - cron: '0 20 * * 0'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Slice-base clone
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
is_pr: 'false'
|
||||
|
||||
build-docker:
|
||||
name: Slice build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
build:
|
||||
name: Slice build
|
||||
needs: [clone, build-docker]
|
||||
uses: ./.github/workflows/_Linux-build.yml
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
is_pr: 'false'
|
||||
|
||||
slice-test:
|
||||
name: Slice test
|
||||
uses: ./.github/workflows/_Slice.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
slice-check: 'true'
|
||||
SLICE_TEST_MODE: insert_baseline
|
||||
MANUALLY_PR_ID: ${{ inputs.PR_ID }}
|
||||
MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
|
||||
@@ -0,0 +1,38 @@
|
||||
name: Slice-baseline-torch
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 0 1 * *'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Slice-base clone
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
is_pr: 'false'
|
||||
|
||||
build-docker:
|
||||
name: Slice build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
slice-test:
|
||||
name: Slice test
|
||||
uses: ./.github/workflows/_Slice.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
slice-check: 'true'
|
||||
SLICE_TEST_MODE: insert_baseline
|
||||
SLICE_BENCHMARK_FRAMEWORKS: torch
|
||||
@@ -0,0 +1,165 @@
|
||||
name: Api-Benchmark
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
baseline:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
MANUALLY_PR_ID:
|
||||
type: string
|
||||
required: false
|
||||
MANUALLY_COMMIT_ID:
|
||||
type: string
|
||||
required: false
|
||||
run-labels:
|
||||
type: string
|
||||
required: false
|
||||
default: "api-bm"
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-api-benchmark
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: api-benchmark
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "api-benchmark"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
data-storage:
|
||||
name: Performance data storage
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true'}}
|
||||
runs-on:
|
||||
group: Api-bm
|
||||
labels: [self-hosted, "${{ inputs.run-labels }}"]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
echo "docker_image=${docker_image}" >> $GITHUB_ENV
|
||||
docker run -i --rm -v ${{ github.workspace }}:/paddle -w /paddle $docker_image /bin/bash -c 'rm -rf * .[^.]*'
|
||||
cp -r /home/PTSTools .
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $4}')`
|
||||
echo $runner_name
|
||||
core_index=-1
|
||||
if [ $runner_name == "paddle-1" ];then
|
||||
core_index=0
|
||||
elif [ $runner_name == "paddle-2" ];then
|
||||
core_index=8
|
||||
fi
|
||||
echo "core_index=${core_index}" >> $GITHUB_ENV
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
python: "python3.10"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker container ls -a --filter "name=paddle-CI-*-api-benchmark-${core_index}*" --format "{{.ID}}" | xargs -r docker rm -f
|
||||
docker container ls -a --filter "name=api_benchmark_ci_baseline_" --format "{{.ID}} {{.CreatedAt}}" | awk '$2 <= "'$(date -d '1 day ago' +'%Y-%m-%d')'" {print $1}' | xargs -r docker rm -f
|
||||
docker run -d -t --name ${container_name} --shm-size=128g \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e python \
|
||||
-e core_index \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e RUN_ID \
|
||||
-e wheel_link \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download PaddleTest.tar.gz
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ldconfig
|
||||
set -e
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
# git config --global --add safe.directory ${work_dir}
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
# git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
mkdir -p ${{ github.workspace }}/../../../pip
|
||||
${python} -m pip config set global.cache-dir ${{ github.workspace }}/../../../pip
|
||||
${python} -m pip install -r ./PaddleTest/framework/e2e/api_benchmark_new/requirement.txt
|
||||
'
|
||||
|
||||
- name: Paddletest check
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-11.8/compat:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}
|
||||
cd ./PaddleTest/framework/e2e/api_benchmark_new
|
||||
cp /paddle/PTSTools/Uploader/apibm_config.yml .
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
if [[ "${{ inputs.baseline }}" == "true" ]];then
|
||||
set -e
|
||||
if [[ "${{ inputs.MANUALLY_PR_ID }}" == "" ]]; then
|
||||
export pr_wheel_link=https://paddle-github-action.bj.bcebos.com/PR/build/$PR_ID/$COMMIT_ID/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
else
|
||||
export pr_wheel_link=https://paddle-github-action.bj.bcebos.com/PR/build/${{ inputs.MANUALLY_PR_ID }}/${{ inputs.MANUALLY_COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
fi
|
||||
${python} -m pip install $pr_wheel_link
|
||||
${python} runner_ci_multipro_action.py --yaml ../yaml/sort_api_benchmark_fp32.yml --core_index ${core_index} --baseline_whl_link $pr_wheel_link
|
||||
exit 0
|
||||
fi
|
||||
${python} -m pip install $wheel_link
|
||||
if [ ${core_index} -eq -1 ];then
|
||||
${python} runner_ci_action.py --yaml ../yaml/api_benchmark_fp32.yml --core_index 2
|
||||
else
|
||||
${python} runner_ci_multipro_action.py --yaml ../yaml/sort_api_benchmark_fp32.yml --core_index ${core_index}
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,152 @@
|
||||
name: Auto-Parallel
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /workspace/Paddle
|
||||
PADDLE_ROOT: /workspace/Paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-Auto-Parallel
|
||||
ci_scripts: /workspace/Paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: auto-parallel
|
||||
no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "auto-parallel"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
parallel-test:
|
||||
name: Parallel test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Auto-Parallel
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_dynamic_static_unified_comm: "True"
|
||||
python_version: "3.10"
|
||||
paddle_whl: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
nvidia-docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/workspace \
|
||||
-v /home/FleetX_CI:/gpt_data \
|
||||
-v /home/Llm_gpt_CI:/llm_gpt_data \
|
||||
-v /home/Llama_CI:/llama_data \
|
||||
-v /home/.cache/pip:/home/.cache/pip \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e paddle_whl \
|
||||
-e FLAGS_dynamic_static_unified_comm \
|
||||
-e python_version \
|
||||
-w /workspace --runtime=nvidia ${docker_image}
|
||||
|
||||
- name: Download Paddle
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/build.tar.gz --no-check-certificate
|
||||
echo "Extracting build.tar.gz"
|
||||
tar --use-compress-program="pzstd" -xpf build.tar.gz
|
||||
mv paddle Paddle
|
||||
rm -f build.tar.gz
|
||||
'
|
||||
|
||||
- name: Download PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "Download and extract PaddleNLP.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
|
||||
tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cd PaddleNLP
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git pull
|
||||
git checkout 6ac04028757dfbcc089916997493611f62de81b2
|
||||
git switch -c 6ac04028757dfbcc089916997493611f62de81b2
|
||||
git cherry-pick bc08aeec91d2c992c3d8d39755bea7c6213b0e82
|
||||
git cherry-pick 7ab35ce94eca977bcf3b44bfb42deb0e0b5ef158
|
||||
git cherry-pick 2ac85997c2dafe3d67a4aac01d553ad76d6024bf
|
||||
git submodule update --init --recursive --force
|
||||
'
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ldconfig
|
||||
set -e
|
||||
pip config set global.cache-dir "/home/.cache/pip"
|
||||
ln -sf $(which python${python_version}) /usr/bin/python
|
||||
python -c "import sys; print(sys.version_info[:])"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
timeout 80m bash Paddle/ci/auto_parallel/ci_auto_parallel.sh ${paddle_whl}
|
||||
'
|
||||
|
||||
- name: Upload and display logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /workspace/case_logs
|
||||
for FILE in /workspace/case_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/Auto-Parallel/${PR_ID}/${COMMIT_ID}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/Auto-Parallel/${PR_ID}/${COMMIT_ID}/logs/$file"
|
||||
done
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,141 @@
|
||||
name: CE-CINN-Framework
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-CINN-Framework
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: ce-cinn-framework
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-cinn-framework
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
cinn:
|
||||
name: CINN
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker container ls -a --filter "name=paddle-CI-*-api-benchmark-${core_index}*" --format "{{.ID}}" | xargs -r docker rm -f
|
||||
docker container ls -a --filter "name=api_benchmark_ci_baseline_" --format "{{.ID}} {{.CreatedAt}}" | awk '$2 <= "'$(date -d '1 day ago' +'%Y-%m-%d')'" {print $1}' | xargs -r docker rm -f
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e python \
|
||||
-e core_index \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
git checkout test
|
||||
rm build.tar.gz
|
||||
cd /
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
'
|
||||
|
||||
- name: Determine ci trigger
|
||||
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/ce_cinn_diff.sh
|
||||
if [ ${sum_num} -eq 0 ];then
|
||||
echo "The modified files does not affect LayerCase in CE-CINN-Framework, so skip this ci."
|
||||
echo "skip_ci=true" >> ${{ github.env }}
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Run check
|
||||
if: ${{ env.skip_ci != 'true' }}
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
pip config set global.cache-dir "$CACHE_DIR/pip"
|
||||
pip install /paddle/build/pr_whl/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
cd /PaddleTest/framework/e2e/PaddleLT_new
|
||||
pip install -r requirement.txt
|
||||
source ./scene/set_ci_dy^dy2stcinn_train^dy2stcinn_eval_inputspec_env.sh
|
||||
python support/dict_to_yml.py --filename apibm_config.yml --data_str "$(cat $CACHE_DIR/cinn_config)"
|
||||
set -e
|
||||
python run.py
|
||||
exit $(head -n 1 "exit_code.txt")
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,312 @@
|
||||
name: CE-Framework
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework
|
||||
CI_name: ce-framework
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass-whl:
|
||||
name: Check bypass (ce-whl)
|
||||
if: ${{ inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-whl
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
check-bypass-infer:
|
||||
name: Check bypass (ce-infer)
|
||||
if: ${{ inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-infer
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
check-bypass-test:
|
||||
name: Check bypass (ce-test)
|
||||
if: ${{ inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-test
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
whl:
|
||||
name: Whl
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass-whl.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass-whl]
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework-whl
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Check whl size
|
||||
env:
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
bash ci/check_whl_size.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
infer:
|
||||
name: Infer
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass-infer.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass-infer]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework-infer
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
FLAGS_enable_eager_mode: 1
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
cd /
|
||||
cp /paddle/build/pr_whl/*.whl .
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
'
|
||||
|
||||
- name: Install paddle and dependencies
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir run_env
|
||||
ln -s $(which python3.10) run_env/python
|
||||
ln -s $(which pip3.10) run_env/pip
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
pip config set global.cache-dir "$CACHE_DIR/pip"
|
||||
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
python -m pip install pip --upgrade
|
||||
pip install /paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --no-index --no-deps
|
||||
pip install -r /PaddleTest/inference/python_api_test/requirements.txt
|
||||
'
|
||||
|
||||
- name: Check
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
cd /PaddleTest/inference/python_api_test
|
||||
bash parallel_run.sh 2
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass-test.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass-test]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework-test
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
FLAGS_enable_eager_mode: 1
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
cd /
|
||||
cp /paddle/build/pr_whl/*.whl .
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
'
|
||||
|
||||
- name: Install paddle and dependencies
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir run_env
|
||||
ln -s $(which python3.10) run_env/python
|
||||
ln -s $(which pip3.10) run_env/pip
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
pip config set global.cache-dir "${CFS_DIR}/.cache/pip"
|
||||
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
pip install sympy
|
||||
pip install /paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --no-index --no-deps
|
||||
'
|
||||
|
||||
- name: Check
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
cd /PaddleTest/framework/api
|
||||
rm ./device/test_cuda_get_device_capability.py
|
||||
bash run_paddle_ci.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,161 @@
|
||||
name: Clone-linux
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
workflow-name:
|
||||
type: string
|
||||
required: false
|
||||
clone_dir:
|
||||
type: string
|
||||
required: false
|
||||
default: "Paddle"
|
||||
is_pr:
|
||||
type: string
|
||||
required: false
|
||||
default: "true"
|
||||
skill-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
outputs:
|
||||
can-skip:
|
||||
value: ${{ jobs.clone.outputs.can-skip }}
|
||||
slice-check:
|
||||
value: ${{ jobs.clone.outputs.slice-check }}
|
||||
xpu-only:
|
||||
value: ${{ jobs.clone.outputs.xpu-only }}
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
ci_scripts: ${{ github.workspace }}/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ${{ inputs.workflow-name }}
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
clone:
|
||||
name: Clone Paddle
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
needs: [check-bypass]
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip == 'true' || inputs.skill-can-skip == 'true' }}
|
||||
slice-check: ${{ steps.check-execution.outputs.slice-check }}
|
||||
xpu-only: ${{ steps.xpu-only-check.outputs.xpu-only }}
|
||||
runs-on:
|
||||
group: HK-Clone
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Debug - Print inputs values
|
||||
run: |
|
||||
echo "=== Debug Info ==="
|
||||
echo "inputs.skill-can-skip: '${{ inputs.skill-can-skip }}'"
|
||||
echo "inputs.skill-can-skip length: $(echo '${{ inputs.skill-can-skip }}' | wc -c)"
|
||||
echo "==="
|
||||
|
||||
- name: Set environment variables and print
|
||||
id: set-vars
|
||||
run: |
|
||||
# Determine if can skip based on check-bypass and skill-can-skip
|
||||
if [[ "${{ needs.check-bypass.outputs.can-skip }}" == "true" ]] || [[ "${{ inputs.skill-can-skip }}" == "true" ]]; then
|
||||
echo "CAN_SKIP=true" >> $GITHUB_ENV
|
||||
else
|
||||
echo "CAN_SKIP=false" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Print can-skip value
|
||||
run: |
|
||||
echo "can-skip output: ${{ env.CAN_SKIP }}"
|
||||
|
||||
- name: Check submodules status
|
||||
run: git submodule foreach --recursive 'git rev-parse HEAD || rm -rf $PWD' || true
|
||||
|
||||
- name: Clone paddle
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
fetch-depth: 1000
|
||||
|
||||
- name: Merge PR to test branch
|
||||
id: check-execution
|
||||
if: ${{ inputs.is_pr == 'true' }}
|
||||
run: |
|
||||
git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
|
||||
git switch ${{ github.event.pull_request.base.ref }}
|
||||
set +e
|
||||
git branch -D test
|
||||
set -e
|
||||
git gc
|
||||
git switch -c test
|
||||
git config user.name "PaddleCI"
|
||||
git config user.email "paddle_ci@example.com"
|
||||
git fetch origin pull/${{ github.event.pull_request.number }}/head:pr
|
||||
git merge --no-ff pr
|
||||
git submodule sync --recursive
|
||||
git submodule update --init --recursive
|
||||
git branch -d pr
|
||||
source ${ci_scripts}/check_execution.sh
|
||||
bash ${ci_scripts}/third_party_tag.sh
|
||||
|
||||
- name: Check if XPU-only PR
|
||||
id: xpu-only-check
|
||||
if: ${{ inputs.is_pr == 'true' }}
|
||||
env:
|
||||
DISABLE_HW_CI_SKIP: ${{ vars.DISABLE_HW_CI_SKIP || 'false' }}
|
||||
run: |
|
||||
if bash ci/check_xpu_only.sh; then
|
||||
echo "xpu-only=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "xpu-only=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download bos client
|
||||
env:
|
||||
home_path: "/home/paddle/actions-runner/"
|
||||
bos_file: "/home/paddle/actions-runner/bos/BosClient.py"
|
||||
run: |
|
||||
if [ ! -f "${bos_file}" ]; then
|
||||
wget -q --no-proxy -O ${home_path}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${home_path}/bos
|
||||
tar xf ${home_path}/bos_new.tar.gz -C ${home_path}/bos
|
||||
fi
|
||||
|
||||
- name: Push paddle-action.tar.gz to bos
|
||||
env:
|
||||
AK: paddle
|
||||
SK: paddle
|
||||
bos_file: "/home/paddle/actions-runner/bos/BosClient.py"
|
||||
run: |
|
||||
cd ..
|
||||
tar -I 'zstd -T0' -cf Paddle.tar.gz Paddle
|
||||
echo "::group::Install bce-python-sdk"
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
python ${bos_file} Paddle.tar.gz paddle-github-action/PR/${{ inputs.clone_dir }}/${PR_ID}/${COMMIT_ID}
|
||||
rm Paddle.tar.gz
|
||||
cd -
|
||||
git switch ${BRANCH}
|
||||
set +e
|
||||
git branch -D test
|
||||
git gc
|
||||
|
||||
# - name: Clean environment
|
||||
# if: always()
|
||||
# run: |
|
||||
# cd ${{ github.workspace }}
|
||||
# rm -rf * .[^.]*
|
||||
@@ -0,0 +1,90 @@
|
||||
name: Clone-windows
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
skill-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
outputs:
|
||||
can-skip:
|
||||
value: ${{ jobs.clone.outputs.can-skip }}
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
WORK_DIR: ${{ github.workspace }}\..
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Clone Paddle
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
outputs:
|
||||
can-skip: ${{ inputs.skill-can-skip == 'true' }}
|
||||
runs-on:
|
||||
group: win-clone
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Git config
|
||||
run: |
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git config --global http.postBuffer 524288000
|
||||
git config --global core.longpaths true
|
||||
|
||||
- name: Clone paddle
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
fetch-depth: 1000
|
||||
|
||||
- name: Merge pr
|
||||
run: |
|
||||
REM git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
|
||||
git switch %BRANCH%
|
||||
git branch -D test
|
||||
git gc
|
||||
git switch -c test
|
||||
git fetch origin pull/%PR_ID%/head:pr
|
||||
git merge --no-ff pr
|
||||
git branch -d pr
|
||||
REM call %ci_scripts%\third_party_tag.bat
|
||||
git config -f .gitmodules submodule.third_party/openvino.update none && git submodule sync third_party/openvino
|
||||
git submodule update --init --recursive
|
||||
|
||||
- name: Upload paddle to bos
|
||||
working-directory: ..
|
||||
env:
|
||||
BCE_FILE: ${{ github.workspace }}\..\bce-python-sdk-new\BosClient.py
|
||||
run: |
|
||||
if not exist %WORK_DIR%\bce-python-sdk-new (
|
||||
echo There is no bce in this PC, will install bce.
|
||||
pip install wget
|
||||
pip install shutil
|
||||
echo Download package from https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz
|
||||
python -c "import wget;wget.download('https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz')"
|
||||
python -c "import shutil;shutil.unpack_archive('bos_new.tar.gz', extract_dir='./bce-python-sdk-new',format='gztar')"
|
||||
)
|
||||
python -m pip install pycryptodome
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
tar -cf Paddle.tar Paddle && zstd Paddle.tar
|
||||
python %BCE_FILE% Paddle.tar.zst paddle-github-action/windows/PR/%PR_ID%/%COMMIT_ID%
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
|
||||
- name: Clean env
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
git switch %BRANCH%
|
||||
git branch -D test
|
||||
git config -f .gitmodules submodule.third_party/openvino.update none && git submodule sync third_party/openvino
|
||||
git submodule update --init --recursive
|
||||
git gc
|
||||
@@ -0,0 +1,206 @@
|
||||
name: DeepMD-Kit Test
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
MANUALLY_PR_ID:
|
||||
type: string
|
||||
required: false
|
||||
MANUALLY_COMMIT_ID:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-deepmd
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: deepmd
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "deepmd"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
deepmd-test:
|
||||
name: DeepMD-Kit Integration Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
container_name="deepmd_test_ci_${RUN_ID}"
|
||||
echo "container_name=${container_name}" >> $GITHUB_ENV
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e RUN_ID \
|
||||
-e wheel_link \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Print PR information
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
echo "Testing PR #${{ env.PR_ID }}"
|
||||
echo "Commit: ${{ env.COMMIT_ID }}"
|
||||
echo "Branch: ${{ env.BRANCH }}"
|
||||
'
|
||||
|
||||
- name: Install PaddlePaddle
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
mkdir -p ${{ github.workspace }}/../../../.cache/pip
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python3.10 -m pip config set global.cache-dir ${{ github.workspace }}/../../../.cache/pip
|
||||
|
||||
if [[ "${{ inputs.MANUALLY_PR_ID }}" == "" ]]; then
|
||||
echo "Installing PaddlePaddle from: $wheel_link"
|
||||
python3.10 -m pip install $wheel_link
|
||||
else
|
||||
echo "Installing PaddlePaddle from manual PR: ${{ inputs.MANUALLY_PR_ID }}"
|
||||
python3.10 -m pip install https://paddle-github-action.bj.bcebos.com/PR/build/${{ inputs.MANUALLY_PR_ID }}/${{ inputs.MANUALLY_COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
fi
|
||||
|
||||
python3.10 -c "import paddle; print(f\"PaddlePaddle version: {paddle.__version__}\")"
|
||||
'
|
||||
|
||||
- name: Clone and setup deepmd-kit
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
cd /paddle
|
||||
rm -rf deepmd-kit
|
||||
echo "Cloning deepmd-kit repository..."
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
git clone https://github.com/deepmodeling/deepmd-kit.git -b v3.1.2
|
||||
ls -lah /paddle/deepmd-kit
|
||||
cd deepmd-kit
|
||||
'
|
||||
|
||||
- name: Install deepmd-kit dependencies
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
cd /paddle/deepmd-kit
|
||||
echo "Installing deepmd-kit..."
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
DP_ENABLE_TENSORFLOW=0 DP_ENABLE_PYTORCH=0 python3.10 -m pip install -v -e . -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
'
|
||||
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
echo "Installing pytest..."
|
||||
python3.10 -m pip install \
|
||||
pytest pytest-xdist pytest-json-report pytest-metadata pytest-mock pytest-rich pytest-sugar pytest-timeout \
|
||||
-i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
'
|
||||
|
||||
- name: Run deepmd-kit pytest
|
||||
id: pytest_run
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
cd /paddle/deepmd-kit
|
||||
echo "=========================================="
|
||||
echo "Running DeepMD-Kit Integration Tests"
|
||||
echo "=========================================="
|
||||
|
||||
python3.10 -m pytest -vvv --tb=long --color=yes \
|
||||
source/tests/pd/test_auto_batch_size.py \
|
||||
source/tests/pd/test_change_bias.py \
|
||||
source/tests/pd/test_decomp.py \
|
||||
source/tests/pd/test_dp_show.py \
|
||||
source/tests/pd/test_finetune.py \
|
||||
source/tests/pd/test_multitask.py \
|
||||
source/tests/pd/test_training.py \
|
||||
source/tests/pd/test_update_sel.py \
|
||||
source/tests/pd/test_utils.py \
|
||||
source/tests/pd/model/test_atomic_model_atomic_stat.py \
|
||||
source/tests/pd/model/test_atomic_model_global_stat.py \
|
||||
source/tests/pd/model/test_autodiff.py \
|
||||
source/tests/pd/model/test_deeppot.py \
|
||||
source/tests/pd/model/test_descriptor_dpa1.py \
|
||||
source/tests/pd/model/test_descriptor_dpa2.py \
|
||||
source/tests/pd/model/test_dp_atomic_model.py \
|
||||
source/tests/pd/model/test_dp_model.py \
|
||||
source/tests/pd/model/test_dpa1.py \
|
||||
source/tests/pd/model/test_dpa2.py \
|
||||
source/tests/pd/model/test_dpa3.py \
|
||||
source/tests/pd/model/test_dynamic_sel.py \
|
||||
source/tests/pd/model/test_ener_fitting.py \
|
||||
source/tests/pd/model/test_env_mat.py \
|
||||
source/tests/pd/model/test_exclusion_mask.py \
|
||||
source/tests/pd/model/test_force_grad.py \
|
||||
source/tests/pd/model/test_forward_lower.py \
|
||||
source/tests/pd/model/test_get_model.py \
|
||||
source/tests/pd/model/test_jit.py \
|
||||
source/tests/pd/model/test_mlp.py \
|
||||
source/tests/pd/model/test_nlist.py \
|
||||
source/tests/pd/model/test_null_input.py \
|
||||
source/tests/pd/model/test_permutation_denoise.py \
|
||||
source/tests/pd/model/test_permutation.py \
|
||||
source/tests/pd/model/test_region.py \
|
||||
source/tests/pd/model/test_rot_denoise.py \
|
||||
source/tests/pd/model/test_rot.py \
|
||||
source/tests/pd/model/test_rotation.py \
|
||||
source/tests/pd/model/test_se_e2_a.py \
|
||||
source/tests/pd/model/test_smooth.py \
|
||||
source/tests/pd/model/test_trans_denoise.py \
|
||||
source/tests/pd/model/test_trans.py \
|
||||
source/tests/pd/model/test_unused_params.py 2>&1 | tee /paddle/pytest_output.log
|
||||
TEST_RESULT=${PIPESTATUS[0]}
|
||||
|
||||
echo "DeepMD-Kit Test Finished with status: $TEST_RESULT"
|
||||
exit $TEST_RESULT
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'cd /paddle && rm -rf deepmd-kit'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,189 @@
|
||||
name: Distribute-stable-Formers
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_distribute_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-distribute-formers
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: distribute
|
||||
no_proxy: bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn,paddlepaddle.org.cn
|
||||
docker_image: ${{ inputs.docker_distribute_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "formers"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
formers-test:
|
||||
name: formers-Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Distribute
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_COVERAGE: "OFF"
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
|
||||
PRECISION_TEST: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
WITH_INCREMENTAL_COVERAGE: "OFF"
|
||||
WITH_ONNXRUNTIME: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PY_VERSION: "3.10"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_CUDNN_FRONTEND: "ON"
|
||||
FLAGS_enable_cudnn_frontend: 1
|
||||
CACHE_DIR: /root/.cache/build
|
||||
CCACHE_DIR: /root/.ccache/formers
|
||||
CFS_DIR: /home/data/cfs
|
||||
paddle_whl: /workspace/dist/*.whl
|
||||
formers_docker: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest
|
||||
run: |
|
||||
export CUDA_SO="$(\ls -d /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls -d /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')"
|
||||
export DEVICES="$(\ls -d /dev/nvidia* | xargs -I{} echo "-v {}:{}") $(\ls /dev/nvidia-caps/* | xargs -I{} echo "-v {}:{}")"
|
||||
export SMI="-v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi"
|
||||
container_name=${TASK}-test-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} ${CUDA_SO} ${DEVICES} ${SMI} --runtime=nvidia --shm-size=32G --privileged \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/ssd1/models:/home/models" \
|
||||
-v "/ssd1/root:/root" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/workspace \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CI_name \
|
||||
-e PF_HOME=/home/models \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_COVERAGE \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
|
||||
-e PRECISION_TEST \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e AGILE_REVISION \
|
||||
-e WITH_INCREMENTAL_COVERAGE \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e GIT_PR_ID \
|
||||
-e PY_VERSION \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_CUDNN_FRONTEND \
|
||||
-e FLAGS_enable_cudnn_frontend \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CFS_DIR \
|
||||
-e paddle_whl \
|
||||
-e no_proxy \
|
||||
-w /workspace --network host ${formers_docker}
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global --add safe.directory /workspace
|
||||
git checkout test
|
||||
'
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN
|
||||
set -ex
|
||||
bash /workspace/ci/formers_test.sh
|
||||
'
|
||||
- name: Upload and display logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
|
||||
bos_prefix="${PR_ID}/${COMMIT_ID}"
|
||||
else
|
||||
bos_prefix="schedule/$(date +%Y%m%d)"
|
||||
fi
|
||||
# api test logs
|
||||
cd /workspace/PaddleFormers/unittest_logs
|
||||
for FILE in /workspace/PaddleFormers/unittest_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs/$file"
|
||||
done
|
||||
# models test logs
|
||||
cd /workspace/PaddleFormers/model_unittest_logs
|
||||
for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file"
|
||||
done
|
||||
'
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,170 @@
|
||||
name: Distribute-stable-Test
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_distribute_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-distribute-test
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: distribute
|
||||
no_proxy: bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn,paddlepaddle.org.cn
|
||||
docker_image: ${{ inputs.docker_distribute_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: distribute-test
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Distribute
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_COVERAGE: "OFF"
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
|
||||
PRECISION_TEST: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
WITH_INCREMENTAL_COVERAGE: "OFF"
|
||||
WITH_ONNXRUNTIME: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PY_VERSION: "3.10"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_CUDNN_FRONTEND: "ON"
|
||||
FLAGS_enable_cudnn_frontend: 1
|
||||
CACHE_DIR: /root/.cache/build
|
||||
CCACHE_DIR: /root/.ccache/gpubox
|
||||
run: |
|
||||
export CUDA_SO="$(\ls -d /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls -d /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')"
|
||||
export DEVICES="$(\ls -d /dev/nvidia* | xargs -I{} echo "-v {}:{}") $(\ls /dev/nvidia-caps/* | xargs -I{} echo "-v {}:{}")"
|
||||
export SMI="-v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi"
|
||||
container_name=${TASK}-test-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} ${CUDA_SO} ${DEVICES} ${SMI} --runtime=nvidia --shm-size=32G \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/ssd1/root:/root" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CI_name \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_COVERAGE \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
|
||||
-e PRECISION_TEST \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e AGILE_REVISION \
|
||||
-e WITH_INCREMENTAL_COVERAGE \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e GIT_PR_ID \
|
||||
-e PY_VERSION \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_CUDNN_FRONTEND \
|
||||
-e FLAGS_enable_cudnn_frontend \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git checkout test
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/distribute_test.sh
|
||||
'
|
||||
|
||||
- name: Upload and display logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /case_logs
|
||||
for FILE in /case_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/Distribute-Stable/${PR_ID}/${COMMIT_ID}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/Distribute-Stable/${PR_ID}/${COMMIT_ID}/logs/$file"
|
||||
done
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,234 @@
|
||||
name: Distribute-stable-Build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_distribute_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-distribute
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: distribute
|
||||
no_proxy: bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn,paddlepaddle.org.cn
|
||||
docker_image: ${{ inputs.docker_distribute_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: distribute
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip }}
|
||||
needs: [check-bypass]
|
||||
if: ${{ inputs.clone-can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_MKL: "OFF"
|
||||
WITH_PYTHON: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
GPUBOX_DEMO_INSTALL_DIR: /root/.cache/build
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/python35
|
||||
PY_VERSION: "3.10"
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: "Ninja"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
CUDA_ARCH_NAME: Ampere
|
||||
WITH_CUDNN_FRONTEND: "ON"
|
||||
FLAGS_enable_cudnn_frontend: 1
|
||||
CACHE_DIR: /root/.cache/build
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_STATSLOG: /paddle/build/.stats.log
|
||||
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CI_name \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e WITH_TENSORRT \
|
||||
-e GENERATOR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e WITH_AVX \
|
||||
-e WITH_PYTHON \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_STATSLOG \
|
||||
-e CCACHE_SLOPPINESS \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e GPUBOX_DEMO_INSTALL_DIR \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_CUDNN_FRONTEND \
|
||||
-e FLAGS_enable_cudnn_frontend \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p /root/.cache/build
|
||||
mkdir -p /root/.ccache/gpubox
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Download flashattn cache
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set +e
|
||||
flashattn_version=$(git submodule status | grep flashattn | awk "{print \$1}" | sed "s#-##g")
|
||||
echo "flashattn_version=${flashattn_version}" >> ${{ github.env }}
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/flashattn_cache/flashattn_libs_${flashattn_version}.tar.gz --no-check-certificate; FACODE=$?
|
||||
if [ $FACODE -ne 0 ]; then
|
||||
echo "flashattn_cached_package=true" >> ${{ github.env }}
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Build
|
||||
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Packaging of products
|
||||
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
if [ "${{ env.flashattn_cached_package }}" == "true" ]; then
|
||||
cd ${work_dir}/build/third_party/install/flashattn/lib
|
||||
mkdir flashattn_libs_${{ env.flashattn_version }} && cd flashattn_libs_${{ env.flashattn_version }}
|
||||
mkdir fa_libs && cp ../lib*.so fa_libs && tar -zcf fa_libs.tar ./fa_libs && rm -rf ./fa_libs
|
||||
md5sum fa_libs.tar |awk "{print \$1}" >MD5.txt
|
||||
cd .. && tar -zcf flashattn_libs_${{ env.flashattn_version }}.tar ./flashattn_libs_${{ env.flashattn_version }}
|
||||
fi
|
||||
bash ${ci_scripts}/compress_build.sh
|
||||
cd ${work_dir}/..
|
||||
tar --use-compress-program="pzstd -1" --warning=no-file-changed -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload product to bos
|
||||
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: "*.whl"
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -x
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
echo "::group::Install bce-python-sdk"
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd ..
|
||||
source ${{ github.workspace }}/../../../unproxy
|
||||
echo "Uploading Paddle.tar.gz to bos"
|
||||
python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading whl to bos"
|
||||
mv ${work_dir}/dist/${{ env.paddle_whl }} .
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
if [ "${{ env.flashattn_cached_package }}" == "true" ]; then
|
||||
echo "Uploading flashattn_libs_${flashattn_version}.tar.gz to bos"
|
||||
mv ${work_dir}/build/third_party/install/flashattn/lib/flashattn_libs_${{ env.flashattn_version }}.tar .
|
||||
python ${{ env.bos_file }} flashattn_libs_${{ env.flashattn_version }}.tar paddle-github-action/PR/gpups/flashattn_cache
|
||||
fi
|
||||
rm -rf Paddle.tar.gz ${{ env.paddle_whl }} flashattn_libs_${flashattn_version}.tar
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,123 @@
|
||||
name: Doc-Preview
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_doc_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /workspace/paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-Doc-Preview
|
||||
CI_name: Doc-Preview
|
||||
CFS_DIR: /home/data/cfs
|
||||
work_dir: /paddle
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "Doc-Preview"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-doc:
|
||||
name: Build doc
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
BUILD_DOC: "true"
|
||||
UPLOAD_DOC: "true"
|
||||
BOSBUCKET: "paddle-site-web-dev"
|
||||
PREVIEW_SITE: "paddle-docs-preview.paddlepaddle.org.cn"
|
||||
AGILE_COMPILE_BRANCH: ${{ startsWith(github.event.pull_request.base.ref, 'fleety_') && 'develop' || github.event.pull_request.base.ref }}
|
||||
BOS_CREDENTIAL_AK: "paddle"
|
||||
BOS_CREDENTIAL_SK: "paddle"
|
||||
PADDLE_WHL: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_doc_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/python35-cpu:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CACHE_DIR \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e AGILE_PULL_ID \
|
||||
-e AGILE_REVISION \
|
||||
-e BUILD_DOC \
|
||||
-e UPLOAD_DOC \
|
||||
-e BOSBUCKET \
|
||||
-e PREVIEW_SITE \
|
||||
-e BRANCH \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e BOS_CREDENTIAL_AK \
|
||||
-e BOS_CREDENTIAL_SK \
|
||||
-e PADDLE_WHL \
|
||||
-e CFS_DIR \
|
||||
-e work_dir \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Doc build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${AGILE_PULL_ID}/${AGILE_REVISION}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
if ! api_doc_spec_diff=$(python tools/diff_api.py paddle/fluid/API_DEV.spec.doc paddle/fluid/API_PR.spec.doc); then
|
||||
echo "diff_api.py failed for API doc specs."
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$api_doc_spec_diff" ]; then
|
||||
echo "API documents no change."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd /
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
pip3 install --progress-bar off -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple --force-reinstall ${work_dir}/build/pr_whl/*.whl
|
||||
bash ${work_dir}/ci/run_docs_preview.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,322 @@
|
||||
name: PR-CI-Inference
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_inference_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
docker_image: ${{ inputs.docker_inference_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "inference"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_build
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: "0.15"
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
|
||||
WITH_PYTHON: "OFF"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_NVCC_LAZY: "OFF"
|
||||
PYTHON_ABI: cp310-cp310
|
||||
CACHE_DIR: /root/.cache/inference
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: "0.8"
|
||||
CUDA_ARCH_NAME: Pascal
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/home/data/cfs/.ccache/inference:/root/.ccache/inference" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:/action-runner" \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_PYTHON \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e WITH_NVCC_LAZY \
|
||||
-e PYTHON_ABI \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e PY_VERSION \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-build/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git config --global --add safe.directory "*"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
source /action-runner/proxy
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Run build
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
source /action-runner/proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/inference_build.sh
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload build.tar.gz
|
||||
env:
|
||||
home_path: /action-runner
|
||||
bos_file: /action-runner/bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source /action-runner/proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
rm -rf `find . -name "*.o"`
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --tries=5 --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
source /action-runner/unproxy
|
||||
cd /
|
||||
tar --use-compress-program="pigz -1" -cpPf build.tar.gz paddle
|
||||
echo "Uploading build.tar.gz to cfs"
|
||||
mkdir -p ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}
|
||||
cp build.tar.gz ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID} && echo success
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
# docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: build
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_test
|
||||
runs-on:
|
||||
group: Inference-P4
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: "0.15"
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
|
||||
WITH_PYTHON: "OFF"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_NVCC_LAZY: "OFF"
|
||||
PYTHON_ABI: cp310-cp310
|
||||
CACHE_DIR: /root/.cache/inference
|
||||
CCACHE_DIR: /root/.ccache/inference
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: "0.8"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:/action-runner" \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_PYTHON \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e WITH_NVCC_LAZY \
|
||||
-e PYTHON_ABI \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e PY_VERSION \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
tar --use-compress-program="pigz -1" -xpf build.tar.gz --strip-components=1
|
||||
rm build.tar.gz
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
echo "export PATH=${PATH}" >> ~/.bashrc
|
||||
echo "::group::Install dependencies"
|
||||
source /action-runner/proxy
|
||||
pip install -r "python/unittest_py/requirements.txt"
|
||||
echo "::endgroup::"
|
||||
'
|
||||
|
||||
- name: Test fluid library for inference
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
test_fluid_lib
|
||||
'
|
||||
|
||||
- name: Test go inference api
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
test_go_inference_api
|
||||
'
|
||||
|
||||
- name: Check approvals of unittest
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
check_approvals_of_unittest 3
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
# docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
@@ -0,0 +1,201 @@
|
||||
name: Linux-CPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_cpu_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether to skip the job"
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-cpu
|
||||
ci_scripts: ${{ github.workspace }}/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: cpu
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: cpu
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip || inputs.clone-can-skip || 'false' }}
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_GPU: "OFF"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_MKL: "OFF"
|
||||
WITH_TESTING: "ON"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PREC_SUFFIX: .py3
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
PROC_RUN: 12
|
||||
FLAGS_enable_eager_mode: 1
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: "Ninja"
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
WITH_AVX: "OFF"
|
||||
CACHE_DIR: "/root/.cache/cpu"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
WITH_OPENVINO: "ON"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_cpu_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/python35-cpu:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e WITH_GPU \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PREC_SUFFIX \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e PROC_RUN \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e WITH_TENSORRT \
|
||||
-e GENERATOR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e GITHUB_ENV \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CI_name \
|
||||
-e no_proxy \
|
||||
-e WITH_OPENVINO \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_linux_cpu_test.sh
|
||||
EXCODE=$?
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload paddle_whl
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar -xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
cd dist
|
||||
echo "Uploading paddle_whl to bos"
|
||||
source ${{ github.workspace }}/../../../unproxy
|
||||
python3.10 ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/cpu_whl/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,308 @@
|
||||
name: Linux-DCU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_dcu_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
docker_image: ${{ inputs.docker_dcu_image }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
ci_scripts_runner: ${{ github.workspace }}/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
PADDLE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: dcu
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "dcu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-dcu_build
|
||||
runs-on:
|
||||
group: HK-CPU
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_DOC: "OFF"
|
||||
WITH_GPU: "OFF"
|
||||
WITH_ROCM: "ON"
|
||||
WITH_RCCL: "ON"
|
||||
WITH_XPU: "OFF"
|
||||
WITH_TENSORRT: "OFF"
|
||||
WITH_COVERAGE: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
CMAKE_BUILD_TYPE: "Release"
|
||||
WITH_AVX: "ON"
|
||||
WITH_MKL: "ON"
|
||||
WITH_ARM: "OFF"
|
||||
WITH_CACHE: "ON"
|
||||
RUN_TEST: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
CI_SKIP_CPP_TEST: "OFF"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
CMAKE_EXPORT_COMPILE_COMMANDS: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
PY_VERSION: "3.10"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --shm-size=128G \
|
||||
--ulimit nofile=102400:102400 -d -t --name ${container_name} \
|
||||
-v $home_dir/.cache:/root/.cache \
|
||||
-v $home_dir/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_DOC \
|
||||
-e WITH_GPU \
|
||||
-e WITH_ROCM \
|
||||
-e WITH_RCCL \
|
||||
-e WITH_XPU \
|
||||
-e WITH_TENSORRT \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e WITH_MKL \
|
||||
-e WITH_AVX \
|
||||
-e WITH_ARM \
|
||||
-e RUN_TEST \
|
||||
-e WITH_TESTING \
|
||||
-e CI_SKIP_CPP_TEST \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e BRANCH \
|
||||
-e PADDLE_BRANCH \
|
||||
-e PADDLE_VERSION \
|
||||
-e no_proxy \
|
||||
-e CMAKE_EXPORT_COMPILE_COMMANDS \
|
||||
-e PY_VERSION \
|
||||
-e GIT_PR_ID \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ulimit -n 102400
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop or target branch"
|
||||
git pull upstream $BRANCH --no-edit
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Run build
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source /opt/dtk-24.04.1/env.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
rm -rf `find . -name "*.a"`
|
||||
rm -rf `find . -name "*.o"`
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload build.tar.gz and paddle_whl to bos
|
||||
env:
|
||||
AK: paddle
|
||||
SK: paddle
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle_dcu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
apt install zstd -y
|
||||
cd ..
|
||||
tar --use-compress-program="pzstd -5" -cf build.tar.gz paddle
|
||||
echo "Uploading build.tar.gz"
|
||||
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/dcu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading paddle_whl"
|
||||
cp /paddle/dist/${{ env.paddle_whl }} .
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/dcu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: build
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-dcu_test
|
||||
runs-on:
|
||||
group: dcu-z100
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
set -x
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
||||
source utils.sh
|
||||
determine_dcu_runner ${runner_name}
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
WITH_ROCM: "ON"
|
||||
WITH_RCCL: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_MKL: "ON"
|
||||
IF_DCU: "OFF"
|
||||
WITH_TENSORRT: "OFF"
|
||||
WITH_XPU: "OFF"
|
||||
WITH_CINN: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_TESTING: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
HIP_VISIBLE_DEVICES: ${{ env.HIP_VISIBLE_DEVICES }}
|
||||
PY_VERSION: "3.10"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --ulimit nofile=102400 --ulimit core=-1 --shm-size=32g -d -t --name ${container_name} \
|
||||
-v /ssd1/cibuild/.cache:/root/.cache \
|
||||
-v /ssd1/cibuild/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--device=/dev/mkfd \
|
||||
--group-add video \
|
||||
--shm-size=32g \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_ROCM \
|
||||
-e WITH_RCCL \
|
||||
-e WITH_AVX \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TENSORRT \
|
||||
-e WITH_XPU \
|
||||
-e IF_DCU \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e WITH_RCCL \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e no_proxy \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download build.tar.gz
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf ./*
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/dcu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
|
||||
tar --use-compress-program="pzstd" -xf build.tar.gz --strip-components=1
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Run test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PATH=/usr/local/bin:${PATH}
|
||||
ln -sf $(which python3.10) /usr/local/bin/python
|
||||
ln -sf $(which pip3.10) /usr/local/bin/pip
|
||||
pip3.10 install ./dist/paddlepaddle_dcu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
wget -q --no-proxy https://paddle-device.bj.bcebos.com/dcu/hyhal-Z100.tar.gz
|
||||
tar -zxf hyhal-Z100.tar.gz -C /opt
|
||||
source /opt/dtk-24.04.1/env.sh
|
||||
bash -x ${ci_scripts}/dcu_test.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
@@ -0,0 +1,466 @@
|
||||
name: Linux-IXUCA
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
concurrency:
|
||||
group: linux-ixuca-pr-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: ixuca
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "ixuca"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test-primary:
|
||||
name: Build and Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on: iluvatar-gpu-2
|
||||
timeout-minutes: 120
|
||||
container:
|
||||
image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:3.3.0
|
||||
env:
|
||||
LD_LIBRARY_PATH: /usr/local/corex/lib
|
||||
LIBRARY_PATH: /usr/local/corex/lib
|
||||
env:
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
outputs:
|
||||
result: ${{ steps.set-result.outputs.result }}
|
||||
|
||||
steps:
|
||||
- name: Check approval
|
||||
id: check-approval
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
AUTHORIZED_REVIEWERS=("YqGe585" "yongqiangma")
|
||||
|
||||
echo "Authorized reviewers: ${AUTHORIZED_REVIEWERS[*]}"
|
||||
echo "Repository: ${{ github.repository }}"
|
||||
echo "PR number: ${{ github.event.pull_request.number }}"
|
||||
|
||||
response=""
|
||||
for i in 1 2 3; do
|
||||
_tmp=$(curl -s --fail --max-time 30 \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" 2>/dev/null) && { response="$_tmp"; break; }
|
||||
echo "Attempt $i to fetch PR reviews failed, retrying..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "Response length: ${#response}"
|
||||
|
||||
can_skip=$(python3 -c '
|
||||
import json, sys
|
||||
|
||||
def debug(msg):
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
debug("=== Review Check Debug Info ===")
|
||||
debug(f"Authorized reviewers: {sys.argv[1:]}")
|
||||
AUTHORIZED = set(sys.argv[1:])
|
||||
|
||||
try:
|
||||
input_data = sys.stdin.read()
|
||||
debug(f"Raw input data length: {len(input_data)}")
|
||||
data = json.loads(input_data)
|
||||
debug(f"Parsed {len(data)} reviews")
|
||||
except Exception as e:
|
||||
debug(f"JSON parse error: {str(e)}")
|
||||
print("false")
|
||||
sys.exit(0)
|
||||
|
||||
latest_state = {}
|
||||
debug("\n=== Review States ===")
|
||||
for r in data:
|
||||
user = r.get("user", {}).get("login")
|
||||
if not user:
|
||||
continue
|
||||
state = r.get("state")
|
||||
latest_state[user] = state
|
||||
commit_id = r.get("commit_id") or ""
|
||||
debug(f"- {user}: {state} (commit: {commit_id[:8]})")
|
||||
|
||||
debug("\n=== Final Check ===")
|
||||
for user, state in latest_state.items():
|
||||
if user in AUTHORIZED and state == "APPROVED":
|
||||
debug(f"Found APPROVED review from authorized user: {user}")
|
||||
print("true")
|
||||
sys.exit(0)
|
||||
|
||||
debug("No APPROVED review found from authorized users")
|
||||
print("false")
|
||||
' "${AUTHORIZED_REVIEWERS[@]}" <<< "$response")
|
||||
|
||||
echo "can-skip=$can_skip" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$can_skip" = "true" ]; then
|
||||
echo "Found valid approval from authorized reviewer, skipping IXUCA CI"
|
||||
else
|
||||
echo "No valid approval found, running full CI"
|
||||
fi
|
||||
|
||||
- name: Download Paddle.tar.gz and setup
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy \
|
||||
https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
||||
rm Paddle.tar.gz
|
||||
cd Paddle
|
||||
git config --global --add safe.directory "*"
|
||||
git config --global http.proxy http://oversea-website-proxy.aistudio.public:8888
|
||||
git config --global https.proxy http://oversea-website-proxy.aistudio.public:8888
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Fetching upstream develop branch"
|
||||
git fetch upstream develop
|
||||
echo "Rebasing test branch onto upstream/develop"
|
||||
git rebase upstream/develop || {
|
||||
echo "Rebase failed, attempting to continue with current state"
|
||||
git rebase --abort || true
|
||||
}
|
||||
echo "Latest 5 commits after rebase:"
|
||||
git log --oneline -5
|
||||
|
||||
- name: Get Paddle-iluvatar PR number
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
id: get-pcd-pr
|
||||
run: |
|
||||
# Get Paddle PR description
|
||||
PR_BODY=$(curl -s \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" \
|
||||
| python3 -c "import json,sys; data=json.load(sys.stdin); print(data.get('body', ''))")
|
||||
|
||||
# Extract [Paddle-iluvatar PR] XXXX from description
|
||||
PCD_PR_NUM=$(echo "$PR_BODY" | grep -oP '\[Paddle-iluvatar PR\]\s*\K\d+' || echo "")
|
||||
|
||||
if [ -n "$PCD_PR_NUM" ]; then
|
||||
echo "Found Paddle-iluvatar PR: $PCD_PR_NUM"
|
||||
echo "pcd_pr_num=$PCD_PR_NUM" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No Paddle-iluvatar PR found in PR description"
|
||||
echo "pcd_pr_num=" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Clone Paddle-iluvatar and setup
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
git config --global --get http.proxy
|
||||
git config --global --get https.proxy
|
||||
for i in 1 2 3; do
|
||||
git clone --depth=1 -b develop https://github.com/PaddlePaddle/Paddle-iluvatar.git && break
|
||||
echo "git clone attempt $i failed, retrying in 10s..."
|
||||
rm -rf Paddle-iluvatar
|
||||
sleep 10
|
||||
done
|
||||
cd Paddle-iluvatar
|
||||
|
||||
# If Paddle-iluvatar PR number exists, fetch and merge it
|
||||
PCD_PR_NUM="${{ steps.get-pcd-pr.outputs.pcd_pr_num }}"
|
||||
if [ -n "$PCD_PR_NUM" ]; then
|
||||
git config user.email "Paddle-CI@example.com"
|
||||
git config user.name "Paddle-CI-Bot"
|
||||
git fetch --unshallow origin
|
||||
echo "Fetching and merging Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
git fetch origin pull/$PCD_PR_NUM/head:pr-$PCD_PR_NUM
|
||||
git merge pr-$PCD_PR_NUM --no-edit
|
||||
echo "Successfully merged Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
fi
|
||||
|
||||
rm -rf Paddle
|
||||
mv ../Paddle .
|
||||
|
||||
- name: Build paddlepaddle-iluvatar
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
export PATH=/usr/local/corex/bin:$PATH
|
||||
cd Paddle-iluvatar
|
||||
apt-get install -y patchelf
|
||||
bash build_paddle.sh
|
||||
|
||||
- name: Install paddlepaddle-iluvatar
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
pip install Paddle-iluvatar/Paddle/build/python/dist/paddlepaddle_iluvatar*.whl
|
||||
retry_or_skip() {
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
echo "WARN: command failed, retrying in 20s: $*"
|
||||
sleep 20
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
echo "WARN: command failed again, skip install: $*"
|
||||
return 0
|
||||
}
|
||||
retry_or_skip pip install parameterized pyyaml
|
||||
|
||||
- name: Run tests
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
export FLAG_SKIP_FLOAT64=1
|
||||
cd Paddle-iluvatar/tests && bash run_test.sh
|
||||
|
||||
- name: Set result
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
id: set-result
|
||||
run: |
|
||||
echo "result=success" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
chmod -R 777 . || true
|
||||
|
||||
# test-fallback:
|
||||
# name: Build and Test (Fallback)
|
||||
# needs: [check-bypass, test-primary]
|
||||
# if: >-
|
||||
# ${{ always()
|
||||
# && needs.check-bypass.result == 'success'
|
||||
# && needs.check-bypass.outputs.can-skip != 'true'
|
||||
# && needs.test-primary.outputs.result != 'success' }}
|
||||
# runs-on:
|
||||
# group: test2
|
||||
# timeout-minutes: 120
|
||||
# env:
|
||||
# TASK: paddle-CI-${{ github.event.pull_request.number }}-ixuca
|
||||
|
||||
# steps:
|
||||
# - name: Check approval
|
||||
# id: check-approval
|
||||
# run: |
|
||||
# set -euo pipefail
|
||||
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
|
||||
# AUTHORIZED_REVIEWERS=("YqGe585" "yongqiangma")
|
||||
|
||||
# echo "Authorized reviewers: ${AUTHORIZED_REVIEWERS[*]}"
|
||||
# echo "Repository: ${{ github.repository }}"
|
||||
# echo "PR number: ${{ github.event.pull_request.number }}"
|
||||
|
||||
# response=""
|
||||
# for i in 1 2 3; do
|
||||
# _tmp=$(curl -s --fail --max-time 30 \
|
||||
# -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
# -H "Accept: application/vnd.github.v3+json" \
|
||||
# "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" 2>/dev/null) && { response="$_tmp"; break; }
|
||||
# echo "Attempt $i to fetch PR reviews failed, retrying..."
|
||||
# sleep 5
|
||||
# done
|
||||
|
||||
# echo "Response length: ${#response}"
|
||||
|
||||
# can_skip=$(python3 -c '
|
||||
# import json, sys
|
||||
|
||||
# def debug(msg):
|
||||
# print(msg, file=sys.stderr)
|
||||
|
||||
# debug("=== Review Check Debug Info ===")
|
||||
# debug(f"Authorized reviewers: {sys.argv[1:]}")
|
||||
# AUTHORIZED = set(sys.argv[1:])
|
||||
|
||||
# try:
|
||||
# input_data = sys.stdin.read()
|
||||
# debug(f"Raw input data length: {len(input_data)}")
|
||||
# data = json.loads(input_data)
|
||||
# debug(f"Parsed {len(data)} reviews")
|
||||
# except Exception as e:
|
||||
# debug(f"JSON parse error: {str(e)}")
|
||||
# print("false")
|
||||
# sys.exit(0)
|
||||
|
||||
# latest_state = {}
|
||||
# debug("\n=== Review States ===")
|
||||
# for r in data:
|
||||
# user = r.get("user", {}).get("login")
|
||||
# if not user:
|
||||
# continue
|
||||
# state = r.get("state")
|
||||
# latest_state[user] = state
|
||||
# commit_id = r.get("commit_id") or ""
|
||||
# debug(f"- {user}: {state} (commit: {commit_id[:8]})")
|
||||
|
||||
# debug("\n=== Final Check ===")
|
||||
# for user, state in latest_state.items():
|
||||
# if user in AUTHORIZED and state == "APPROVED":
|
||||
# debug(f"Found APPROVED review from authorized user: {user}")
|
||||
# print("true")
|
||||
# sys.exit(0)
|
||||
|
||||
# debug("No APPROVED review found from authorized users")
|
||||
# print("false")
|
||||
# ' "${AUTHORIZED_REVIEWERS[@]}" <<< "$response")
|
||||
|
||||
# echo "can-skip=$can_skip" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# if [ "$can_skip" = "true" ]; then
|
||||
# echo "Found valid approval from authorized reviewer, skipping IXUCA CI"
|
||||
# else
|
||||
# echo "No valid approval found, running full CI"
|
||||
# fi
|
||||
|
||||
# - name: Download Paddle.tar.gz and setup
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# timeout-minutes: 30
|
||||
# run: |
|
||||
# set -e
|
||||
# echo "Downloading Paddle.tar.gz"
|
||||
# wget -q --tries=5 --no-proxy \
|
||||
# https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
# echo "Extracting Paddle.tar.gz"
|
||||
# tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
||||
# rm Paddle.tar.gz
|
||||
# cd Paddle
|
||||
# git config --global --add safe.directory "*"
|
||||
# git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
# git config pull.rebase false
|
||||
# git checkout test
|
||||
# echo "Fetching upstream develop branch"
|
||||
# git config --global --unset-all http.proxy || true
|
||||
# git config --global --unset-all https.proxy || true
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# git config --global http.proxy "$http_proxy"
|
||||
# git config --global https.proxy "$https_proxy"
|
||||
# git fetch upstream develop
|
||||
# echo "Rebasing test branch onto upstream/develop"
|
||||
# git rebase upstream/develop || {
|
||||
# echo "Rebase failed, attempting to continue with current state"
|
||||
# git rebase --abort || true
|
||||
# }
|
||||
# echo "Latest 5 commits after rebase:"
|
||||
# git log --oneline -5
|
||||
|
||||
# - name: Get Paddle-iluvatar PR number
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# id: get-pcd-pr
|
||||
# run: |
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
|
||||
# # Get Paddle PR description
|
||||
# PR_BODY=$(curl -s \
|
||||
# -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
# -H "Accept: application/vnd.github.v3+json" \
|
||||
# "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" \
|
||||
# | python3 -c "import json,sys; data=json.load(sys.stdin); print(data.get('body', ''))")
|
||||
|
||||
# # Extract [Paddle-iluvatar PR] XXXX from description
|
||||
# PCD_PR_NUM=$(echo "$PR_BODY" | grep -oP '\[Paddle-iluvatar PR\]\s*\K\d+' || echo "")
|
||||
|
||||
# if [ -n "$PCD_PR_NUM" ]; then
|
||||
# echo "Found Paddle-iluvatar PR: $PCD_PR_NUM"
|
||||
# echo "pcd_pr_num=$PCD_PR_NUM" >> "$GITHUB_OUTPUT"
|
||||
# else
|
||||
# echo "No Paddle-iluvatar PR found in PR description"
|
||||
# echo "pcd_pr_num=" >> "$GITHUB_OUTPUT"
|
||||
# fi
|
||||
|
||||
# - name: Clone Paddle-iluvatar and setup
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# run: |
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
|
||||
# rm -rf Paddle-iluvatar
|
||||
# git clone --depth=1 -b develop https://github.com/PaddlePaddle/Paddle-iluvatar.git
|
||||
# cd Paddle-iluvatar
|
||||
|
||||
# # If Paddle-iluvatar PR number exists, fetch and merge it
|
||||
# PCD_PR_NUM="${{ steps.get-pcd-pr.outputs.pcd_pr_num }}"
|
||||
# if [ -n "$PCD_PR_NUM" ]; then
|
||||
# git config user.email "Paddle-CI@example.com"
|
||||
# git config user.name "Paddle-CI-Bot"
|
||||
# git fetch --unshallow origin
|
||||
# echo "Fetching and merging Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
# git fetch origin pull/$PCD_PR_NUM/head:pr-$PCD_PR_NUM
|
||||
# git merge pr-$PCD_PR_NUM --no-edit
|
||||
# echo "Successfully merged Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
# fi
|
||||
|
||||
# rm -rf Paddle
|
||||
# mv ../Paddle .
|
||||
|
||||
# - name: Start docker container
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# run: |
|
||||
# container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
# echo "container_name=${container_name}" >> $GITHUB_ENV
|
||||
# docker run --network host --privileged --cap-add=ALL \
|
||||
# --pid=host --shm-size=128G -d -t --name ${container_name} \
|
||||
# -v /usr/src:/usr/src -v /lib/modules:/lib/modules \
|
||||
# -v /dev:/dev -v /home:/home -v /data:/data \
|
||||
# -v ${{ github.workspace }}/Paddle-iluvatar:/paddle \
|
||||
# -e LD_LIBRARY_PATH=/usr/local/corex/lib \
|
||||
# -e LIBRARY_PATH=/usr/local/corex/lib \
|
||||
# -v /root/.cache:/root/.cache \
|
||||
# -v /root/.cache:/root/.ccache \
|
||||
# -e BRANCH -e PR_ID -e COMMIT_ID \
|
||||
# -w /paddle \
|
||||
# ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:3.3.0 /bin/bash
|
||||
|
||||
# - name: Build and test
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# run: |
|
||||
# docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
# retry_or_skip() {
|
||||
# if "$@"; then
|
||||
# return 0
|
||||
# fi
|
||||
# echo "WARN: command failed, retrying in 20s: $*"
|
||||
# sleep 20
|
||||
# if "$@"; then
|
||||
# return 0
|
||||
# fi
|
||||
# echo "WARN: command failed again, skip install: $*"
|
||||
# return 0
|
||||
# }
|
||||
# export PATH=/usr/local/corex/bin:$PATH
|
||||
# apt-get install -y patchelf
|
||||
# bash build_paddle.sh
|
||||
# pip install Paddle/build/python/dist/paddlepaddle_iluvatar*.whl
|
||||
# retry_or_skip pip install parameterized pyyaml
|
||||
# export FLAG_SKIP_FLOAT64=1
|
||||
# cd tests && bash run_test.sh
|
||||
# '
|
||||
|
||||
# - name: Cleanup
|
||||
# if: always()
|
||||
# run: |
|
||||
# docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*' || true
|
||||
# docker stop ${{ env.container_name }} || true
|
||||
# docker rm ${{ env.container_name }} || true
|
||||
@@ -0,0 +1,197 @@
|
||||
name: Linux-NPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
docker_npu_image:
|
||||
type: string
|
||||
required: true
|
||||
|
||||
env:
|
||||
docker_image: ${{ inputs.docker_npu_image }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
ci_scripts_runner: ${{ github.workspace }}/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-npu
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: npu
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "npu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: NPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker run -i --rm \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e BRANCH \
|
||||
-w /paddle $docker_image /bin/bash -c 'rm -rf * .[^.]*'
|
||||
source ~/.bashrc
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
||||
rm Paddle.tar.gz
|
||||
cd Paddle
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
source ${{ github.workspace }}/Paddle/ci/utils.sh
|
||||
determine_npu_runner ${runner_name}
|
||||
echo no_proxy="localhost,127.0.0.1,localaddress,.bj.bcebos.com,.localdomain.com,.cdn.bcebos.com,.baidu.com,.bcebos.com" >> ${{ github.env }}
|
||||
|
||||
- name: Clone PaddleCustomDevice repository
|
||||
run: |
|
||||
source ~/.bashrc
|
||||
git clone --depth=1000 -b develop https://github.com/PaddlePaddle/PaddleCustomDevice.git
|
||||
cd PaddleCustomDevice
|
||||
cp -r ../Paddle .
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_COVERAGE: "OFF"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PYTHON_VERSION: "3.10"
|
||||
no_proxy: bcebos.com
|
||||
USE_910B: 1
|
||||
FLAGS_set_to_1d: "False"
|
||||
NVIDIA_TF32_OVERRIDE: 0
|
||||
paddle_submodule: "ON"
|
||||
MC2: 1
|
||||
HCCL_OP_BASE_FFTS_MODE_ENABLE: "TRUE"
|
||||
PADDLE_XCCL_BACKEND: npu
|
||||
HCCL_SOCKET_IFNAME: =xgbe0
|
||||
FLAGS_eager_communication_connection: 1
|
||||
FLAGS_use_stride_kernel: 0
|
||||
FLAGS_allocator_strategy: naive_best_fit
|
||||
FLAGS_npu_storage_format: 0
|
||||
TEST_IMPORTANT: "ON"
|
||||
PADDLE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
echo ${ASCEND_RT_VISIBLE_DEVICES}
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --privileged --pids-limit 409600 --shm-size=128G -d -t \
|
||||
--cap-add=SYS_PTRACE --security-opt seccomp=unconfined --name ${container_name} \
|
||||
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/dcmi:/usr/local/dcmi \
|
||||
-v /ssd2/workspace/npu-dev/.cache:/root/.cache \
|
||||
-v /ssd2/workspace/npu-dev/.ccache:/root/.ccache \
|
||||
-v $home_dir/actions-runner:$home_dir/actions-runner \
|
||||
-v ${{ github.workspace }}/PaddleCustomDevice:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e PADDLE_VERSION \
|
||||
-e ASCEND_RT_VISIBLE_DEVICES \
|
||||
-e WITH_COVERAGE \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e AGILE_PULL_ID \
|
||||
-e AGILE_REVISION \
|
||||
-e GIT_PR_ID \
|
||||
-e PYTHON_VERSION \
|
||||
-e no_proxy \
|
||||
-e USE_910B \
|
||||
-e FLAGS_set_to_1d \
|
||||
-e NVIDIA_TF32_OVERRIDE \
|
||||
-e paddle_submodule \
|
||||
-e MC2 \
|
||||
-e HCCL_OP_BASE_FFTS_MODE_ENABLE \
|
||||
-e PADDLE_XCCL_BACKEND \
|
||||
-e HCCL_SOCKET_IFNAME \
|
||||
-e FLAGS_eager_communication_connection \
|
||||
-e FLAGS_use_stride_kernel \
|
||||
-e FLAGS_allocator_strategy \
|
||||
-e FLAGS_npu_storage_format \
|
||||
-e TEST_IMPORTANT \
|
||||
-e PADDLE_BRANCH \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Install Paddle-CPU
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -x
|
||||
source ~/.bashrc
|
||||
wget -q https://sys-p0.bj.bcebos.com/libstdc%2B%2B6_13.1.0-8ubuntu1_20.04.2_amd64.deb
|
||||
dpkg -i *deb
|
||||
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/cpu_whl/${PR_ID}/${COMMIT_ID}/paddlepaddle-0.0.0-cp310-cp310-linux_x86_64.whl --no-check-certificate
|
||||
PATH=/usr/local/bin:${PATH}
|
||||
echo "export PATH=$PATH" >> ~/.bashrc
|
||||
ln -sf $(which python3.10) /usr/local/bin/python
|
||||
ln -sf $(which pip3.10) /usr/local/bin/pip
|
||||
ln -sf $(which python3.10) /usr/bin/python
|
||||
ln -sf $(which pip3.10) /usr/bin/pip
|
||||
echo "::group::Install Paddle"
|
||||
pip install paddlepaddle-*.whl && rm -rf paddlepaddle-*.whl
|
||||
python -c "import paddle; print(paddle.__version__)"
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
echo "::endgroup::"
|
||||
'
|
||||
|
||||
- name: Build and test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
set -x
|
||||
echo "::group::Install dependencies"
|
||||
python -m pip install PyGithub
|
||||
python -m pip install wheel
|
||||
echo "::endgroup::"
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
pip install -U numpy==1.26.4
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
git config --global --add safe.directory ${work_dir}/Paddle
|
||||
cd Paddle
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
cd ..
|
||||
bash backends/npu/tools/pr_ci_npu.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
@@ -0,0 +1,283 @@
|
||||
name: Linux-XPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_xpu_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
docker_image: ${{ inputs.docker_xpu_image }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
ci_scripts_runner: ${{ github.workspace }}/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: xpu
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "xpu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-xpu_build
|
||||
runs-on:
|
||||
group: XPU-build
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_XPU: "ON"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
WITH_AVX: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_TESTING: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
XPU_VISIBLE_DEVICES: "0,1"
|
||||
WITH_XPU_BKCL: "ON"
|
||||
WITH_XPU_FFT: "ON"
|
||||
WITH_XPU_XRE5: "ON"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
IF_KUNLUN3: "OFF"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --ulimit nofile=102400:102400 -d -t --name ${container_name} \
|
||||
-v $home_dir/.cache:/root/.cache \
|
||||
-v $home_dir/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_XPU \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e XPU_VISIBLE_DEVICES \
|
||||
-e WITH_XPU_BKCL \
|
||||
-e WITH_XPU_FFT \
|
||||
-e WITH_XPU_XRE5 \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e IF_KUNLUN3 \
|
||||
-e no_proxy \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ulimit -n 102400
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop or target branch"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Run build
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
apt install zstd -y
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
bash ${ci_scripts}/compress_build.sh
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload build.tar.gz and paddle_whl to bos
|
||||
env:
|
||||
AK: paddle
|
||||
SK: paddle
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle_xpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
cd ..
|
||||
tar --use-compress-program="pzstd -5" -cf build.tar.gz paddle
|
||||
# source /home/opt/deck/1.0/etc/bashrc
|
||||
echo "Uploading build.tar.gz"
|
||||
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading paddle_whl"
|
||||
cp /paddle/dist/${{ env.paddle_whl }} .
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${container_name}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: build
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-xpu_test
|
||||
runs-on:
|
||||
group: Kunlun
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
||||
source utils.sh
|
||||
determine_kunlun_runner ${runner_name}
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_XPU: "ON"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
WITH_AVX: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_TESTING: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
XPU_VISIBLE_DEVICES: ${{ env.CUDA_VISIBLE_DEVICES }}
|
||||
CUDA_VISIBLE_DEVICES: ${{ env.CUDA_VISIBLE_DEVICES }}
|
||||
WITH_XPU_BKCL: "ON"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
IF_KUNLUN3: "OFF"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
FLAGS_use_stride_kernel: "0"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --ulimit nofile=102400 --ulimit core=-1 --shm-size=32g -d -t --name ${container_name} \
|
||||
-v /ssd1/cibuild/.cache:/root/.cache \
|
||||
-v /ssd1/cibuild/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-v /ssd1/cibuild/tmp:/tmp \
|
||||
--device ${XPU_CODE_1} \
|
||||
--device ${XPU_CODE_2} \
|
||||
--device /dev/xpuctrl \
|
||||
--shm-size=32g \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_XPU \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e XPU_VISIBLE_DEVICES \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e WITH_XPU_BKCL \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e IF_KUNLUN3 \
|
||||
-e no_proxy \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e FLAGS_use_stride_kernel \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download build.tar.gz
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/xpu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
|
||||
tar --use-compress-program="pzstd" -xf build.tar.gz --strip-components=1
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Run test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/kunlun_test.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${container_name}
|
||||
@@ -0,0 +1,236 @@
|
||||
name: Linux-build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
is_pr:
|
||||
type: string
|
||||
required: false
|
||||
default: "true"
|
||||
clone-can-skip:
|
||||
default: "false"
|
||||
required: false
|
||||
type: string
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether to skip the job"
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-build
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: build
|
||||
CFS_DIR: /home/data/cfs
|
||||
is_pr: ${{ inputs.is_pr }}
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: build
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_COVERAGE: "OFF"
|
||||
PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
PRECISION_TEST: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /home/data/cfs/.cache/build
|
||||
WITH_INCREMENTAL_COVERAGE: "OFF"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_STATSLOG: /paddle/build/.stats.log
|
||||
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
||||
CACHE_DIR: /home/data/cfs/.cache/build
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e is_pr \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_COVERAGE \
|
||||
-e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e PRECISION_TEST \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_INCREMENTAL_COVERAGE \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_SLOPPINESS \
|
||||
-e CCACHE_STATSLOG \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-build/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
'
|
||||
|
||||
- name: Merge target branch
|
||||
if: ${{ inputs.is_pr == 'true' }}
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
if: needs.check-bypass.outputs.can-skip != 'true'
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p /root/.cache/build
|
||||
mkdir -p /root/.ccache/build
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Check sequence op
|
||||
if: needs.check-bypass.outputs.can-skip != 'true'
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${ci_scripts}/utils.sh; init
|
||||
bash ${work_dir}/tools/check_sequence_op.sh
|
||||
clean_build_files
|
||||
'
|
||||
|
||||
- name: Upload paddle_whl
|
||||
if: needs.check-bypass.outputs.can-skip != 'true'
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
bash ${ci_scripts}/compress_build.sh
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
cd /paddle/build/paddle
|
||||
ls | grep -v cinn | xargs rm -rf
|
||||
rm -rf /paddle/build/third_party
|
||||
rm -rf /paddle/build/dist
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf build.tar.gz paddle
|
||||
echo "Uploading build.tar.gz to bos"
|
||||
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading build.tar.gz to cfs"
|
||||
mkdir -p ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}
|
||||
mv build.tar.gz ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID} && echo success
|
||||
mv /paddle/${{ env.paddle_whl }} .
|
||||
echo "Uploading paddle_whl to bos"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && always() }}
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,106 @@
|
||||
name: Mac-CPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: "${{ github.workspace }}/Paddle"
|
||||
MACOSX_DEPLOYMENT_TARGET: 10.11
|
||||
PADDLE_DEV_NAME: "paddlepaddle/paddle:dev"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
WITH_DEB: "OFF"
|
||||
RUN_TEST: "OFF"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_FLUID_ONLY: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
INSTALL_PREFIX: "/Users/paddle/CI_install_path"
|
||||
PYTHON_ABI: "cp310-cp310"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_AVX: "OFF"
|
||||
WITH_ARM: "ON"
|
||||
PROC_RUN: 8
|
||||
WITH_CACHE: "ON"
|
||||
NIGHTLY_MODE: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PRECISION_TEST: "OFF"
|
||||
CI_SKIP_CPP_TEST: "OFF"
|
||||
WITH_ONNXRUNTIME: "OFF"
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: "Ninja"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: mac
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
if: ${{ inputs.clone-can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: Mac-CI
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Check environment
|
||||
run: |
|
||||
set -x
|
||||
cd ${work_dir}
|
||||
[ -d "${work_dir}/Paddle" ] && rm -rf ${work_dir}/Paddle
|
||||
[ -f "${work_dir}/Paddle.tar.gz" ] && rm -rf ${work_dir}/Paddle.tar.gz
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
tar xf Paddle.tar.gz && cd Paddle
|
||||
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
|
||||
- name: Build with mac python3.10
|
||||
run: |
|
||||
set -x
|
||||
cd ${work_dir}/Paddle
|
||||
source ~/.zshrc
|
||||
bash -x ${work_dir}/Paddle/ci/run_setup.sh bdist_wheel ${parallel_number:-""}
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
|
||||
- name: Test with mac python3.10
|
||||
run: |
|
||||
set -ex
|
||||
cd ${work_dir}/Paddle
|
||||
pip3.10 install -r ${work_dir}/Paddle/python/unittest_py/requirements.txt
|
||||
pip3.10 install -U ${work_dir}/Paddle/dist/*whl
|
||||
bash -x ${work_dir}/Paddle/ci/run_mac_test.sh ${PYTHON_ABI:-""} ${PROC_RUN:-""}
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
@@ -0,0 +1,168 @@
|
||||
name: Model-benchmark
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /workspace/paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-Model-benchmark
|
||||
CI_name: model-benchmark
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "model-benchmark"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
model-benchmark:
|
||||
name: Benchmark test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: model-benchmark
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PRID: ${{ github.event.pull_request.number }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
CUDA_ARCH_NAME: Volta
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
benchmark_ci_cache: /home/data/cfs/.cache/model_benchmark/new_benchmark
|
||||
ROOT_DIR: /workspace
|
||||
bos_path: modelBK_CI
|
||||
CACHE_DIR: /root/.cache/model_benchmark
|
||||
work_dir: /workspace/paddle
|
||||
AGILE_COMMENTS: '[{"Commit":"${{ github.event.pull_request.head.sha }}","author":"${{ github.event.pull_request.user.login }}"}]'
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/workspace \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e BRANCH \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e AGILE_PULL_ID \
|
||||
-e GIT_PR_ID \
|
||||
-e PRID \
|
||||
-e AGILE_REVISION \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e benchmark_ci_cache \
|
||||
-e bos_path \
|
||||
-e ROOT_DIR \
|
||||
-e CACHE_DIR \
|
||||
-e work_dir \
|
||||
-e AGILE_COMMENTS \
|
||||
-w /workspace --network host ${docker_image}
|
||||
|
||||
- name: Prepare environment and download paddle
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo $AGILE_COMMENTS
|
||||
DEVICE_ID=$(cat /sys/class/dmi/id/product_uuid |awk -F "-" "{print \$5}")
|
||||
echo "export DEVICE_ID=$DEVICE_ID" >> ~/.bashrc
|
||||
echo $DEVICE_ID
|
||||
echo "Downloading build.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
|
||||
echo "Extracting build.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz
|
||||
rm build.tar.gz
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
cd paddle && git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
check_paddle ${{ github.env }}
|
||||
'
|
||||
|
||||
- name: Download model_benchmark_ci
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
wget_model_benchmark_ci
|
||||
'
|
||||
|
||||
- name: Download paddlescope
|
||||
timeout-minutes: 30
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
get_paddlescope_tar
|
||||
'
|
||||
|
||||
- name: Check skip
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
check_skip ${{ github.env }}
|
||||
'
|
||||
|
||||
- name: Check model benchmark
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
export EXCODE=0
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
check_model_benchmark
|
||||
if [[ $EXCODE -eq 0 ]];then
|
||||
echo "Congratulations! Your PR passed the model benchmark ci."
|
||||
elif [[ $EXCODE -eq 10 ]];then
|
||||
echo "Sorry, some models failed to run, Please check the result_html.html."
|
||||
elif [[ $EXCODE -eq 11 ]];then
|
||||
echo "Sorry, some models have abnormal performance, Please check the result_html.html."
|
||||
else
|
||||
echo "Sorry, There are environmental issues, please contact QA(guomengmeng01) for exemption."
|
||||
fi
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,318 @@
|
||||
name: PR-CI-SOT
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_sot_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-sot
|
||||
ci_scripts: /paddle/ci
|
||||
CI_name: sot
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "sot"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-and-test:
|
||||
name: Build and Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: run container
|
||||
env:
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_MKL: "OFF"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
PREC_SUFFIX: .py3
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PROC_RUN: 12
|
||||
FLAGS_enable_eager_mode: 1
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: Ninja
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
WITH_AVX: "OFF"
|
||||
CACHE_DIR: "/root/.cache/sot"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
run: |
|
||||
set -x
|
||||
container_name=${TASK}-$(date +%s)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_sot_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:${{ github.workspace }}/../../.." \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e PREC_SUFFIX \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PROC_RUN \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e WITH_TENSORRT \
|
||||
-e GENERATOR \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e GITHUB_ENV \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Determine sot ci trigger
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/determine_sot_ci_trigger.sh
|
||||
determine_excode=$?
|
||||
echo "determine_excode=$determine_excode" >> ${{ github.env }}
|
||||
'
|
||||
|
||||
- name: CMake Pre-download
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.10
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.10
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.10
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.10
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.11
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.11
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.11
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.11
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.12
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.12
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.12
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.12
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.13
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.13
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.13
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.13
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.14
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.14
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.14
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.14
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,140 @@
|
||||
name: Slice
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
slice-check:
|
||||
type: string
|
||||
required: false
|
||||
SLICE_TEST_MODE:
|
||||
type: string
|
||||
required: false
|
||||
default: "test_ci"
|
||||
SLICE_BENCHMARK_FRAMEWORKS:
|
||||
type: string
|
||||
required: false
|
||||
default: "paddle"
|
||||
MANUALLY_PR_ID:
|
||||
type: string
|
||||
required: false
|
||||
MANUALLY_COMMIT_ID:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-slice
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: slice
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass-slice.yml
|
||||
with:
|
||||
workflow-name: "slice"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
slice:
|
||||
name: Slice test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: slice
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
SLICE_TEST_MODE: ${{ inputs.SLICE_TEST_MODE }}
|
||||
SLICE_BENCHMARK_FRAMEWORKS: ${{ inputs.SLICE_BENCHMARK_FRAMEWORKS }}
|
||||
run: |
|
||||
container_name="api_benchmark_ci_${RUN_ID}"
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e core_index \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e RUN_ID \
|
||||
-e wheel_link \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e SLICE_TEST_MODE \
|
||||
-e SLICE_BENCHMARK_FRAMEWORKS \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading PaddleTest"
|
||||
wget -q https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-proxy
|
||||
echo "Extracting PaddleTest"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xzf PaddleTest.tar.gz && cd PaddleTest
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
'
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p ${{ github.workspace }}/../../../.cache/pip
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python3.10 -m pip config set global.cache-dir ${{ github.workspace }}/../../../.cache/pip
|
||||
if [[ "${{ inputs.SLICE_BENCHMARK_FRAMEWORKS }}" == "torch" ]];then
|
||||
python3.10 -m pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu118
|
||||
else
|
||||
if [[ "${{ inputs.MANUALLY_PR_ID }}" == "" ]]; then
|
||||
python3.10 -m pip install $wheel_link
|
||||
else
|
||||
python3.10 -m pip install https://paddle-github-action.bj.bcebos.com/PR/build/${{ inputs.MANUALLY_PR_ID }}/${{ inputs.MANUALLY_COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
fi
|
||||
fi
|
||||
python3.10 -m pip install -r PaddleTest/framework/e2e/api_benchmark/requirement.txt
|
||||
cd PaddleTest/framework/slice_benchmark
|
||||
cp ${{ github.workspace }}/../../../apibm_config.yml .
|
||||
bash run.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,172 @@
|
||||
name: Static-check
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-static-check
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: static-check
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: static-check
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build:
|
||||
name: Test
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Debug - Print inputs
|
||||
run: |
|
||||
echo "=== Static-Check Debug Info ==="
|
||||
echo "inputs.can-skip: ${{ inputs.can-skip }}"
|
||||
echo "needs.check-bypass.outputs.can-skip: ${{ needs.check-bypass.outputs.can-skip }}"
|
||||
echo "needs.check-bypass.result: ${{ needs.check-bypass.result }}"
|
||||
echo "==============================="
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
WITH_COVERAGE: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_MKL: "ON"
|
||||
SAMPLE_CODE_EXEC_THREADS: 1
|
||||
CACHE_DIR: /home/data/cfs/.cache/static-check
|
||||
CCACHE_DIR: /home/data/cfs/.ccache/static-check
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-v /root/.cache/paddle/dataset/ESC-50-master \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e WITH_MKL \
|
||||
-e SAMPLE_CODE_EXEC_THREADS \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_COVERAGE \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
git checkout test
|
||||
'
|
||||
|
||||
- name: Static check
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/static_check.sh
|
||||
'
|
||||
|
||||
- name: Check api approval
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ~/.bashrc
|
||||
unset GREP_OPTIONS
|
||||
bash ${work_dir}/tools/check_api_approvals.sh;approval_error=$?
|
||||
if [ "$approval_error" != 0 ];then
|
||||
exit 6
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Check GPU Kernel approval
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${work_dir}/tools/check_gpu_kernel_approval.sh;approval_error=$?
|
||||
if [ "$approval_error" != 0 ];then
|
||||
exit 6
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,114 @@
|
||||
name: Windows-GPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
ACTION_DIR: ${{ github.workspace }}\..\..\..
|
||||
WORKSPACE_DIR: ${{ github.workspace }}\..
|
||||
cache_dir: C:\action-cache
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "win-gpu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: win-gpu
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
NIGHTLY_MODE: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_TPCACHE: "OFF"
|
||||
WITH_SCCACHE: "ON"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
WITH_TESTING: "ON"
|
||||
PRECISION_TEST: "OFF"
|
||||
PYTHON_ROOT: C:\Python310
|
||||
vcvars64_dir: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat'
|
||||
CUDA_TOOLKIT_ROOT_DIR: 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.0'
|
||||
TENSORRT_ROOT: D:\TensorRT
|
||||
CTEST_PARALLEL_LEVEL: 1
|
||||
GENERATOR: "Ninja"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
MSVC_STATIC_CRT: "OFF"
|
||||
WITH_MKL: "ON"
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
ON_INFER: "ON"
|
||||
WITH_TENSORRT: "ON"
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
WIN_UNITTEST_LEVEL: 0
|
||||
CUDA_ARCH_NAME: "Auto"
|
||||
steps:
|
||||
- name: Download Paddle and clean env
|
||||
working-directory: ..
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
whoami
|
||||
cd
|
||||
rmdir /s/q %WORKSPACE_DIR%\Paddle
|
||||
echo "path=%path%;C:\Program Files\NVIDIA Corporation\NVSMI" >> %GITHUB_ENV%
|
||||
python -m pip install wget
|
||||
python -c "import wget;wget.download('https://paddle-github-action.bj.bcebos.com/windows/PR/%PR_ID%/%COMMIT_ID%/Paddle.tar.zst')"
|
||||
zstd -d Paddle.tar.zst && tar -xf Paddle.tar
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
call %ci_scripts%\clean_env.bat
|
||||
|
||||
- name: Config env
|
||||
run: |
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\config_env.bat
|
||||
|
||||
- name: Build paddle
|
||||
run: |
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\build.bat
|
||||
|
||||
- name: Test whl package
|
||||
run: |
|
||||
call %ci_scripts%\test_whl_package.bat
|
||||
|
||||
- name: Test paddle
|
||||
run: |
|
||||
call %ci_scripts%\test.bat
|
||||
|
||||
- name: Test fluid library
|
||||
run: |
|
||||
call %ci_scripts%\test_fluid_library.bat
|
||||
|
||||
- name: Clean up dir
|
||||
if: always()
|
||||
run: |
|
||||
call %ci_scripts%\clean_env.bat
|
||||
if "%WITH_TESTING%"=="ON" (
|
||||
for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
|
||||
) || ver >nul
|
||||
@@ -0,0 +1,112 @@
|
||||
name: Windows-GPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
ACTION_DIR: ${{ github.workspace }}\..\..\..
|
||||
WORKSPACE_DIR: ${{ github.workspace }}\..
|
||||
cache_dir: C:\action-cache
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "win-inference"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: win-infer
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
NIGHTLY_MODE: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_TPCACHE: "OFF"
|
||||
WITH_SCCACHE: "ON"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
WITH_TESTING: "ON"
|
||||
PRECISION_TEST: "OFF"
|
||||
PYTHON_ROOT: C:\Python310
|
||||
vcvars64_dir: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat'
|
||||
CUDA_TOOLKIT_ROOT_DIR: 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7'
|
||||
TENSORRT_ROOT: D:/TensorRT-8.0.1.6
|
||||
CTEST_PARALLEL_LEVEL: 1
|
||||
GENERATOR: "Ninja"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WITH_MKL: "ON"
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
MSVC_STATIC_CRT: "ON"
|
||||
ON_INFER: "ON"
|
||||
WITH_TENSORRT: "ON"
|
||||
WITH_INFERENCE_API_TEST: "ON"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WIN_UNITTEST_LEVEL: 2
|
||||
CUDA_ARCH_NAME: "Auto"
|
||||
steps:
|
||||
- name: Download Paddle and clean env
|
||||
working-directory: ..
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
whoami
|
||||
cd
|
||||
rmdir /s/q %WORKSPACE_DIR%\Paddle
|
||||
echo "path=%path%;C:\Program Files\NVIDIA Corporation\NVSMI" >> %GITHUB_ENV%
|
||||
python -m pip install wget
|
||||
python -c "import wget;wget.download('https://paddle-github-action.bj.bcebos.com/windows/PR/%PR_ID%/%COMMIT_ID%/Paddle.tar.zst')"
|
||||
zstd -d Paddle.tar.zst && tar -xf Paddle.tar
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
call %ci_scripts%\clean_env.bat
|
||||
|
||||
- name: Config env
|
||||
run: |
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\config_env.bat
|
||||
|
||||
- name: Build paddle
|
||||
run: |
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
call %ci_scripts%\pre_download.bat
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\build.bat
|
||||
|
||||
- name: Test whl package
|
||||
run: |
|
||||
call %ci_scripts%\test_whl_package.bat
|
||||
|
||||
- name: Test paddle
|
||||
run: |
|
||||
call %ci_scripts%\test.bat
|
||||
|
||||
- name: Clean up dir
|
||||
if: always()
|
||||
run: |
|
||||
call %ci_scripts%\clean_env.bat
|
||||
if "%WITH_TESTING%"=="ON" (
|
||||
for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
|
||||
) || ver >nul
|
||||
@@ -0,0 +1,104 @@
|
||||
name: Windows-GPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
ACTION_DIR: ${{ github.workspace }}\..\..\..
|
||||
WORKSPACE_DIR: ${{ github.workspace }}\..
|
||||
cache_dir: C:\action-cache
|
||||
SCCACHE_ROOT: C:\sccache
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "win-openblas"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: win-openblas
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
NIGHTLY_MODE: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_CACHE: "OFF"
|
||||
WITH_TPCACHE: "OFF"
|
||||
WITH_SCCACHE: "ON"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
FLAGS_enable_eager_mode: 1
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
WITH_TESTING: "ON"
|
||||
PRECISION_TEST: "OFF"
|
||||
PYTHON_ROOT: C:\Python310
|
||||
CTEST_PARALLEL_LEVEL: 1
|
||||
GENERATOR: "Ninja"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WITH_MKL: "OFF"
|
||||
WITH_GPU: "OFF"
|
||||
WITH_AVX: "OFF"
|
||||
MSVC_STATIC_CRT: "ON"
|
||||
ON_INFER: "ON"
|
||||
WIN_UNITTEST_LEVEL: 2
|
||||
CUDA_ARCH_NAME: "Auto"
|
||||
steps:
|
||||
- name: Download Paddle and clean env
|
||||
working-directory: ..
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
whoami
|
||||
cd
|
||||
rmdir /s/q %WORKSPACE_DIR%\Paddle
|
||||
python -m pip install wget
|
||||
python -c "import wget;wget.download('https://paddle-github-action.bj.bcebos.com/windows/PR/%PR_ID%/%COMMIT_ID%/Paddle.tar.zst')"
|
||||
zstd -d Paddle.tar.zst && tar -xf Paddle.tar
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
call %ci_scripts%\clean_env.bat
|
||||
|
||||
- name: Config env
|
||||
run: |
|
||||
call %ci_scripts%\config_env.bat
|
||||
|
||||
- name: Build paddle
|
||||
run: |
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
call %ci_scripts%\build.bat
|
||||
|
||||
- name: Test whl package
|
||||
run: |
|
||||
call %ci_scripts%\test_whl_package.bat
|
||||
|
||||
- name: Test paddle
|
||||
run: |
|
||||
call %ci_scripts%\test.bat
|
||||
|
||||
- name: Clean up dir
|
||||
if: always()
|
||||
run: |
|
||||
call %ci_scripts%\clean_env.bat
|
||||
if "%WITH_TESTING%"=="ON" (
|
||||
for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
|
||||
) || ver >nul
|
||||
@@ -0,0 +1,29 @@
|
||||
name: CI-Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel CI-Build for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel CI-build
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,29 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel CI for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel CI
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,29 @@
|
||||
name: Coverage
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel Coverage for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel Coverage
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,29 @@
|
||||
name: CI-Windows
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel CI-Windows for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel CI-Windows
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,51 @@
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
workflow-name:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
github-token:
|
||||
required: true
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether the workflow can be skipped."
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
CI_TEAM_MEMBERS: '["swgu98", "risemeup1", "xiaoguoguo626807", "luotao1"]'
|
||||
outputs:
|
||||
can-skip: ${{ steps.check-bypass.outputs.can-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- id: check-bypass
|
||||
name: Check Bypass
|
||||
uses: PFCCLab/ci-bypass@v2
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
non-pull-request-event-strategy: 'never-skipped'
|
||||
type: 'composite'
|
||||
composite-rule: |
|
||||
{
|
||||
"any": [
|
||||
{
|
||||
"type": "labeled",
|
||||
"label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
},
|
||||
{
|
||||
"type": "commented",
|
||||
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
workflow-name:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
github-token:
|
||||
required: true
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether the workflow can be skipped."
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
CI_TEAM_MEMBERS: '["swgu98", "risemeup1" , "XieYunshen", "luotao1", "SigureMo"]'
|
||||
outputs:
|
||||
can-skip: ${{ steps.check-bypass.outputs.can-skip }}
|
||||
timeout-minutes: 3
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- id: check-bypass
|
||||
name: Check Bypass
|
||||
uses: PFCCLab/ci-bypass@v2
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
non-pull-request-event-strategy: "never-skipped"
|
||||
type: "composite"
|
||||
composite-rule: |
|
||||
{
|
||||
"any": [
|
||||
{
|
||||
"type": "labeled",
|
||||
"label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
},
|
||||
{
|
||||
"type": "commented",
|
||||
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
name: Cherry Pick
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
branches: [develop]
|
||||
types: [closed, labeled]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
cherry-pick:
|
||||
if: >
|
||||
github.repository_owner == 'PaddlePaddle' &&
|
||||
github.event.pull_request.merged == true &&
|
||||
(
|
||||
github.event.action == 'labeled' ||
|
||||
contains(join(github.event.pull_request.labels.*.name, ' '), 'cherry-pick')
|
||||
)
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Cherry Pick
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.CHERRY_PICK_BOT_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
MERGE_COMMIT_SHA: ${{ github.event.pull_request.merge_commit_sha }}
|
||||
BOT_USERNAME: risemeup1111
|
||||
REPO_NAME: risemeup1111/Paddle
|
||||
run: |
|
||||
# Function to post comment
|
||||
post_comment() {
|
||||
gh pr comment "$PR_NUMBER" --body "$1"
|
||||
}
|
||||
|
||||
# Configure git for the original author
|
||||
echo "Fetching author info for $PR_AUTHOR..."
|
||||
AUTHOR_INFO=$(gh api "/users/$PR_AUTHOR" --jq '{email: .email, name: .name}')
|
||||
AUTHOR_EMAIL=$(echo "$AUTHOR_INFO" | jq -r '.email')
|
||||
AUTHOR_NAME=$(echo "$AUTHOR_INFO" | jq -r '.name')
|
||||
|
||||
if [ "$AUTHOR_EMAIL" = "null" ] || [ -z "$AUTHOR_EMAIL" ]; then
|
||||
AUTHOR_EMAIL="${PR_AUTHOR}@users.noreply.github.com"
|
||||
echo "Author email not found, using default: $AUTHOR_EMAIL"
|
||||
fi
|
||||
if [ "$AUTHOR_NAME" = "null" ] || [ -z "$AUTHOR_NAME" ]; then
|
||||
AUTHOR_NAME="${PR_AUTHOR}"
|
||||
echo "Author name not found, using username: $AUTHOR_NAME"
|
||||
fi
|
||||
|
||||
git config user.name "$AUTHOR_NAME"
|
||||
git config user.email "$AUTHOR_EMAIL"
|
||||
|
||||
# Capture current SHA to return to later
|
||||
ORIGINAL_HEAD_SHA=$(git rev-parse HEAD)
|
||||
|
||||
# Get labels
|
||||
LABELS=$(gh pr view "$PR_NUMBER" --json labels --jq '.labels[].name')
|
||||
|
||||
if [ -z "$LABELS" ]; then
|
||||
echo "No labels found."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Loop through labels
|
||||
while read -r label; do
|
||||
if [[ "$label" == cherry-pick:* ]]; then
|
||||
TARGET_BRANCH=$(echo "${label#cherry-pick:}" | xargs)
|
||||
|
||||
if [ -z "$TARGET_BRANCH" ]; then
|
||||
echo "Empty target branch for label '$label', skipping."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Processing cherry-pick to $TARGET_BRANCH"
|
||||
|
||||
# Check if target branch exists on remote
|
||||
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH"; then
|
||||
echo "Target branch $TARGET_BRANCH does not exist."
|
||||
post_comment "❌ Cherry-pick failed: Target branch \`$TARGET_BRANCH\` does not exist."
|
||||
continue
|
||||
fi
|
||||
|
||||
# Create a new branch for the cherry-pick
|
||||
NEW_BRANCH="cherry-pick/$PR_NUMBER/$TARGET_BRANCH"
|
||||
|
||||
# Clean up local branch if it exists (from previous run)
|
||||
if git show-ref --verify --quiet "refs/heads/$NEW_BRANCH"; then
|
||||
git branch -D "$NEW_BRANCH"
|
||||
fi
|
||||
|
||||
# Fetch the target branch and checkout a new branch from it
|
||||
git fetch origin "$TARGET_BRANCH"
|
||||
git checkout -b "$NEW_BRANCH" "origin/$TARGET_BRANCH"
|
||||
|
||||
# Cherry pick
|
||||
# Try standard cherry-pick first (for squash merges or single commits)
|
||||
if git cherry-pick "$MERGE_COMMIT_SHA"; then
|
||||
echo "Cherry-pick successful."
|
||||
else
|
||||
echo "Standard cherry-pick failed, trying with -m 1 (for merge commits)..."
|
||||
git cherry-pick --abort
|
||||
if git cherry-pick -m 1 "$MERGE_COMMIT_SHA"; then
|
||||
echo "Cherry-pick with -m 1 successful."
|
||||
else
|
||||
echo "Cherry-pick failed."
|
||||
git cherry-pick --abort
|
||||
post_comment "❌ Cherry-pick failed: Conflicts detected when cherry-picking to \`$TARGET_BRANCH\`. Please resolve manually."
|
||||
|
||||
# Cleanup
|
||||
git checkout "$ORIGINAL_HEAD_SHA"
|
||||
git branch -D "$NEW_BRANCH"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Push
|
||||
# Construct authenticated URL for the fork
|
||||
FORK_URL_AUTH="https://${BOT_USERNAME}:${GH_TOKEN}@github.com/${REPO_NAME}.git"
|
||||
|
||||
echo "Pushing to fork..."
|
||||
git push "$FORK_URL_AUTH" "$NEW_BRANCH" --force
|
||||
|
||||
# Create PR
|
||||
# If PR_TITLE starts with "[", don't insert an extra space.
|
||||
if [ "${PR_TITLE:0:1}" = "[" ]; then
|
||||
NEW_TITLE="[$TARGET_BRANCH]$PR_TITLE"
|
||||
else
|
||||
NEW_TITLE="[$TARGET_BRANCH] $PR_TITLE"
|
||||
fi
|
||||
|
||||
NEW_BODY="$PR_BODY
|
||||
|
||||
---
|
||||
|
||||
Cherry-pick of #$PR_NUMBER (authored by @${PR_AUTHOR}) to \`$TARGET_BRANCH\`.
|
||||
|
||||
devPR:https://github.com/PaddlePaddle/Paddle/pull/$PR_NUMBER <!-- For pass CI -->"
|
||||
|
||||
# Prepare head ref for PR creation (owner:branch)
|
||||
HEAD_REF="${BOT_USERNAME}:${NEW_BRANCH}"
|
||||
|
||||
# Check if PR already exists
|
||||
EXISTING_PR=$(gh pr list --base "$TARGET_BRANCH" --head "$NEW_BRANCH" --json url --jq '.[0].url')
|
||||
|
||||
if [ -n "$EXISTING_PR" ]; then
|
||||
echo "PR already exists: $EXISTING_PR"
|
||||
post_comment "ℹ️ Cherry-pick PR already exists: $EXISTING_PR"
|
||||
else
|
||||
# Create PR using gh CLI, ignoring errors because of "Resource not accessible" false positives
|
||||
gh pr create --base "$TARGET_BRANCH" --head "$HEAD_REF" --title "$NEW_TITLE" --body "$NEW_BODY" || true
|
||||
|
||||
# Wait a bit for eventual consistency
|
||||
sleep 2
|
||||
|
||||
# Search for the created PR
|
||||
CREATED_PR_URL=$(gh pr list --head "$NEW_BRANCH" --state all --json url --jq '.[0].url')
|
||||
|
||||
if [ -n "$CREATED_PR_URL" ]; then
|
||||
echo "Created PR: $CREATED_PR_URL"
|
||||
post_comment "✅ Cherry-pick successful! Created PR: $CREATED_PR_URL"
|
||||
|
||||
# Request review
|
||||
gh pr review-request "$CREATED_PR_URL" --reviewer "$PR_AUTHOR" || true
|
||||
else
|
||||
echo "Failed to create PR."
|
||||
post_comment "❌ Cherry-pick failed: Could not create PR to \`$TARGET_BRANCH\`."
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Cleanup for next loop
|
||||
git checkout "$ORIGINAL_HEAD_SHA"
|
||||
git branch -D "$NEW_BRANCH"
|
||||
fi
|
||||
done <<< "$LABELS"
|
||||
|
||||
- name: Remove Cherry Pick Labels
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO_NAME: ${{ github.repository }}
|
||||
run: |
|
||||
LABELS=$(gh pr view "$PR_NUMBER" --repo "$REPO_NAME" --json labels --jq '.labels[].name')
|
||||
if [ -z "$LABELS" ]; then
|
||||
exit 0
|
||||
fi
|
||||
while read -r label; do
|
||||
if [[ "$label" == cherry-pick:* ]]; then
|
||||
echo "Removing label: $label"
|
||||
gh pr edit "$PR_NUMBER" --repo "$REPO_NAME" --remove-label "$label"
|
||||
fi
|
||||
done <<< "$LABELS"
|
||||
@@ -0,0 +1,72 @@
|
||||
name: Cleanup ccache
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 18 * * *" # Daily at 02:00 UTC+8
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
TASK: cleanup-ccache-nightly
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
cleanup-ccache:
|
||||
name: Cleanup L2 ccache
|
||||
if: github.repository == 'PaddlePaddle/Paddle'
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
|
||||
steps:
|
||||
- name: Launch container
|
||||
env:
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
run: |
|
||||
set -x
|
||||
container_name=${TASK}-$(date +%s)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-coverage-test
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:${{ github.workspace }}/../../.." \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GITHUB_ENV \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Cleanup L2 ccache in CFS
|
||||
run: |
|
||||
docker exec -t ${container_name} bash -c '
|
||||
CFS_CCACHE_DIR="${CCACHE_SECONDARY_STORAGE#file://}"
|
||||
ccache --trim-dir ${CFS_CCACHE_DIR} --trim-max-size 500G
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,136 @@
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone_dir:
|
||||
required: false
|
||||
type: string
|
||||
default: "Paddle"
|
||||
task:
|
||||
required: false
|
||||
type: string
|
||||
default: "cpu"
|
||||
is_merge:
|
||||
required: false
|
||||
type: string
|
||||
default: "false"
|
||||
outputs:
|
||||
docker_cpu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_cpu_image }}
|
||||
|
||||
docker_sot_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_sot_image }}
|
||||
|
||||
docker_coverage_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_coverage_image }}
|
||||
|
||||
docker_build_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_build_image }}
|
||||
|
||||
docker_distribute_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_distribute_image }}
|
||||
|
||||
docker_xpu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_xpu_image }}
|
||||
|
||||
docker_npu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_npu_image }}
|
||||
|
||||
docker_dcu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_dcu_image }}
|
||||
|
||||
docker_doc_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_doc_image }}
|
||||
|
||||
jobs:
|
||||
build-docker-images:
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
name: Build docker
|
||||
runs-on:
|
||||
group: Docker-build
|
||||
outputs:
|
||||
docker_cpu_image: ${{ steps.build-docker-images.outputs.docker_cpu_image }}
|
||||
docker_sot_image: ${{ steps.build-docker-images.outputs.docker_sot_image }}
|
||||
docker_coverage_image: ${{ steps.build-docker-images.outputs.docker_coverage_image }}
|
||||
docker_build_image: ${{ steps.build-docker-images.outputs.docker_build_image }}
|
||||
docker_distribute_image: ${{ steps.build-docker-images.outputs.docker_distribute_image }}
|
||||
docker_xpu_image: ${{ steps.build-docker-images.outputs.docker_xpu_image }}
|
||||
docker_npu_image: ${{ steps.build-docker-images.outputs.docker_npu_image }}
|
||||
docker_dcu_image: ${{ steps.build-docker-images.outputs.docker_dcu_image }}
|
||||
docker_doc_image: ${{ steps.build-docker-images.outputs.docker_doc_image }}
|
||||
steps:
|
||||
- id: build-docker-images
|
||||
name: Build docker images
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
docker_cpu_file: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6
|
||||
docker_sot_file: Dockerfile.gcc133_ubuntu24_cpu_sot
|
||||
docker_coverage_file: Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage
|
||||
docker_build_file: Dockerfile.cuda11.2_cudnn8_gcc82_trt8
|
||||
docker_distribute_file: Dockerfile.cuda123_cudnn9_gcc122_ubuntu20
|
||||
docker_xpu_file: Dockerfile.develop.xre.ubuntu22
|
||||
docker_npu_file: Dockerfile.develop.npu
|
||||
docker_dcu_file: Dockerfile.develop.dtk
|
||||
docker_doc_file: Dockerfile.doc
|
||||
dockerfile_script: https://raw.githubusercontent.com/PaddlePaddle/Paddle/refs/heads/develop/tools/dockerfile/ci_dockerfile.sh
|
||||
dockerfile_origin: https://raw.githubusercontent.com/PaddlePaddle/Paddle/refs/heads/develop/tools/dockerfile/Dockerfile.ubuntu20
|
||||
run: |
|
||||
set -x
|
||||
rm -rf * .[^.]*
|
||||
if [ "${{ inputs.is_merge }}" == "true" ]; then
|
||||
mkdir -p tools/dockerfile && cd tools/dockerfile
|
||||
wget -q --tries=5 --no-proxy ${dockerfile_script} ${dockerfile_origin} --no-check-certificate
|
||||
else
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/${{ inputs.clone_dir }}/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
tar xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
cd tools/dockerfile
|
||||
fi
|
||||
bash ci_dockerfile.sh
|
||||
cd ../..
|
||||
|
||||
# docker build images
|
||||
if [ "${{ inputs.task }}" == "cpu" ]; then
|
||||
declare -A docker_files=(["docker_cpu"]="$docker_cpu_file" ["docker_distribute"]="$docker_distribute_file" ["docker_xpu"]="$docker_xpu_file" ["docker_npu"]="$docker_npu_file" ["docker_dcu"]="$docker_dcu_file" ["docker_sot"]="$docker_sot_file")
|
||||
elif [ "${{ inputs.task }}" == "build" ]; then
|
||||
declare -A docker_files=(["docker_build"]="$docker_build_file" ["docker_doc"]="$docker_doc_file")
|
||||
else
|
||||
declare -A docker_files=(["docker_coverage"]="$docker_coverage_file")
|
||||
fi
|
||||
for name in "${!docker_files[@]}"
|
||||
do
|
||||
md5_value=`md5sum tools/dockerfile/${docker_files[$name]} | awk '{print $1}'`
|
||||
if [ $name == "docker_dcu" ]; then
|
||||
md5_value="76937a563116f6008c8ca4cb4f592759"
|
||||
fi
|
||||
if [ $name == "docker_npu" ]; then
|
||||
md5_value="a3793bdeea5ae881a0c1eaf4d7c30c64"
|
||||
fi
|
||||
docker_image="ccr-2vdh3abv-pub.cnc.bj.baidubce.com/ci/paddle:${md5_value}"
|
||||
declare "${name}_image=${docker_image}"
|
||||
echo "${name}_image=${docker_image}" >> $GITHUB_OUTPUT
|
||||
set +e
|
||||
docker pull $docker_image
|
||||
pull_status=$?
|
||||
set -e
|
||||
if [ $pull_status -eq 0 ];then
|
||||
echo use docker cache
|
||||
else
|
||||
docker build -t $docker_image -f tools/dockerfile/${docker_files[$name]} .
|
||||
docker push $docker_image
|
||||
echo end docker build
|
||||
fi
|
||||
done
|
||||
|
||||
# clean workspace
|
||||
cd ${{ github.workspace }}
|
||||
rm -rf * .[^.]*
|
||||
@@ -0,0 +1,58 @@
|
||||
name: Remove Skip-CI Labels
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [synchronize]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
remove-skip-ci-labels:
|
||||
name: Remove skip-ci labels on new commits
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Get PR labels
|
||||
id: get-labels
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const { data: labels } = await github.rest.issues.listLabelsOnIssue({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number
|
||||
});
|
||||
|
||||
const skipCiLabels = labels
|
||||
.filter(label => label.name.startsWith('skip-ci:'))
|
||||
.map(label => label.name);
|
||||
|
||||
console.log('Found skip-ci labels:', skipCiLabels);
|
||||
core.setOutput('skip-ci-labels', JSON.stringify(skipCiLabels));
|
||||
core.setOutput('has-skip-ci-labels', skipCiLabels.length > 0 ? 'true' : 'false');
|
||||
|
||||
- name: Remove skip-ci labels
|
||||
if: steps.get-labels.outputs.has-skip-ci-labels == 'true'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const skipCiLabels = JSON.parse('${{ steps.get-labels.outputs.skip-ci-labels }}');
|
||||
|
||||
for (const label of skipCiLabels) {
|
||||
console.log(`Removing label: ${label}`);
|
||||
await github.rest.issues.removeLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
name: label
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Successfully removed ${skipCiLabels.length} skip-ci label(s)`);
|
||||
@@ -0,0 +1,347 @@
|
||||
name: Re-run
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
re-run:
|
||||
if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/re-run') && github.event.comment.user.login == github.event.issue.user.login }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Rerun all failed jobs
|
||||
if: ${{ contains(github.event.comment.body, 'all-failed') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'all-failed'
|
||||
|
||||
- name: Rerun Approval
|
||||
if: ${{ contains(github.event.comment.body, 'approval') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Check approval'
|
||||
|
||||
- name: Rerun Codestyle-check
|
||||
if: ${{ contains(github.event.comment.body, 'codestyle') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Pre Commit'
|
||||
|
||||
- name: Rerun Clone
|
||||
if: ${{ contains(github.event.comment.body, 'clone') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Clone-linux / Clone Paddle'
|
||||
|
||||
- name: Rerun sot
|
||||
if: ${{ contains(github.event.comment.body, 'sot') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-SOT / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun CPU
|
||||
if: ${{ contains(github.event.comment.body, 'cpu') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-CPU / Build and test'
|
||||
|
||||
- name: Rerun Mac
|
||||
if: ${{ contains(github.event.comment.body, 'mac') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Mac-CPU / Build and test'
|
||||
|
||||
- name: Rerun NPU
|
||||
if: ${{ contains(github.event.comment.body, 'npu')}}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-NPU / Test'
|
||||
|
||||
- name: Rerun IXUCA
|
||||
if: ${{ contains(github.event.comment.body, 'ixuca') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-IXUCA / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Inference
|
||||
if: ${{ contains(github.event.comment.body, 'inference') && !contains(github.event.comment.body, 'build') && !contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-Inference / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Inference build
|
||||
if: ${{ contains(github.event.comment.body, 'inference') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-Inference / Build'
|
||||
|
||||
- name: Rerun Inference test
|
||||
if: ${{ contains(github.event.comment.body, 'inference') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-Inference / Test'
|
||||
|
||||
- name: Rerun Coverage clone
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage clone / Clone Paddle'
|
||||
|
||||
- name: Rerun Coverage docker build
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') && contains(github.event.comment.body, 'docker') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage build docker / Build docker'
|
||||
|
||||
- name: Rerun Coverage build
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage build'
|
||||
|
||||
- name: Rerun Coverage test
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage test'
|
||||
|
||||
- name: Rerun XPU
|
||||
if: ${{ contains(github.event.comment.body, 'xpu') && !contains(github.event.comment.body, 'build') && !contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-XPU / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun XPU build
|
||||
if: ${{ contains(github.event.comment.body, 'xpu') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-XPU / Build'
|
||||
|
||||
- name: Rerun XPU test
|
||||
if: ${{ contains(github.event.comment.body, 'xpu') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-XPU / Test'
|
||||
|
||||
- name: Rerun Distribute-stable build
|
||||
if: ${{ contains(github.event.comment.body, 'distribute') && !contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Distribute-stable-build / Build'
|
||||
|
||||
- name: Rerun Distribute-stable test
|
||||
if: ${{ contains(github.event.comment.body, 'distribute') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Distribute-stable-test / Test'
|
||||
|
||||
- name: Rerun build
|
||||
if: ${{ contains(github.event.comment.body, 'linux') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-build / Build'
|
||||
|
||||
- name: Rerun Static-check test
|
||||
if: ${{ contains(github.event.comment.body, 'static-check') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Static-Check / Test'
|
||||
|
||||
- name: Rerun CE-CINN-Framework
|
||||
if: ${{ contains(github.event.comment.body, 'ce-cinn') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-CINN-Framework / CINN'
|
||||
|
||||
- name: Rerun CE-Framework Whl
|
||||
if: ${{ contains(github.event.comment.body, 'ce') && contains(github.event.comment.body, 'whl') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-Framework / Whl'
|
||||
|
||||
- name: Rerun CE-Framework Infer
|
||||
if: ${{ contains(github.event.comment.body, 'ce') && contains(github.event.comment.body, 'infer') && !contains(github.event.comment.body, 'inference') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-Framework / Infer'
|
||||
|
||||
- name: Rerun CE-Framework Test
|
||||
if: ${{ contains(github.event.comment.body, 'ce') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-Framework / Test'
|
||||
|
||||
- name: Rerun Api-Benchmark
|
||||
if: ${{ contains(github.event.comment.body, 'api-benchmark') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Api-Benchmark / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Doc-Preview
|
||||
if: ${{ contains(github.event.comment.body, 'doc') && contains(github.event.comment.body, 'preview') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Doc-Preview / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Slice
|
||||
if: ${{ contains(github.event.comment.body, 'slice') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Slice / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Windows-GPU
|
||||
if: ${{ contains(github.event.comment.body, 'win') && contains(github.event.comment.body, 'gpu') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Windows-GPU / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Windows-Inference
|
||||
if: ${{ contains(github.event.comment.body, 'win') && contains(github.event.comment.body, 'infer')}}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Windows-Inference / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Windows-OPENBLAS
|
||||
if: ${{ contains(github.event.comment.body, 'win') && contains(github.event.comment.body, 'openblas')}}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Windows-OPENBLAS / Check bypass / Check bypass'
|
||||
@@ -0,0 +1,82 @@
|
||||
name: Check version equality on release branch
|
||||
|
||||
on:
|
||||
create:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
check-version-and-create-issue:
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
if: github.ref_type == 'branch' && startsWith(github.ref, 'refs/heads/release/') && github.repository_owner == 'PaddlePaddle'
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Check DEVELOP_VERSION and RELEASE_VERSION equality
|
||||
id: check_versions
|
||||
run: |
|
||||
FILE="paddle/fluid/pir/serialize_deserialize/CMakeLists.txt"
|
||||
if [ ! -f "$FILE" ]; then
|
||||
echo "file_exists=false" >> "$GITHUB_OUTPUT"
|
||||
echo "equal=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
else
|
||||
echo "file_exists=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
develop_version=$(grep -oP '(?<=add_definitions\(-DDEVELOP_VERSION=)[0-9]+' "$FILE")
|
||||
release_version=$(grep -oP '(?<=add_definitions\(-DRELEASE_VERSION=)[0-9]+' "$FILE")
|
||||
|
||||
echo "Develop version: $develop_version"
|
||||
echo "Release version: $release_version"
|
||||
|
||||
echo "develop_version=$develop_version" >> "$GITHUB_OUTPUT"
|
||||
echo "release_version=$release_version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$develop_version" = "$release_version" ]; then
|
||||
echo "equal=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "equal=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create Issue if versions not equal
|
||||
if: steps.check_versions.outputs.equal == 'false'
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
DEVELOP_VERSION: ${{ steps.check_versions.outputs.develop_version }}
|
||||
RELEASE_VERSION: ${{ steps.check_versions.outputs.release_version }}
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const branchName = context.ref.replace('refs/heads/', '');
|
||||
const developVersion = parseInt(process.env.DEVELOP_VERSION);
|
||||
const releaseVersion = parseInt(process.env.RELEASE_VERSION);
|
||||
const issueTitle = `DEVELOP_VERSION and RELEASE_VERSION mismatch in paddle/fluid/pir/serialize_deserialize/CMakeLists.txt (${branchName})`;
|
||||
const issueBody =
|
||||
`Hello, in the new branch \`${branchName}\`, the \`DEVELOP_VERSION\` and \`RELEASE_VERSION\` values in \`paddle/fluid/pir/serialize_deserialize/CMakeLists.txt\` are inconsistent. Please check.\n\n` +
|
||||
`**Current values:**\n` +
|
||||
`- DEVELOP_VERSION: \`${developVersion}\`\n` +
|
||||
`- RELEASE_VERSION: \`${releaseVersion}\`\n\n` +
|
||||
`**Recommended update (apply to both \`develop\` and \`release\` branches):**\n` +
|
||||
`\`\`\`\n` +
|
||||
`DEVELOP_VERSION=${releaseVersion + 1}\n` +
|
||||
`RELEASE_VERSION=${releaseVersion + 1}\n` +
|
||||
`\`\`\`\n\n` +
|
||||
`Also, please remember to update the corresponding YAML files under:\n` +
|
||||
`\`paddle/fluid/pir/serialize_deserialize/patch\`\n`;
|
||||
const issueResult = await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: issueTitle,
|
||||
body: issueBody,
|
||||
});
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueResult.data.number,
|
||||
body: 'cc @changeyoung98 @xiaoguoguo626807 @goocody 请尽快处理上述版本不一致问题，谢谢。'
|
||||
});
|
||||
Reference in New Issue
Block a user