chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,79 @@
|
||||
# This file is migrated from CI script, it's an effort of modenizing our dev infra.
|
||||
# Code owners are expected to take responsibility for review patches to respective file.
|
||||
|
||||
/CMakeLists.txt @wanghuancoder @XiaoguangHu01
|
||||
paddle/fluid/distributed/collective @sneaxiy @ForFishes
|
||||
paddle/fluid/eager/autograd_meta.cc @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/autograd_meta.h @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/backward.cc @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/backward.h @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/grad_node_info.cc @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/grad_node_info.h @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/grad_tensor_holder.cc @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/grad_tensor_holder.h @JiabinYang @xiaoguoguo626807
|
||||
paddle/fluid/eager/tensor_wrapper.h @JiabinYang @zhangbo9674
|
||||
paddle/fluid/framework/block_desc.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/details/op_registry.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/framework.proto @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/grad_op_desc_maker.h @XiaoguangHu01 @xiaoguoguo626807 @Xreki
|
||||
paddle/fluid/framework/ir/graph.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/ir/node.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/lod_tensor.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/op_desc.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/operator.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/scope.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/selected_rows.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/tensor.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/framework/unused_var_check.cc @zhangbo9674
|
||||
paddle/fluid/framework/var_desc.h @XiaoguangHu01 @zhangbo9674 @Xreki
|
||||
paddle/fluid/operators/distributed/send_recv.proto.in @gongweibao @seiriosPlus
|
||||
paddle/fluid/prim/api/api.yaml @xiaoguoguo626807 @JiabinYang @zhangbo9674
|
||||
paddle/fluid/prim/api/composite_backward/composite_backward_api.h @xiaoguoguo626807 @JiabinYang
|
||||
paddle/fluid/prim/api/composite_backward/composite_double_backward_api.h @xiaoguoguo626807 @JiabinYang
|
||||
paddle/fluid/prim/api/manual_prim/prim_manual_api.h @xiaoguoguo626807 @JiabinYang
|
||||
paddle/phi/api/include/tensor.h @zhangbo9674
|
||||
paddle/phi/core/attribute.h @zhangbo9674
|
||||
paddle/phi/core/dense_tensor.h @zhangbo9674
|
||||
paddle/phi/core/device_context.h @zhangbo9674
|
||||
paddle/phi/core/infermeta_utils.h @zhangbo9674
|
||||
paddle/phi/core/kernel_context.h @zhangbo9674
|
||||
paddle/phi/core/kernel_factory.h @zhangbo9674
|
||||
paddle/phi/core/kernel_registry.h @zhangbo9674
|
||||
paddle/phi/core/kernel_utils.h @zhangbo9674
|
||||
paddle/phi/core/meta_tensor.h @zhangbo9674
|
||||
paddle/phi/core/tensor_base.h @zhangbo9674
|
||||
paddle/phi/core/tensor_meta.h @zhangbo9674
|
||||
paddle/phi/infermeta/spmd_rules @LiYuRio @ForFishes @From00
|
||||
paddle/scripts/paddle_build.bat @zhwesky2010 @wanghuancoder
|
||||
paddle/scripts/paddle_build.sh @risemeup1 @zhangbo9674 @XieYunshen
|
||||
pyproject.toml @SigureMo @gouzil
|
||||
python/paddle/autograd/backward_utils.py @xiaoguoguo626807 @changeyoung98
|
||||
python/paddle/autograd/ir_backward.py @xiaoguoguo626807 @changeyoung98
|
||||
python/paddle/base/backward.py @XiaoguangHu01 @xiaoguoguo626807 @Xreki @qili93
|
||||
python/paddle/base/compiler.py @XiaoguangHu01 @zhangbo9674 @Xreki @qili93
|
||||
python/paddle/base/dygraph/layers.py @JiabinYang @zhangbo9674
|
||||
python/paddle/base/framework.py @XiaoguangHu01 @zhangbo9674 @Xreki @qili93
|
||||
python/paddle/base/__init__.py @zhangbo9674 @qili93
|
||||
test/white_list/check_op_sequence_batch_1_input_white_list.py @zhangbo9674
|
||||
test/white_list/check_op_sequence_instance_0_input_white_list.py @zhangbo9674
|
||||
test/white_list/check_shape_white_list.py @hong19860320 @zhangbo9674
|
||||
test/white_list/compile_vs_runtime_white_list.py @zhangbo9674
|
||||
test/white_list/no_check_set_white_list.py @zhangbo9674
|
||||
test/white_list/no_grad_set_white_list.py @zhangbo9674
|
||||
test/white_list/op_accuracy_white_list.py @juncaipeng @zhangting2020
|
||||
test/white_list/op_threshold_white_list.py @juncaipeng @zhangting2020
|
||||
python/paddle/distributed/fleet/__init__.py @sneaxiy @raindrops2sea
|
||||
python/paddle/distributed/fleet/launch.py @sneaxiy @raindrops2sea
|
||||
python/paddle/distributed/__init__.py @sneaxiy @raindrops2sea
|
||||
python/paddle/incubate/autograd/composite_rules.py @xiaoguoguo626807 @JiabinYang
|
||||
python/paddle/incubate/autograd/primitives.py @xiaoguoguo626807 @JiabinYang
|
||||
python/paddle/_typing @SigureMo @zrr1999 @gouzil
|
||||
python/requirements.txt @zhangbo9674 @jzhang533 @kolinwei
|
||||
test/dygraph_to_static @SigureMo @DrRyanHuang @zrr1999 @gouzil
|
||||
test/sot @SigureMo @DrRyanHuang @zrr1999 @gouzil
|
||||
tools/parallel_UT_rule.py @zhwesky2010 @wanghuancoder
|
||||
tools/windows/run_unittests.sh @zhwesky2010 @wanghuancoder
|
||||
.pre-commit-config.yaml @SigureMo @gouzil
|
||||
_typos.toml @SigureMo
|
||||
ci/rules @SigureMo @zrr1999
|
||||
ci/rule-tests @SigureMo @zrr1999
|
||||
@@ -0,0 +1,66 @@
|
||||
|
||||
name: 🐛 报BUG Bug Report
|
||||
description: 报告一个可复现的BUG帮助我们修复框架。 Report a bug to help us reproduce and fix it.
|
||||
labels: [type/bug-report, status/new-issue]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 在向Paddle报bug之前，请先查询[历史issue](https://github.com/PaddlePaddle/Paddle/issues)是否报过同样的bug。
|
||||
|
||||
#### Before submitting a bug, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/PaddlePaddle/Paddle/issues).
|
||||
|
||||
- type: textarea
|
||||
id: code
|
||||
attributes:
|
||||
label: bug描述 Describe the Bug
|
||||
description: |
|
||||
请清晰简洁的描述这个bug，最好附上bug复现环境、bug复现步骤及最小代码集，以便我们可以通过运行代码来重现错误。代码片段需要尽可能简洁，请花些时间去掉不相关的代码以帮助我们有效地调试。我们希望通过复制代码并运行得到与你相同的结果，请避免任何外部数据或包含相关的导入等。例如：
|
||||
```python
|
||||
# 导入所有必要的库。 All necessary imports at the beginning.
|
||||
# paddlepaddle <= 2.1.2
|
||||
import paddle
|
||||
|
||||
# 一个简洁的片段，能够定位到bug。 A succinct reproducing example trimmed down to the essential parts.
|
||||
a = paddle.rand(shape=[1,4])
|
||||
b = paddle.rand(shape=[1,4])
|
||||
a.stop_gradient = False
|
||||
b.stop_gradient = False
|
||||
|
||||
c = paddle.zeros((4, 4))
|
||||
c[0, :] = a/b
|
||||
|
||||
print('Is c requires grad: ', not c.stop_gradient) # 注意：这里出现了bug，期望requires_grad=True
|
||||
```
|
||||
如果代码太长，请将可执行代码放到[AIStudio](https://aistudio.baidu.com/aistudio/index)中并将项目设置为公开（或者放到github gist上），请在项目中描述清楚bug复现步骤，在issue中描述期望结果与实际结果。
|
||||
如果你报告的是一个报错信息，请将完整回溯的报错贴在这里，并使用 ` ```三引号块``` `展示错误信息。
|
||||
|
||||
|
||||
placeholder: |
|
||||
请清晰简洁的描述这个bug。A clear and concise description of what the bug is.
|
||||
|
||||
```python
|
||||
# 最小可复现代码。 Sample code to reproduce the problem.
|
||||
```
|
||||
|
||||
```shell
|
||||
带有完整回溯的报错信息。 The error message you got, with the full traceback.
|
||||
```
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: others
|
||||
attributes:
|
||||
label: 其他补充信息 Additional Supplementary Information
|
||||
description: |
|
||||
如果你还有其他需要补充的内容，请写在这里。
|
||||
If you have anything else to add, please write it here.
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
感谢你的贡献 🎉！Thanks for your contribution 🎉!
|
||||
@@ -0,0 +1,37 @@
|
||||
name: 🚀 新需求 Feature Request
|
||||
description: 提交一个你对Paddle的新需求。 Submit a request for a new Paddle feature.
|
||||
labels: [type/feature-request, status/new-issue]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 你可以在这里提出你对Paddle框架的新需求，包括但不限于：功能或模型缺失、功能不全或无法使用、精度/性能不符合预期等。
|
||||
|
||||
#### You could submit a request for a new Paddle feature here, including but not limited to: new features or models, incomplete or unusable features, accuracy/performance not as expected, etc.
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: 需求描述 Feature Description
|
||||
description: |
|
||||
请尽可能包含任务目标、需求场景、功能描述等信息，全面的信息有利于我们准确评估你的需求。
|
||||
Please include as much information as possible, such as mission objectives, requirement scenarios, functional descriptions, etc. Comprehensive information will help us accurately assess your feature request.
|
||||
value: "任务目标（请描述你正在做的项目是什么，如模型、论文、项目是什么？）; <br /> 需求场景（请描述你的项目中为什么需要用此功能）; <br /> 功能描述（请简单描述或设计这个功能）"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: 替代实现 Alternatives
|
||||
description: |
|
||||
如果你考虑过的任何替代解决方案或功能，请简要描述下，我们会综合评估。
|
||||
A description of any alternative solutions or features you've considered, if any.
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
感谢你的贡献 🎉！Thanks for your contribution 🎉!
|
||||
@@ -0,0 +1,68 @@
|
||||
name: 🗂 安装 Build/Installation Issue
|
||||
description: 报告一个安装问题。 Report an issue related to build or install Paddle.
|
||||
labels: [type/build, status/new-issue]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 安装请参考[官网文档](https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/develop/install/pip/linux-pip.html)，若未能解决你的问题，你可以在这里提issue。
|
||||
|
||||
#### Before submitting a Build/Installation Issue, please make sure you have visited the [official website](https://www.paddlepaddle.org.cn/documentation/docs/en/install/index_en.html).
|
||||
|
||||
- type: textarea
|
||||
id: error
|
||||
attributes:
|
||||
label: 问题描述 Issue Description
|
||||
description: |
|
||||
请详细描述你的问题，同步贴出报错信息、日志/代码关键片段、复现步骤，以便我们快速排查问题。
|
||||
Please describe your problem in detail, and synchronously post the error message, key log/code snippet, and reproduction steps, so that we can quickly troubleshoot the problem.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: environment
|
||||
attributes:
|
||||
label: 版本&环境信息 Version & Environment Information
|
||||
description: |
|
||||
请参考以下命令运行脚本[summary_env.py](https://github.com/PaddlePaddle/Paddle/blob/develop/tools/summary_env.py)获取版本&环境信息，并将输出拷贝在这里。
|
||||
Please run the following and paste the output below.
|
||||
```shell
|
||||
wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/tools/summary_env.py
|
||||
python3 -m pip install distro
|
||||
python3 summary_env.py
|
||||
```
|
||||
若运行脚本出现问题，请在issue中说明，并提供以下信息：
|
||||
1. PaddlePaddle版本：请提供你的PaddlePaddle版本号（如2.0.0）或CommitID。
|
||||
2. CPU（可选）：请提供CPU型号，MKL/OpenBlas/MKLDNN/等数学库的使用情况，是否支持AVX指令集。
|
||||
3. GPU：请提供GPU型号，CUDA（如cuda10.2）和CUDNN版本号（如cudnn7.6.5）。
|
||||
4. 系统环境：请说明系统类型、版本（如Mac OS 10.14）。
|
||||
5. Python版本（如python 3.8）。
|
||||
6. （可选）若安装过程遇到问题，请提供安装方式（pip/conda/docker/源码编译）和相应的安装命令。
|
||||
7. （可选）若使用paddle过程中，遇到了无法使用gpu相关问题，请在命令行中键入`nvidia-smi`和`nvcc -V`，提供这两个命令输出的截图。
|
||||
8. （可选）若使用特殊硬件，请单独注明。
|
||||
|
||||
placeholder: |
|
||||
****************************************
|
||||
Paddle version:
|
||||
Paddle With CUDA:
|
||||
|
||||
OS:
|
||||
GCC version:
|
||||
Clang version:
|
||||
CMake version:
|
||||
Libc version:
|
||||
Python version:
|
||||
|
||||
CUDA version:
|
||||
cuDNN version:
|
||||
Nvidia driver version:
|
||||
Nvidia driver List:
|
||||
****************************************
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
感谢你的贡献 🎉！Thanks for your contribution 🎉!
|
||||
@@ -0,0 +1,37 @@
|
||||
name: 📚 文档 Documentation Issue
|
||||
description: 反馈一个官网文档错误。 Report an issue related to https://www.paddlepaddle.org.cn/.
|
||||
labels: [type/docs, status/new-issue]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 请确认反馈的问题来自PaddlePaddle官网文档：https://www.paddlepaddle.org.cn/ 。
|
||||
|
||||
#### Before submitting a Documentation Issue, Please make sure that issue is related to https://www.paddlepaddle.org.cn/.
|
||||
|
||||
- type: textarea
|
||||
id: link
|
||||
attributes:
|
||||
label: 文档链接&描述 Document Links & Description
|
||||
description: |
|
||||
请说明有问题的文档链接以及该文档存在的问题。
|
||||
Please fill in the link to the document and describe the question.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: error
|
||||
attributes:
|
||||
label: 请提出你的建议 Please give your suggestion
|
||||
description: |
|
||||
请告诉我们，你希望如何改进这个文档。或者你可以提个PR修复这个问题。[教程参考](https://github.com/PaddlePaddle/docs/wiki#%E8%B4%A1%E7%8C%AE%E6%96%87%E6%A1%A3)
|
||||
Please tell us how you would like to improve this document. Or you can submit a PR to fix this problem.
|
||||
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
感谢你的贡献 🎉！Thanks for your contribution 🎉!
|
||||
@@ -0,0 +1,33 @@
|
||||
name: 🙋🏼♀️🙋🏻♂️提问 Ask a Question
|
||||
description: 提出一个使用/咨询问题。 Ask a usage or consultation question.
|
||||
labels: [type/question, status/new-issue]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 你可以在这里提出一个使用/咨询问题，提问之前请确保：
|
||||
|
||||
- 1）已经百度/谷歌搜索过你的问题，但是没有找到解答；
|
||||
|
||||
- 2）已经在官网查询过[API文档](https://www.paddlepaddle.org.cn/documentation/docs/zh/api/index_cn.html)与[FAQ](https://www.paddlepaddle.org.cn/documentation/docs/zh/faq/index_cn.html)，但是没有找到解答；
|
||||
|
||||
- 3）已经在[历史issue](https://github.com/PaddlePaddle/Paddle/issues)中搜索过，没有找到同类issue或issue未被解答。
|
||||
|
||||
|
||||
#### You could ask a usage or consultation question here, before your start, please make sure:
|
||||
|
||||
- 1) You have searched your question on Baidu/Google, but found no answer;
|
||||
|
||||
- 2) You have checked the [API documentation](https://www.paddlepaddle.org.cn/documentation/docs/en/api/index_en.html), but found no answer;
|
||||
|
||||
- 3) You have searched [the existing and past issues](https://github.com/PaddlePaddle/Paddle/issues), but found no similar issue or the issue has not been answered.
|
||||
|
||||
|
||||
|
||||
- type: textarea
|
||||
id: question
|
||||
attributes:
|
||||
label: 请提出你的问题 Please ask your question
|
||||
validations:
|
||||
required: true
|
||||
@@ -0,0 +1,23 @@
|
||||
name: 🧩 其他 Others
|
||||
description: 提出其他问题。 Report any other non-support related issues.
|
||||
labels: [type/others, status/new-issue]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
#### 你可以在这里提出任何前面几类模板不适用的问题，包括但不限于：优化性建议、框架使用体验反馈、版本兼容性问题、报错信息不清楚等。
|
||||
|
||||
#### You can report any issues that are not applicable to the previous types of templates, including but not limited to: enhancement suggestions, feedback on the use of the framework, version compatibility issues, unclear error information, etc.
|
||||
|
||||
- type: textarea
|
||||
id: others
|
||||
attributes:
|
||||
label: 问题描述 Please describe your issue
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
感谢你的贡献 🎉！ Thanks for your contribution 🎉!
|
||||
@@ -0,0 +1,17 @@
|
||||
<!-- TemplateReference: https://github.com/PaddlePaddle/Paddle/wiki/PULL-REQUEST-TEMPLATE--REFERENCE -->
|
||||
<!-- Demo: https://github.com/PaddlePaddle/Paddle/pull/24810 -->
|
||||
|
||||
### PR Category
|
||||
<!-- One of [ User Experience | Execute Infrastructure | Operator Mechanism | CINN | Custom Device | Performance Optimization | Distributed Strategy | Parameter Server | Communication Library | Auto Parallel | Inference | Environment Adaptation ] -->
|
||||
|
||||
|
||||
### PR Types
|
||||
<!-- One of [ New features | Bug fixes | Improvements | Performance | BC Breaking | Deprecations | Docs | Devs | Not User Facing | Security | Others ] -->
|
||||
|
||||
|
||||
### Description
|
||||
<!-- Describe what you’ve done -->
|
||||
|
||||
|
||||
### 是否引起精度变化
|
||||
<!-- one of the following [ 是 | 否 ]-->
|
||||
@@ -0,0 +1,30 @@
|
||||
name: 'Rerun Workflow'
|
||||
description: 'Re-run GitHub Actions workflow for a given Pull Request'
|
||||
inputs:
|
||||
GITHUB_TOKEN:
|
||||
description: 'GitHub token with repo scope'
|
||||
required: true
|
||||
OWNER:
|
||||
description: 'Repository owner'
|
||||
required: true
|
||||
REPO:
|
||||
description: 'Repository name'
|
||||
required: true
|
||||
PR_ID:
|
||||
description: 'Pull Request ID'
|
||||
required: true
|
||||
JOB_NAME:
|
||||
description: 'Job name to rerun'
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- run: bash ./.github/actions/rerun-workflow/rerun.sh
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }}
|
||||
OWNER: ${{ inputs.OWNER }}
|
||||
REPO: ${{ inputs.REPO }}
|
||||
PR_ID: ${{ inputs.PR_ID }}
|
||||
JOB_NAME: ${{ inputs.JOB_NAME }}
|
||||
@@ -0,0 +1,77 @@
|
||||
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
|
||||
COMMIT_SHA=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/pulls/$PR_ID" | jq -r '.head.sha')
|
||||
|
||||
echo "Commit SHA: $COMMIT_SHA"
|
||||
|
||||
response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/runs?head_sha=$COMMIT_SHA&per_page=100")
|
||||
|
||||
echo "Response: $response"
|
||||
|
||||
run_ids=$(echo "$response" | jq -r '.workflow_runs[].id')
|
||||
|
||||
if [ -n "$run_ids" ]; then
|
||||
echo "Found run_ids for commit $COMMIT_SHA: $run_ids"
|
||||
|
||||
for run_id in $run_ids; do
|
||||
if [ "$JOB_NAME" = "all-failed" ]; then
|
||||
echo "Rerunning all failed jobs for run_id: $run_id"
|
||||
|
||||
rerun_response=$(curl -X POST -s -w "%{http_code}" -o /dev/null \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/rerun-failed-jobs")
|
||||
if [ "$rerun_response" -eq 201 ]; then
|
||||
echo "Successfully requested rerun for all blocked jobs in run_id: $run_id"
|
||||
else
|
||||
echo "Failed to request rerun for run_id: $run_id with status code $rerun_response"
|
||||
fi
|
||||
|
||||
else
|
||||
jobs_response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/runs/$run_id/jobs")
|
||||
|
||||
echo "Jobs Response for run_id $run_id: $jobs_response"
|
||||
|
||||
# if [[ "$JOB_NAME" == *"bypass"* ]]; then
|
||||
block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
|
||||
'.jobs[] | select(.name == $job_name) | .id')
|
||||
# else
|
||||
# block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
|
||||
# '.jobs[] | select(.name == $job_name and .conclusion != "success") | .id')
|
||||
# fi
|
||||
|
||||
if [ -n "$block_jobs" ]; then
|
||||
echo "Found block jobs for run_id $run_id: $block_jobs"
|
||||
|
||||
for job_id in $block_jobs; do
|
||||
echo "Rerunning job_id: $job_id"
|
||||
curl -X POST -H "Accept: application/vnd.github.v3+json" \
|
||||
-H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$OWNER/$REPO/actions/jobs/$job_id/rerun"
|
||||
done
|
||||
else
|
||||
echo "No block jobs found for run_id $run_id with name $JOB_NAME."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "No matching workflow runs found for commit $COMMIT_SHA."
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,47 @@
|
||||
name: 'PR Skip Check'
|
||||
description: 'Check if PR should skip CI based on file paths'
|
||||
author: 'Paddle CI Team'
|
||||
|
||||
inputs:
|
||||
ignore-paths:
|
||||
description: 'Comma-separated paths to ignore (e.g., "skill/**,docs/**,**.md")'
|
||||
required: true
|
||||
default: '.agents/skills/**'
|
||||
base-ref:
|
||||
description: 'Base branch (defaults to PR target branch)'
|
||||
required: false
|
||||
default: ${{ github.event.pull_request.base.ref }}
|
||||
|
||||
outputs:
|
||||
should-skip:
|
||||
description: 'Whether CI should be skipped (true/false)'
|
||||
value: ${{ steps.check.outputs.should-skip }}
|
||||
is-pr:
|
||||
description: 'Whether it is a PR event'
|
||||
value: ${{ steps.check.outputs.is-pr }}
|
||||
changed-files:
|
||||
description: 'List of all changed files (newline separated)'
|
||||
value: ${{ steps.check.outputs.changed-files }}
|
||||
non-ignored-files:
|
||||
description: 'List of files not in ignored paths'
|
||||
value: ${{ steps.check.outputs.non-ignored-files }}
|
||||
total-count:
|
||||
description: 'Total number of changed files'
|
||||
value: ${{ steps.check.outputs.total-count }}
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run PR skip check script
|
||||
id: check
|
||||
shell: bash
|
||||
run: bash ${{ github.action_path }}/check_skip.sh
|
||||
env:
|
||||
IGNORE_PATHS: ${{ inputs.ignore-paths }}
|
||||
BASE_REF: ${{ inputs.base-ref }}
|
||||
GITHUB_EVENT_NAME: ${{ github.event_name }}
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
|
||||
echo "🔍 [PR Skip Check] Starting..."
|
||||
|
||||
# Parse input parameters
|
||||
IFS=',' read -ra IGNORE_PATHS <<< "${IGNORE_PATHS:-skill/**}"
|
||||
BASE_REF="${BASE_REF:-$GITHUB_BASE_REF}"
|
||||
|
||||
echo "📋 Event type: $GITHUB_EVENT_NAME"
|
||||
echo "📋 Ignore path patterns:"
|
||||
printf ' - %s\n' "${IGNORE_PATHS[@]}"
|
||||
|
||||
# Initialize output
|
||||
IS_PR="false"
|
||||
SHOULD_SKIP="false"
|
||||
CHANGED_FILES=""
|
||||
NON_IGNORED_FILES=""
|
||||
TOTAL_COUNT=0
|
||||
|
||||
# Check if PR event
|
||||
if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
|
||||
IS_PR="true"
|
||||
echo "🔄 Detected PR event, running skip check"
|
||||
|
||||
# Get PR changed files
|
||||
get_pr_changed_files() {
|
||||
local files=""
|
||||
|
||||
if [ -n "$BASE_REF" ]; then
|
||||
echo "📌 Base branch: $BASE_REF" >&2
|
||||
files=$(git diff --name-only "origin/$BASE_REF...HEAD" 2>/dev/null)
|
||||
else
|
||||
echo "⚠️ No base branch specified, using HEAD^..HEAD" >&2
|
||||
files=$(git diff --name-only HEAD^ HEAD 2>/dev/null || echo "")
|
||||
fi
|
||||
|
||||
echo "$files" | grep -v '^$' || echo ""
|
||||
}
|
||||
|
||||
CHANGED_FILES=$(get_pr_changed_files)
|
||||
|
||||
if [ -n "$CHANGED_FILES" ]; then
|
||||
echo "📋 PR changed files:"
|
||||
echo "$CHANGED_FILES" | sed 's/^/ - /'
|
||||
|
||||
# Count total
|
||||
TOTAL_COUNT=$(echo "$CHANGED_FILES" | wc -l | xargs)
|
||||
|
||||
# Check if files match ignore patterns
|
||||
check_files() {
|
||||
local non_ignored=""
|
||||
|
||||
echo "🔍 [DEBUG] Starting to check files..." >&2
|
||||
|
||||
while IFS= read -r file; do
|
||||
[ -z "$file" ] && continue
|
||||
echo "🔍 [DEBUG] Checking file: '$file'" >&2
|
||||
|
||||
is_ignored=false
|
||||
echo "🔍 [DEBUG] IGNORE_PATHS length: ${#IGNORE_PATHS[@]}" >&2
|
||||
for pattern in "${IGNORE_PATHS[@]}"; do
|
||||
pattern=$(echo "$pattern" | xargs)
|
||||
echo "🔍 [DEBUG] Original pattern: '$pattern'" >&2
|
||||
if [ -n "$pattern" ]; then
|
||||
# Replace ** with * for bash pattern matching
|
||||
bash_pattern="${pattern//\**/*}"
|
||||
echo "🔍 [DEBUG] bash_pattern: '$bash_pattern'" >&2
|
||||
# Debug: show pattern matching
|
||||
if [[ "$file" == $bash_pattern ]]; then
|
||||
is_ignored=true
|
||||
echo " ✅ Ignored: $file (pattern: $pattern -> $bash_pattern)" >&2
|
||||
break
|
||||
else
|
||||
echo " 🔍 Check: $file (pattern: $pattern -> $bash_pattern) - no match" >&2
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$is_ignored" = false ]; then
|
||||
echo " ❌ Not ignored: $file" >&2
|
||||
if [ -z "$non_ignored" ]; then
|
||||
non_ignored="$file"
|
||||
else
|
||||
non_ignored="$non_ignored"$'\n'"$file"
|
||||
fi
|
||||
fi
|
||||
done <<< "$CHANGED_FILES"
|
||||
|
||||
echo "$non_ignored"
|
||||
}
|
||||
|
||||
NON_IGNORED_FILES=$(check_files)
|
||||
|
||||
echo "📋 Non-ignored files:"
|
||||
echo "$NON_IGNORED_FILES" | sed 's/^/ - /'
|
||||
|
||||
# Determine if should skip
|
||||
if [ -z "$NON_IGNORED_FILES" ]; then
|
||||
SHOULD_SKIP="true"
|
||||
echo "✅ All PR changes are in ignored paths, can skip CI"
|
||||
else
|
||||
SHOULD_SKIP="false"
|
||||
echo "❌ Some files not in ignored paths, continue CI"
|
||||
fi
|
||||
else
|
||||
echo "📭 No changed files detected in PR"
|
||||
SHOULD_SKIP="false"
|
||||
fi
|
||||
else
|
||||
# Non-PR event (push, workflow_dispatch, etc.)
|
||||
echo "🔄 not a PR event ($GITHUB_EVENT_NAME), run CI"
|
||||
SHOULD_SKIP="false"
|
||||
fi
|
||||
|
||||
# Set output variables
|
||||
echo "should-skip=$SHOULD_SKIP" >> $GITHUB_OUTPUT
|
||||
echo "is-pr=$IS_PR" >> $GITHUB_OUTPUT
|
||||
echo "total-count=$TOTAL_COUNT" >> $GITHUB_OUTPUT
|
||||
|
||||
# Handle multiline output (changed-files)
|
||||
if [ -n "$CHANGED_FILES" ]; then
|
||||
echo 'changed-files<<EOF' >> $GITHUB_OUTPUT
|
||||
echo "$CHANGED_FILES" >> $GITHUB_OUTPUT
|
||||
echo 'EOF' >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "changed-files=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
# Handle multiline output (non-ignored-files)
|
||||
if [ -n "$NON_IGNORED_FILES" ]; then
|
||||
echo 'non-ignored-files<<EOF' >> $GITHUB_OUTPUT
|
||||
echo "$NON_IGNORED_FILES" >> $GITHUB_OUTPUT
|
||||
echo 'EOF' >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "non-ignored-files=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
echo "📊 total=$TOTAL_COUNT, can skip=$SHOULD_SKIP"
|
||||
echo "✅ [PR Skip Check] finished"
|
||||
@@ -0,0 +1,13 @@
|
||||
codecov:
|
||||
notify:
|
||||
wait_for_ci: false
|
||||
|
||||
coverage:
|
||||
status:
|
||||
project:
|
||||
default:
|
||||
target: 50% # overall project Coverage
|
||||
threshold: 1% # Allow the coverage to drop by 1%, and posting a success status.
|
||||
patch:
|
||||
default:
|
||||
target: 90% # lines adjusted Coverage < 80% CI will fail
|
||||
@@ -0,0 +1,87 @@
|
||||
name: Api-benchmark-baseline
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
PR_ID:
|
||||
required: false
|
||||
type: string
|
||||
COMMIT_ID:
|
||||
required: false
|
||||
type: string
|
||||
job-name:
|
||||
required: true
|
||||
default: 'api-benchmark'
|
||||
type: choice
|
||||
options:
|
||||
- api-benchmark
|
||||
- others
|
||||
schedule:
|
||||
- cron: '0 21 * * *'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Api benchmark clone
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
is_pr: 'false'
|
||||
|
||||
build-docker:
|
||||
name: Api benchmark build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
build:
|
||||
name: Api benchmark build
|
||||
if: github.event_name == 'schedule' && github.event.schedule == '0 21 * * *'
|
||||
needs: [clone, build-docker]
|
||||
uses: ./.github/workflows/_Linux-build.yml
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
is_pr: 'false'
|
||||
|
||||
api-benchmark-baseline-schedule:
|
||||
name: Api benchmark baseline with schedule
|
||||
strategy:
|
||||
matrix:
|
||||
run-labels: [api-bm-20, api-bm-27]
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
baseline: 'true'
|
||||
run-labels: ${{ matrix.run-labels }}
|
||||
|
||||
api-benchmark-baseline-pr-20:
|
||||
name: Api benchmark baseline with PR on 20
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.job-name == 'api-benchmark'
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
baseline: 'true'
|
||||
MANUALLY_PR_ID: ${{ inputs.PR_ID }}
|
||||
MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
|
||||
run-labels: api-bm-20
|
||||
|
||||
api-benchmark-baseline-pr-27:
|
||||
name: Api benchmark baseline with PR on 27
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.job-name == 'api-benchmark'
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
baseline: 'true'
|
||||
MANUALLY_PR_ID: ${{ inputs.PR_ID }}
|
||||
MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
|
||||
run-labels: api-bm-27
|
||||
@@ -0,0 +1,53 @@
|
||||
name: Approval
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
|
||||
env:
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
token: ${{ vars.ACTION_GITHUB_TOKEN }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: 'approval'
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
check-approvers:
|
||||
name: Check approval
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 200
|
||||
|
||||
- name: Update paddle
|
||||
run: |
|
||||
git switch -c test
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git fetch upstream $BRANCH
|
||||
git checkout $BRANCH
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git merge test
|
||||
|
||||
- name: Display Required Approvers
|
||||
run: |
|
||||
bash ci/check_approval.sh
|
||||
@@ -0,0 +1,154 @@
|
||||
name: CI-Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**, fleety_*]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip || 'false' }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Clone-linux
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
workflow-name: 'CI-build'
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
build-docker:
|
||||
name: build docker images
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
inference:
|
||||
name: PR-CI-Inference
|
||||
uses: ./.github/workflows/_Inference.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_inference_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
build:
|
||||
name: Linux-build
|
||||
uses: ./.github/workflows/_Linux-build.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
static-check:
|
||||
name: Static-Check
|
||||
uses: ./.github/workflows/_Static-Check.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
ce-framework:
|
||||
name: CE-Framework
|
||||
uses: ./.github/workflows/_CE-Framework.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
ce-cinn-framework:
|
||||
name: CE-CINN-Framework
|
||||
uses: ./.github/workflows/_CE-CINN-Framework.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
api-benchmark:
|
||||
name: Api-Benchmark
|
||||
uses: ./.github/workflows/_Api-Benchmark.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
# auto-parallel:
|
||||
# name: Auto-Parallel
|
||||
# uses: ./.github/workflows/_Auto-Parallel.yml
|
||||
# needs: [clone, build-docker, build]
|
||||
# with:
|
||||
# can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
# docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
model-benchmark:
|
||||
name: Model-Benchmark
|
||||
uses: ./.github/workflows/_Model-Benchmark.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
|
||||
doc-preview:
|
||||
name: Doc-Preview
|
||||
uses: ./.github/workflows/_Doc-Preview.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_doc_image: ${{ needs.build-docker.outputs.docker_doc_image }}
|
||||
|
||||
slice:
|
||||
name: Slice
|
||||
uses: ./.github/workflows/_Slice.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
slice-check: ${{ needs.clone.outputs.slice-check }}
|
||||
|
||||
deepmd-test:
|
||||
name: DeepMD-Kit-Test
|
||||
uses: ./.github/workflows/_Deepmd-pd-test.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
build-can-skip: ${{ needs.build.outputs.can-skip }}
|
||||
can-skip: ${{ needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
@@ -0,0 +1,66 @@
|
||||
name: CI-Windows
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**, fleety_*]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Clone-windows
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-windows.yml
|
||||
with:
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
win-openblas:
|
||||
name: Windows-OPENBLAS
|
||||
uses: ./.github/workflows/_Windows-OPENBLAS.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
win-gpu:
|
||||
name: Windows-GPU
|
||||
uses: ./.github/workflows/_Windows-GPU.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
win-inference:
|
||||
name: Windows-Inference
|
||||
uses: ./.github/workflows/_Windows-Inference.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
@@ -0,0 +1,129 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**, fleety_*]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Clone-linux
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
workflow-name: 'CI'
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
build-docker:
|
||||
name: build docker images
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
|
||||
sot:
|
||||
name: PR-CI-SOT
|
||||
uses: ./.github/workflows/_SOT.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_sot_image: ${{ needs.build-docker.outputs.docker_sot_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
mac:
|
||||
name: Mac-CPU
|
||||
uses: ./.github/workflows/_Mac.yml
|
||||
needs: clone
|
||||
with:
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
xpu:
|
||||
name: Linux-XPU
|
||||
uses: ./.github/workflows/_Linux-XPU.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_xpu_image: ${{ needs.build-docker.outputs.docker_xpu_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
dcu:
|
||||
name: Linux-DCU
|
||||
uses: ./.github/workflows/_Linux-DCU.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_dcu_image: ${{ needs.build-docker.outputs.docker_dcu_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
cpu:
|
||||
name: Linux-CPU
|
||||
uses: ./.github/workflows/_Linux-CPU.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_cpu_image: ${{ needs.build-docker.outputs.docker_cpu_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
npu:
|
||||
name: Linux-NPU
|
||||
uses: ./.github/workflows/_Linux-NPU.yml
|
||||
needs: [clone, cpu, build-docker]
|
||||
with:
|
||||
can-skip: ${{ needs.cpu.outputs.can-skip == 'true' || needs.clone.outputs.can-skip == 'true' }}
|
||||
docker_npu_image: ${{ needs.build-docker.outputs.docker_npu_image }}
|
||||
|
||||
# ixuca:
|
||||
# name: Linux-IXUCA
|
||||
# uses: ./.github/workflows/_Linux-IXUCA.yml
|
||||
# needs: [clone]
|
||||
# with:
|
||||
# can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
distribute:
|
||||
name: Distribute-stable-build
|
||||
uses: ./.github/workflows/_Distribute-stable.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_distribute_image: ${{ needs.build-docker.outputs.docker_distribute_image }}
|
||||
clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
# distribute-test:
|
||||
# name: Distribute-stable-test
|
||||
# uses: ./.github/workflows/_Distribute-stable-Test.yml
|
||||
# needs: [clone, build-docker, distribute]
|
||||
# with:
|
||||
# docker_distribute_image: ${{ needs.build-docker.outputs.docker_distribute_image }}
|
||||
# clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
|
||||
# distribute-formers:
|
||||
# name: Distribute-stable-formers
|
||||
# uses: ./.github/workflows/_Distribute-stable-Formers.yml
|
||||
# needs: [clone, build-docker, distribute]
|
||||
# with:
|
||||
# docker_distribute_image: ${{ needs.build-docker.outputs.docker_distribute_image }}
|
||||
# clone-can-skip: ${{ needs.clone.outputs.can-skip }}
|
||||
@@ -0,0 +1,62 @@
|
||||
name: Check PR Template
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [develop, release/**]
|
||||
types: [opened, synchronize, reopened, edited]
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: 'template'
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
check:
|
||||
name: Check
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Template
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Clone paddle
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Check PR Template
|
||||
env:
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
AGILE_COMPILE_BRANCH: ${{ github.base_ref }}
|
||||
AGILE_CHECKIN_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
method: check_pr
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
run: |
|
||||
set +e
|
||||
python3 tools/CheckPRTemplate.py
|
||||
EXCODE=$?
|
||||
set -e
|
||||
echo "EXCODE: $EXCODE"
|
||||
echo "ipipe_log_param_EXCODE: $EXCODE"
|
||||
if [[ "$EXCODE" != "0" ]];then
|
||||
echo -e "######################################################"
|
||||
echo -e "If you encounter a situation where the PR template does not match the error message, please use the following link to update your PR: [ https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/.github/PULL_REQUEST_TEMPLATE.md ]"
|
||||
echo -e "## Reference Documentation: ##"
|
||||
echo -e "[ https://github.com/PaddlePaddle/Paddle/wiki/PULL-REQUEST-TEMPLATE--REFERENCE ]"
|
||||
echo -e "[ https://github.com/PaddlePaddle/Paddle/wiki/paddle_ci_manual ]"
|
||||
echo -e "######################################################"
|
||||
exit $EXCODE
|
||||
fi
|
||||
export method=$method
|
||||
echo $method
|
||||
set +x
|
||||
source ~/.icafe
|
||||
wget -q --no-check-certificate https://paddle-qa.bj.bcebos.com/baidu/cloud/modify_icafe.py
|
||||
set -x
|
||||
python3 modify_icafe.py
|
||||
@@ -0,0 +1,62 @@
|
||||
name: Codestyle-Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [develop, release/**]
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "codestyle"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
pre-commit:
|
||||
name: Pre Commit
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
BRANCH: develop
|
||||
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout base repo
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
fetch-depth: 1000
|
||||
|
||||
- name: Merge PR to test branch
|
||||
run: |
|
||||
git fetch origin pull/${PR_ID}/merge
|
||||
git checkout -b test FETCH_HEAD
|
||||
|
||||
- name: Setup python3.12
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install pre-commit==2.17.0
|
||||
pip install ast-grep-cli==0.44.0 # This version should be consistent with the one in .pre-commit-config.yaml
|
||||
|
||||
- name: Run ast-grep unit tests
|
||||
run: |
|
||||
ast-grep test
|
||||
|
||||
- name: Check pre-commit
|
||||
run: |
|
||||
set +e
|
||||
bash -x tools/codestyle/pre_commit.sh;EXCODE=$?
|
||||
exit $EXCODE
|
||||
@@ -0,0 +1,426 @@
|
||||
name: Coverage
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip || 'false' }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
|
||||
clone:
|
||||
name: Coverage clone
|
||||
needs: check-skill
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
workflow-name: "coverage"
|
||||
clone_dir: Paddle-coverage
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
build-docker:
|
||||
name: Coverage build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-coverage
|
||||
task: coverage
|
||||
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
needs: [clone]
|
||||
if: needs.clone.outputs.can-skip != 'true'
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: coverage
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [clone, build-docker, check-bypass]
|
||||
if: ${{ needs.clone.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip }}
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_PIP_CUDA_LIBRARIES: "OFF"
|
||||
WITH_FLAGCX: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
ON_INFER: "ON"
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
UT_RUN_TYPE_SETTING: WITHOUT_HYBRID
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_PIP_CUDA_LIBRARIES \
|
||||
-e WITH_FLAGCX \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ON_INFER \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e UT_RUN_TYPE_SETTING \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream $BRANCH"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash $ci_scripts/coverage_build.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Check added unittests
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/check_added_ut.sh
|
||||
'
|
||||
|
||||
- name: Check coverage build size requires approval
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/coverage_build_size_approval.sh
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
Build_Size=$(du -h --max-depth=0 ${work_dir}/build |awk '"'"'{print $1}'"'"')
|
||||
echo "Build_Size=${Build_Size}" > ${work_dir}/dist/coverage_build_size
|
||||
find ./ -type f -size +200M | xargs du -lh
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf paddle_inference_install_dir
|
||||
rm -rf paddle_inference_c_install_dir
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle/dist
|
||||
mkdir -p ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
|
||||
echo "Uploading coverage build size"
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
cd /
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
cp Paddle.tar.gz ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, build-docker]
|
||||
if: needs.build.outputs.can-skip != 'true'
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz .
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/coverage_test.sh
|
||||
TEST_EXIT_CODE=$?
|
||||
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> ${{ github.env }}
|
||||
if [[ "$TEST_EXIT_CODE" -ne 0 && "$TEST_EXIT_CODE" -ne 9 ]]; then
|
||||
exit $TEST_EXIT_CODE
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Generate coverage information
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
unset GREP_OPTIONS
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/utils.sh; check_coverage
|
||||
mkdir -p /home/data/cfs/coverage/${PR_ID}/${COMMIT_ID}
|
||||
cp build/coverage_files/* /home/data/cfs/coverage/${PR_ID}/${COMMIT_ID}
|
||||
'
|
||||
|
||||
- name: Upload coverage report
|
||||
uses: codecov/codecov-action@v6
|
||||
with:
|
||||
directory: build/coverage_files
|
||||
verbose: true
|
||||
|
||||
- name: Determine whether the coverage rate reaches 90%
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
unset GREP_OPTIONS
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/utils.sh
|
||||
bash $ci_scripts/coverage_judge.sh
|
||||
COVERAGE_EXIT_CODE=$?
|
||||
echo $COVERAGE_EXIT_CODE
|
||||
if [ "$COVERAGE_EXIT_CODE" -ne 0 ]; then
|
||||
echo "Coverage check failed, unit tests have all passed, please do not rerun, check whether the newly added code lines are fully covered by unit tests. If you have any questions, please contact XieYunshen."
|
||||
exit $COVERAGE_EXIT_CODE
|
||||
else
|
||||
echo "Coverage passed"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,736 @@
|
||||
name: CI-H
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: h-coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
fleet_branch: "release/0.2"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-skill:
|
||||
name: Check file paths
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
can-skip: ${{ steps.skip-skill.outputs.should-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Run skip check
|
||||
id: skip-skill
|
||||
uses: ./.github/actions/skip-skill
|
||||
with:
|
||||
ignore-paths: '.agents/skills/**'
|
||||
- name: Print outputs
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== check-skill outputs ==="
|
||||
echo "steps.skip-skill.outputs.should-skip: '${{ steps.skip-skill.outputs.should-skip }}'"
|
||||
echo "============================"
|
||||
|
||||
clone:
|
||||
name: Coverage clone
|
||||
needs: [check-skill]
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
workflow-name: "coverage"
|
||||
clone_dir: h-ci
|
||||
skill-can-skip: ${{ needs.check-skill.outputs.can-skip }}
|
||||
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
needs: [clone]
|
||||
if: ${{ needs.clone.outputs.can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: h-ci
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [clone, check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && needs.clone.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip == 'true' || needs.clone.outputs.can-skip == 'true' || 'false' }}
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Hopper
|
||||
WITH_AVX: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_FA_BUILD_WITH_CACHE: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e WITH_FA_BUILD_WITH_CACHE \
|
||||
-e PY_VERSION \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-ci/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global --add safe.directory "*"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream $BRANCH"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
flashattn_version=$(git submodule status|grep flashattn|awk "{print \$1}"|sed "s#-##g")
|
||||
echo flashattn_version:$flashattn_version
|
||||
url="https://xly-devops.bj.bcebos.com/gpups/flash-attention/cu90/flashattn_libs_${flashattn_version}.tar"
|
||||
echo url:$url
|
||||
url_return=`curl -s -o /dev/null -w "%{http_code}" $url`
|
||||
if [ "$url_return" != "200" ];then
|
||||
echo "flashattn cache not found, please contact umiswing"
|
||||
exit 7
|
||||
fi
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/cmake_preload_data_file.sh
|
||||
pip install -r python/requirements.txt
|
||||
mkdir -p build && cd build
|
||||
ccache -z
|
||||
cmake .. -DPY_VERSION=3.12 -DWITH_GPU=ON -DWITH_DISTRIBUTE=ON -DWITH_TESTING=ON -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN="90" -DFA_JOB_POOLS_COMPILE=1 -DWITH_CUDNN_FRONTEND=ON -DON_INFER=OFF -DWITH_NVSHMEM=ON
|
||||
make -j20
|
||||
EXIT_CODE=$?
|
||||
ccache -s
|
||||
exit $EXIT_CODE
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
: "${PY_VERSION:?PY_VERSION is required}"
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle
|
||||
mv /Paddle.tar.gz .
|
||||
cp ./build/python/dist/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl .
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/h-coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "End Upload"
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: H-Coverage
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
||||
source utils.sh
|
||||
determine_gpu_runner ${runner_name}
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Hopper
|
||||
WITH_AVX: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev
|
||||
docker run -d -t --gpus "\"device=${GPU_DEVICES}\"" --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
id: unit_test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
pip install build//python/dist/*.whl --no-deps
|
||||
pip install -r python/unittest_py/requirements.txt
|
||||
bash $ci_scripts/h-test.sh
|
||||
'
|
||||
|
||||
- name: FA Test
|
||||
if: (success() || failure()) && steps.unit_test.conclusion != 'skipped'
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install pytest einops
|
||||
cd test/test_flashmask_ci
|
||||
bash run.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
fleet-build:
|
||||
name: Build Fleet whl
|
||||
needs: [build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
TASK: fleet-ci-paddle-build-whl-${{ github.event.pull_request.number }}
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_STATSLOG: /paddle/build/.stats.log
|
||||
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e fleet_branch \
|
||||
-e CI_name \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e CCACHE_STATSLOG \
|
||||
-e CCACHE_SLOPPINESS \
|
||||
-e PY_VERSION \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: uv build whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
set -x
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
apt-get update
|
||||
apt-get install -y libnvidia-compute-535
|
||||
python -m pip install --upgrade pip wheel setuptools
|
||||
git clone https://github.com/PaddlePaddle/PaddleFleet.git .
|
||||
git config --global --add safe.directory /paddle
|
||||
git config user.name "PaddleCI"
|
||||
git config user.email "paddle_ci@example.com"
|
||||
git config pull.rebase false
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
mkdir -p /home/.cache/pip
|
||||
pip cache dir
|
||||
echo "Install uv"
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt
|
||||
echo "uv build"
|
||||
git submodule update --init --recursive
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/compat:$LD_LIBRARY_PATH
|
||||
export IS_NVIDIA=True
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
pip install paddle-nvidia-nvshmem-cu12 --extra-index-url https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
uv build --wheel --no-build-isolation -v --python $(which python)
|
||||
'
|
||||
|
||||
- name: download ops whl from latest
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
set -x
|
||||
OPS_DST=paddlefleet_ops-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
wget -q --no-proxy "https://paddle-whl.bj.bcebos.com/nightly/cu129/paddlefleet-ops/${OPS_DST}" -O dist/${OPS_DST}
|
||||
'
|
||||
|
||||
- name: upload whl to BOS
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
cd /
|
||||
if [ ! -f "bos/BosClient.py" ]; then
|
||||
wget -q --no-proxy -O bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir bos
|
||||
tar xf bos_new.tar.gz -C bos
|
||||
fi
|
||||
target_path="paddle-github-action/PR/paddlefleet/${{ env.PR_ID }}/${{ env.COMMIT_ID }}"
|
||||
tar zcf paddldfleet.tar.gz paddle
|
||||
python /bos/BosClient.py paddldfleet.tar.gz ${target_path}
|
||||
'
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'bash ci/clean_uv_cache.sh; rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
fleet_single_card_test:
|
||||
name: Fleet Unit test (single card)
|
||||
needs: [build, fleet-build]
|
||||
if: needs.build.outputs.can-skip != 'true'
|
||||
runs-on:
|
||||
group: Fleet-H-single-card
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
PIP_CACHE_DIR: /root/.cache/pip
|
||||
CACHE_DIR: /root/.cache
|
||||
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-single-card-test
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
gpu_id=$(( $(echo $PWD | awk -F'/' '{print $3}' | awk -F'-' '{print $2}') - 1 ))
|
||||
echo GPU_DEVICES="$gpu_id" >> $GITHUB_ENV
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
GPU_DEVICES: ${{ env.GPU_DEVICES }}
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --name ${container_name} --gpus "\"device=${GPU_DEVICES}\"" --shm-size=32G \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
||||
-v /ssd1/paddle-1/action_cache:/root/.cache \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CACHE_DIR \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e PIP_CACHE_DIR \
|
||||
-e PY_VERSION \
|
||||
-e work_dir \
|
||||
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
||||
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
||||
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
||||
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
||||
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
||||
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
||||
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Install PaddleFleet
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
source /root/proxy
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
||||
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
||||
tar -xf paddldfleet.tar.gz --strip-components=1
|
||||
git config --global --add safe.directory /paddle
|
||||
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
||||
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
||||
chmod +x /usr/local/bin/yq
|
||||
'
|
||||
|
||||
- name: Download paddle.tar.gz and install paddle whl
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
mkdir -p /PaddlePaddle
|
||||
cd /PaddlePaddle
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
echo "paddlefleet commit:"
|
||||
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
||||
'
|
||||
|
||||
- name: Single card test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -xce '
|
||||
pwd
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
bash ci/single_card_test.sh
|
||||
single_card_exit_code=$?
|
||||
if [[ "$single_card_exit_code" != "0" ]]; then
|
||||
echo -e "::error:: \033[31mSingle card test failed.\033[0m"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\033[32mSingle card test succeeded.\033[0m"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
fleet-multi-card_test:
|
||||
name: Fleet Unit test (multi-card)
|
||||
needs: [build, fleet-build, check-bypass, check-skill]
|
||||
if: ${{ needs.build.outputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' && needs.check-skill.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Fleet-H-multi-card
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
PIP_CACHE_DIR: /root/.cache/pip
|
||||
TASK: paddle-fleet-CI-${{ github.event.pull_request.number }}-multi-card_test
|
||||
docker_image: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:ubuntu24-cuda129-py312-dev"
|
||||
PY_VERSION: "3.12"
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}/../../../proxy:/root/proxy \
|
||||
-v ${{ github.workspace }}/../../../.cache:/root/.cache \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CACHE_DIR \
|
||||
-e fleet_branch \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e PIP_CACHE_DIR \
|
||||
-e PY_VERSION \
|
||||
-e work_dir \
|
||||
-e GITHUB_SHA="${{ github.event.pull_request.head.sha }}" \
|
||||
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
|
||||
-e GITHUB_BASE_SHA="${{ github.event.pull_request.base.sha }}" \
|
||||
-e GITHUB_REPO_NAME="${{ github.repository }}" \
|
||||
-e GITHUB_EVENT_NAME="${{ github.event_name }}" \
|
||||
-e GITHUB_EVENT_PULL_REQUEST_NUMBER="${{ github.event.pull_request.number }}" \
|
||||
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
|
||||
-e GITHUB_RUN_ID="${{ github.run_id }}" \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Install PaddleFleet
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
rm -rf * .[^.]*
|
||||
source /root/proxy
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
pip install uv coverage==7.6.1 bce-python-sdk==0.8.74 wrapt pytest matplotlib parameterized
|
||||
wget -q --tries=5 --no-proxy --no-check-certificate https://paddle-github-action.cdn.bcebos.com/PR/paddlefleet/${PR_ID}/${COMMIT_ID}/paddldfleet.tar.gz
|
||||
tar -xf paddldfleet.tar.gz --strip-components=1
|
||||
git config --global --add safe.directory /paddle
|
||||
pip install dist/paddlefleet-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/
|
||||
pip install dist/paddlefleet_ops-*.whl --extra-index-url=https://www.paddlepaddle.org.cn/packages/stable/cu129/ --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq
|
||||
chmod +x /usr/local/bin/yq
|
||||
'
|
||||
|
||||
- name: Download paddle.tar.gz and install paddle whl
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
mkdir -p /PaddlePaddle
|
||||
cd /PaddlePaddle
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/h-coverage/${PR_ID}/${COMMIT_ID}/paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --no-check-certificate
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
pip uninstall paddlepaddle-gpu -y
|
||||
pip install paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl --force-reinstall --extra-index-url=https://www.paddlepaddle.org.cn/packages/nightly/cu129/
|
||||
echo "paddlefleet commit:"
|
||||
python -c "import paddlefleet; print(paddlefleet.version.commit)"
|
||||
'
|
||||
|
||||
- name: Multi-card test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -ce '
|
||||
export PYTHONPATH=$(pwd)
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-12.9/targets/x86_64-linux/lib:/usr/local/cuda/lib64
|
||||
if [ "${BRANCH}" != "develop" ]; then
|
||||
git checkout $fleet_branch
|
||||
echo "Checked out fleet branch: $fleet_branch"
|
||||
else
|
||||
git checkout develop
|
||||
echo "Checked out fleet branch: develop"
|
||||
fi
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
export UV_SKIP_WHEEL_FILENAME_CHECK=1 #This environment variable allows installing the latest commit-level whl package of Paddle.
|
||||
export UV_NO_SYNC=1 # This environment variable prevents uv sync from being executed when running un run.
|
||||
export UV_HTTP_TIMEOUT=300
|
||||
bash ci/multi-card_test.sh
|
||||
multi_card_exit_code=$?
|
||||
if [[ "$multi_card_exit_code" != "0" ]]; then
|
||||
echo -e "::error:: \033[31mMulti card test failed.\033[0m"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\033[32mMulti card test succeeded.\033[0m"
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,393 @@
|
||||
name: Night-Coverage
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 18 * * *"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
CI_name: coverage
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
name: Coverage build docker
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
outputs:
|
||||
docker_coverage_image: ${{ steps.build-docker-images.outputs.docker_coverage_image }}
|
||||
runs-on:
|
||||
group: HK-Clone
|
||||
steps:
|
||||
- name: build-docker-images
|
||||
id: build-docker-images
|
||||
run: |
|
||||
set -x
|
||||
cd ${{ github.workspace }}
|
||||
pwd
|
||||
git clone --depth=1000 https://github.com/PaddlePaddle/Paddle.git
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
|
||||
cd Paddle/tools/dockerfile
|
||||
bash ci_dockerfile.sh
|
||||
md5_value=`md5sum Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage |awk '{print $1}'`
|
||||
echo "docker_coverage_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/ci/paddle:${md5_value}" >> $GITHUB_OUTPUT
|
||||
|
||||
# clean workspace
|
||||
cd ${{ github.workspace }}
|
||||
rm -rf * .[^.]*
|
||||
|
||||
build:
|
||||
name: Coverage build
|
||||
needs: [build-docker]
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_PIP_CUDA_LIBRARIES: "OFF"
|
||||
WITH_FLAGCX: "ON"
|
||||
LITE_GIT_TAG: develop
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
ON_INFER: "ON"
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
UT_RUN_TYPE_SETTING: WITHOUT_HYBRID
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_PIP_CUDA_LIBRARIES \
|
||||
-e WITH_FLAGCX \
|
||||
-e LITE_GIT_TAG \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ON_INFER \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e UT_RUN_TYPE_SETTING \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
echo "Clone Paddle"
|
||||
git clone --depth=1000 https://github.com/PaddlePaddle/Paddle.git .
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
echo "Extracting Paddle"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
git checkout test
|
||||
git submodule update --init --recursive
|
||||
echo "Pull upstream $BRANCH"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p ${CFS_DIR}/.cache/coverage
|
||||
mkdir -p ${CFS_DIR}/.ccache/coverage
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash $ci_scripts/coverage_build.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Clean up env
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
Build_Size=$(du -h --max-depth=0 ${work_dir}/build |awk '"'"'{print $1}'"'"')
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
echo "Build_Size=${Build_Size}" > ${work_dir}/dist/coverage_build_size
|
||||
find ./ -type f -size +200M | xargs du -lh
|
||||
rm -rf $(find . -name "*.a")
|
||||
rm -rf $(find . -name "*.o")
|
||||
rm -rf paddle_inference_install_dir
|
||||
rm -rf paddle_inference_c_install_dir
|
||||
rm -rf lib.linux-x86_64-${PY_VERSION}
|
||||
find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf
|
||||
rm -rf ./python/build/lib.linux-x86_64-${PY_VERSION}/
|
||||
cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /paddle/dist
|
||||
coverage_tag=$(date +%Y-%m-%d)
|
||||
mkdir -p ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
echo "Uploading coverage build size"
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/night/coverage/${coverage_tag}
|
||||
python ${{ env.bos_file }} coverage_build_size paddle-github-action/night/coverage
|
||||
echo "Uploading coverage wheel"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/night/coverage/${coverage_tag}
|
||||
cd /
|
||||
echo "Uploading Paddle.tar.gz"
|
||||
cp Paddle.tar.gz ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Coverage test
|
||||
needs: [build, build-docker]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: "/root/.cache/coverage"
|
||||
CCACHE_DIR: "/root/.ccache/coverage"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_AVX: "ON"
|
||||
WITH_COVERAGE: "ON"
|
||||
WITH_ALL_COVERAGE: "ON"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.12"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
|
||||
CCACHE_MAXSIZE: 200G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
FLAGS_PIR_OPTEST: "TRUE"
|
||||
ON_INFER: "ON"
|
||||
COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ needs.build-docker.outputs.docker_coverage_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GIT_PR_ID \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e ci_scripts \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_AVX \
|
||||
-e WITH_COVERAGE \
|
||||
-e WITH_ALL_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e FLAGS_PIR_OPTEST \
|
||||
-e ON_INFER \
|
||||
-e COVERAGE_FILE \
|
||||
-e GITHUB_TOKEN \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz from cfs"
|
||||
coverage_tag=$(date +%Y-%m-%d)
|
||||
cp ${CFS_DIR}/coverage_night/${coverage_tag}/Paddle.tar.gz .
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash $ci_scripts/coverage_test.sh
|
||||
TEST_EXIT_CODE=$?
|
||||
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> ${{ github.env }}
|
||||
if [[ "$TEST_EXIT_CODE" -ne 0 && "$TEST_EXIT_CODE" -ne 9 ]]; then
|
||||
exit $TEST_EXIT_CODE
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Generate coverage information
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
commit_info=$(git log --format=fuller |head -1|awk "{print \$2}")
|
||||
touch ${PADDLE_ROOT}/night_coverage.txt
|
||||
echo "commit:${commit_info}" >>${PADDLE_ROOT}/night_coverage.txt
|
||||
unset GREP_OPTIONS
|
||||
export WITH_ALL_COVERAGE=ON
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/utils.sh; check_coverage
|
||||
coverage_tag=$(date +"%m-%d")
|
||||
mkdir -p ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
cp build/coverage_files/* ${CFS_DIR}/coverage_night/${coverage_tag}
|
||||
'
|
||||
|
||||
- name: Upload coverage product
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp312-cp312-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
echo "Uploading night_coverage.txt"
|
||||
coverage_time=$(date +%Y-%m-%d)
|
||||
python ${{ env.bos_file }} night_coverage.txt paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded night_coverage.txt"
|
||||
echo "Uploading coverage-full.info"
|
||||
python ${{ env.bos_file }} build/coverage-full.info paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded coverage-full.info"
|
||||
echo "Uploading python-coverage-full.info"
|
||||
python ${{ env.bos_file }} build/python-coverage-full.info paddle-github-action/night/coverage/${coverage_time}
|
||||
echo "Uploaded python-coverage-full.info"
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
rm Paddle.tar.gz
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,54 @@
|
||||
name: Slice-baseline-paddle
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
PR_ID:
|
||||
required: false
|
||||
type: string
|
||||
COMMIT_ID:
|
||||
required: false
|
||||
type: string
|
||||
# schedule:
|
||||
# - cron: '0 20 * * 0'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Slice-base clone
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
is_pr: 'false'
|
||||
|
||||
build-docker:
|
||||
name: Slice build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
build:
|
||||
name: Slice build
|
||||
needs: [clone, build-docker]
|
||||
uses: ./.github/workflows/_Linux-build.yml
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
is_pr: 'false'
|
||||
|
||||
slice-test:
|
||||
name: Slice test
|
||||
uses: ./.github/workflows/_Slice.yml
|
||||
needs: [clone, build-docker, build]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
slice-check: 'true'
|
||||
SLICE_TEST_MODE: insert_baseline
|
||||
MANUALLY_PR_ID: ${{ inputs.PR_ID }}
|
||||
MANUALLY_COMMIT_ID: ${{ inputs.COMMIT_ID }}
|
||||
@@ -0,0 +1,38 @@
|
||||
name: Slice-baseline-torch
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 0 1 * *'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Slice-base clone
|
||||
uses: ./.github/workflows/_Clone-linux.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
is_pr: 'false'
|
||||
|
||||
build-docker:
|
||||
name: Slice build docker
|
||||
needs: clone
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
clone_dir: Paddle-build
|
||||
task: build
|
||||
|
||||
slice-test:
|
||||
name: Slice test
|
||||
uses: ./.github/workflows/_Slice.yml
|
||||
needs: [clone, build-docker]
|
||||
with:
|
||||
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
|
||||
slice-check: 'true'
|
||||
SLICE_TEST_MODE: insert_baseline
|
||||
SLICE_BENCHMARK_FRAMEWORKS: torch
|
||||
@@ -0,0 +1,165 @@
|
||||
name: Api-Benchmark
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
baseline:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
MANUALLY_PR_ID:
|
||||
type: string
|
||||
required: false
|
||||
MANUALLY_COMMIT_ID:
|
||||
type: string
|
||||
required: false
|
||||
run-labels:
|
||||
type: string
|
||||
required: false
|
||||
default: "api-bm"
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-api-benchmark
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: api-benchmark
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "api-benchmark"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
data-storage:
|
||||
name: Performance data storage
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true'}}
|
||||
runs-on:
|
||||
group: Api-bm
|
||||
labels: [self-hosted, "${{ inputs.run-labels }}"]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
echo "docker_image=${docker_image}" >> $GITHUB_ENV
|
||||
docker run -i --rm -v ${{ github.workspace }}:/paddle -w /paddle $docker_image /bin/bash -c 'rm -rf * .[^.]*'
|
||||
cp -r /home/PTSTools .
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $4}')`
|
||||
echo $runner_name
|
||||
core_index=-1
|
||||
if [ $runner_name == "paddle-1" ];then
|
||||
core_index=0
|
||||
elif [ $runner_name == "paddle-2" ];then
|
||||
core_index=8
|
||||
fi
|
||||
echo "core_index=${core_index}" >> $GITHUB_ENV
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
python: "python3.10"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker container ls -a --filter "name=paddle-CI-*-api-benchmark-${core_index}*" --format "{{.ID}}" | xargs -r docker rm -f
|
||||
docker container ls -a --filter "name=api_benchmark_ci_baseline_" --format "{{.ID}} {{.CreatedAt}}" | awk '$2 <= "'$(date -d '1 day ago' +'%Y-%m-%d')'" {print $1}' | xargs -r docker rm -f
|
||||
docker run -d -t --name ${container_name} --shm-size=128g \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e python \
|
||||
-e core_index \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e RUN_ID \
|
||||
-e wheel_link \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download PaddleTest.tar.gz
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ldconfig
|
||||
set -e
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
# git config --global --add safe.directory ${work_dir}
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
# git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
mkdir -p ${{ github.workspace }}/../../../pip
|
||||
${python} -m pip config set global.cache-dir ${{ github.workspace }}/../../../pip
|
||||
${python} -m pip install -r ./PaddleTest/framework/e2e/api_benchmark_new/requirement.txt
|
||||
'
|
||||
|
||||
- name: Paddletest check
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export LD_LIBRARY_PATH=/usr/local/cuda-11.8/compat:/usr/local/cuda/compat:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}
|
||||
cd ./PaddleTest/framework/e2e/api_benchmark_new
|
||||
cp /paddle/PTSTools/Uploader/apibm_config.yml .
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
if [[ "${{ inputs.baseline }}" == "true" ]];then
|
||||
set -e
|
||||
if [[ "${{ inputs.MANUALLY_PR_ID }}" == "" ]]; then
|
||||
export pr_wheel_link=https://paddle-github-action.bj.bcebos.com/PR/build/$PR_ID/$COMMIT_ID/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
else
|
||||
export pr_wheel_link=https://paddle-github-action.bj.bcebos.com/PR/build/${{ inputs.MANUALLY_PR_ID }}/${{ inputs.MANUALLY_COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
fi
|
||||
${python} -m pip install $pr_wheel_link
|
||||
${python} runner_ci_multipro_action.py --yaml ../yaml/sort_api_benchmark_fp32.yml --core_index ${core_index} --baseline_whl_link $pr_wheel_link
|
||||
exit 0
|
||||
fi
|
||||
${python} -m pip install $wheel_link
|
||||
if [ ${core_index} -eq -1 ];then
|
||||
${python} runner_ci_action.py --yaml ../yaml/api_benchmark_fp32.yml --core_index 2
|
||||
else
|
||||
${python} runner_ci_multipro_action.py --yaml ../yaml/sort_api_benchmark_fp32.yml --core_index ${core_index}
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,152 @@
|
||||
name: Auto-Parallel
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /workspace/Paddle
|
||||
PADDLE_ROOT: /workspace/Paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-Auto-Parallel
|
||||
ci_scripts: /workspace/Paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: auto-parallel
|
||||
no_proxy: "localhost,bj.bcebos.com,su.bcebos.com,bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "auto-parallel"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
parallel-test:
|
||||
name: Parallel test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Auto-Parallel
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_dynamic_static_unified_comm: "True"
|
||||
python_version: "3.10"
|
||||
paddle_whl: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
nvidia-docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/workspace \
|
||||
-v /home/FleetX_CI:/gpt_data \
|
||||
-v /home/Llm_gpt_CI:/llm_gpt_data \
|
||||
-v /home/Llama_CI:/llama_data \
|
||||
-v /home/.cache/pip:/home/.cache/pip \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e paddle_whl \
|
||||
-e FLAGS_dynamic_static_unified_comm \
|
||||
-e python_version \
|
||||
-w /workspace --runtime=nvidia ${docker_image}
|
||||
|
||||
- name: Download Paddle
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/build.tar.gz --no-check-certificate
|
||||
echo "Extracting build.tar.gz"
|
||||
tar --use-compress-program="pzstd" -xpf build.tar.gz
|
||||
mv paddle Paddle
|
||||
rm -f build.tar.gz
|
||||
'
|
||||
|
||||
- name: Download PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "Download and extract PaddleNLP.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate
|
||||
tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cd PaddleNLP
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git pull
|
||||
git checkout 6ac04028757dfbcc089916997493611f62de81b2
|
||||
git switch -c 6ac04028757dfbcc089916997493611f62de81b2
|
||||
git cherry-pick bc08aeec91d2c992c3d8d39755bea7c6213b0e82
|
||||
git cherry-pick 7ab35ce94eca977bcf3b44bfb42deb0e0b5ef158
|
||||
git cherry-pick 2ac85997c2dafe3d67a4aac01d553ad76d6024bf
|
||||
git submodule update --init --recursive --force
|
||||
'
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ldconfig
|
||||
set -e
|
||||
pip config set global.cache-dir "/home/.cache/pip"
|
||||
ln -sf $(which python${python_version}) /usr/bin/python
|
||||
python -c "import sys; print(sys.version_info[:])"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
timeout 80m bash Paddle/ci/auto_parallel/ci_auto_parallel.sh ${paddle_whl}
|
||||
'
|
||||
|
||||
- name: Upload and display logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /workspace/case_logs
|
||||
for FILE in /workspace/case_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/Auto-Parallel/${PR_ID}/${COMMIT_ID}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/Auto-Parallel/${PR_ID}/${COMMIT_ID}/logs/$file"
|
||||
done
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,141 @@
|
||||
name: CE-CINN-Framework
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-CINN-Framework
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: ce-cinn-framework
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-cinn-framework
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
cinn:
|
||||
name: CINN
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker container ls -a --filter "name=paddle-CI-*-api-benchmark-${core_index}*" --format "{{.ID}}" | xargs -r docker rm -f
|
||||
docker container ls -a --filter "name=api_benchmark_ci_baseline_" --format "{{.ID}} {{.CreatedAt}}" | awk '$2 <= "'$(date -d '1 day ago' +'%Y-%m-%d')'" {print $1}' | xargs -r docker rm -f
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e python \
|
||||
-e core_index \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
git checkout test
|
||||
rm build.tar.gz
|
||||
cd /
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
'
|
||||
|
||||
- name: Determine ci trigger
|
||||
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${ci_scripts}/ce_cinn_diff.sh
|
||||
if [ ${sum_num} -eq 0 ];then
|
||||
echo "The modified files does not affect LayerCase in CE-CINN-Framework, so skip this ci."
|
||||
echo "skip_ci=true" >> ${{ github.env }}
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Run check
|
||||
if: ${{ env.skip_ci != 'true' }}
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
pip config set global.cache-dir "$CACHE_DIR/pip"
|
||||
pip install /paddle/build/pr_whl/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
cd /PaddleTest/framework/e2e/PaddleLT_new
|
||||
pip install -r requirement.txt
|
||||
source ./scene/set_ci_dy^dy2stcinn_train^dy2stcinn_eval_inputspec_env.sh
|
||||
python support/dict_to_yml.py --filename apibm_config.yml --data_str "$(cat $CACHE_DIR/cinn_config)"
|
||||
set -e
|
||||
python run.py
|
||||
exit $(head -n 1 "exit_code.txt")
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,312 @@
|
||||
name: CE-Framework
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework
|
||||
CI_name: ce-framework
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass-whl:
|
||||
name: Check bypass (ce-whl)
|
||||
if: ${{ inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-whl
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
check-bypass-infer:
|
||||
name: Check bypass (ce-infer)
|
||||
if: ${{ inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-infer
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
check-bypass-test:
|
||||
name: Check bypass (ce-test)
|
||||
if: ${{ inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ce-test
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
whl:
|
||||
name: Whl
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass-whl.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass-whl]
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework-whl
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Check whl size
|
||||
env:
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
bash ci/check_whl_size.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
infer:
|
||||
name: Infer
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass-infer.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass-infer]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework-infer
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
FLAGS_enable_eager_mode: 1
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
cd /
|
||||
cp /paddle/build/pr_whl/*.whl .
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
'
|
||||
|
||||
- name: Install paddle and dependencies
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir run_env
|
||||
ln -s $(which python3.10) run_env/python
|
||||
ln -s $(which pip3.10) run_env/pip
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
pip config set global.cache-dir "$CACHE_DIR/pip"
|
||||
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
python -m pip install pip --upgrade
|
||||
pip install /paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --no-index --no-deps
|
||||
pip install -r /PaddleTest/inference/python_api_test/requirements.txt
|
||||
'
|
||||
|
||||
- name: Check
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
cd /PaddleTest/inference/python_api_test
|
||||
bash parallel_run.sh 2
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass-test.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass-test]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework-test
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
FLAGS_enable_eager_mode: 1
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e CACHE_DIR \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download Paddle and PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
cd /
|
||||
cp /paddle/build/pr_whl/*.whl .
|
||||
echo "Downloading PaddleTest.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-check-certificate
|
||||
echo "Extracting PaddleTest.tar.gz"
|
||||
tar -zvxf PaddleTest.tar.gz 1>/dev/null 2>&1
|
||||
'
|
||||
|
||||
- name: Install paddle and dependencies
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir run_env
|
||||
ln -s $(which python3.10) run_env/python
|
||||
ln -s $(which pip3.10) run_env/pip
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
pip config set global.cache-dir "${CFS_DIR}/.cache/pip"
|
||||
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||
pip install sympy
|
||||
pip install /paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl --no-index --no-deps
|
||||
'
|
||||
|
||||
- name: Check
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
cd /PaddleTest/framework/api
|
||||
rm ./device/test_cuda_get_device_capability.py
|
||||
bash run_paddle_ci.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,161 @@
|
||||
name: Clone-linux
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
workflow-name:
|
||||
type: string
|
||||
required: false
|
||||
clone_dir:
|
||||
type: string
|
||||
required: false
|
||||
default: "Paddle"
|
||||
is_pr:
|
||||
type: string
|
||||
required: false
|
||||
default: "true"
|
||||
skill-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
outputs:
|
||||
can-skip:
|
||||
value: ${{ jobs.clone.outputs.can-skip }}
|
||||
slice-check:
|
||||
value: ${{ jobs.clone.outputs.slice-check }}
|
||||
xpu-only:
|
||||
value: ${{ jobs.clone.outputs.xpu-only }}
|
||||
|
||||
permissions: read-all
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
ci_scripts: ${{ github.workspace }}/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: ${{ inputs.workflow-name }}
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
clone:
|
||||
name: Clone Paddle
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
needs: [check-bypass]
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip == 'true' || inputs.skill-can-skip == 'true' }}
|
||||
slice-check: ${{ steps.check-execution.outputs.slice-check }}
|
||||
xpu-only: ${{ steps.xpu-only-check.outputs.xpu-only }}
|
||||
runs-on:
|
||||
group: HK-Clone
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Debug - Print inputs values
|
||||
run: |
|
||||
echo "=== Debug Info ==="
|
||||
echo "inputs.skill-can-skip: '${{ inputs.skill-can-skip }}'"
|
||||
echo "inputs.skill-can-skip length: $(echo '${{ inputs.skill-can-skip }}' | wc -c)"
|
||||
echo "==="
|
||||
|
||||
- name: Set environment variables and print
|
||||
id: set-vars
|
||||
run: |
|
||||
# Determine if can skip based on check-bypass and skill-can-skip
|
||||
if [[ "${{ needs.check-bypass.outputs.can-skip }}" == "true" ]] || [[ "${{ inputs.skill-can-skip }}" == "true" ]]; then
|
||||
echo "CAN_SKIP=true" >> $GITHUB_ENV
|
||||
else
|
||||
echo "CAN_SKIP=false" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Print can-skip value
|
||||
run: |
|
||||
echo "can-skip output: ${{ env.CAN_SKIP }}"
|
||||
|
||||
- name: Check submodules status
|
||||
run: git submodule foreach --recursive 'git rev-parse HEAD || rm -rf $PWD' || true
|
||||
|
||||
- name: Clone paddle
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
fetch-depth: 1000
|
||||
|
||||
- name: Merge PR to test branch
|
||||
id: check-execution
|
||||
if: ${{ inputs.is_pr == 'true' }}
|
||||
run: |
|
||||
git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
|
||||
git switch ${{ github.event.pull_request.base.ref }}
|
||||
set +e
|
||||
git branch -D test
|
||||
set -e
|
||||
git gc
|
||||
git switch -c test
|
||||
git config user.name "PaddleCI"
|
||||
git config user.email "paddle_ci@example.com"
|
||||
git fetch origin pull/${{ github.event.pull_request.number }}/head:pr
|
||||
git merge --no-ff pr
|
||||
git submodule sync --recursive
|
||||
git submodule update --init --recursive
|
||||
git branch -d pr
|
||||
source ${ci_scripts}/check_execution.sh
|
||||
bash ${ci_scripts}/third_party_tag.sh
|
||||
|
||||
- name: Check if XPU-only PR
|
||||
id: xpu-only-check
|
||||
if: ${{ inputs.is_pr == 'true' }}
|
||||
env:
|
||||
DISABLE_HW_CI_SKIP: ${{ vars.DISABLE_HW_CI_SKIP || 'false' }}
|
||||
run: |
|
||||
if bash ci/check_xpu_only.sh; then
|
||||
echo "xpu-only=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "xpu-only=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download bos client
|
||||
env:
|
||||
home_path: "/home/paddle/actions-runner/"
|
||||
bos_file: "/home/paddle/actions-runner/bos/BosClient.py"
|
||||
run: |
|
||||
if [ ! -f "${bos_file}" ]; then
|
||||
wget -q --no-proxy -O ${home_path}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${home_path}/bos
|
||||
tar xf ${home_path}/bos_new.tar.gz -C ${home_path}/bos
|
||||
fi
|
||||
|
||||
- name: Push paddle-action.tar.gz to bos
|
||||
env:
|
||||
AK: paddle
|
||||
SK: paddle
|
||||
bos_file: "/home/paddle/actions-runner/bos/BosClient.py"
|
||||
run: |
|
||||
cd ..
|
||||
tar -I 'zstd -T0' -cf Paddle.tar.gz Paddle
|
||||
echo "::group::Install bce-python-sdk"
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
python ${bos_file} Paddle.tar.gz paddle-github-action/PR/${{ inputs.clone_dir }}/${PR_ID}/${COMMIT_ID}
|
||||
rm Paddle.tar.gz
|
||||
cd -
|
||||
git switch ${BRANCH}
|
||||
set +e
|
||||
git branch -D test
|
||||
git gc
|
||||
|
||||
# - name: Clean environment
|
||||
# if: always()
|
||||
# run: |
|
||||
# cd ${{ github.workspace }}
|
||||
# rm -rf * .[^.]*
|
||||
@@ -0,0 +1,90 @@
|
||||
name: Clone-windows
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
skill-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
outputs:
|
||||
can-skip:
|
||||
value: ${{ jobs.clone.outputs.can-skip }}
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
WORK_DIR: ${{ github.workspace }}\..
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
clone:
|
||||
name: Clone Paddle
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
outputs:
|
||||
can-skip: ${{ inputs.skill-can-skip == 'true' }}
|
||||
runs-on:
|
||||
group: win-clone
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Git config
|
||||
run: |
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git config --global http.postBuffer 524288000
|
||||
git config --global core.longpaths true
|
||||
|
||||
- name: Clone paddle
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
fetch-depth: 1000
|
||||
|
||||
- name: Merge pr
|
||||
run: |
|
||||
REM git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
|
||||
git switch %BRANCH%
|
||||
git branch -D test
|
||||
git gc
|
||||
git switch -c test
|
||||
git fetch origin pull/%PR_ID%/head:pr
|
||||
git merge --no-ff pr
|
||||
git branch -d pr
|
||||
REM call %ci_scripts%\third_party_tag.bat
|
||||
git config -f .gitmodules submodule.third_party/openvino.update none && git submodule sync third_party/openvino
|
||||
git submodule update --init --recursive
|
||||
|
||||
- name: Upload paddle to bos
|
||||
working-directory: ..
|
||||
env:
|
||||
BCE_FILE: ${{ github.workspace }}\..\bce-python-sdk-new\BosClient.py
|
||||
run: |
|
||||
if not exist %WORK_DIR%\bce-python-sdk-new (
|
||||
echo There is no bce in this PC, will install bce.
|
||||
pip install wget
|
||||
pip install shutil
|
||||
echo Download package from https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz
|
||||
python -c "import wget;wget.download('https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz')"
|
||||
python -c "import shutil;shutil.unpack_archive('bos_new.tar.gz', extract_dir='./bce-python-sdk-new',format='gztar')"
|
||||
)
|
||||
python -m pip install pycryptodome
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
tar -cf Paddle.tar Paddle && zstd Paddle.tar
|
||||
python %BCE_FILE% Paddle.tar.zst paddle-github-action/windows/PR/%PR_ID%/%COMMIT_ID%
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
|
||||
- name: Clean env
|
||||
run: |
|
||||
cd ${{ github.workspace }}
|
||||
git switch %BRANCH%
|
||||
git branch -D test
|
||||
git config -f .gitmodules submodule.third_party/openvino.update none && git submodule sync third_party/openvino
|
||||
git submodule update --init --recursive
|
||||
git gc
|
||||
@@ -0,0 +1,206 @@
|
||||
name: DeepMD-Kit Test
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
MANUALLY_PR_ID:
|
||||
type: string
|
||||
required: false
|
||||
MANUALLY_COMMIT_ID:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-deepmd
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: deepmd
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "deepmd"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
deepmd-test:
|
||||
name: DeepMD-Kit Integration Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
container_name="deepmd_test_ci_${RUN_ID}"
|
||||
echo "container_name=${container_name}" >> $GITHUB_ENV
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e RUN_ID \
|
||||
-e wheel_link \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Print PR information
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
echo "Testing PR #${{ env.PR_ID }}"
|
||||
echo "Commit: ${{ env.COMMIT_ID }}"
|
||||
echo "Branch: ${{ env.BRANCH }}"
|
||||
'
|
||||
|
||||
- name: Install PaddlePaddle
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
mkdir -p ${{ github.workspace }}/../../../.cache/pip
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python3.10 -m pip config set global.cache-dir ${{ github.workspace }}/../../../.cache/pip
|
||||
|
||||
if [[ "${{ inputs.MANUALLY_PR_ID }}" == "" ]]; then
|
||||
echo "Installing PaddlePaddle from: $wheel_link"
|
||||
python3.10 -m pip install $wheel_link
|
||||
else
|
||||
echo "Installing PaddlePaddle from manual PR: ${{ inputs.MANUALLY_PR_ID }}"
|
||||
python3.10 -m pip install https://paddle-github-action.bj.bcebos.com/PR/build/${{ inputs.MANUALLY_PR_ID }}/${{ inputs.MANUALLY_COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
fi
|
||||
|
||||
python3.10 -c "import paddle; print(f\"PaddlePaddle version: {paddle.__version__}\")"
|
||||
'
|
||||
|
||||
- name: Clone and setup deepmd-kit
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
cd /paddle
|
||||
rm -rf deepmd-kit
|
||||
echo "Cloning deepmd-kit repository..."
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
git clone https://github.com/deepmodeling/deepmd-kit.git -b v3.1.2
|
||||
ls -lah /paddle/deepmd-kit
|
||||
cd deepmd-kit
|
||||
'
|
||||
|
||||
- name: Install deepmd-kit dependencies
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
cd /paddle/deepmd-kit
|
||||
echo "Installing deepmd-kit..."
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
DP_ENABLE_TENSORFLOW=0 DP_ENABLE_PYTORCH=0 python3.10 -m pip install -v -e . -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
'
|
||||
|
||||
- name: Install test dependencies
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
echo "Installing pytest..."
|
||||
python3.10 -m pip install \
|
||||
pytest pytest-xdist pytest-json-report pytest-metadata pytest-mock pytest-rich pytest-sugar pytest-timeout \
|
||||
-i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
'
|
||||
|
||||
- name: Run deepmd-kit pytest
|
||||
id: pytest_run
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
cd /paddle/deepmd-kit
|
||||
echo "=========================================="
|
||||
echo "Running DeepMD-Kit Integration Tests"
|
||||
echo "=========================================="
|
||||
|
||||
python3.10 -m pytest -vvv --tb=long --color=yes \
|
||||
source/tests/pd/test_auto_batch_size.py \
|
||||
source/tests/pd/test_change_bias.py \
|
||||
source/tests/pd/test_decomp.py \
|
||||
source/tests/pd/test_dp_show.py \
|
||||
source/tests/pd/test_finetune.py \
|
||||
source/tests/pd/test_multitask.py \
|
||||
source/tests/pd/test_training.py \
|
||||
source/tests/pd/test_update_sel.py \
|
||||
source/tests/pd/test_utils.py \
|
||||
source/tests/pd/model/test_atomic_model_atomic_stat.py \
|
||||
source/tests/pd/model/test_atomic_model_global_stat.py \
|
||||
source/tests/pd/model/test_autodiff.py \
|
||||
source/tests/pd/model/test_deeppot.py \
|
||||
source/tests/pd/model/test_descriptor_dpa1.py \
|
||||
source/tests/pd/model/test_descriptor_dpa2.py \
|
||||
source/tests/pd/model/test_dp_atomic_model.py \
|
||||
source/tests/pd/model/test_dp_model.py \
|
||||
source/tests/pd/model/test_dpa1.py \
|
||||
source/tests/pd/model/test_dpa2.py \
|
||||
source/tests/pd/model/test_dpa3.py \
|
||||
source/tests/pd/model/test_dynamic_sel.py \
|
||||
source/tests/pd/model/test_ener_fitting.py \
|
||||
source/tests/pd/model/test_env_mat.py \
|
||||
source/tests/pd/model/test_exclusion_mask.py \
|
||||
source/tests/pd/model/test_force_grad.py \
|
||||
source/tests/pd/model/test_forward_lower.py \
|
||||
source/tests/pd/model/test_get_model.py \
|
||||
source/tests/pd/model/test_jit.py \
|
||||
source/tests/pd/model/test_mlp.py \
|
||||
source/tests/pd/model/test_nlist.py \
|
||||
source/tests/pd/model/test_null_input.py \
|
||||
source/tests/pd/model/test_permutation_denoise.py \
|
||||
source/tests/pd/model/test_permutation.py \
|
||||
source/tests/pd/model/test_region.py \
|
||||
source/tests/pd/model/test_rot_denoise.py \
|
||||
source/tests/pd/model/test_rot.py \
|
||||
source/tests/pd/model/test_rotation.py \
|
||||
source/tests/pd/model/test_se_e2_a.py \
|
||||
source/tests/pd/model/test_smooth.py \
|
||||
source/tests/pd/model/test_trans_denoise.py \
|
||||
source/tests/pd/model/test_trans.py \
|
||||
source/tests/pd/model/test_unused_params.py 2>&1 | tee /paddle/pytest_output.log
|
||||
TEST_RESULT=${PIPESTATUS[0]}
|
||||
|
||||
echo "DeepMD-Kit Test Finished with status: $TEST_RESULT"
|
||||
exit $TEST_RESULT
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'cd /paddle && rm -rf deepmd-kit'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,189 @@
|
||||
name: Distribute-stable-Formers
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_distribute_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-distribute-formers
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: distribute
|
||||
no_proxy: bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn,paddlepaddle.org.cn
|
||||
docker_image: ${{ inputs.docker_distribute_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "formers"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
formers-test:
|
||||
name: formers-Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Distribute
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_COVERAGE: "OFF"
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
|
||||
PRECISION_TEST: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
WITH_INCREMENTAL_COVERAGE: "OFF"
|
||||
WITH_ONNXRUNTIME: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PY_VERSION: "3.10"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_CUDNN_FRONTEND: "ON"
|
||||
FLAGS_enable_cudnn_frontend: 1
|
||||
CACHE_DIR: /root/.cache/build
|
||||
CCACHE_DIR: /root/.ccache/formers
|
||||
CFS_DIR: /home/data/cfs
|
||||
paddle_whl: /workspace/dist/*.whl
|
||||
formers_docker: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest
|
||||
run: |
|
||||
export CUDA_SO="$(\ls -d /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls -d /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')"
|
||||
export DEVICES="$(\ls -d /dev/nvidia* | xargs -I{} echo "-v {}:{}") $(\ls /dev/nvidia-caps/* | xargs -I{} echo "-v {}:{}")"
|
||||
export SMI="-v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi"
|
||||
container_name=${TASK}-test-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} ${CUDA_SO} ${DEVICES} ${SMI} --runtime=nvidia --shm-size=32G --privileged \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/ssd1/models:/home/models" \
|
||||
-v "/ssd1/root:/root" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/workspace \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CI_name \
|
||||
-e PF_HOME=/home/models \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_COVERAGE \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
|
||||
-e PRECISION_TEST \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e AGILE_REVISION \
|
||||
-e WITH_INCREMENTAL_COVERAGE \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e GIT_PR_ID \
|
||||
-e PY_VERSION \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_CUDNN_FRONTEND \
|
||||
-e FLAGS_enable_cudnn_frontend \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CFS_DIR \
|
||||
-e paddle_whl \
|
||||
-e no_proxy \
|
||||
-w /workspace --network host ${formers_docker}
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global --add safe.directory /workspace
|
||||
git checkout test
|
||||
'
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN
|
||||
set -ex
|
||||
bash /workspace/ci/formers_test.sh
|
||||
'
|
||||
- name: Upload and display logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then
|
||||
bos_prefix="${PR_ID}/${COMMIT_ID}"
|
||||
else
|
||||
bos_prefix="schedule/$(date +%Y%m%d)"
|
||||
fi
|
||||
# api test logs
|
||||
cd /workspace/PaddleFormers/unittest_logs
|
||||
for FILE in /workspace/PaddleFormers/unittest_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs/$file"
|
||||
done
|
||||
# models test logs
|
||||
cd /workspace/PaddleFormers/model_unittest_logs
|
||||
for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file"
|
||||
done
|
||||
'
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,170 @@
|
||||
name: Distribute-stable-Test
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_distribute_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-distribute-test
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: distribute
|
||||
no_proxy: bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn,paddlepaddle.org.cn
|
||||
docker_image: ${{ inputs.docker_distribute_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: distribute-test
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: Distribute
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_COVERAGE: "OFF"
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
|
||||
PRECISION_TEST: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
WITH_INCREMENTAL_COVERAGE: "OFF"
|
||||
WITH_ONNXRUNTIME: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PY_VERSION: "3.10"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
WITH_CUDNN_FRONTEND: "ON"
|
||||
FLAGS_enable_cudnn_frontend: 1
|
||||
CACHE_DIR: /root/.cache/build
|
||||
CCACHE_DIR: /root/.ccache/gpubox
|
||||
run: |
|
||||
export CUDA_SO="$(\ls -d /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls -d /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')"
|
||||
export DEVICES="$(\ls -d /dev/nvidia* | xargs -I{} echo "-v {}:{}") $(\ls /dev/nvidia-caps/* | xargs -I{} echo "-v {}:{}")"
|
||||
export SMI="-v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi"
|
||||
container_name=${TASK}-test-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} ${CUDA_SO} ${DEVICES} ${SMI} --runtime=nvidia --shm-size=32G \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/ssd1/root:/root" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CI_name \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_COVERAGE \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
|
||||
-e PRECISION_TEST \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e AGILE_REVISION \
|
||||
-e WITH_INCREMENTAL_COVERAGE \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e PADDLE_VERSION \
|
||||
-e GIT_PR_ID \
|
||||
-e PY_VERSION \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_CUDNN_FRONTEND \
|
||||
-e FLAGS_enable_cudnn_frontend \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program="pzstd" -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git checkout test
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/distribute_test.sh
|
||||
'
|
||||
|
||||
- name: Upload and display logs
|
||||
if: always()
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd /case_logs
|
||||
for FILE in /case_logs/*; do
|
||||
file=$(basename "$FILE")
|
||||
python ${{ env.bos_file }} $file paddle-github-action/PR/Distribute-Stable/${PR_ID}/${COMMIT_ID}/logs
|
||||
echo "$file: https://paddle-github-action.bj.bcebos.com/PR/Distribute-Stable/${PR_ID}/${COMMIT_ID}/logs/$file"
|
||||
done
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,234 @@
|
||||
name: Distribute-stable-Build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_distribute_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-distribute
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: distribute
|
||||
no_proxy: bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn,paddlepaddle.org.cn
|
||||
docker_image: ${{ inputs.docker_distribute_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: distribute
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip }}
|
||||
needs: [check-bypass]
|
||||
if: ${{ inputs.clone-can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_MKL: "OFF"
|
||||
WITH_PYTHON: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
GPUBOX_DEMO_INSTALL_DIR: /root/.cache/build
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/python35
|
||||
PY_VERSION: "3.10"
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: "Ninja"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
CUDA_ARCH_NAME: Ampere
|
||||
WITH_CUDNN_FRONTEND: "ON"
|
||||
FLAGS_enable_cudnn_frontend: 1
|
||||
CACHE_DIR: /root/.cache/build
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_STATSLOG: /paddle/build/.stats.log
|
||||
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
||||
run: |
|
||||
container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e CI_name \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e WITH_TENSORRT \
|
||||
-e GENERATOR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e WITH_AVX \
|
||||
-e WITH_PYTHON \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_STATSLOG \
|
||||
-e CCACHE_SLOPPINESS \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e GPUBOX_DEMO_INSTALL_DIR \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_CUDNN_FRONTEND \
|
||||
-e FLAGS_enable_cudnn_frontend \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p /root/.cache/build
|
||||
mkdir -p /root/.ccache/gpubox
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Download flashattn cache
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set +e
|
||||
flashattn_version=$(git submodule status | grep flashattn | awk "{print \$1}" | sed "s#-##g")
|
||||
echo "flashattn_version=${flashattn_version}" >> ${{ github.env }}
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/flashattn_cache/flashattn_libs_${flashattn_version}.tar.gz --no-check-certificate; FACODE=$?
|
||||
if [ $FACODE -ne 0 ]; then
|
||||
echo "flashattn_cached_package=true" >> ${{ github.env }}
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Build
|
||||
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Packaging of products
|
||||
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
if [ "${{ env.flashattn_cached_package }}" == "true" ]; then
|
||||
cd ${work_dir}/build/third_party/install/flashattn/lib
|
||||
mkdir flashattn_libs_${{ env.flashattn_version }} && cd flashattn_libs_${{ env.flashattn_version }}
|
||||
mkdir fa_libs && cp ../lib*.so fa_libs && tar -zcf fa_libs.tar ./fa_libs && rm -rf ./fa_libs
|
||||
md5sum fa_libs.tar |awk "{print \$1}" >MD5.txt
|
||||
cd .. && tar -zcf flashattn_libs_${{ env.flashattn_version }}.tar ./flashattn_libs_${{ env.flashattn_version }}
|
||||
fi
|
||||
bash ${ci_scripts}/compress_build.sh
|
||||
cd ${work_dir}/..
|
||||
tar --use-compress-program="pzstd -1" --warning=no-file-changed -cf Paddle.tar.gz paddle
|
||||
'
|
||||
|
||||
- name: Upload product to bos
|
||||
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py
|
||||
paddle_whl: "*.whl"
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -x
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
echo "::group::Install bce-python-sdk"
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
cd ..
|
||||
source ${{ github.workspace }}/../../../unproxy
|
||||
echo "Uploading Paddle.tar.gz to bos"
|
||||
python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading whl to bos"
|
||||
mv ${work_dir}/dist/${{ env.paddle_whl }} .
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
if [ "${{ env.flashattn_cached_package }}" == "true" ]; then
|
||||
echo "Uploading flashattn_libs_${flashattn_version}.tar.gz to bos"
|
||||
mv ${work_dir}/build/third_party/install/flashattn/lib/flashattn_libs_${{ env.flashattn_version }}.tar .
|
||||
python ${{ env.bos_file }} flashattn_libs_${{ env.flashattn_version }}.tar paddle-github-action/PR/gpups/flashattn_cache
|
||||
fi
|
||||
rm -rf Paddle.tar.gz ${{ env.paddle_whl }} flashattn_libs_${flashattn_version}.tar
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,123 @@
|
||||
name: Doc-Preview
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_doc_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /workspace/paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-Doc-Preview
|
||||
CI_name: Doc-Preview
|
||||
CFS_DIR: /home/data/cfs
|
||||
work_dir: /paddle
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "Doc-Preview"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-doc:
|
||||
name: Build doc
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CACHE_DIR: /home/data/cfs/.cache
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
BUILD_DOC: "true"
|
||||
UPLOAD_DOC: "true"
|
||||
BOSBUCKET: "paddle-site-web-dev"
|
||||
PREVIEW_SITE: "paddle-docs-preview.paddlepaddle.org.cn"
|
||||
AGILE_COMPILE_BRANCH: ${{ startsWith(github.event.pull_request.base.ref, 'fleety_') && 'develop' || github.event.pull_request.base.ref }}
|
||||
BOS_CREDENTIAL_AK: "paddle"
|
||||
BOS_CREDENTIAL_SK: "paddle"
|
||||
PADDLE_WHL: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_doc_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/python35-cpu:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CACHE_DIR \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e AGILE_PULL_ID \
|
||||
-e AGILE_REVISION \
|
||||
-e BUILD_DOC \
|
||||
-e UPLOAD_DOC \
|
||||
-e BOSBUCKET \
|
||||
-e PREVIEW_SITE \
|
||||
-e BRANCH \
|
||||
-e AGILE_COMPILE_BRANCH \
|
||||
-e BOS_CREDENTIAL_AK \
|
||||
-e BOS_CREDENTIAL_SK \
|
||||
-e PADDLE_WHL \
|
||||
-e CFS_DIR \
|
||||
-e work_dir \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Doc build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${AGILE_PULL_ID}/${AGILE_REVISION}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
if ! api_doc_spec_diff=$(python tools/diff_api.py paddle/fluid/API_DEV.spec.doc paddle/fluid/API_PR.spec.doc); then
|
||||
echo "diff_api.py failed for API doc specs."
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$api_doc_spec_diff" ]; then
|
||||
echo "API documents no change."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd /
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
pip3 install --progress-bar off -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple --force-reinstall ${work_dir}/build/pr_whl/*.whl
|
||||
bash ${work_dir}/ci/run_docs_preview.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,322 @@
|
||||
name: PR-CI-Inference
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_inference_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
docker_image: ${{ inputs.docker_inference_image }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "inference"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_build
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: "0.15"
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
|
||||
WITH_PYTHON: "OFF"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_NVCC_LAZY: "OFF"
|
||||
PYTHON_ABI: cp310-cp310
|
||||
CACHE_DIR: /root/.cache/inference
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: "0.8"
|
||||
CUDA_ARCH_NAME: Pascal
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/home/data/cfs/.ccache/inference:/root/.ccache/inference" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:/action-runner" \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_PYTHON \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e WITH_NVCC_LAZY \
|
||||
-e PYTHON_ABI \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e PY_VERSION \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-build/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git config --global --add safe.directory "*"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
source /action-runner/proxy
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Run build
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c '
|
||||
source /action-runner/proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/inference_build.sh
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload build.tar.gz
|
||||
env:
|
||||
home_path: /action-runner
|
||||
bos_file: /action-runner/bos_retry/BosClient.py
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source /action-runner/proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
rm -rf `find . -name "*.o"`
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --tries=5 --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos_retry
|
||||
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
|
||||
fi
|
||||
source /action-runner/unproxy
|
||||
cd /
|
||||
tar --use-compress-program="pigz -1" -cpPf build.tar.gz paddle
|
||||
echo "Uploading build.tar.gz to cfs"
|
||||
mkdir -p ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}
|
||||
cp build.tar.gz ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID} && echo success
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
# docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: build
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-inference_test
|
||||
runs-on:
|
||||
group: Inference-P4
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
FLAGS_fraction_of_gpu_memory_to_use: "0.15"
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 4
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "ON"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
INFERENCE_DEMO_INSTALL_DIR: /root/.cache/inference_demo
|
||||
WITH_PYTHON: "OFF"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_NVCC_LAZY: "OFF"
|
||||
PYTHON_ABI: cp310-cp310
|
||||
CACHE_DIR: /root/.cache/inference
|
||||
CCACHE_DIR: /root/.ccache/inference
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: "0.8"
|
||||
CUDA_ARCH_NAME: Auto
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker pull $docker_image
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:/action-runner" \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e WITH_AVX \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_PYTHON \
|
||||
-e WITH_ONNXRUNTIME \
|
||||
-e WITH_NVCC_LAZY \
|
||||
-e PYTHON_ABI \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e PY_VERSION \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
tar --use-compress-program="pigz -1" -xpf build.tar.gz --strip-components=1
|
||||
rm build.tar.gz
|
||||
export PATH=$(pwd)/run_env:${PATH}
|
||||
echo "export PATH=${PATH}" >> ~/.bashrc
|
||||
echo "::group::Install dependencies"
|
||||
source /action-runner/proxy
|
||||
pip install -r "python/unittest_py/requirements.txt"
|
||||
echo "::endgroup::"
|
||||
'
|
||||
|
||||
- name: Test fluid library for inference
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
test_fluid_lib
|
||||
'
|
||||
|
||||
- name: Test go inference api
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
test_go_inference_api
|
||||
'
|
||||
|
||||
- name: Check approvals of unittest
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/utils.sh
|
||||
init
|
||||
source /action-runner/proxy
|
||||
check_approvals_of_unittest 3
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
# docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
@@ -0,0 +1,201 @@
|
||||
name: Linux-CPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_cpu_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether to skip the job"
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-cpu
|
||||
ci_scripts: ${{ github.workspace }}/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: cpu
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: cpu
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
outputs:
|
||||
can-skip: ${{ needs.check-bypass.outputs.can-skip || inputs.clone-can-skip || 'false' }}
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_GPU: "OFF"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_MKL: "OFF"
|
||||
WITH_TESTING: "ON"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PREC_SUFFIX: .py3
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
PROC_RUN: 12
|
||||
FLAGS_enable_eager_mode: 1
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: "Ninja"
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
WITH_AVX: "OFF"
|
||||
CACHE_DIR: "/root/.cache/cpu"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
WITH_OPENVINO: "ON"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_cpu_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache/python35-cpu:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e WITH_GPU \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PREC_SUFFIX \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e PROC_RUN \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e WITH_TENSORRT \
|
||||
-e GENERATOR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e GITHUB_ENV \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CI_name \
|
||||
-e no_proxy \
|
||||
-e WITH_OPENVINO \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_linux_cpu_test.sh
|
||||
EXCODE=$?
|
||||
source ${ci_scripts}/utils.sh; clean_build_files
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload paddle_whl
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
echo "::group::Install bce-python-sdk"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
echo "::endgroup::"
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar -xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
cd dist
|
||||
echo "Uploading paddle_whl to bos"
|
||||
source ${{ github.workspace }}/../../../unproxy
|
||||
python3.10 ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/cpu_whl/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,308 @@
|
||||
name: Linux-DCU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_dcu_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
docker_image: ${{ inputs.docker_dcu_image }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
ci_scripts_runner: ${{ github.workspace }}/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
PADDLE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: dcu
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "dcu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-dcu_build
|
||||
runs-on:
|
||||
group: HK-CPU
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_DOC: "OFF"
|
||||
WITH_GPU: "OFF"
|
||||
WITH_ROCM: "ON"
|
||||
WITH_RCCL: "ON"
|
||||
WITH_XPU: "OFF"
|
||||
WITH_TENSORRT: "OFF"
|
||||
WITH_COVERAGE: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
CMAKE_BUILD_TYPE: "Release"
|
||||
WITH_AVX: "ON"
|
||||
WITH_MKL: "ON"
|
||||
WITH_ARM: "OFF"
|
||||
WITH_CACHE: "ON"
|
||||
RUN_TEST: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
CI_SKIP_CPP_TEST: "OFF"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
CMAKE_EXPORT_COMPILE_COMMANDS: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
PY_VERSION: "3.10"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --shm-size=128G \
|
||||
--ulimit nofile=102400:102400 -d -t --name ${container_name} \
|
||||
-v $home_dir/.cache:/root/.cache \
|
||||
-v $home_dir/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_DOC \
|
||||
-e WITH_GPU \
|
||||
-e WITH_ROCM \
|
||||
-e WITH_RCCL \
|
||||
-e WITH_XPU \
|
||||
-e WITH_TENSORRT \
|
||||
-e WITH_COVERAGE \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e WITH_MKL \
|
||||
-e WITH_AVX \
|
||||
-e WITH_ARM \
|
||||
-e RUN_TEST \
|
||||
-e WITH_TESTING \
|
||||
-e CI_SKIP_CPP_TEST \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e BRANCH \
|
||||
-e PADDLE_BRANCH \
|
||||
-e PADDLE_VERSION \
|
||||
-e no_proxy \
|
||||
-e CMAKE_EXPORT_COMPILE_COMMANDS \
|
||||
-e PY_VERSION \
|
||||
-e GIT_PR_ID \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ulimit -n 102400
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop or target branch"
|
||||
git pull upstream $BRANCH --no-edit
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Run build
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source /opt/dtk-24.04.1/env.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
rm -rf `find . -name "*.a"`
|
||||
rm -rf `find . -name "*.o"`
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload build.tar.gz and paddle_whl to bos
|
||||
env:
|
||||
AK: paddle
|
||||
SK: paddle
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle_dcu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
apt install zstd -y
|
||||
cd ..
|
||||
tar --use-compress-program="pzstd -5" -cf build.tar.gz paddle
|
||||
echo "Uploading build.tar.gz"
|
||||
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/dcu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading paddle_whl"
|
||||
cp /paddle/dist/${{ env.paddle_whl }} .
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/dcu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: build
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-dcu_test
|
||||
runs-on:
|
||||
group: dcu-z100
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
set -x
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
||||
source utils.sh
|
||||
determine_dcu_runner ${runner_name}
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
WITH_ROCM: "ON"
|
||||
WITH_RCCL: "ON"
|
||||
WITH_AVX: "ON"
|
||||
WITH_MKL: "ON"
|
||||
IF_DCU: "OFF"
|
||||
WITH_TENSORRT: "OFF"
|
||||
WITH_XPU: "OFF"
|
||||
WITH_CINN: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_TESTING: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
HIP_VISIBLE_DEVICES: ${{ env.HIP_VISIBLE_DEVICES }}
|
||||
PY_VERSION: "3.10"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --ulimit nofile=102400 --ulimit core=-1 --shm-size=32g -d -t --name ${container_name} \
|
||||
-v /ssd1/cibuild/.cache:/root/.cache \
|
||||
-v /ssd1/cibuild/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
--device=/dev/kfd \
|
||||
--device=/dev/dri \
|
||||
--device=/dev/mkfd \
|
||||
--group-add video \
|
||||
--shm-size=32g \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_ROCM \
|
||||
-e WITH_RCCL \
|
||||
-e WITH_AVX \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TENSORRT \
|
||||
-e WITH_XPU \
|
||||
-e IF_DCU \
|
||||
-e HIP_VISIBLE_DEVICES \
|
||||
-e WITH_RCCL \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e no_proxy \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download build.tar.gz
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf ./*
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/dcu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
|
||||
tar --use-compress-program="pzstd" -xf build.tar.gz --strip-components=1
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Run test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PATH=/usr/local/bin:${PATH}
|
||||
ln -sf $(which python3.10) /usr/local/bin/python
|
||||
ln -sf $(which pip3.10) /usr/local/bin/pip
|
||||
pip3.10 install ./dist/paddlepaddle_dcu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
wget -q --no-proxy https://paddle-device.bj.bcebos.com/dcu/hyhal-Z100.tar.gz
|
||||
tar -zxf hyhal-Z100.tar.gz -C /opt
|
||||
source /opt/dtk-24.04.1/env.sh
|
||||
bash -x ${ci_scripts}/dcu_test.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
@@ -0,0 +1,466 @@
|
||||
name: Linux-IXUCA
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
concurrency:
|
||||
group: linux-ixuca-pr-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: ixuca
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "ixuca"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test-primary:
|
||||
name: Build and Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on: iluvatar-gpu-2
|
||||
timeout-minutes: 120
|
||||
container:
|
||||
image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:3.3.0
|
||||
env:
|
||||
LD_LIBRARY_PATH: /usr/local/corex/lib
|
||||
LIBRARY_PATH: /usr/local/corex/lib
|
||||
env:
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
outputs:
|
||||
result: ${{ steps.set-result.outputs.result }}
|
||||
|
||||
steps:
|
||||
- name: Check approval
|
||||
id: check-approval
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
AUTHORIZED_REVIEWERS=("YqGe585" "yongqiangma")
|
||||
|
||||
echo "Authorized reviewers: ${AUTHORIZED_REVIEWERS[*]}"
|
||||
echo "Repository: ${{ github.repository }}"
|
||||
echo "PR number: ${{ github.event.pull_request.number }}"
|
||||
|
||||
response=""
|
||||
for i in 1 2 3; do
|
||||
_tmp=$(curl -s --fail --max-time 30 \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" 2>/dev/null) && { response="$_tmp"; break; }
|
||||
echo "Attempt $i to fetch PR reviews failed, retrying..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "Response length: ${#response}"
|
||||
|
||||
can_skip=$(python3 -c '
|
||||
import json, sys
|
||||
|
||||
def debug(msg):
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
debug("=== Review Check Debug Info ===")
|
||||
debug(f"Authorized reviewers: {sys.argv[1:]}")
|
||||
AUTHORIZED = set(sys.argv[1:])
|
||||
|
||||
try:
|
||||
input_data = sys.stdin.read()
|
||||
debug(f"Raw input data length: {len(input_data)}")
|
||||
data = json.loads(input_data)
|
||||
debug(f"Parsed {len(data)} reviews")
|
||||
except Exception as e:
|
||||
debug(f"JSON parse error: {str(e)}")
|
||||
print("false")
|
||||
sys.exit(0)
|
||||
|
||||
latest_state = {}
|
||||
debug("\n=== Review States ===")
|
||||
for r in data:
|
||||
user = r.get("user", {}).get("login")
|
||||
if not user:
|
||||
continue
|
||||
state = r.get("state")
|
||||
latest_state[user] = state
|
||||
commit_id = r.get("commit_id") or ""
|
||||
debug(f"- {user}: {state} (commit: {commit_id[:8]})")
|
||||
|
||||
debug("\n=== Final Check ===")
|
||||
for user, state in latest_state.items():
|
||||
if user in AUTHORIZED and state == "APPROVED":
|
||||
debug(f"Found APPROVED review from authorized user: {user}")
|
||||
print("true")
|
||||
sys.exit(0)
|
||||
|
||||
debug("No APPROVED review found from authorized users")
|
||||
print("false")
|
||||
' "${AUTHORIZED_REVIEWERS[@]}" <<< "$response")
|
||||
|
||||
echo "can-skip=$can_skip" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$can_skip" = "true" ]; then
|
||||
echo "Found valid approval from authorized reviewer, skipping IXUCA CI"
|
||||
else
|
||||
echo "No valid approval found, running full CI"
|
||||
fi
|
||||
|
||||
- name: Download Paddle.tar.gz and setup
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy \
|
||||
https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
||||
rm Paddle.tar.gz
|
||||
cd Paddle
|
||||
git config --global --add safe.directory "*"
|
||||
git config --global http.proxy http://oversea-website-proxy.aistudio.public:8888
|
||||
git config --global https.proxy http://oversea-website-proxy.aistudio.public:8888
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Fetching upstream develop branch"
|
||||
git fetch upstream develop
|
||||
echo "Rebasing test branch onto upstream/develop"
|
||||
git rebase upstream/develop || {
|
||||
echo "Rebase failed, attempting to continue with current state"
|
||||
git rebase --abort || true
|
||||
}
|
||||
echo "Latest 5 commits after rebase:"
|
||||
git log --oneline -5
|
||||
|
||||
- name: Get Paddle-iluvatar PR number
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
id: get-pcd-pr
|
||||
run: |
|
||||
# Get Paddle PR description
|
||||
PR_BODY=$(curl -s \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" \
|
||||
| python3 -c "import json,sys; data=json.load(sys.stdin); print(data.get('body', ''))")
|
||||
|
||||
# Extract [Paddle-iluvatar PR] XXXX from description
|
||||
PCD_PR_NUM=$(echo "$PR_BODY" | grep -oP '\[Paddle-iluvatar PR\]\s*\K\d+' || echo "")
|
||||
|
||||
if [ -n "$PCD_PR_NUM" ]; then
|
||||
echo "Found Paddle-iluvatar PR: $PCD_PR_NUM"
|
||||
echo "pcd_pr_num=$PCD_PR_NUM" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No Paddle-iluvatar PR found in PR description"
|
||||
echo "pcd_pr_num=" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Clone Paddle-iluvatar and setup
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
git config --global --get http.proxy
|
||||
git config --global --get https.proxy
|
||||
for i in 1 2 3; do
|
||||
git clone --depth=1 -b develop https://github.com/PaddlePaddle/Paddle-iluvatar.git && break
|
||||
echo "git clone attempt $i failed, retrying in 10s..."
|
||||
rm -rf Paddle-iluvatar
|
||||
sleep 10
|
||||
done
|
||||
cd Paddle-iluvatar
|
||||
|
||||
# If Paddle-iluvatar PR number exists, fetch and merge it
|
||||
PCD_PR_NUM="${{ steps.get-pcd-pr.outputs.pcd_pr_num }}"
|
||||
if [ -n "$PCD_PR_NUM" ]; then
|
||||
git config user.email "Paddle-CI@example.com"
|
||||
git config user.name "Paddle-CI-Bot"
|
||||
git fetch --unshallow origin
|
||||
echo "Fetching and merging Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
git fetch origin pull/$PCD_PR_NUM/head:pr-$PCD_PR_NUM
|
||||
git merge pr-$PCD_PR_NUM --no-edit
|
||||
echo "Successfully merged Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
fi
|
||||
|
||||
rm -rf Paddle
|
||||
mv ../Paddle .
|
||||
|
||||
- name: Build paddlepaddle-iluvatar
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
export PATH=/usr/local/corex/bin:$PATH
|
||||
cd Paddle-iluvatar
|
||||
apt-get install -y patchelf
|
||||
bash build_paddle.sh
|
||||
|
||||
- name: Install paddlepaddle-iluvatar
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
pip install Paddle-iluvatar/Paddle/build/python/dist/paddlepaddle_iluvatar*.whl
|
||||
retry_or_skip() {
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
echo "WARN: command failed, retrying in 20s: $*"
|
||||
sleep 20
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
echo "WARN: command failed again, skip install: $*"
|
||||
return 0
|
||||
}
|
||||
retry_or_skip pip install parameterized pyyaml
|
||||
|
||||
- name: Run tests
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
run: |
|
||||
export FLAG_SKIP_FLOAT64=1
|
||||
cd Paddle-iluvatar/tests && bash run_test.sh
|
||||
|
||||
- name: Set result
|
||||
if: steps.check-approval.outputs.can-skip != 'true'
|
||||
id: set-result
|
||||
run: |
|
||||
echo "result=success" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
chmod -R 777 . || true
|
||||
|
||||
# test-fallback:
|
||||
# name: Build and Test (Fallback)
|
||||
# needs: [check-bypass, test-primary]
|
||||
# if: >-
|
||||
# ${{ always()
|
||||
# && needs.check-bypass.result == 'success'
|
||||
# && needs.check-bypass.outputs.can-skip != 'true'
|
||||
# && needs.test-primary.outputs.result != 'success' }}
|
||||
# runs-on:
|
||||
# group: test2
|
||||
# timeout-minutes: 120
|
||||
# env:
|
||||
# TASK: paddle-CI-${{ github.event.pull_request.number }}-ixuca
|
||||
|
||||
# steps:
|
||||
# - name: Check approval
|
||||
# id: check-approval
|
||||
# run: |
|
||||
# set -euo pipefail
|
||||
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
|
||||
# AUTHORIZED_REVIEWERS=("YqGe585" "yongqiangma")
|
||||
|
||||
# echo "Authorized reviewers: ${AUTHORIZED_REVIEWERS[*]}"
|
||||
# echo "Repository: ${{ github.repository }}"
|
||||
# echo "PR number: ${{ github.event.pull_request.number }}"
|
||||
|
||||
# response=""
|
||||
# for i in 1 2 3; do
|
||||
# _tmp=$(curl -s --fail --max-time 30 \
|
||||
# -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
# -H "Accept: application/vnd.github.v3+json" \
|
||||
# "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" 2>/dev/null) && { response="$_tmp"; break; }
|
||||
# echo "Attempt $i to fetch PR reviews failed, retrying..."
|
||||
# sleep 5
|
||||
# done
|
||||
|
||||
# echo "Response length: ${#response}"
|
||||
|
||||
# can_skip=$(python3 -c '
|
||||
# import json, sys
|
||||
|
||||
# def debug(msg):
|
||||
# print(msg, file=sys.stderr)
|
||||
|
||||
# debug("=== Review Check Debug Info ===")
|
||||
# debug(f"Authorized reviewers: {sys.argv[1:]}")
|
||||
# AUTHORIZED = set(sys.argv[1:])
|
||||
|
||||
# try:
|
||||
# input_data = sys.stdin.read()
|
||||
# debug(f"Raw input data length: {len(input_data)}")
|
||||
# data = json.loads(input_data)
|
||||
# debug(f"Parsed {len(data)} reviews")
|
||||
# except Exception as e:
|
||||
# debug(f"JSON parse error: {str(e)}")
|
||||
# print("false")
|
||||
# sys.exit(0)
|
||||
|
||||
# latest_state = {}
|
||||
# debug("\n=== Review States ===")
|
||||
# for r in data:
|
||||
# user = r.get("user", {}).get("login")
|
||||
# if not user:
|
||||
# continue
|
||||
# state = r.get("state")
|
||||
# latest_state[user] = state
|
||||
# commit_id = r.get("commit_id") or ""
|
||||
# debug(f"- {user}: {state} (commit: {commit_id[:8]})")
|
||||
|
||||
# debug("\n=== Final Check ===")
|
||||
# for user, state in latest_state.items():
|
||||
# if user in AUTHORIZED and state == "APPROVED":
|
||||
# debug(f"Found APPROVED review from authorized user: {user}")
|
||||
# print("true")
|
||||
# sys.exit(0)
|
||||
|
||||
# debug("No APPROVED review found from authorized users")
|
||||
# print("false")
|
||||
# ' "${AUTHORIZED_REVIEWERS[@]}" <<< "$response")
|
||||
|
||||
# echo "can-skip=$can_skip" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# if [ "$can_skip" = "true" ]; then
|
||||
# echo "Found valid approval from authorized reviewer, skipping IXUCA CI"
|
||||
# else
|
||||
# echo "No valid approval found, running full CI"
|
||||
# fi
|
||||
|
||||
# - name: Download Paddle.tar.gz and setup
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# timeout-minutes: 30
|
||||
# run: |
|
||||
# set -e
|
||||
# echo "Downloading Paddle.tar.gz"
|
||||
# wget -q --tries=5 --no-proxy \
|
||||
# https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
# echo "Extracting Paddle.tar.gz"
|
||||
# tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
||||
# rm Paddle.tar.gz
|
||||
# cd Paddle
|
||||
# git config --global --add safe.directory "*"
|
||||
# git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
# git config pull.rebase false
|
||||
# git checkout test
|
||||
# echo "Fetching upstream develop branch"
|
||||
# git config --global --unset-all http.proxy || true
|
||||
# git config --global --unset-all https.proxy || true
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# git config --global http.proxy "$http_proxy"
|
||||
# git config --global https.proxy "$https_proxy"
|
||||
# git fetch upstream develop
|
||||
# echo "Rebasing test branch onto upstream/develop"
|
||||
# git rebase upstream/develop || {
|
||||
# echo "Rebase failed, attempting to continue with current state"
|
||||
# git rebase --abort || true
|
||||
# }
|
||||
# echo "Latest 5 commits after rebase:"
|
||||
# git log --oneline -5
|
||||
|
||||
# - name: Get Paddle-iluvatar PR number
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# id: get-pcd-pr
|
||||
# run: |
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
|
||||
# # Get Paddle PR description
|
||||
# PR_BODY=$(curl -s \
|
||||
# -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
# -H "Accept: application/vnd.github.v3+json" \
|
||||
# "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" \
|
||||
# | python3 -c "import json,sys; data=json.load(sys.stdin); print(data.get('body', ''))")
|
||||
|
||||
# # Extract [Paddle-iluvatar PR] XXXX from description
|
||||
# PCD_PR_NUM=$(echo "$PR_BODY" | grep -oP '\[Paddle-iluvatar PR\]\s*\K\d+' || echo "")
|
||||
|
||||
# if [ -n "$PCD_PR_NUM" ]; then
|
||||
# echo "Found Paddle-iluvatar PR: $PCD_PR_NUM"
|
||||
# echo "pcd_pr_num=$PCD_PR_NUM" >> "$GITHUB_OUTPUT"
|
||||
# else
|
||||
# echo "No Paddle-iluvatar PR found in PR description"
|
||||
# echo "pcd_pr_num=" >> "$GITHUB_OUTPUT"
|
||||
# fi
|
||||
|
||||
# - name: Clone Paddle-iluvatar and setup
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# run: |
|
||||
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
||||
|
||||
# rm -rf Paddle-iluvatar
|
||||
# git clone --depth=1 -b develop https://github.com/PaddlePaddle/Paddle-iluvatar.git
|
||||
# cd Paddle-iluvatar
|
||||
|
||||
# # If Paddle-iluvatar PR number exists, fetch and merge it
|
||||
# PCD_PR_NUM="${{ steps.get-pcd-pr.outputs.pcd_pr_num }}"
|
||||
# if [ -n "$PCD_PR_NUM" ]; then
|
||||
# git config user.email "Paddle-CI@example.com"
|
||||
# git config user.name "Paddle-CI-Bot"
|
||||
# git fetch --unshallow origin
|
||||
# echo "Fetching and merging Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
# git fetch origin pull/$PCD_PR_NUM/head:pr-$PCD_PR_NUM
|
||||
# git merge pr-$PCD_PR_NUM --no-edit
|
||||
# echo "Successfully merged Paddle-iluvatar PR #$PCD_PR_NUM"
|
||||
# fi
|
||||
|
||||
# rm -rf Paddle
|
||||
# mv ../Paddle .
|
||||
|
||||
# - name: Start docker container
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# run: |
|
||||
# container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
# echo "container_name=${container_name}" >> $GITHUB_ENV
|
||||
# docker run --network host --privileged --cap-add=ALL \
|
||||
# --pid=host --shm-size=128G -d -t --name ${container_name} \
|
||||
# -v /usr/src:/usr/src -v /lib/modules:/lib/modules \
|
||||
# -v /dev:/dev -v /home:/home -v /data:/data \
|
||||
# -v ${{ github.workspace }}/Paddle-iluvatar:/paddle \
|
||||
# -e LD_LIBRARY_PATH=/usr/local/corex/lib \
|
||||
# -e LIBRARY_PATH=/usr/local/corex/lib \
|
||||
# -v /root/.cache:/root/.cache \
|
||||
# -v /root/.cache:/root/.ccache \
|
||||
# -e BRANCH -e PR_ID -e COMMIT_ID \
|
||||
# -w /paddle \
|
||||
# ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:3.3.0 /bin/bash
|
||||
|
||||
# - name: Build and test
|
||||
# if: steps.check-approval.outputs.can-skip != 'true'
|
||||
# run: |
|
||||
# docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
# retry_or_skip() {
|
||||
# if "$@"; then
|
||||
# return 0
|
||||
# fi
|
||||
# echo "WARN: command failed, retrying in 20s: $*"
|
||||
# sleep 20
|
||||
# if "$@"; then
|
||||
# return 0
|
||||
# fi
|
||||
# echo "WARN: command failed again, skip install: $*"
|
||||
# return 0
|
||||
# }
|
||||
# export PATH=/usr/local/corex/bin:$PATH
|
||||
# apt-get install -y patchelf
|
||||
# bash build_paddle.sh
|
||||
# pip install Paddle/build/python/dist/paddlepaddle_iluvatar*.whl
|
||||
# retry_or_skip pip install parameterized pyyaml
|
||||
# export FLAG_SKIP_FLOAT64=1
|
||||
# cd tests && bash run_test.sh
|
||||
# '
|
||||
|
||||
# - name: Cleanup
|
||||
# if: always()
|
||||
# run: |
|
||||
# docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*' || true
|
||||
# docker stop ${{ env.container_name }} || true
|
||||
# docker rm ${{ env.container_name }} || true
|
||||
@@ -0,0 +1,197 @@
|
||||
name: Linux-NPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
docker_npu_image:
|
||||
type: string
|
||||
required: true
|
||||
|
||||
env:
|
||||
docker_image: ${{ inputs.docker_npu_image }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
ci_scripts_runner: ${{ github.workspace }}/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-npu
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: npu
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "npu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: NPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker run -i --rm \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e BRANCH \
|
||||
-w /paddle $docker_image /bin/bash -c 'rm -rf * .[^.]*'
|
||||
source ~/.bashrc
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
||||
rm Paddle.tar.gz
|
||||
cd Paddle
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop"
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
source ${{ github.workspace }}/Paddle/ci/utils.sh
|
||||
determine_npu_runner ${runner_name}
|
||||
echo no_proxy="localhost,127.0.0.1,localaddress,.bj.bcebos.com,.localdomain.com,.cdn.bcebos.com,.baidu.com,.bcebos.com" >> ${{ github.env }}
|
||||
|
||||
- name: Clone PaddleCustomDevice repository
|
||||
run: |
|
||||
source ~/.bashrc
|
||||
git clone --depth=1000 -b develop https://github.com/PaddlePaddle/PaddleCustomDevice.git
|
||||
cd PaddleCustomDevice
|
||||
cp -r ../Paddle .
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_COVERAGE: "OFF"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PYTHON_VERSION: "3.10"
|
||||
no_proxy: bcebos.com
|
||||
USE_910B: 1
|
||||
FLAGS_set_to_1d: "False"
|
||||
NVIDIA_TF32_OVERRIDE: 0
|
||||
paddle_submodule: "ON"
|
||||
MC2: 1
|
||||
HCCL_OP_BASE_FFTS_MODE_ENABLE: "TRUE"
|
||||
PADDLE_XCCL_BACKEND: npu
|
||||
HCCL_SOCKET_IFNAME: =xgbe0
|
||||
FLAGS_eager_communication_connection: 1
|
||||
FLAGS_use_stride_kernel: 0
|
||||
FLAGS_allocator_strategy: naive_best_fit
|
||||
FLAGS_npu_storage_format: 0
|
||||
TEST_IMPORTANT: "ON"
|
||||
PADDLE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
echo ${ASCEND_RT_VISIBLE_DEVICES}
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --privileged --pids-limit 409600 --shm-size=128G -d -t \
|
||||
--cap-add=SYS_PTRACE --security-opt seccomp=unconfined --name ${container_name} \
|
||||
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/dcmi:/usr/local/dcmi \
|
||||
-v /ssd2/workspace/npu-dev/.cache:/root/.cache \
|
||||
-v /ssd2/workspace/npu-dev/.ccache:/root/.ccache \
|
||||
-v $home_dir/actions-runner:$home_dir/actions-runner \
|
||||
-v ${{ github.workspace }}/PaddleCustomDevice:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e PADDLE_VERSION \
|
||||
-e ASCEND_RT_VISIBLE_DEVICES \
|
||||
-e WITH_COVERAGE \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e AGILE_PULL_ID \
|
||||
-e AGILE_REVISION \
|
||||
-e GIT_PR_ID \
|
||||
-e PYTHON_VERSION \
|
||||
-e no_proxy \
|
||||
-e USE_910B \
|
||||
-e FLAGS_set_to_1d \
|
||||
-e NVIDIA_TF32_OVERRIDE \
|
||||
-e paddle_submodule \
|
||||
-e MC2 \
|
||||
-e HCCL_OP_BASE_FFTS_MODE_ENABLE \
|
||||
-e PADDLE_XCCL_BACKEND \
|
||||
-e HCCL_SOCKET_IFNAME \
|
||||
-e FLAGS_eager_communication_connection \
|
||||
-e FLAGS_use_stride_kernel \
|
||||
-e FLAGS_allocator_strategy \
|
||||
-e FLAGS_npu_storage_format \
|
||||
-e TEST_IMPORTANT \
|
||||
-e PADDLE_BRANCH \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Install Paddle-CPU
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -x
|
||||
source ~/.bashrc
|
||||
wget -q https://sys-p0.bj.bcebos.com/libstdc%2B%2B6_13.1.0-8ubuntu1_20.04.2_amd64.deb
|
||||
dpkg -i *deb
|
||||
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/cpu_whl/${PR_ID}/${COMMIT_ID}/paddlepaddle-0.0.0-cp310-cp310-linux_x86_64.whl --no-check-certificate
|
||||
PATH=/usr/local/bin:${PATH}
|
||||
echo "export PATH=$PATH" >> ~/.bashrc
|
||||
ln -sf $(which python3.10) /usr/local/bin/python
|
||||
ln -sf $(which pip3.10) /usr/local/bin/pip
|
||||
ln -sf $(which python3.10) /usr/bin/python
|
||||
ln -sf $(which pip3.10) /usr/bin/pip
|
||||
echo "::group::Install Paddle"
|
||||
pip install paddlepaddle-*.whl && rm -rf paddlepaddle-*.whl
|
||||
python -c "import paddle; print(paddle.__version__)"
|
||||
python -c "import paddle; print(paddle.version.commit)"
|
||||
echo "::endgroup::"
|
||||
'
|
||||
|
||||
- name: Build and test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ~/.bashrc
|
||||
set -x
|
||||
echo "::group::Install dependencies"
|
||||
python -m pip install PyGithub
|
||||
python -m pip install wheel
|
||||
echo "::endgroup::"
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
pip install -U numpy==1.26.4
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
git config --global --add safe.directory ${work_dir}/Paddle
|
||||
cd Paddle
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
cd ..
|
||||
bash backends/npu/tools/pr_ci_npu.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${container_name}
|
||||
docker rm ${container_name}
|
||||
@@ -0,0 +1,283 @@
|
||||
name: Linux-XPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_xpu_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
docker_image: ${{ inputs.docker_xpu_image }}
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /paddle/ci
|
||||
ci_scripts_runner: ${{ github.workspace }}/ci
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: xpu
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "xpu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-xpu_build
|
||||
runs-on:
|
||||
group: XPU-build
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_XPU: "ON"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
WITH_AVX: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_TESTING: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
XPU_VISIBLE_DEVICES: "0,1"
|
||||
WITH_XPU_BKCL: "ON"
|
||||
WITH_XPU_FFT: "ON"
|
||||
WITH_XPU_XRE5: "ON"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
IF_KUNLUN3: "OFF"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --ulimit nofile=102400:102400 -d -t --name ${container_name} \
|
||||
-v $home_dir/.cache:/root/.cache \
|
||||
-v $home_dir/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_XPU \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e XPU_VISIBLE_DEVICES \
|
||||
-e WITH_XPU_BKCL \
|
||||
-e WITH_XPU_FFT \
|
||||
-e WITH_XPU_XRE5 \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e IF_KUNLUN3 \
|
||||
-e no_proxy \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download paddle.tar.gz and update test branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
ulimit -n 102400
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
git config pull.rebase false
|
||||
git checkout test
|
||||
echo "Pull upstream develop or target branch"
|
||||
bash ci/git_pull.sh $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Run build
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
apt install zstd -y
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
bash ${ci_scripts}/compress_build.sh
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Upload build.tar.gz and paddle_whl to bos
|
||||
env:
|
||||
AK: paddle
|
||||
SK: paddle
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle_xpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
set -e
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
cd ..
|
||||
tar --use-compress-program="pzstd -5" -cf build.tar.gz paddle
|
||||
# source /home/opt/deck/1.0/etc/bashrc
|
||||
echo "Uploading build.tar.gz"
|
||||
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading paddle_whl"
|
||||
cp /paddle/dist/${{ env.paddle_whl }} .
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${container_name}
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs: build
|
||||
env:
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-xpu_test
|
||||
runs-on:
|
||||
group: Kunlun
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: Determine the runner
|
||||
run: |
|
||||
runner_name=`(echo $PWD|awk -F '/' '{print $3}')`
|
||||
echo $runner_name
|
||||
wget -q https://xly-devops.bj.bcebos.com/utils.sh
|
||||
source utils.sh
|
||||
determine_kunlun_runner ${runner_name}
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_XPU: "ON"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
CMAKE_BUILD_TYPE: Release
|
||||
WITH_AVX: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_TESTING: "ON"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
XPU_VISIBLE_DEVICES: ${{ env.CUDA_VISIBLE_DEVICES }}
|
||||
CUDA_VISIBLE_DEVICES: ${{ env.CUDA_VISIBLE_DEVICES }}
|
||||
WITH_XPU_BKCL: "ON"
|
||||
CACHE_DIR: /root/.cache
|
||||
CCACHE_DIR: /root/.ccache
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
IF_KUNLUN3: "OFF"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
home_dir: ${{ github.workspace }}/../../../..
|
||||
FLAGS_use_stride_kernel: "0"
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker run --ulimit nofile=102400 --ulimit core=-1 --shm-size=32g -d -t --name ${container_name} \
|
||||
-v /ssd1/cibuild/.cache:/root/.cache \
|
||||
-v /ssd1/cibuild/.ccache:/root/.ccache \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-v /ssd1/cibuild/tmp:/tmp \
|
||||
--device ${XPU_CODE_1} \
|
||||
--device ${XPU_CODE_2} \
|
||||
--device /dev/xpuctrl \
|
||||
--shm-size=32g \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_XPU \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e CMAKE_BUILD_TYPE \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_TESTING \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e XPU_VISIBLE_DEVICES \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e WITH_XPU_BKCL \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e IF_KUNLUN3 \
|
||||
-e no_proxy \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e FLAGS_use_stride_kernel \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image} /bin/bash
|
||||
|
||||
- name: Download build.tar.gz
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/xpu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
|
||||
tar --use-compress-program="pzstd" -xf build.tar.gz --strip-components=1
|
||||
rm build.tar.gz
|
||||
'
|
||||
|
||||
- name: Run test
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/kunlun_test.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
docker exec -t ${container_name} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${container_name}
|
||||
@@ -0,0 +1,236 @@
|
||||
name: Linux-build
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
is_pr:
|
||||
type: string
|
||||
required: false
|
||||
default: "true"
|
||||
clone-can-skip:
|
||||
default: "false"
|
||||
required: false
|
||||
type: string
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether to skip the job"
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-build
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: build
|
||||
CFS_DIR: /home/data/cfs
|
||||
is_pr: ${{ inputs.is_pr }}
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: build
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build:
|
||||
name: Build
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
|
||||
FLAGS_fraction_of_gpu_memory_to_use: 0.15
|
||||
CTEST_OUTPUT_ON_FAILURE: 1
|
||||
CTEST_PARALLEL_LEVEL: 2
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_COVERAGE: "OFF"
|
||||
PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
PRECISION_TEST: "ON"
|
||||
WITH_CINN: "ON"
|
||||
INFERENCE_DEMO_INSTALL_DIR: /home/data/cfs/.cache/build
|
||||
WITH_INCREMENTAL_COVERAGE: "OFF"
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_STATSLOG: /paddle/build/.stats.log
|
||||
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
|
||||
CACHE_DIR: /home/data/cfs/.cache/build
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e BRANCH \
|
||||
-e is_pr \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
|
||||
-e FLAGS_fraction_of_gpu_memory_to_use \
|
||||
-e CTEST_OUTPUT_ON_FAILURE \
|
||||
-e CTEST_PARALLEL_LEVEL \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_COVERAGE \
|
||||
-e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
|
||||
-e CUDA_VISIBLE_DEVICES \
|
||||
-e PRECISION_TEST \
|
||||
-e WITH_CINN \
|
||||
-e INFERENCE_DEMO_INSTALL_DIR \
|
||||
-e WITH_INCREMENTAL_COVERAGE \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PY_VERSION \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_SLOPPINESS \
|
||||
-e CCACHE_STATSLOG \
|
||||
-e CFS_DIR \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-build/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
'
|
||||
|
||||
- name: Merge target branch
|
||||
if: ${{ inputs.is_pr == 'true' }}
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Build
|
||||
if: needs.check-bypass.outputs.can-skip != 'true'
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p /root/.cache/build
|
||||
mkdir -p /root/.ccache/build
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
'
|
||||
|
||||
- name: Check sequence op
|
||||
if: needs.check-bypass.outputs.can-skip != 'true'
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${ci_scripts}/utils.sh; init
|
||||
bash ${work_dir}/tools/check_sequence_op.sh
|
||||
clean_build_files
|
||||
'
|
||||
|
||||
- name: Upload paddle_whl
|
||||
if: needs.check-bypass.outputs.can-skip != 'true'
|
||||
env:
|
||||
home_path: ${{ github.workspace }}/..
|
||||
bos_file: ${{ github.workspace }}/../bos/BosClient.py
|
||||
paddle_whl: paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
bash ${ci_scripts}/compress_build.sh
|
||||
export AK=paddle
|
||||
export SK=paddle
|
||||
if [ ! -f "${{ env.bos_file }}" ]; then
|
||||
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
|
||||
mkdir ${{ env.home_path }}/bos
|
||||
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
|
||||
fi
|
||||
cd /paddle/build/paddle
|
||||
ls | grep -v cinn | xargs rm -rf
|
||||
rm -rf /paddle/build/third_party
|
||||
rm -rf /paddle/build/dist
|
||||
cd /
|
||||
tar --use-compress-program="pzstd -1" -cf build.tar.gz paddle
|
||||
echo "Uploading build.tar.gz to bos"
|
||||
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
echo "Uploading build.tar.gz to cfs"
|
||||
mkdir -p ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}
|
||||
mv build.tar.gz ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID} && echo success
|
||||
mv /paddle/${{ env.paddle_whl }} .
|
||||
echo "Uploading paddle_whl to bos"
|
||||
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && always() }}
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,106 @@
|
||||
name: Mac-CPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: "${{ github.workspace }}/Paddle"
|
||||
MACOSX_DEPLOYMENT_TARGET: 10.11
|
||||
PADDLE_DEV_NAME: "paddlepaddle/paddle:dev"
|
||||
PADDLE_VERSION: 0.0.0
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
WITH_DEB: "OFF"
|
||||
RUN_TEST: "OFF"
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
WITH_FLUID_ONLY: "ON"
|
||||
WITH_TESTING: "ON"
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
INSTALL_PREFIX: "/Users/paddle/CI_install_path"
|
||||
PYTHON_ABI: "cp310-cp310"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_AVX: "OFF"
|
||||
WITH_ARM: "ON"
|
||||
PROC_RUN: 8
|
||||
WITH_CACHE: "ON"
|
||||
NIGHTLY_MODE: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PRECISION_TEST: "OFF"
|
||||
CI_SKIP_CPP_TEST: "OFF"
|
||||
WITH_ONNXRUNTIME: "OFF"
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: "Ninja"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: mac
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
if: ${{ inputs.clone-can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: Mac-CI
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Check environment
|
||||
run: |
|
||||
set -x
|
||||
cd ${work_dir}
|
||||
[ -d "${work_dir}/Paddle" ] && rm -rf ${work_dir}/Paddle
|
||||
[ -f "${work_dir}/Paddle.tar.gz" ] && rm -rf ${work_dir}/Paddle.tar.gz
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
tar xf Paddle.tar.gz && cd Paddle
|
||||
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote -v
|
||||
set +e
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
set -e
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
|
||||
- name: Build with mac python3.10
|
||||
run: |
|
||||
set -x
|
||||
cd ${work_dir}/Paddle
|
||||
source ~/.zshrc
|
||||
bash -x ${work_dir}/Paddle/ci/run_setup.sh bdist_wheel ${parallel_number:-""}
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
|
||||
- name: Test with mac python3.10
|
||||
run: |
|
||||
set -ex
|
||||
cd ${work_dir}/Paddle
|
||||
pip3.10 install -r ${work_dir}/Paddle/python/unittest_py/requirements.txt
|
||||
pip3.10 install -U ${work_dir}/Paddle/dist/*whl
|
||||
bash -x ${work_dir}/Paddle/ci/run_mac_test.sh ${PYTHON_ABI:-""} ${PROC_RUN:-""}
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
@@ -0,0 +1,168 @@
|
||||
name: Model-benchmark
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
ci_scripts: /workspace/paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-Model-benchmark
|
||||
CI_name: model-benchmark
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "model-benchmark"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
model-benchmark:
|
||||
name: Benchmark test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: model-benchmark
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
AGILE_PULL_ID: ${{ github.event.pull_request.number }}
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PRID: ${{ github.event.pull_request.number }}
|
||||
AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
|
||||
CUDA_ARCH_NAME: Volta
|
||||
CCACHE_MAXSIZE: 150G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
benchmark_ci_cache: /home/data/cfs/.cache/model_benchmark/new_benchmark
|
||||
ROOT_DIR: /workspace
|
||||
bos_path: modelBK_CI
|
||||
CACHE_DIR: /root/.cache/model_benchmark
|
||||
work_dir: /workspace/paddle
|
||||
AGILE_COMMENTS: '[{"Commit":"${{ github.event.pull_request.head.sha }}","author":"${{ github.event.pull_request.user.login }}"}]'
|
||||
run: |
|
||||
container_name=${TASK}-${core_index}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/workspace \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e BRANCH \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e AGILE_PULL_ID \
|
||||
-e GIT_PR_ID \
|
||||
-e PRID \
|
||||
-e AGILE_REVISION \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e benchmark_ci_cache \
|
||||
-e bos_path \
|
||||
-e ROOT_DIR \
|
||||
-e CACHE_DIR \
|
||||
-e work_dir \
|
||||
-e AGILE_COMMENTS \
|
||||
-w /workspace --network host ${docker_image}
|
||||
|
||||
- name: Prepare environment and download paddle
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo $AGILE_COMMENTS
|
||||
DEVICE_ID=$(cat /sys/class/dmi/id/product_uuid |awk -F "-" "{print \$5}")
|
||||
echo "export DEVICE_ID=$DEVICE_ID" >> ~/.bashrc
|
||||
echo $DEVICE_ID
|
||||
echo "Downloading build.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
|
||||
echo "Extracting build.tar.gz"
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz
|
||||
rm build.tar.gz
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
cd paddle && git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
check_paddle ${{ github.env }}
|
||||
'
|
||||
|
||||
- name: Download model_benchmark_ci
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
wget_model_benchmark_ci
|
||||
'
|
||||
|
||||
- name: Download paddlescope
|
||||
timeout-minutes: 30
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
get_paddlescope_tar
|
||||
'
|
||||
|
||||
- name: Check skip
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
check_skip ${{ github.env }}
|
||||
'
|
||||
|
||||
- name: Check model benchmark
|
||||
if: ${{ env.can-skip != 'true' }}
|
||||
run: |
|
||||
docker exec ${container_name} bash -c '
|
||||
source ~/.bashrc
|
||||
export EXCODE=0
|
||||
source ${ci_scripts}/model_benchmark.sh
|
||||
check_model_benchmark
|
||||
if [[ $EXCODE -eq 0 ]];then
|
||||
echo "Congratulations! Your PR passed the model benchmark ci."
|
||||
elif [[ $EXCODE -eq 10 ]];then
|
||||
echo "Sorry, some models failed to run, Please check the result_html.html."
|
||||
elif [[ $EXCODE -eq 11 ]];then
|
||||
echo "Sorry, some models have abnormal performance, Please check the result_html.html."
|
||||
else
|
||||
echo "Sorry, There are environmental issues, please contact QA(guomengmeng01) for exemption."
|
||||
fi
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,318 @@
|
||||
name: PR-CI-SOT
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_sot_image:
|
||||
type: string
|
||||
required: true
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-sot
|
||||
ci_scripts: /paddle/ci
|
||||
CI_name: sot
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "sot"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-and-test:
|
||||
name: Build and Test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
- name: run container
|
||||
env:
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_MKL: "OFF"
|
||||
WITH_TESTING: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
PREC_SUFFIX: .py3
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
PROC_RUN: 12
|
||||
FLAGS_enable_eager_mode: 1
|
||||
WITH_TENSORRT: "OFF"
|
||||
GENERATOR: Ninja
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
WITH_AVX: "OFF"
|
||||
CACHE_DIR: "/root/.cache/sot"
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
run: |
|
||||
set -x
|
||||
container_name=${TASK}-$(date +%s)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_sot_image }}
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:${{ github.workspace }}/../../.." \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e CI_name \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_MKL \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e PREC_SUFFIX \
|
||||
-e WITH_UNITY_BUILD \
|
||||
-e PROC_RUN \
|
||||
-e FLAGS_enable_eager_mode \
|
||||
-e WITH_TENSORRT \
|
||||
-e GENERATOR \
|
||||
-e WITH_INFERENCE_API_TEST \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e GITHUB_ENV \
|
||||
-e ci_scripts \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading Paddle.tar.gz"
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
echo "Extracting Paddle.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar xf Paddle.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm Paddle.tar.gz
|
||||
git config --global user.name "PaddleCI"
|
||||
git config --global user.email "paddle_ci@example.com"
|
||||
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
cp ci/git_pull.sh ci/git_pull_copy.sh
|
||||
git checkout $BRANCH
|
||||
bash ci/git_pull_copy.sh $BRANCH
|
||||
git fetch upstream $BRANCH
|
||||
git checkout test
|
||||
git merge --no-edit $BRANCH
|
||||
git submodule update
|
||||
'
|
||||
|
||||
- name: Determine sot ci trigger
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/determine_sot_ci_trigger.sh
|
||||
determine_excode=$?
|
||||
echo "determine_excode=$determine_excode" >> ${{ github.env }}
|
||||
'
|
||||
|
||||
- name: CMake Pre-download
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/cmake-predownload.sh
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.10
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.10
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.10
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.10
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.11
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.11
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.11
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.11
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.12
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.12
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.12
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.12
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.13
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.13
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.13
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.13
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Build with python3.14
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
export PY_VERSION=3.14
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_setup.sh bdist_wheel
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Test with python3.14
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
PADDLE_ROOT: ${{ github.workspace }}
|
||||
if: ${{ env.determine_excode == 0 }}
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/run_sot_test.sh 3.14
|
||||
EXCODE=$?
|
||||
rm -rf ${PADDLE_ROOT}/build/CMakeCache.txt
|
||||
exit $EXCODE
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,140 @@
|
||||
name: Slice
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
slice-check:
|
||||
type: string
|
||||
required: false
|
||||
SLICE_TEST_MODE:
|
||||
type: string
|
||||
required: false
|
||||
default: "test_ci"
|
||||
SLICE_BENCHMARK_FRAMEWORKS:
|
||||
type: string
|
||||
required: false
|
||||
default: "paddle"
|
||||
MANUALLY_PR_ID:
|
||||
type: string
|
||||
required: false
|
||||
MANUALLY_COMMIT_ID:
|
||||
type: string
|
||||
required: false
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-slice
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
|
||||
CI_name: slice
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass-slice.yml
|
||||
with:
|
||||
workflow-name: "slice"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
slice:
|
||||
name: Slice test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: slice
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
SLICE_TEST_MODE: ${{ inputs.SLICE_TEST_MODE }}
|
||||
SLICE_BENCHMARK_FRAMEWORKS: ${{ inputs.SLICE_BENCHMARK_FRAMEWORKS }}
|
||||
run: |
|
||||
container_name="api_benchmark_ci_${RUN_ID}"
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} --shm-size=128g \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e core_index \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e RUN_ID \
|
||||
-e wheel_link \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e SLICE_TEST_MODE \
|
||||
-e SLICE_BENCHMARK_FRAMEWORKS \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download PaddleTest
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
echo "Downloading PaddleTest"
|
||||
wget -q https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-proxy
|
||||
echo "Extracting PaddleTest"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar -xzf PaddleTest.tar.gz && cd PaddleTest
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
'
|
||||
|
||||
- name: Test
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
mkdir -p ${{ github.workspace }}/../../../.cache/pip
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
python3.10 -m pip config set global.cache-dir ${{ github.workspace }}/../../../.cache/pip
|
||||
if [[ "${{ inputs.SLICE_BENCHMARK_FRAMEWORKS }}" == "torch" ]];then
|
||||
python3.10 -m pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu118
|
||||
else
|
||||
if [[ "${{ inputs.MANUALLY_PR_ID }}" == "" ]]; then
|
||||
python3.10 -m pip install $wheel_link
|
||||
else
|
||||
python3.10 -m pip install https://paddle-github-action.bj.bcebos.com/PR/build/${{ inputs.MANUALLY_PR_ID }}/${{ inputs.MANUALLY_COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
|
||||
fi
|
||||
fi
|
||||
python3.10 -m pip install -r PaddleTest/framework/e2e/api_benchmark/requirement.txt
|
||||
cd PaddleTest/framework/slice_benchmark
|
||||
cp ${{ github.workspace }}/../../../apibm_config.yml .
|
||||
bash run.sh
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker rm -f ${{ env.container_name }}
|
||||
@@ -0,0 +1,172 @@
|
||||
name: Static-check
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_build_image:
|
||||
type: string
|
||||
required: true
|
||||
can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
build-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
TASK: paddle-CI-${{ github.event.pull_request.number }}-static-check
|
||||
ci_scripts: /paddle/ci
|
||||
BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
CI_name: static-check
|
||||
CFS_DIR: /home/data/cfs
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' && inputs.build-can-skip != 'true' }}
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: static-check
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build:
|
||||
name: Test
|
||||
if: ${{ inputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
|
||||
needs: [check-bypass]
|
||||
runs-on:
|
||||
group: BD_BJ-V100
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Debug - Print inputs
|
||||
run: |
|
||||
echo "=== Static-Check Debug Info ==="
|
||||
echo "inputs.can-skip: ${{ inputs.can-skip }}"
|
||||
echo "needs.check-bypass.outputs.can-skip: ${{ needs.check-bypass.outputs.can-skip }}"
|
||||
echo "needs.check-bypass.result: ${{ needs.check-bypass.result }}"
|
||||
echo "==============================="
|
||||
|
||||
- name: Check docker image and run container
|
||||
env:
|
||||
WITH_GPU: "ON"
|
||||
CUDA_ARCH_NAME: Volta
|
||||
WITH_AVX: "ON"
|
||||
WITH_TESTING: "OFF"
|
||||
WITH_COVERAGE: "OFF"
|
||||
COVERALLS_UPLOAD: "OFF"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
PADDLE_VERSION: 0.0.0
|
||||
WITH_DISTRIBUTE: "ON"
|
||||
PY_VERSION: "3.10"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
WITH_MKL: "ON"
|
||||
SAMPLE_CODE_EXEC_THREADS: 1
|
||||
CACHE_DIR: /home/data/cfs/.cache/static-check
|
||||
CCACHE_DIR: /home/data/cfs/.ccache/static-check
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=${{ inputs.docker_build_image }}
|
||||
docker run -d -t --gpus all --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/cfs/.ccache:/root/.ccache" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-v /root/.cache/paddle/dataset/ESC-50-master \
|
||||
-e BRANCH \
|
||||
-e PR_ID \
|
||||
-e COMMIT_ID \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e ci_scripts \
|
||||
-e WITH_MKL \
|
||||
-e SAMPLE_CODE_EXEC_THREADS \
|
||||
-e WITH_GPU \
|
||||
-e CUDA_ARCH_NAME \
|
||||
-e WITH_COVERAGE \
|
||||
-e WITH_SHARED_PHI \
|
||||
-e WITH_TESTING \
|
||||
-e COVERALLS_UPLOAD \
|
||||
-e GIT_PR_ID \
|
||||
-e PADDLE_VERSION \
|
||||
-e WITH_DISTRIBUTE \
|
||||
-e PY_VERSION \
|
||||
-e WITH_AVX \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e no_proxy \
|
||||
-e CI_name \
|
||||
-e GITHUB_API_TOKEN \
|
||||
-e CFS_DIR \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Download paddle.tar.gz and merge target branch
|
||||
env:
|
||||
work_dir: ${{ github.workspace }}
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
rm -rf * .[^.]*
|
||||
set -e
|
||||
echo "Downloading build.tar.gz from cfs"
|
||||
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
|
||||
echo "Extracting build.tar.gz"
|
||||
git config --global --add safe.directory ${work_dir}
|
||||
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
|
||||
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
|
||||
rm build.tar.gz
|
||||
git checkout test
|
||||
'
|
||||
|
||||
- name: Static check
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${ci_scripts}/static_check.sh
|
||||
'
|
||||
|
||||
- name: Check api approval
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
source ~/.bashrc
|
||||
unset GREP_OPTIONS
|
||||
bash ${work_dir}/tools/check_api_approvals.sh;approval_error=$?
|
||||
if [ "$approval_error" != 0 ];then
|
||||
exit 6
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Check GPU Kernel approval
|
||||
run: |
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c '
|
||||
source ${{ github.workspace }}/../../../proxy
|
||||
bash ${work_dir}/tools/check_gpu_kernel_approval.sh;approval_error=$?
|
||||
if [ "$approval_error" != 0 ];then
|
||||
exit 6
|
||||
fi
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,114 @@
|
||||
name: Windows-GPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
ACTION_DIR: ${{ github.workspace }}\..\..\..
|
||||
WORKSPACE_DIR: ${{ github.workspace }}\..
|
||||
cache_dir: C:\action-cache
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "win-gpu"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: win-gpu
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
NIGHTLY_MODE: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_TPCACHE: "OFF"
|
||||
WITH_SCCACHE: "ON"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
WITH_TESTING: "ON"
|
||||
PRECISION_TEST: "OFF"
|
||||
PYTHON_ROOT: C:\Python310
|
||||
vcvars64_dir: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat'
|
||||
CUDA_TOOLKIT_ROOT_DIR: 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.0'
|
||||
TENSORRT_ROOT: D:\TensorRT
|
||||
CTEST_PARALLEL_LEVEL: 1
|
||||
GENERATOR: "Ninja"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
MSVC_STATIC_CRT: "OFF"
|
||||
WITH_MKL: "ON"
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
ON_INFER: "ON"
|
||||
WITH_TENSORRT: "ON"
|
||||
WITH_INFERENCE_API_TEST: "OFF"
|
||||
WIN_UNITTEST_LEVEL: 0
|
||||
CUDA_ARCH_NAME: "Auto"
|
||||
steps:
|
||||
- name: Download Paddle and clean env
|
||||
working-directory: ..
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
whoami
|
||||
cd
|
||||
rmdir /s/q %WORKSPACE_DIR%\Paddle
|
||||
echo "path=%path%;C:\Program Files\NVIDIA Corporation\NVSMI" >> %GITHUB_ENV%
|
||||
python -m pip install wget
|
||||
python -c "import wget;wget.download('https://paddle-github-action.bj.bcebos.com/windows/PR/%PR_ID%/%COMMIT_ID%/Paddle.tar.zst')"
|
||||
zstd -d Paddle.tar.zst && tar -xf Paddle.tar
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
call %ci_scripts%\clean_env.bat
|
||||
|
||||
- name: Config env
|
||||
run: |
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\config_env.bat
|
||||
|
||||
- name: Build paddle
|
||||
run: |
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\build.bat
|
||||
|
||||
- name: Test whl package
|
||||
run: |
|
||||
call %ci_scripts%\test_whl_package.bat
|
||||
|
||||
- name: Test paddle
|
||||
run: |
|
||||
call %ci_scripts%\test.bat
|
||||
|
||||
- name: Test fluid library
|
||||
run: |
|
||||
call %ci_scripts%\test_fluid_library.bat
|
||||
|
||||
- name: Clean up dir
|
||||
if: always()
|
||||
run: |
|
||||
call %ci_scripts%\clean_env.bat
|
||||
if "%WITH_TESTING%"=="ON" (
|
||||
for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
|
||||
) || ver >nul
|
||||
@@ -0,0 +1,112 @@
|
||||
name: Windows-GPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
ACTION_DIR: ${{ github.workspace }}\..\..\..
|
||||
WORKSPACE_DIR: ${{ github.workspace }}\..
|
||||
cache_dir: C:\action-cache
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "win-inference"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: win-infer
|
||||
timeout-minutes: 240
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
NIGHTLY_MODE: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_TPCACHE: "OFF"
|
||||
WITH_SCCACHE: "ON"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
WITH_TESTING: "ON"
|
||||
PRECISION_TEST: "OFF"
|
||||
PYTHON_ROOT: C:\Python310
|
||||
vcvars64_dir: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat'
|
||||
CUDA_TOOLKIT_ROOT_DIR: 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7'
|
||||
TENSORRT_ROOT: D:/TensorRT-8.0.1.6
|
||||
CTEST_PARALLEL_LEVEL: 1
|
||||
GENERATOR: "Ninja"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WITH_MKL: "ON"
|
||||
WITH_GPU: "ON"
|
||||
WITH_AVX: "ON"
|
||||
MSVC_STATIC_CRT: "ON"
|
||||
ON_INFER: "ON"
|
||||
WITH_TENSORRT: "ON"
|
||||
WITH_INFERENCE_API_TEST: "ON"
|
||||
WITH_ONNXRUNTIME: "ON"
|
||||
WIN_UNITTEST_LEVEL: 2
|
||||
CUDA_ARCH_NAME: "Auto"
|
||||
steps:
|
||||
- name: Download Paddle and clean env
|
||||
working-directory: ..
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
whoami
|
||||
cd
|
||||
rmdir /s/q %WORKSPACE_DIR%\Paddle
|
||||
echo "path=%path%;C:\Program Files\NVIDIA Corporation\NVSMI" >> %GITHUB_ENV%
|
||||
python -m pip install wget
|
||||
python -c "import wget;wget.download('https://paddle-github-action.bj.bcebos.com/windows/PR/%PR_ID%/%COMMIT_ID%/Paddle.tar.zst')"
|
||||
zstd -d Paddle.tar.zst && tar -xf Paddle.tar
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
call %ci_scripts%\clean_env.bat
|
||||
|
||||
- name: Config env
|
||||
run: |
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\config_env.bat
|
||||
|
||||
- name: Build paddle
|
||||
run: |
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
call %ci_scripts%\pre_download.bat
|
||||
call %ACTION_DIR%\proxy.bat
|
||||
call %ci_scripts%\build.bat
|
||||
|
||||
- name: Test whl package
|
||||
run: |
|
||||
call %ci_scripts%\test_whl_package.bat
|
||||
|
||||
- name: Test paddle
|
||||
run: |
|
||||
call %ci_scripts%\test.bat
|
||||
|
||||
- name: Clean up dir
|
||||
if: always()
|
||||
run: |
|
||||
call %ci_scripts%\clean_env.bat
|
||||
if "%WITH_TESTING%"=="ON" (
|
||||
for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
|
||||
) || ver >nul
|
||||
@@ -0,0 +1,104 @@
|
||||
name: Windows-GPU
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone-can-skip:
|
||||
type: string
|
||||
required: false
|
||||
default: "false"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
BRANCH: ${{ github.base_ref }}
|
||||
ci_scripts: ${{ github.workspace }}\ci\windows
|
||||
ACTION_DIR: ${{ github.workspace }}\..\..\..
|
||||
WORKSPACE_DIR: ${{ github.workspace }}\..
|
||||
cache_dir: C:\action-cache
|
||||
SCCACHE_ROOT: C:\sccache
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: cmd
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
uses: ./.github/workflows/check-bypass.yml
|
||||
with:
|
||||
workflow-name: "win-openblas"
|
||||
secrets:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-and-test:
|
||||
name: Build and test
|
||||
needs: [check-bypass]
|
||||
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.clone-can-skip != 'true' }}
|
||||
runs-on:
|
||||
group: win-openblas
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
PADDLE_VERSION: 0.0.0
|
||||
NIGHTLY_MODE: "OFF"
|
||||
WITH_UNITY_BUILD: "ON"
|
||||
WITH_CACHE: "OFF"
|
||||
WITH_TPCACHE: "OFF"
|
||||
WITH_SCCACHE: "ON"
|
||||
WITH_SHARED_PHI: "ON"
|
||||
FLAGS_enable_eager_mode: 1
|
||||
GIT_PR_ID: ${{ github.event.pull_request.number }}
|
||||
WITH_TESTING: "ON"
|
||||
PRECISION_TEST: "OFF"
|
||||
PYTHON_ROOT: C:\Python310
|
||||
CTEST_PARALLEL_LEVEL: 1
|
||||
GENERATOR: "Ninja"
|
||||
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WITH_MKL: "OFF"
|
||||
WITH_GPU: "OFF"
|
||||
WITH_AVX: "OFF"
|
||||
MSVC_STATIC_CRT: "ON"
|
||||
ON_INFER: "ON"
|
||||
WIN_UNITTEST_LEVEL: 2
|
||||
CUDA_ARCH_NAME: "Auto"
|
||||
steps:
|
||||
- name: Download Paddle and clean env
|
||||
working-directory: ..
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
whoami
|
||||
cd
|
||||
rmdir /s/q %WORKSPACE_DIR%\Paddle
|
||||
python -m pip install wget
|
||||
python -c "import wget;wget.download('https://paddle-github-action.bj.bcebos.com/windows/PR/%PR_ID%/%COMMIT_ID%/Paddle.tar.zst')"
|
||||
zstd -d Paddle.tar.zst && tar -xf Paddle.tar
|
||||
del Paddle.tar Paddle.tar.zst
|
||||
call %ci_scripts%\clean_env.bat
|
||||
|
||||
- name: Config env
|
||||
run: |
|
||||
call %ci_scripts%\config_env.bat
|
||||
|
||||
- name: Build paddle
|
||||
run: |
|
||||
python -m pip install bce-python-sdk==0.8.74
|
||||
call %ci_scripts%\build.bat
|
||||
|
||||
- name: Test whl package
|
||||
run: |
|
||||
call %ci_scripts%\test_whl_package.bat
|
||||
|
||||
- name: Test paddle
|
||||
run: |
|
||||
call %ci_scripts%\test.bat
|
||||
|
||||
- name: Clean up dir
|
||||
if: always()
|
||||
run: |
|
||||
call %ci_scripts%\clean_env.bat
|
||||
if "%WITH_TESTING%"=="ON" (
|
||||
for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
|
||||
) || ver >nul
|
||||
@@ -0,0 +1,29 @@
|
||||
name: CI-Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel CI-Build for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel CI-build
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,29 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel CI for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel CI
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,29 @@
|
||||
name: Coverage
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel Coverage for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel Coverage
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,29 @@
|
||||
name: CI-Windows
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop, release/**]
|
||||
|
||||
permissions: read-all
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
jobs:
|
||||
cancel:
|
||||
name: Cancel CI-Windows for ${{ github.event.pull_request.number }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Cancel CI-Windows
|
||||
run: |
|
||||
exit 0
|
||||
@@ -0,0 +1,51 @@
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
workflow-name:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
github-token:
|
||||
required: true
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether the workflow can be skipped."
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
CI_TEAM_MEMBERS: '["swgu98", "risemeup1", "xiaoguoguo626807", "luotao1"]'
|
||||
outputs:
|
||||
can-skip: ${{ steps.check-bypass.outputs.can-skip }}
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- id: check-bypass
|
||||
name: Check Bypass
|
||||
uses: PFCCLab/ci-bypass@v2
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
non-pull-request-event-strategy: 'never-skipped'
|
||||
type: 'composite'
|
||||
composite-rule: |
|
||||
{
|
||||
"any": [
|
||||
{
|
||||
"type": "labeled",
|
||||
"label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
},
|
||||
{
|
||||
"type": "commented",
|
||||
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
workflow-name:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
github-token:
|
||||
required: true
|
||||
outputs:
|
||||
can-skip:
|
||||
description: "Whether the workflow can be skipped."
|
||||
value: ${{ jobs.check-bypass.outputs.can-skip }}
|
||||
|
||||
jobs:
|
||||
check-bypass:
|
||||
name: Check bypass
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
CI_TEAM_MEMBERS: '["swgu98", "risemeup1" , "XieYunshen", "luotao1", "SigureMo"]'
|
||||
outputs:
|
||||
can-skip: ${{ steps.check-bypass.outputs.can-skip }}
|
||||
timeout-minutes: 3
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- id: check-bypass
|
||||
name: Check Bypass
|
||||
uses: PFCCLab/ci-bypass@v2
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
non-pull-request-event-strategy: "never-skipped"
|
||||
type: "composite"
|
||||
composite-rule: |
|
||||
{
|
||||
"any": [
|
||||
{
|
||||
"type": "labeled",
|
||||
"label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
},
|
||||
{
|
||||
"type": "commented",
|
||||
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
|
||||
"username": ${{ env.CI_TEAM_MEMBERS }}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
name: Cherry Pick
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
branches: [develop]
|
||||
types: [closed, labeled]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
cherry-pick:
|
||||
if: >
|
||||
github.repository_owner == 'PaddlePaddle' &&
|
||||
github.event.pull_request.merged == true &&
|
||||
(
|
||||
github.event.action == 'labeled' ||
|
||||
contains(join(github.event.pull_request.labels.*.name, ' '), 'cherry-pick')
|
||||
)
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Cherry Pick
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.CHERRY_PICK_BOT_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
MERGE_COMMIT_SHA: ${{ github.event.pull_request.merge_commit_sha }}
|
||||
BOT_USERNAME: risemeup1111
|
||||
REPO_NAME: risemeup1111/Paddle
|
||||
run: |
|
||||
# Function to post comment
|
||||
post_comment() {
|
||||
gh pr comment "$PR_NUMBER" --body "$1"
|
||||
}
|
||||
|
||||
# Configure git for the original author
|
||||
echo "Fetching author info for $PR_AUTHOR..."
|
||||
AUTHOR_INFO=$(gh api "/users/$PR_AUTHOR" --jq '{email: .email, name: .name}')
|
||||
AUTHOR_EMAIL=$(echo "$AUTHOR_INFO" | jq -r '.email')
|
||||
AUTHOR_NAME=$(echo "$AUTHOR_INFO" | jq -r '.name')
|
||||
|
||||
if [ "$AUTHOR_EMAIL" = "null" ] || [ -z "$AUTHOR_EMAIL" ]; then
|
||||
AUTHOR_EMAIL="${PR_AUTHOR}@users.noreply.github.com"
|
||||
echo "Author email not found, using default: $AUTHOR_EMAIL"
|
||||
fi
|
||||
if [ "$AUTHOR_NAME" = "null" ] || [ -z "$AUTHOR_NAME" ]; then
|
||||
AUTHOR_NAME="${PR_AUTHOR}"
|
||||
echo "Author name not found, using username: $AUTHOR_NAME"
|
||||
fi
|
||||
|
||||
git config user.name "$AUTHOR_NAME"
|
||||
git config user.email "$AUTHOR_EMAIL"
|
||||
|
||||
# Capture current SHA to return to later
|
||||
ORIGINAL_HEAD_SHA=$(git rev-parse HEAD)
|
||||
|
||||
# Get labels
|
||||
LABELS=$(gh pr view "$PR_NUMBER" --json labels --jq '.labels[].name')
|
||||
|
||||
if [ -z "$LABELS" ]; then
|
||||
echo "No labels found."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Loop through labels
|
||||
while read -r label; do
|
||||
if [[ "$label" == cherry-pick:* ]]; then
|
||||
TARGET_BRANCH=$(echo "${label#cherry-pick:}" | xargs)
|
||||
|
||||
if [ -z "$TARGET_BRANCH" ]; then
|
||||
echo "Empty target branch for label '$label', skipping."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Processing cherry-pick to $TARGET_BRANCH"
|
||||
|
||||
# Check if target branch exists on remote
|
||||
if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH"; then
|
||||
echo "Target branch $TARGET_BRANCH does not exist."
|
||||
post_comment "❌ Cherry-pick failed: Target branch \`$TARGET_BRANCH\` does not exist."
|
||||
continue
|
||||
fi
|
||||
|
||||
# Create a new branch for the cherry-pick
|
||||
NEW_BRANCH="cherry-pick/$PR_NUMBER/$TARGET_BRANCH"
|
||||
|
||||
# Clean up local branch if it exists (from previous run)
|
||||
if git show-ref --verify --quiet "refs/heads/$NEW_BRANCH"; then
|
||||
git branch -D "$NEW_BRANCH"
|
||||
fi
|
||||
|
||||
# Fetch the target branch and checkout a new branch from it
|
||||
git fetch origin "$TARGET_BRANCH"
|
||||
git checkout -b "$NEW_BRANCH" "origin/$TARGET_BRANCH"
|
||||
|
||||
# Cherry pick
|
||||
# Try standard cherry-pick first (for squash merges or single commits)
|
||||
if git cherry-pick "$MERGE_COMMIT_SHA"; then
|
||||
echo "Cherry-pick successful."
|
||||
else
|
||||
echo "Standard cherry-pick failed, trying with -m 1 (for merge commits)..."
|
||||
git cherry-pick --abort
|
||||
if git cherry-pick -m 1 "$MERGE_COMMIT_SHA"; then
|
||||
echo "Cherry-pick with -m 1 successful."
|
||||
else
|
||||
echo "Cherry-pick failed."
|
||||
git cherry-pick --abort
|
||||
post_comment "❌ Cherry-pick failed: Conflicts detected when cherry-picking to \`$TARGET_BRANCH\`. Please resolve manually."
|
||||
|
||||
# Cleanup
|
||||
git checkout "$ORIGINAL_HEAD_SHA"
|
||||
git branch -D "$NEW_BRANCH"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Push
|
||||
# Construct authenticated URL for the fork
|
||||
FORK_URL_AUTH="https://${BOT_USERNAME}:${GH_TOKEN}@github.com/${REPO_NAME}.git"
|
||||
|
||||
echo "Pushing to fork..."
|
||||
git push "$FORK_URL_AUTH" "$NEW_BRANCH" --force
|
||||
|
||||
# Create PR
|
||||
# If PR_TITLE starts with "[", don't insert an extra space.
|
||||
if [ "${PR_TITLE:0:1}" = "[" ]; then
|
||||
NEW_TITLE="[$TARGET_BRANCH]$PR_TITLE"
|
||||
else
|
||||
NEW_TITLE="[$TARGET_BRANCH] $PR_TITLE"
|
||||
fi
|
||||
|
||||
NEW_BODY="$PR_BODY
|
||||
|
||||
---
|
||||
|
||||
Cherry-pick of #$PR_NUMBER (authored by @${PR_AUTHOR}) to \`$TARGET_BRANCH\`.
|
||||
|
||||
devPR:https://github.com/PaddlePaddle/Paddle/pull/$PR_NUMBER <!-- For pass CI -->"
|
||||
|
||||
# Prepare head ref for PR creation (owner:branch)
|
||||
HEAD_REF="${BOT_USERNAME}:${NEW_BRANCH}"
|
||||
|
||||
# Check if PR already exists
|
||||
EXISTING_PR=$(gh pr list --base "$TARGET_BRANCH" --head "$NEW_BRANCH" --json url --jq '.[0].url')
|
||||
|
||||
if [ -n "$EXISTING_PR" ]; then
|
||||
echo "PR already exists: $EXISTING_PR"
|
||||
post_comment "ℹ️ Cherry-pick PR already exists: $EXISTING_PR"
|
||||
else
|
||||
# Create PR using gh CLI, ignoring errors because of "Resource not accessible" false positives
|
||||
gh pr create --base "$TARGET_BRANCH" --head "$HEAD_REF" --title "$NEW_TITLE" --body "$NEW_BODY" || true
|
||||
|
||||
# Wait a bit for eventual consistency
|
||||
sleep 2
|
||||
|
||||
# Search for the created PR
|
||||
CREATED_PR_URL=$(gh pr list --head "$NEW_BRANCH" --state all --json url --jq '.[0].url')
|
||||
|
||||
if [ -n "$CREATED_PR_URL" ]; then
|
||||
echo "Created PR: $CREATED_PR_URL"
|
||||
post_comment "✅ Cherry-pick successful! Created PR: $CREATED_PR_URL"
|
||||
|
||||
# Request review
|
||||
gh pr review-request "$CREATED_PR_URL" --reviewer "$PR_AUTHOR" || true
|
||||
else
|
||||
echo "Failed to create PR."
|
||||
post_comment "❌ Cherry-pick failed: Could not create PR to \`$TARGET_BRANCH\`."
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Cleanup for next loop
|
||||
git checkout "$ORIGINAL_HEAD_SHA"
|
||||
git branch -D "$NEW_BRANCH"
|
||||
fi
|
||||
done <<< "$LABELS"
|
||||
|
||||
- name: Remove Cherry Pick Labels
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO_NAME: ${{ github.repository }}
|
||||
run: |
|
||||
LABELS=$(gh pr view "$PR_NUMBER" --repo "$REPO_NAME" --json labels --jq '.labels[].name')
|
||||
if [ -z "$LABELS" ]; then
|
||||
exit 0
|
||||
fi
|
||||
while read -r label; do
|
||||
if [[ "$label" == cherry-pick:* ]]; then
|
||||
echo "Removing label: $label"
|
||||
gh pr edit "$PR_NUMBER" --repo "$REPO_NAME" --remove-label "$label"
|
||||
fi
|
||||
done <<< "$LABELS"
|
||||
@@ -0,0 +1,72 @@
|
||||
name: Cleanup ccache
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 18 * * *" # Daily at 02:00 UTC+8
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
TASK: cleanup-ccache-nightly
|
||||
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
cleanup-ccache:
|
||||
name: Cleanup L2 ccache
|
||||
if: github.repository == 'PaddlePaddle/Paddle'
|
||||
runs-on:
|
||||
group: GZ_BD-CPU
|
||||
|
||||
steps:
|
||||
- name: Launch container
|
||||
env:
|
||||
work_dir: /paddle
|
||||
PADDLE_ROOT: /paddle
|
||||
CCACHE_MAXSIZE: 50G
|
||||
CCACHE_LIMIT_MULTIPLE: 0.8
|
||||
CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
|
||||
CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
|
||||
run: |
|
||||
set -x
|
||||
container_name=${TASK}-$(date +%s)
|
||||
echo "container_name=${container_name}" >> ${{ github.env }}
|
||||
docker_image=ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda129-coverage-test
|
||||
docker run -d -t --name ${container_name} \
|
||||
-v "/home/data/cfs:/home/data/cfs" \
|
||||
-v "/home/data/cfs/.cache:/root/.cache" \
|
||||
-v "/home/data/shared:/home/data/shared" \
|
||||
-v "/dev/shm:/dev/shm" \
|
||||
-v "${{ github.workspace }}/../../..:${{ github.workspace }}/../../.." \
|
||||
-v ${{ github.workspace }}:/paddle \
|
||||
-e work_dir \
|
||||
-e PADDLE_ROOT \
|
||||
-e GITHUB_ENV \
|
||||
-e CACHE_DIR \
|
||||
-e CCACHE_DIR \
|
||||
-e CCACHE_SECONDARY_STORAGE \
|
||||
-e CCACHE_MAXSIZE \
|
||||
-e CCACHE_LIMIT_MULTIPLE \
|
||||
-e no_proxy \
|
||||
-w /paddle --network host ${docker_image}
|
||||
|
||||
- name: Cleanup L2 ccache in CFS
|
||||
run: |
|
||||
docker exec -t ${container_name} bash -c '
|
||||
CFS_CCACHE_DIR="${CCACHE_SECONDARY_STORAGE#file://}"
|
||||
ccache --trim-dir ${CFS_CCACHE_DIR} --trim-max-size 500G
|
||||
'
|
||||
|
||||
- name: Terminate and delete the container
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
|
||||
docker stop ${{ env.container_name }}
|
||||
docker rm ${{ env.container_name }}
|
||||
@@ -0,0 +1,136 @@
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
clone_dir:
|
||||
required: false
|
||||
type: string
|
||||
default: "Paddle"
|
||||
task:
|
||||
required: false
|
||||
type: string
|
||||
default: "cpu"
|
||||
is_merge:
|
||||
required: false
|
||||
type: string
|
||||
default: "false"
|
||||
outputs:
|
||||
docker_cpu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_cpu_image }}
|
||||
|
||||
docker_sot_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_sot_image }}
|
||||
|
||||
docker_coverage_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_coverage_image }}
|
||||
|
||||
docker_build_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_build_image }}
|
||||
|
||||
docker_distribute_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_distribute_image }}
|
||||
|
||||
docker_xpu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_xpu_image }}
|
||||
|
||||
docker_npu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_npu_image }}
|
||||
|
||||
docker_dcu_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_dcu_image }}
|
||||
|
||||
docker_doc_image:
|
||||
description: "Generate images for all CI usage"
|
||||
value: ${{ jobs.build-docker-images.outputs.docker_doc_image }}
|
||||
|
||||
jobs:
|
||||
build-docker-images:
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
name: Build docker
|
||||
runs-on:
|
||||
group: Docker-build
|
||||
outputs:
|
||||
docker_cpu_image: ${{ steps.build-docker-images.outputs.docker_cpu_image }}
|
||||
docker_sot_image: ${{ steps.build-docker-images.outputs.docker_sot_image }}
|
||||
docker_coverage_image: ${{ steps.build-docker-images.outputs.docker_coverage_image }}
|
||||
docker_build_image: ${{ steps.build-docker-images.outputs.docker_build_image }}
|
||||
docker_distribute_image: ${{ steps.build-docker-images.outputs.docker_distribute_image }}
|
||||
docker_xpu_image: ${{ steps.build-docker-images.outputs.docker_xpu_image }}
|
||||
docker_npu_image: ${{ steps.build-docker-images.outputs.docker_npu_image }}
|
||||
docker_dcu_image: ${{ steps.build-docker-images.outputs.docker_dcu_image }}
|
||||
docker_doc_image: ${{ steps.build-docker-images.outputs.docker_doc_image }}
|
||||
steps:
|
||||
- id: build-docker-images
|
||||
name: Build docker images
|
||||
env:
|
||||
PR_ID: ${{ github.event.pull_request.number || '0' }}
|
||||
COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
docker_cpu_file: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6
|
||||
docker_sot_file: Dockerfile.gcc133_ubuntu24_cpu_sot
|
||||
docker_coverage_file: Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage
|
||||
docker_build_file: Dockerfile.cuda11.2_cudnn8_gcc82_trt8
|
||||
docker_distribute_file: Dockerfile.cuda123_cudnn9_gcc122_ubuntu20
|
||||
docker_xpu_file: Dockerfile.develop.xre.ubuntu22
|
||||
docker_npu_file: Dockerfile.develop.npu
|
||||
docker_dcu_file: Dockerfile.develop.dtk
|
||||
docker_doc_file: Dockerfile.doc
|
||||
dockerfile_script: https://raw.githubusercontent.com/PaddlePaddle/Paddle/refs/heads/develop/tools/dockerfile/ci_dockerfile.sh
|
||||
dockerfile_origin: https://raw.githubusercontent.com/PaddlePaddle/Paddle/refs/heads/develop/tools/dockerfile/Dockerfile.ubuntu20
|
||||
run: |
|
||||
set -x
|
||||
rm -rf * .[^.]*
|
||||
if [ "${{ inputs.is_merge }}" == "true" ]; then
|
||||
mkdir -p tools/dockerfile && cd tools/dockerfile
|
||||
wget -q --tries=5 --no-proxy ${dockerfile_script} ${dockerfile_origin} --no-check-certificate
|
||||
else
|
||||
wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/${{ inputs.clone_dir }}/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
||||
tar xf Paddle.tar.gz --strip-components=1
|
||||
rm Paddle.tar.gz
|
||||
cd tools/dockerfile
|
||||
fi
|
||||
bash ci_dockerfile.sh
|
||||
cd ../..
|
||||
|
||||
# docker build images
|
||||
if [ "${{ inputs.task }}" == "cpu" ]; then
|
||||
declare -A docker_files=(["docker_cpu"]="$docker_cpu_file" ["docker_distribute"]="$docker_distribute_file" ["docker_xpu"]="$docker_xpu_file" ["docker_npu"]="$docker_npu_file" ["docker_dcu"]="$docker_dcu_file" ["docker_sot"]="$docker_sot_file")
|
||||
elif [ "${{ inputs.task }}" == "build" ]; then
|
||||
declare -A docker_files=(["docker_build"]="$docker_build_file" ["docker_doc"]="$docker_doc_file")
|
||||
else
|
||||
declare -A docker_files=(["docker_coverage"]="$docker_coverage_file")
|
||||
fi
|
||||
for name in "${!docker_files[@]}"
|
||||
do
|
||||
md5_value=`md5sum tools/dockerfile/${docker_files[$name]} | awk '{print $1}'`
|
||||
if [ $name == "docker_dcu" ]; then
|
||||
md5_value="76937a563116f6008c8ca4cb4f592759"
|
||||
fi
|
||||
if [ $name == "docker_npu" ]; then
|
||||
md5_value="a3793bdeea5ae881a0c1eaf4d7c30c64"
|
||||
fi
|
||||
docker_image="ccr-2vdh3abv-pub.cnc.bj.baidubce.com/ci/paddle:${md5_value}"
|
||||
declare "${name}_image=${docker_image}"
|
||||
echo "${name}_image=${docker_image}" >> $GITHUB_OUTPUT
|
||||
set +e
|
||||
docker pull $docker_image
|
||||
pull_status=$?
|
||||
set -e
|
||||
if [ $pull_status -eq 0 ];then
|
||||
echo use docker cache
|
||||
else
|
||||
docker build -t $docker_image -f tools/dockerfile/${docker_files[$name]} .
|
||||
docker push $docker_image
|
||||
echo end docker build
|
||||
fi
|
||||
done
|
||||
|
||||
# clean workspace
|
||||
cd ${{ github.workspace }}
|
||||
rm -rf * .[^.]*
|
||||
@@ -0,0 +1,58 @@
|
||||
name: Remove Skip-CI Labels
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [synchronize]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
remove-skip-ci-labels:
|
||||
name: Remove skip-ci labels on new commits
|
||||
if: ${{ github.repository_owner == 'PaddlePaddle' }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Get PR labels
|
||||
id: get-labels
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const { data: labels } = await github.rest.issues.listLabelsOnIssue({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number
|
||||
});
|
||||
|
||||
const skipCiLabels = labels
|
||||
.filter(label => label.name.startsWith('skip-ci:'))
|
||||
.map(label => label.name);
|
||||
|
||||
console.log('Found skip-ci labels:', skipCiLabels);
|
||||
core.setOutput('skip-ci-labels', JSON.stringify(skipCiLabels));
|
||||
core.setOutput('has-skip-ci-labels', skipCiLabels.length > 0 ? 'true' : 'false');
|
||||
|
||||
- name: Remove skip-ci labels
|
||||
if: steps.get-labels.outputs.has-skip-ci-labels == 'true'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const skipCiLabels = JSON.parse('${{ steps.get-labels.outputs.skip-ci-labels }}');
|
||||
|
||||
for (const label of skipCiLabels) {
|
||||
console.log(`Removing label: ${label}`);
|
||||
await github.rest.issues.removeLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
name: label
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Successfully removed ${skipCiLabels.length} skip-ci label(s)`);
|
||||
@@ -0,0 +1,347 @@
|
||||
name: Re-run
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
re-run:
|
||||
if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/re-run') && github.event.comment.user.login == github.event.issue.user.login }}
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Rerun all failed jobs
|
||||
if: ${{ contains(github.event.comment.body, 'all-failed') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'all-failed'
|
||||
|
||||
- name: Rerun Approval
|
||||
if: ${{ contains(github.event.comment.body, 'approval') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Check approval'
|
||||
|
||||
- name: Rerun Codestyle-check
|
||||
if: ${{ contains(github.event.comment.body, 'codestyle') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Pre Commit'
|
||||
|
||||
- name: Rerun Clone
|
||||
if: ${{ contains(github.event.comment.body, 'clone') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Clone-linux / Clone Paddle'
|
||||
|
||||
- name: Rerun sot
|
||||
if: ${{ contains(github.event.comment.body, 'sot') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-SOT / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun CPU
|
||||
if: ${{ contains(github.event.comment.body, 'cpu') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-CPU / Build and test'
|
||||
|
||||
- name: Rerun Mac
|
||||
if: ${{ contains(github.event.comment.body, 'mac') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Mac-CPU / Build and test'
|
||||
|
||||
- name: Rerun NPU
|
||||
if: ${{ contains(github.event.comment.body, 'npu')}}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-NPU / Test'
|
||||
|
||||
- name: Rerun IXUCA
|
||||
if: ${{ contains(github.event.comment.body, 'ixuca') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-IXUCA / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Inference
|
||||
if: ${{ contains(github.event.comment.body, 'inference') && !contains(github.event.comment.body, 'build') && !contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-Inference / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Inference build
|
||||
if: ${{ contains(github.event.comment.body, 'inference') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-Inference / Build'
|
||||
|
||||
- name: Rerun Inference test
|
||||
if: ${{ contains(github.event.comment.body, 'inference') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'PR-CI-Inference / Test'
|
||||
|
||||
- name: Rerun Coverage clone
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage clone / Clone Paddle'
|
||||
|
||||
- name: Rerun Coverage docker build
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') && contains(github.event.comment.body, 'docker') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage build docker / Build docker'
|
||||
|
||||
- name: Rerun Coverage build
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage build'
|
||||
|
||||
- name: Rerun Coverage test
|
||||
if: ${{ contains(github.event.comment.body, 'coverage') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Coverage test'
|
||||
|
||||
- name: Rerun XPU
|
||||
if: ${{ contains(github.event.comment.body, 'xpu') && !contains(github.event.comment.body, 'build') && !contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-XPU / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun XPU build
|
||||
if: ${{ contains(github.event.comment.body, 'xpu') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-XPU / Build'
|
||||
|
||||
- name: Rerun XPU test
|
||||
if: ${{ contains(github.event.comment.body, 'xpu') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-XPU / Test'
|
||||
|
||||
- name: Rerun Distribute-stable build
|
||||
if: ${{ contains(github.event.comment.body, 'distribute') && !contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Distribute-stable-build / Build'
|
||||
|
||||
- name: Rerun Distribute-stable test
|
||||
if: ${{ contains(github.event.comment.body, 'distribute') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Distribute-stable-test / Test'
|
||||
|
||||
- name: Rerun build
|
||||
if: ${{ contains(github.event.comment.body, 'linux') && contains(github.event.comment.body, 'build') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Linux-build / Build'
|
||||
|
||||
- name: Rerun Static-check test
|
||||
if: ${{ contains(github.event.comment.body, 'static-check') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Static-Check / Test'
|
||||
|
||||
- name: Rerun CE-CINN-Framework
|
||||
if: ${{ contains(github.event.comment.body, 'ce-cinn') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-CINN-Framework / CINN'
|
||||
|
||||
- name: Rerun CE-Framework Whl
|
||||
if: ${{ contains(github.event.comment.body, 'ce') && contains(github.event.comment.body, 'whl') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-Framework / Whl'
|
||||
|
||||
- name: Rerun CE-Framework Infer
|
||||
if: ${{ contains(github.event.comment.body, 'ce') && contains(github.event.comment.body, 'infer') && !contains(github.event.comment.body, 'inference') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-Framework / Infer'
|
||||
|
||||
- name: Rerun CE-Framework Test
|
||||
if: ${{ contains(github.event.comment.body, 'ce') && contains(github.event.comment.body, 'test') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'CE-Framework / Test'
|
||||
|
||||
- name: Rerun Api-Benchmark
|
||||
if: ${{ contains(github.event.comment.body, 'api-benchmark') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Api-Benchmark / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Doc-Preview
|
||||
if: ${{ contains(github.event.comment.body, 'doc') && contains(github.event.comment.body, 'preview') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Doc-Preview / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Slice
|
||||
if: ${{ contains(github.event.comment.body, 'slice') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Slice / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Windows-GPU
|
||||
if: ${{ contains(github.event.comment.body, 'win') && contains(github.event.comment.body, 'gpu') }}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Windows-GPU / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Windows-Inference
|
||||
if: ${{ contains(github.event.comment.body, 'win') && contains(github.event.comment.body, 'infer')}}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Windows-Inference / Check bypass / Check bypass'
|
||||
|
||||
- name: Rerun Windows-OPENBLAS
|
||||
if: ${{ contains(github.event.comment.body, 'win') && contains(github.event.comment.body, 'openblas')}}
|
||||
uses: ./.github/actions/rerun-workflow
|
||||
with:
|
||||
PR_ID: ${{ github.event.issue.number }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
REPO: ${{ github.event.repository.name }}
|
||||
JOB_NAME: 'Windows-OPENBLAS / Check bypass / Check bypass'
|
||||
@@ -0,0 +1,82 @@
|
||||
name: Check version equality on release branch
|
||||
|
||||
on:
|
||||
create:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
check-version-and-create-issue:
|
||||
runs-on:
|
||||
group: APPROVAL
|
||||
if: github.ref_type == 'branch' && startsWith(github.ref, 'refs/heads/release/') && github.repository_owner == 'PaddlePaddle'
|
||||
steps:
|
||||
- name: Cleanup
|
||||
run: |
|
||||
rm -rf * .[^.]*
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Check DEVELOP_VERSION and RELEASE_VERSION equality
|
||||
id: check_versions
|
||||
run: |
|
||||
FILE="paddle/fluid/pir/serialize_deserialize/CMakeLists.txt"
|
||||
if [ ! -f "$FILE" ]; then
|
||||
echo "file_exists=false" >> "$GITHUB_OUTPUT"
|
||||
echo "equal=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
else
|
||||
echo "file_exists=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
develop_version=$(grep -oP '(?<=add_definitions\(-DDEVELOP_VERSION=)[0-9]+' "$FILE")
|
||||
release_version=$(grep -oP '(?<=add_definitions\(-DRELEASE_VERSION=)[0-9]+' "$FILE")
|
||||
|
||||
echo "Develop version: $develop_version"
|
||||
echo "Release version: $release_version"
|
||||
|
||||
echo "develop_version=$develop_version" >> "$GITHUB_OUTPUT"
|
||||
echo "release_version=$release_version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$develop_version" = "$release_version" ]; then
|
||||
echo "equal=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "equal=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create Issue if versions not equal
|
||||
if: steps.check_versions.outputs.equal == 'false'
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
DEVELOP_VERSION: ${{ steps.check_versions.outputs.develop_version }}
|
||||
RELEASE_VERSION: ${{ steps.check_versions.outputs.release_version }}
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const branchName = context.ref.replace('refs/heads/', '');
|
||||
const developVersion = parseInt(process.env.DEVELOP_VERSION);
|
||||
const releaseVersion = parseInt(process.env.RELEASE_VERSION);
|
||||
const issueTitle = `DEVELOP_VERSION and RELEASE_VERSION mismatch in paddle/fluid/pir/serialize_deserialize/CMakeLists.txt (${branchName})`;
|
||||
const issueBody =
|
||||
`Hello, in the new branch \`${branchName}\`, the \`DEVELOP_VERSION\` and \`RELEASE_VERSION\` values in \`paddle/fluid/pir/serialize_deserialize/CMakeLists.txt\` are inconsistent. Please check.\n\n` +
|
||||
`**Current values:**\n` +
|
||||
`- DEVELOP_VERSION: \`${developVersion}\`\n` +
|
||||
`- RELEASE_VERSION: \`${releaseVersion}\`\n\n` +
|
||||
`**Recommended update (apply to both \`develop\` and \`release\` branches):**\n` +
|
||||
`\`\`\`\n` +
|
||||
`DEVELOP_VERSION=${releaseVersion + 1}\n` +
|
||||
`RELEASE_VERSION=${releaseVersion + 1}\n` +
|
||||
`\`\`\`\n\n` +
|
||||
`Also, please remember to update the corresponding YAML files under:\n` +
|
||||
`\`paddle/fluid/pir/serialize_deserialize/patch\`\n`;
|
||||
const issueResult = await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: issueTitle,
|
||||
body: issueBody,
|
||||
});
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueResult.data.number,
|
||||
body: 'cc @changeyoung98 @xiaoguoguo626807 @goocody 请尽快处理上述版本不一致问题，谢谢。'
|
||||
});
|
||||
Reference in New Issue
Block a user