wehub-resource-sync
467 lines
18 KiB
YAML
467 lines
18 KiB
YAML
name: Linux-IXUCA
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
can-skip:
|
|
type: string
|
|
required: false
|
|
default: "false"
|
|
|
|
concurrency:
|
|
group: linux-ixuca-pr-${{ github.event.pull_request.number }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
PR_ID: ${{ github.event.pull_request.number }}
|
|
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
|
|
BRANCH: ${{ github.event.pull_request.base.ref }}
|
|
CI_name: ixuca
|
|
|
|
jobs:
|
|
check-bypass:
|
|
name: Check bypass
|
|
uses: ./.github/workflows/check-bypass.yml
|
|
with:
|
|
workflow-name: "ixuca"
|
|
secrets:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
test-primary:
|
|
name: Build and Test
|
|
needs: [check-bypass]
|
|
if: ${{ needs.check-bypass.outputs.can-skip != 'true' && inputs.can-skip != 'true' }}
|
|
runs-on: iluvatar-gpu-2
|
|
timeout-minutes: 120
|
|
container:
|
|
image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:3.3.0
|
|
env:
|
|
LD_LIBRARY_PATH: /usr/local/corex/lib
|
|
LIBRARY_PATH: /usr/local/corex/lib
|
|
env:
|
|
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
outputs:
|
|
result: ${{ steps.set-result.outputs.result }}
|
|
|
|
steps:
|
|
- name: Check approval
|
|
id: check-approval
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
AUTHORIZED_REVIEWERS=("YqGe585" "yongqiangma")
|
|
|
|
echo "Authorized reviewers: ${AUTHORIZED_REVIEWERS[*]}"
|
|
echo "Repository: ${{ github.repository }}"
|
|
echo "PR number: ${{ github.event.pull_request.number }}"
|
|
|
|
response=""
|
|
for i in 1 2 3; do
|
|
_tmp=$(curl -s --fail --max-time 30 \
|
|
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" 2>/dev/null) && { response="$_tmp"; break; }
|
|
echo "Attempt $i to fetch PR reviews failed, retrying..."
|
|
sleep 5
|
|
done
|
|
|
|
echo "Response length: ${#response}"
|
|
|
|
can_skip=$(python3 -c '
|
|
import json, sys
|
|
|
|
def debug(msg):
|
|
print(msg, file=sys.stderr)
|
|
|
|
debug("=== Review Check Debug Info ===")
|
|
debug(f"Authorized reviewers: {sys.argv[1:]}")
|
|
AUTHORIZED = set(sys.argv[1:])
|
|
|
|
try:
|
|
input_data = sys.stdin.read()
|
|
debug(f"Raw input data length: {len(input_data)}")
|
|
data = json.loads(input_data)
|
|
debug(f"Parsed {len(data)} reviews")
|
|
except Exception as e:
|
|
debug(f"JSON parse error: {str(e)}")
|
|
print("false")
|
|
sys.exit(0)
|
|
|
|
latest_state = {}
|
|
debug("\n=== Review States ===")
|
|
for r in data:
|
|
user = r.get("user", {}).get("login")
|
|
if not user:
|
|
continue
|
|
state = r.get("state")
|
|
latest_state[user] = state
|
|
commit_id = r.get("commit_id") or ""
|
|
debug(f"- {user}: {state} (commit: {commit_id[:8]})")
|
|
|
|
debug("\n=== Final Check ===")
|
|
for user, state in latest_state.items():
|
|
if user in AUTHORIZED and state == "APPROVED":
|
|
debug(f"Found APPROVED review from authorized user: {user}")
|
|
print("true")
|
|
sys.exit(0)
|
|
|
|
debug("No APPROVED review found from authorized users")
|
|
print("false")
|
|
' "${AUTHORIZED_REVIEWERS[@]}" <<< "$response")
|
|
|
|
echo "can-skip=$can_skip" >> "$GITHUB_OUTPUT"
|
|
|
|
if [ "$can_skip" = "true" ]; then
|
|
echo "Found valid approval from authorized reviewer, skipping IXUCA CI"
|
|
else
|
|
echo "No valid approval found, running full CI"
|
|
fi
|
|
|
|
- name: Download Paddle.tar.gz and setup
|
|
if: steps.check-approval.outputs.can-skip != 'true'
|
|
timeout-minutes: 30
|
|
run: |
|
|
set -e
|
|
echo "Downloading Paddle.tar.gz"
|
|
wget -q --tries=5 --no-proxy \
|
|
https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
|
echo "Extracting Paddle.tar.gz"
|
|
tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
|
rm Paddle.tar.gz
|
|
cd Paddle
|
|
git config --global --add safe.directory "*"
|
|
git config --global http.proxy http://oversea-website-proxy.aistudio.public:8888
|
|
git config --global https.proxy http://oversea-website-proxy.aistudio.public:8888
|
|
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
|
git config pull.rebase false
|
|
git checkout test
|
|
echo "Fetching upstream develop branch"
|
|
git fetch upstream develop
|
|
echo "Rebasing test branch onto upstream/develop"
|
|
git rebase upstream/develop || {
|
|
echo "Rebase failed, attempting to continue with current state"
|
|
git rebase --abort || true
|
|
}
|
|
echo "Latest 5 commits after rebase:"
|
|
git log --oneline -5
|
|
|
|
- name: Get Paddle-iluvatar PR number
|
|
if: steps.check-approval.outputs.can-skip != 'true'
|
|
id: get-pcd-pr
|
|
run: |
|
|
# Get Paddle PR description
|
|
PR_BODY=$(curl -s \
|
|
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" \
|
|
| python3 -c "import json,sys; data=json.load(sys.stdin); print(data.get('body', ''))")
|
|
|
|
# Extract [Paddle-iluvatar PR] XXXX from description
|
|
PCD_PR_NUM=$(echo "$PR_BODY" | grep -oP '\[Paddle-iluvatar PR\]\s*\K\d+' || echo "")
|
|
|
|
if [ -n "$PCD_PR_NUM" ]; then
|
|
echo "Found Paddle-iluvatar PR: $PCD_PR_NUM"
|
|
echo "pcd_pr_num=$PCD_PR_NUM" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "No Paddle-iluvatar PR found in PR description"
|
|
echo "pcd_pr_num=" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Clone Paddle-iluvatar and setup
|
|
if: steps.check-approval.outputs.can-skip != 'true'
|
|
run: |
|
|
git config --global --get http.proxy
|
|
git config --global --get https.proxy
|
|
for i in 1 2 3; do
|
|
git clone --depth=1 -b develop https://github.com/PaddlePaddle/Paddle-iluvatar.git && break
|
|
echo "git clone attempt $i failed, retrying in 10s..."
|
|
rm -rf Paddle-iluvatar
|
|
sleep 10
|
|
done
|
|
cd Paddle-iluvatar
|
|
|
|
# If Paddle-iluvatar PR number exists, fetch and merge it
|
|
PCD_PR_NUM="${{ steps.get-pcd-pr.outputs.pcd_pr_num }}"
|
|
if [ -n "$PCD_PR_NUM" ]; then
|
|
git config user.email "Paddle-CI@example.com"
|
|
git config user.name "Paddle-CI-Bot"
|
|
git fetch --unshallow origin
|
|
echo "Fetching and merging Paddle-iluvatar PR #$PCD_PR_NUM"
|
|
git fetch origin pull/$PCD_PR_NUM/head:pr-$PCD_PR_NUM
|
|
git merge pr-$PCD_PR_NUM --no-edit
|
|
echo "Successfully merged Paddle-iluvatar PR #$PCD_PR_NUM"
|
|
fi
|
|
|
|
rm -rf Paddle
|
|
mv ../Paddle .
|
|
|
|
- name: Build paddlepaddle-iluvatar
|
|
if: steps.check-approval.outputs.can-skip != 'true'
|
|
run: |
|
|
export PATH=/usr/local/corex/bin:$PATH
|
|
cd Paddle-iluvatar
|
|
apt-get install -y patchelf
|
|
bash build_paddle.sh
|
|
|
|
- name: Install paddlepaddle-iluvatar
|
|
if: steps.check-approval.outputs.can-skip != 'true'
|
|
run: |
|
|
pip install Paddle-iluvatar/Paddle/build/python/dist/paddlepaddle_iluvatar*.whl
|
|
retry_or_skip() {
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
echo "WARN: command failed, retrying in 20s: $*"
|
|
sleep 20
|
|
if "$@"; then
|
|
return 0
|
|
fi
|
|
echo "WARN: command failed again, skip install: $*"
|
|
return 0
|
|
}
|
|
retry_or_skip pip install parameterized pyyaml
|
|
|
|
- name: Run tests
|
|
if: steps.check-approval.outputs.can-skip != 'true'
|
|
run: |
|
|
export FLAG_SKIP_FLOAT64=1
|
|
cd Paddle-iluvatar/tests && bash run_test.sh
|
|
|
|
- name: Set result
|
|
if: steps.check-approval.outputs.can-skip != 'true'
|
|
id: set-result
|
|
run: |
|
|
echo "result=success" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Cleanup
|
|
if: always()
|
|
run: |
|
|
rm -rf * .[^.]*
|
|
chmod -R 777 . || true
|
|
|
|
# test-fallback:
|
|
# name: Build and Test (Fallback)
|
|
# needs: [check-bypass, test-primary]
|
|
# if: >-
|
|
# ${{ always()
|
|
# && needs.check-bypass.result == 'success'
|
|
# && needs.check-bypass.outputs.can-skip != 'true'
|
|
# && needs.test-primary.outputs.result != 'success' }}
|
|
# runs-on:
|
|
# group: test2
|
|
# timeout-minutes: 120
|
|
# env:
|
|
# TASK: paddle-CI-${{ github.event.pull_request.number }}-ixuca
|
|
|
|
# steps:
|
|
# - name: Check approval
|
|
# id: check-approval
|
|
# run: |
|
|
# set -euo pipefail
|
|
|
|
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
|
|
# AUTHORIZED_REVIEWERS=("YqGe585" "yongqiangma")
|
|
|
|
# echo "Authorized reviewers: ${AUTHORIZED_REVIEWERS[*]}"
|
|
# echo "Repository: ${{ github.repository }}"
|
|
# echo "PR number: ${{ github.event.pull_request.number }}"
|
|
|
|
# response=""
|
|
# for i in 1 2 3; do
|
|
# _tmp=$(curl -s --fail --max-time 30 \
|
|
# -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
|
# -H "Accept: application/vnd.github.v3+json" \
|
|
# "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" 2>/dev/null) && { response="$_tmp"; break; }
|
|
# echo "Attempt $i to fetch PR reviews failed, retrying..."
|
|
# sleep 5
|
|
# done
|
|
|
|
# echo "Response length: ${#response}"
|
|
|
|
# can_skip=$(python3 -c '
|
|
# import json, sys
|
|
|
|
# def debug(msg):
|
|
# print(msg, file=sys.stderr)
|
|
|
|
# debug("=== Review Check Debug Info ===")
|
|
# debug(f"Authorized reviewers: {sys.argv[1:]}")
|
|
# AUTHORIZED = set(sys.argv[1:])
|
|
|
|
# try:
|
|
# input_data = sys.stdin.read()
|
|
# debug(f"Raw input data length: {len(input_data)}")
|
|
# data = json.loads(input_data)
|
|
# debug(f"Parsed {len(data)} reviews")
|
|
# except Exception as e:
|
|
# debug(f"JSON parse error: {str(e)}")
|
|
# print("false")
|
|
# sys.exit(0)
|
|
|
|
# latest_state = {}
|
|
# debug("\n=== Review States ===")
|
|
# for r in data:
|
|
# user = r.get("user", {}).get("login")
|
|
# if not user:
|
|
# continue
|
|
# state = r.get("state")
|
|
# latest_state[user] = state
|
|
# commit_id = r.get("commit_id") or ""
|
|
# debug(f"- {user}: {state} (commit: {commit_id[:8]})")
|
|
|
|
# debug("\n=== Final Check ===")
|
|
# for user, state in latest_state.items():
|
|
# if user in AUTHORIZED and state == "APPROVED":
|
|
# debug(f"Found APPROVED review from authorized user: {user}")
|
|
# print("true")
|
|
# sys.exit(0)
|
|
|
|
# debug("No APPROVED review found from authorized users")
|
|
# print("false")
|
|
# ' "${AUTHORIZED_REVIEWERS[@]}" <<< "$response")
|
|
|
|
# echo "can-skip=$can_skip" >> "$GITHUB_OUTPUT"
|
|
|
|
# if [ "$can_skip" = "true" ]; then
|
|
# echo "Found valid approval from authorized reviewer, skipping IXUCA CI"
|
|
# else
|
|
# echo "No valid approval found, running full CI"
|
|
# fi
|
|
|
|
# - name: Download Paddle.tar.gz and setup
|
|
# if: steps.check-approval.outputs.can-skip != 'true'
|
|
# timeout-minutes: 30
|
|
# run: |
|
|
# set -e
|
|
# echo "Downloading Paddle.tar.gz"
|
|
# wget -q --tries=5 --no-proxy \
|
|
# https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
|
|
# echo "Extracting Paddle.tar.gz"
|
|
# tar --use-compress-program='pzstd' -xpf Paddle.tar.gz
|
|
# rm Paddle.tar.gz
|
|
# cd Paddle
|
|
# git config --global --add safe.directory "*"
|
|
# git remote add upstream https://github.com/PaddlePaddle/Paddle.git
|
|
# git config pull.rebase false
|
|
# git checkout test
|
|
# echo "Fetching upstream develop branch"
|
|
# git config --global --unset-all http.proxy || true
|
|
# git config --global --unset-all https.proxy || true
|
|
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
# git config --global http.proxy "$http_proxy"
|
|
# git config --global https.proxy "$https_proxy"
|
|
# git fetch upstream develop
|
|
# echo "Rebasing test branch onto upstream/develop"
|
|
# git rebase upstream/develop || {
|
|
# echo "Rebase failed, attempting to continue with current state"
|
|
# git rebase --abort || true
|
|
# }
|
|
# echo "Latest 5 commits after rebase:"
|
|
# git log --oneline -5
|
|
|
|
# - name: Get Paddle-iluvatar PR number
|
|
# if: steps.check-approval.outputs.can-skip != 'true'
|
|
# id: get-pcd-pr
|
|
# run: |
|
|
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
|
|
# # Get Paddle PR description
|
|
# PR_BODY=$(curl -s \
|
|
# -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
|
# -H "Accept: application/vnd.github.v3+json" \
|
|
# "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" \
|
|
# | python3 -c "import json,sys; data=json.load(sys.stdin); print(data.get('body', ''))")
|
|
|
|
# # Extract [Paddle-iluvatar PR] XXXX from description
|
|
# PCD_PR_NUM=$(echo "$PR_BODY" | grep -oP '\[Paddle-iluvatar PR\]\s*\K\d+' || echo "")
|
|
|
|
# if [ -n "$PCD_PR_NUM" ]; then
|
|
# echo "Found Paddle-iluvatar PR: $PCD_PR_NUM"
|
|
# echo "pcd_pr_num=$PCD_PR_NUM" >> "$GITHUB_OUTPUT"
|
|
# else
|
|
# echo "No Paddle-iluvatar PR found in PR description"
|
|
# echo "pcd_pr_num=" >> "$GITHUB_OUTPUT"
|
|
# fi
|
|
|
|
# - name: Clone Paddle-iluvatar and setup
|
|
# if: steps.check-approval.outputs.can-skip != 'true'
|
|
# run: |
|
|
# export https_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
# export http_proxy=http://oversea-website-proxy.aistudio.public:8888
|
|
|
|
# rm -rf Paddle-iluvatar
|
|
# git clone --depth=1 -b develop https://github.com/PaddlePaddle/Paddle-iluvatar.git
|
|
# cd Paddle-iluvatar
|
|
|
|
# # If Paddle-iluvatar PR number exists, fetch and merge it
|
|
# PCD_PR_NUM="${{ steps.get-pcd-pr.outputs.pcd_pr_num }}"
|
|
# if [ -n "$PCD_PR_NUM" ]; then
|
|
# git config user.email "Paddle-CI@example.com"
|
|
# git config user.name "Paddle-CI-Bot"
|
|
# git fetch --unshallow origin
|
|
# echo "Fetching and merging Paddle-iluvatar PR #$PCD_PR_NUM"
|
|
# git fetch origin pull/$PCD_PR_NUM/head:pr-$PCD_PR_NUM
|
|
# git merge pr-$PCD_PR_NUM --no-edit
|
|
# echo "Successfully merged Paddle-iluvatar PR #$PCD_PR_NUM"
|
|
# fi
|
|
|
|
# rm -rf Paddle
|
|
# mv ../Paddle .
|
|
|
|
# - name: Start docker container
|
|
# if: steps.check-approval.outputs.can-skip != 'true'
|
|
# run: |
|
|
# container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
|
|
# echo "container_name=${container_name}" >> $GITHUB_ENV
|
|
# docker run --network host --privileged --cap-add=ALL \
|
|
# --pid=host --shm-size=128G -d -t --name ${container_name} \
|
|
# -v /usr/src:/usr/src -v /lib/modules:/lib/modules \
|
|
# -v /dev:/dev -v /home:/home -v /data:/data \
|
|
# -v ${{ github.workspace }}/Paddle-iluvatar:/paddle \
|
|
# -e LD_LIBRARY_PATH=/usr/local/corex/lib \
|
|
# -e LIBRARY_PATH=/usr/local/corex/lib \
|
|
# -v /root/.cache:/root/.cache \
|
|
# -v /root/.cache:/root/.ccache \
|
|
# -e BRANCH -e PR_ID -e COMMIT_ID \
|
|
# -w /paddle \
|
|
# ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-ixuca:3.3.0 /bin/bash
|
|
|
|
# - name: Build and test
|
|
# if: steps.check-approval.outputs.can-skip != 'true'
|
|
# run: |
|
|
# docker exec -t ${{ env.container_name }} /bin/bash -c '
|
|
# retry_or_skip() {
|
|
# if "$@"; then
|
|
# return 0
|
|
# fi
|
|
# echo "WARN: command failed, retrying in 20s: $*"
|
|
# sleep 20
|
|
# if "$@"; then
|
|
# return 0
|
|
# fi
|
|
# echo "WARN: command failed again, skip install: $*"
|
|
# return 0
|
|
# }
|
|
# export PATH=/usr/local/corex/bin:$PATH
|
|
# apt-get install -y patchelf
|
|
# bash build_paddle.sh
|
|
# pip install Paddle/build/python/dist/paddlepaddle_iluvatar*.whl
|
|
# retry_or_skip pip install parameterized pyyaml
|
|
# export FLAG_SKIP_FLOAT64=1
|
|
# cd tests && bash run_test.sh
|
|
# '
|
|
|
|
# - name: Cleanup
|
|
# if: always()
|
|
# run: |
|
|
# docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*' || true
|
|
# docker stop ${{ env.container_name }} || true
|
|
# docker rm ${{ env.container_name }} || true
|