chore: import upstream snapshot with attribution
PR Test (NPU) / check-changes (push) Has been cancelled
PR Test (NPU) / pr-gate (push) Has been cancelled
PR Test (NPU) / set-image-config (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-4-npu-a3 (push) Has been cancelled
PR Test (NPU) / stage-b-test-16-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-1-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-2-npu-a3 (push) Has been cancelled
PR Test (Arm64) / pr-gate (push) Has been cancelled
PR Test (Arm64) / check-changes (push) Has been cancelled
PR Test (Arm64) / build-test (push) Has been cancelled
PR Test (sgl-router) / gate (push) Has been cancelled
PR Test (sgl-router) / tier-1 — lint (push) Has been cancelled
PR Test (sgl-router) / tier-2 — build + test (push) Has been cancelled
PR Test (sgl-router) / tier-3 — docker (placeholder) (push) Has been cancelled
PR Test (sgl-router) / tier-3 — k8s integration (push) Has been cancelled
PR Test (sgl-router) / tier-3 — e2e (push) Has been cancelled
PR Test (sgl-router) / finish (push) Has been cancelled
PR Test (NPU) / single-node-poc (map[name:qwen3_6_27b_w8a8_1p_in64k_out1k_50ms runner:linux-aarch64-a3-2 test_case:test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type:perf]) (push) Has been cancelled
PR Test (NPU) / pr-test-npu-finish (push) Has been cancelled
PR Test (Xeon) / pr-gate (push) Has been cancelled
PR Test (Xeon) / check-changes (push) Has been cancelled
PR Test (Xeon) / build-test (, xeon-gnr, base-b-test-cpu) (push) Has been cancelled
PR Test (XPU) / check-changes (push) Has been cancelled
PR Test (XPU) / pr-gate (push) Has been cancelled
PR Test (XPU) / stage-a-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / wait-for-stage-a (push) Has been cancelled
PR Test (XPU) / stage-b-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / finish (push) Has been cancelled
CI Model Inventory / build-inventory (push) Has been cancelled
Lint / lint (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Compilation Check (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Manual Policy (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Request Processing (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Summary (push) Has been cancelled
PR Test (SMG) / build-wheel (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on windows (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (x86_64 - auto) (push) Has been cancelled
PR Test (SMG) / python-unit-tests (push) Has been cancelled
PR Test (SMG) / unit-tests (push) Has been cancelled
PR Test (SMG) / benchmarks (push) Has been cancelled
PR Test (SMG) / chat-completions (push) Has been cancelled
PR Test (SMG) / chat-completions-4gpu (push) Has been cancelled
PR Test (SMG) / e2e (push) Has been cancelled
PR Test (SMG) / docker-build-test (push) Has been cancelled
PR Test (SMG) / k8s-integration (push) Has been cancelled
PR Test (SMG) / finish (push) Has been cancelled
PR Test (SMG) / summarize-benchmarks (push) Has been cancelled
Release SGLang Model Gateway Docker Image / publish (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Build SDist (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Upload to PyPI (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (aarch64, 12.9, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (x86_64, 12.9, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu129 (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (aarch64, 13.0, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (x86_64, 13.0, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu130 (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 700) (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 720) (push) Has been cancelled
Release SGLang Kernels / release-rocm700 (push) Has been cancelled
Release SGLang Kernels / release-rocm720 (push) Has been cancelled
Release SGLang Kernels / build-musa43 (43, 3.10) (push) Has been cancelled
Release SGLang Kernels / release-musa43 (push) Has been cancelled
PR Test (NPU) / check-changes (push) Has been cancelled
PR Test (NPU) / pr-gate (push) Has been cancelled
PR Test (NPU) / set-image-config (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-4-npu-a3 (push) Has been cancelled
PR Test (NPU) / stage-b-test-16-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-1-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-2-npu-a3 (push) Has been cancelled
PR Test (Arm64) / pr-gate (push) Has been cancelled
PR Test (Arm64) / check-changes (push) Has been cancelled
PR Test (Arm64) / build-test (push) Has been cancelled
PR Test (sgl-router) / gate (push) Has been cancelled
PR Test (sgl-router) / tier-1 — lint (push) Has been cancelled
PR Test (sgl-router) / tier-2 — build + test (push) Has been cancelled
PR Test (sgl-router) / tier-3 — docker (placeholder) (push) Has been cancelled
PR Test (sgl-router) / tier-3 — k8s integration (push) Has been cancelled
PR Test (sgl-router) / tier-3 — e2e (push) Has been cancelled
PR Test (sgl-router) / finish (push) Has been cancelled
PR Test (NPU) / single-node-poc (map[name:qwen3_6_27b_w8a8_1p_in64k_out1k_50ms runner:linux-aarch64-a3-2 test_case:test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type:perf]) (push) Has been cancelled
PR Test (NPU) / pr-test-npu-finish (push) Has been cancelled
PR Test (Xeon) / pr-gate (push) Has been cancelled
PR Test (Xeon) / check-changes (push) Has been cancelled
PR Test (Xeon) / build-test (, xeon-gnr, base-b-test-cpu) (push) Has been cancelled
PR Test (XPU) / check-changes (push) Has been cancelled
PR Test (XPU) / pr-gate (push) Has been cancelled
PR Test (XPU) / stage-a-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / wait-for-stage-a (push) Has been cancelled
PR Test (XPU) / stage-b-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / finish (push) Has been cancelled
CI Model Inventory / build-inventory (push) Has been cancelled
Lint / lint (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Compilation Check (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Manual Policy (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Request Processing (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Summary (push) Has been cancelled
PR Test (SMG) / build-wheel (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on windows (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (x86_64 - auto) (push) Has been cancelled
PR Test (SMG) / python-unit-tests (push) Has been cancelled
PR Test (SMG) / unit-tests (push) Has been cancelled
PR Test (SMG) / benchmarks (push) Has been cancelled
PR Test (SMG) / chat-completions (push) Has been cancelled
PR Test (SMG) / chat-completions-4gpu (push) Has been cancelled
PR Test (SMG) / e2e (push) Has been cancelled
PR Test (SMG) / docker-build-test (push) Has been cancelled
PR Test (SMG) / k8s-integration (push) Has been cancelled
PR Test (SMG) / finish (push) Has been cancelled
PR Test (SMG) / summarize-benchmarks (push) Has been cancelled
Release SGLang Model Gateway Docker Image / publish (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Build SDist (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Upload to PyPI (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (aarch64, 12.9, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (x86_64, 12.9, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu129 (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (aarch64, 13.0, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (x86_64, 13.0, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu130 (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 700) (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 720) (push) Has been cancelled
Release SGLang Kernels / release-rocm700 (push) Has been cancelled
Release SGLang Kernels / release-rocm720 (push) Has been cancelled
Release SGLang Kernels / build-musa43 (43, 3.10) (push) Has been cancelled
Release SGLang Kernels / release-musa43 (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
name: Check Maintenance Mode
|
||||
description: Blocks CI in two independent modes driven by issue #21065. (1) Full-pause: when the issue is open. (2) Rebase-required: whenever the issue body contains a `MIN_BASE_SHA: <sha>` directive — enforced regardless of whether the issue is open or closed, so maintainers can require all PRs to rebase past a specific commit without having to open the maintenance issue. Both modes are bypassed by the `bypass-maintenance` label on the PR, or by env PR_TEST_BYPASS_MAINTENANCE_ON_MAIN=true (PR Test workflow on main only). Merging non-CI-fix PRs is prohibited during full-pause; in severe cases, merge permissions may be revoked.
|
||||
|
||||
inputs:
|
||||
github-token:
|
||||
description: GitHub token for API access
|
||||
required: false
|
||||
default: ${{ github.token }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Check maintenance mode
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ inputs.github-token }}
|
||||
run: |
|
||||
MAINTENANCE_ISSUE=21065
|
||||
REPO="${{ github.repository }}"
|
||||
PR_NUMBER="${{ github.event.pull_request.number }}"
|
||||
PR_HEAD_SHA="${{ github.event.pull_request.head.sha }}"
|
||||
|
||||
# PR Test workflow only: scheduled runs and runs on main (dispatch / workflow_call) set this env
|
||||
if [[ "${PR_TEST_BYPASS_MAINTENANCE_ON_MAIN:-}" == "true" ]]; then
|
||||
echo "✅ PR Test on main branch; bypassing maintenance gate."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Use curl + jq instead of `gh` because self-hosted GPU runners
|
||||
# don't have the gh CLI installed. Without this, the action
|
||||
# silently fell through to "Proceeding with CI" on every GPU
|
||||
# job — the rebase gate only fired on ubuntu-latest jobs that
|
||||
# had gh pre-installed. curl+jq are reliably available on every
|
||||
# Linux runner.
|
||||
gh_api() {
|
||||
local path="$1"
|
||||
local err_file="$2"
|
||||
curl --silent --show-error --fail \
|
||||
--max-time 30 \
|
||||
-H "Authorization: Bearer $GH_TOKEN" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
"https://api.github.com/$path" 2>"$err_file"
|
||||
}
|
||||
|
||||
# Fetch issue state and body. Fail-open: if we can't read the issue
|
||||
# (network blip, missing token scope), CI proceeds.
|
||||
ERR_FILE=$(mktemp)
|
||||
ISSUE_JSON=$(gh_api "repos/$REPO/issues/$MAINTENANCE_ISSUE" "$ERR_FILE" || true)
|
||||
if [[ -z "$ISSUE_JSON" ]]; then
|
||||
echo "⚠️ Issue fetch returned empty. curl stderr was:"
|
||||
cat "$ERR_FILE" || true
|
||||
fi
|
||||
rm -f "$ERR_FILE"
|
||||
ISSUE_STATE=$(printf '%s' "$ISSUE_JSON" | jq -r '.state // "UNKNOWN"' 2>/dev/null || echo "UNKNOWN")
|
||||
# Issues API returns state in lowercase ("open"/"closed"); normalize
|
||||
# to uppercase so existing comparisons against "OPEN" still work.
|
||||
ISSUE_STATE=$(printf '%s' "$ISSUE_STATE" | tr '[:lower:]' '[:upper:]')
|
||||
ISSUE_BODY=$(printf '%s' "$ISSUE_JSON" | jq -r '.body // ""' 2>/dev/null || echo "")
|
||||
echo "DEBUG: ISSUE_STATE=$ISSUE_STATE body_length=${#ISSUE_BODY}"
|
||||
|
||||
# Parse optional `MIN_BASE_SHA: <sha>` directive from the issue body
|
||||
# (first occurrence wins). Whenever this directive is present, the
|
||||
# rebase check is enforced regardless of whether the issue is open
|
||||
# or closed — so maintainers can require all PRs to rebase past a
|
||||
# specific commit without having to open the maintenance issue.
|
||||
# `grep` exits 1 on no-match, which under `set -eo pipefail` (the
|
||||
# default for `shell: bash` composite steps) would abort the whole
|
||||
# script — silently failing every PR whose issue body has no
|
||||
# MIN_BASE_SHA line. Wrap grep in a brace group with `|| true` so
|
||||
# an empty match falls through cleanly to "no directive set".
|
||||
MIN_BASE_SHA=$(
|
||||
{ printf '%s' "$ISSUE_BODY" | tr -d '\r' | grep -iE '^[[:space:]]*`?MIN_BASE_SHA`?[[:space:]]*[:=]' || true; } \
|
||||
| head -n1 | sed -E 's/.*[:=][[:space:]]*//; s/`//g' | awk '{print $1}'
|
||||
)
|
||||
if [[ -n "$MIN_BASE_SHA" ]] && ! [[ "$MIN_BASE_SHA" =~ ^[a-fA-F0-9]{7,40}$ ]]; then
|
||||
WARN="⚠️ Ignoring malformed MIN_BASE_SHA directive in issue #$MAINTENANCE_ISSUE: '$MIN_BASE_SHA' (must be 7-40 hex chars)"
|
||||
echo "$WARN"
|
||||
echo "$WARN" >> "$GITHUB_STEP_SUMMARY"
|
||||
MIN_BASE_SHA=""
|
||||
fi
|
||||
|
||||
# If neither gate is active (no MIN_BASE_SHA, issue not open), nothing to do.
|
||||
if [[ -z "$MIN_BASE_SHA" && "$ISSUE_STATE" != "OPEN" ]]; then
|
||||
echo "✅ Maintenance mode is OFF and no MIN_BASE_SHA directive. Proceeding with CI."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# bypass-maintenance label bypasses both gates. PR labels live on
|
||||
# the issue resource (PRs are issues with extra metadata in the
|
||||
# GH API), so we hit the same /issues/{n} endpoint.
|
||||
if [[ -n "$PR_NUMBER" ]]; then
|
||||
ERR_FILE=$(mktemp)
|
||||
PR_JSON=$(gh_api "repos/$REPO/issues/$PR_NUMBER" "$ERR_FILE" || true)
|
||||
rm -f "$ERR_FILE"
|
||||
HAS_BYPASS=$(printf '%s' "$PR_JSON" | jq -r '[.labels[]?.name] | map(select(. == "bypass-maintenance")) | length' 2>/dev/null || echo "0")
|
||||
if [[ "${HAS_BYPASS:-0}" -gt 0 ]]; then
|
||||
echo "✅ PR #$PR_NUMBER has 'bypass-maintenance' label. Bypassing maintenance + rebase checks."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Rebase-required gate (independent of issue open/closed state).
|
||||
if [[ -n "$MIN_BASE_SHA" ]]; then
|
||||
if [[ -z "$PR_NUMBER" || -z "$PR_HEAD_SHA" ]]; then
|
||||
echo "✅ Not a PR context; skipping rebase check."
|
||||
else
|
||||
# Use GitHub compare API: status is "ahead"/"identical" when MIN_BASE_SHA is reachable from PR head.
|
||||
ERR_FILE=$(mktemp)
|
||||
COMPARE_JSON=$(gh_api "repos/$REPO/compare/$MIN_BASE_SHA...$PR_HEAD_SHA" "$ERR_FILE" || true)
|
||||
if [[ -z "$COMPARE_JSON" ]]; then
|
||||
echo "⚠️ Compare API failed. curl stderr was:"
|
||||
cat "$ERR_FILE" || true
|
||||
fi
|
||||
rm -f "$ERR_FILE"
|
||||
COMPARE_STATUS=$(printf '%s' "$COMPARE_JSON" | jq -r '.status // "UNKNOWN"' 2>/dev/null || echo "UNKNOWN")
|
||||
COMPARE_STATUS="${COMPARE_STATUS:-UNKNOWN}"
|
||||
|
||||
case "$COMPARE_STATUS" in
|
||||
ahead|identical)
|
||||
echo "✅ PR #$PR_NUMBER contains required base ${MIN_BASE_SHA:0:12} ($COMPARE_STATUS)."
|
||||
;;
|
||||
UNKNOWN)
|
||||
echo "⚠️ Could not determine rebase status via GitHub API; fail-open, allowing rebase check to pass."
|
||||
;;
|
||||
*)
|
||||
MSG=$(printf "%s\n" \
|
||||
"## ⚠️ Rebase Required Before CI Can Run" \
|
||||
"A major update has landed on \`main\`. All PRs must rebase onto the latest \`main\` before CI will run." \
|
||||
"Required base commit: \`${MIN_BASE_SHA:0:12}\` (your PR is \`$COMPARE_STATUS\` relative to this commit)." \
|
||||
"" \
|
||||
"What should you do?" \
|
||||
"- Rebase your branch onto the latest \`main\` and push again" \
|
||||
"- Follow CI Maintenance Mode issue: https://github.com/$REPO/issues/$MAINTENANCE_ISSUE for context" \
|
||||
"- CI-fix PRs may request the \`bypass-maintenance\` label to skip this check")
|
||||
echo "$MSG" >> "$GITHUB_STEP_SUMMARY"
|
||||
while IFS= read -r line; do
|
||||
echo "::error::$line"
|
||||
done <<< "$MSG"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
# Full-pause maintenance gate (only when issue is open).
|
||||
if [[ "$ISSUE_STATE" == "OPEN" ]]; then
|
||||
MSG=$(printf "%s\n" \
|
||||
"## ⚠️ CI Maintenance Mode is Active" \
|
||||
"The CI infrastructure is currently under maintenance." \
|
||||
"All PR CI runs are paused until maintenance is complete." \
|
||||
"**Merging non-CI-fix PRs is prohibited during maintenance mode.** In severe cases, merge permissions may be revoked." \
|
||||
"You might also experience unexpected failures during this period." \
|
||||
"The team is working on the issue and will update the status as soon as possible." \
|
||||
"" \
|
||||
"What should you do?" \
|
||||
"- **Do NOT merge non-CI-fix PRs** until maintenance mode is lifted" \
|
||||
"- Check back later (~12 hours)" \
|
||||
"- Follow CI Maintenance Mode issue: https://github.com/$REPO/issues/$MAINTENANCE_ISSUE for status updates")
|
||||
|
||||
echo "$MSG" >> "$GITHUB_STEP_SUMMARY"
|
||||
while IFS= read -r line; do
|
||||
echo "::error::$line"
|
||||
done <<< "$MSG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Rebase check passed; full-pause not active. Proceeding with CI."
|
||||
@@ -0,0 +1,98 @@
|
||||
name: Check PR Test Health
|
||||
description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fastfail` label.
|
||||
|
||||
inputs:
|
||||
github-token:
|
||||
description: 'GitHub token for API calls'
|
||||
required: false
|
||||
default: ${{ github.token }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token }}
|
||||
script: |
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.info('Skipping health check (SKIP_PR_TEST_HEALTH_CHECK=true)');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip for scheduled runs — they should collect all failures, not fast-fail
|
||||
if (context.eventName === 'schedule') {
|
||||
core.info('Skipping health check for scheduled run');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check lint status from the separate Lint workflow (lint.yml).
|
||||
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
||||
// checks.listForRef which queries by commit SHA across ALL workflows.
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
const { data } = await github.rest.checks.listForRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: ref,
|
||||
check_name: 'lint',
|
||||
});
|
||||
const lintRun = data.check_runs.find(
|
||||
cr => cr.app?.slug === 'github-actions'
|
||||
);
|
||||
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
||||
core.setFailed('Fast-fail: lint check failed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
|
||||
// Lint check above still runs.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
}
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
core.info('Skipping jobs-failed check (bypass-fastfail label present)');
|
||||
return;
|
||||
}
|
||||
|
||||
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
});
|
||||
// Find jobs that failed from a real error, not from fast-fail cascade
|
||||
const rootCauseFailures = jobs.filter(j => {
|
||||
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
||||
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
||||
// should not cascade fast-fail to other stages. j.name shape from
|
||||
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
||||
// + optional " (<matrix>)". Split off the base job key before exact
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
if (rootCauseFailures.length > 0) {
|
||||
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
name: Upload CUDA Coredumps
|
||||
description: Upload CUDA coredump files as artifacts, optionally signal to a tracker issue, and clean up.
|
||||
|
||||
inputs:
|
||||
artifact-suffix:
|
||||
description: Suffix appended to the artifact name (e.g. matrix partition id)
|
||||
required: false
|
||||
default: ""
|
||||
retention-days:
|
||||
description: Number of days to retain the artifact
|
||||
required: false
|
||||
default: "7"
|
||||
tracker-issue:
|
||||
description: |
|
||||
If set, post a one-line comment to sgl-project/sglang issue
|
||||
#<tracker-issue> when at least one coredump is detected. Requires
|
||||
`bot-token` with issues:write on sgl-project/sglang.
|
||||
required: false
|
||||
default: ""
|
||||
bot-token:
|
||||
description: PAT with issues:write on sgl-project/sglang. Required when tracker-issue is set.
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Check for coredumps
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
# Mirror get_dump_dir() in python/sglang/srt/debug_utils/cuda_coredump.py:
|
||||
# explicit override > per-job RUNNER_TEMP > /tmp default, then a
|
||||
# per-(run, attempt) subdir so dumps are never mis-attributed across CI
|
||||
# jobs that share a self-hosted runner's filesystem.
|
||||
if [ -n "$SGLANG_CUDA_COREDUMP_DIR" ]; then
|
||||
base="$SGLANG_CUDA_COREDUMP_DIR"
|
||||
elif [ -n "$RUNNER_TEMP" ]; then
|
||||
base="$RUNNER_TEMP/sglang_cuda_coredumps"
|
||||
else
|
||||
base="/tmp/sglang_cuda_coredumps"
|
||||
fi
|
||||
if [ -n "$GITHUB_RUN_ID" ]; then
|
||||
dir="$base/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||
else
|
||||
dir="$base"
|
||||
fi
|
||||
echo "dump_dir=$dir" >> "$GITHUB_OUTPUT"
|
||||
if [ -d "$dir" ] && [ -n "$(ls -A "$dir" 2>/dev/null)" ]; then
|
||||
echo "has_dumps=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_dumps=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Upload CUDA coredumps
|
||||
if: steps.check.outputs.has_dumps == 'true'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cuda-coredumps-${{ github.job }}${{ inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) }}
|
||||
path: ${{ steps.check.outputs.dump_dir }}/
|
||||
retention-days: ${{ inputs.retention-days }}
|
||||
|
||||
- name: Signal coredump to tracker issue
|
||||
if: steps.check.outputs.has_dumps == 'true' && inputs.tracker-issue != '' && inputs.bot-token != ''
|
||||
shell: bash
|
||||
env:
|
||||
BOT_TOKEN: ${{ inputs.bot-token }}
|
||||
PR_NUM: ${{ github.event.pull_request.number }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
TRACKER_ISSUE: ${{ inputs.tracker-issue }}
|
||||
run: |
|
||||
if [ -n "$PR_NUM" ]; then
|
||||
ref_label="PR #${PR_NUM}"
|
||||
else
|
||||
ref_label="$EVENT_NAME"
|
||||
fi
|
||||
# Resolve own job_id via REST API: match by runner_name + status
|
||||
# in_progress (the current job is the only one in_progress on this
|
||||
# runner). Robust across matrix expansions and artifact-suffix shapes.
|
||||
job_id=$(curl -sS \
|
||||
-H "Authorization: Bearer ${BOT_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs?per_page=100" \
|
||||
| python3 -c 'import json,sys,os; print(next((j["id"] for j in json.load(sys.stdin)["jobs"] if j.get("runner_name")==os.environ["RUNNER_NAME"] and j.get("status")=="in_progress"), ""))')
|
||||
if [ -n "$job_id" ]; then
|
||||
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/job/${job_id}"
|
||||
else
|
||||
# Fallback to run-attempt URL if job_id lookup failed
|
||||
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}"
|
||||
fi
|
||||
body_json=$(printf '{"body":"@hnyls2002 [Coredump Tracker] %s - %s"}' "${ref_label}" "${run_url}")
|
||||
curl -sS -X POST \
|
||||
-H "Authorization: Bearer ${BOT_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
"https://api.github.com/repos/sgl-project/sglang/issues/${TRACKER_ISSUE}/comments" \
|
||||
-d "${body_json}"
|
||||
|
||||
- name: Cleanup CUDA coredumps
|
||||
shell: bash
|
||||
run: rm -rf "${{ steps.check.outputs.dump_dir }}"
|
||||
@@ -0,0 +1,229 @@
|
||||
name: Wait for Jobs
|
||||
description: Poll and wait for specified jobs in the current workflow run to complete. Returns success immediately when the PR carries the `bypass-fastfail` label, letting downstream stages dispatch in parallel (same effect as scheduled runs).
|
||||
|
||||
inputs:
|
||||
stage-name:
|
||||
description: 'Human-readable stage name for log messages (e.g. "base-a")'
|
||||
required: true
|
||||
jobs:
|
||||
description: |
|
||||
JSON array of job specs to wait for. Each element is either:
|
||||
- a string: exact job name (e.g. "base-a-test-1-gpu-small")
|
||||
- an object { "prefix": "...", "expected_count": N }: for matrix jobs
|
||||
required: true
|
||||
max-wait-minutes:
|
||||
description: 'Maximum time to wait before timing out'
|
||||
required: false
|
||||
default: '240'
|
||||
poll-interval-seconds:
|
||||
description: 'Seconds between polling attempts'
|
||||
required: false
|
||||
default: '60'
|
||||
github-token:
|
||||
description: 'GitHub token for API calls'
|
||||
required: false
|
||||
default: ${{ github.token }}
|
||||
|
||||
outputs:
|
||||
result:
|
||||
description: 'Overall result: success, failure, or timeout'
|
||||
value: ${{ steps.wait.outputs.result }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Wait for jobs to complete
|
||||
id: wait
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
INPUT_STAGE_NAME: ${{ inputs.stage-name }}
|
||||
INPUT_JOBS: ${{ inputs.jobs }}
|
||||
INPUT_MAX_WAIT_MINUTES: ${{ inputs.max-wait-minutes }}
|
||||
INPUT_POLL_INTERVAL_SECONDS: ${{ inputs.poll-interval-seconds }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token }}
|
||||
script: |
|
||||
const stageName = process.env.INPUT_STAGE_NAME;
|
||||
const jobSpecs = JSON.parse(process.env.INPUT_JOBS);
|
||||
const maxWaitMinutes = parseInt(process.env.INPUT_MAX_WAIT_MINUTES);
|
||||
const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS);
|
||||
const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds;
|
||||
|
||||
// bypass-fastfail label opts the PR out of stage-to-stage waiting,
|
||||
// letting all stages dispatch in parallel like scheduled runs do.
|
||||
let labels = [];
|
||||
if (context.payload.pull_request?.labels) {
|
||||
labels = context.payload.pull_request.labels.map(l => l.name);
|
||||
} else {
|
||||
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
||||
try {
|
||||
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: ref,
|
||||
});
|
||||
if (prs.length > 0) {
|
||||
labels = prs[0].labels.map(l => l.name);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(`Could not fetch PR labels for ${ref}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
if (labels.includes('bypass-fastfail')) {
|
||||
console.log(`Skipping ${stageName} wait (bypass-fastfail label present)`);
|
||||
core.setOutput('result', 'success');
|
||||
return;
|
||||
}
|
||||
|
||||
// Normalize job specs into a uniform format
|
||||
const normalizedSpecs = jobSpecs.map(spec => {
|
||||
if (typeof spec === 'string') {
|
||||
return { prefix: spec, expected_count: 1, exact: true };
|
||||
}
|
||||
return { ...spec, exact: false };
|
||||
});
|
||||
|
||||
const totalExpectedJobs = normalizedSpecs.reduce((sum, s) => sum + s.expected_count, 0);
|
||||
|
||||
const matchesSpec = (jobName, spec) => {
|
||||
if (spec.exact) {
|
||||
return jobName === spec.prefix;
|
||||
}
|
||||
// Match bare prefix or any GHA-rendered shard suffix. Inline matrix
|
||||
// produces `<prefix> (<shard>)`; reusable-workflow callers produce
|
||||
// `<prefix> / <called-job> (<shard>)`. Both have a space after the
|
||||
// prefix, so a single ' '-delimited check covers both.
|
||||
return jobName === spec.prefix || jobName.startsWith(spec.prefix + ' ');
|
||||
};
|
||||
|
||||
// Use ETag conditional requests to avoid consuming rate limit when nothing changed.
|
||||
// GitHub returns 304 Not Modified for unchanged data, which is FREE (no rate limit cost).
|
||||
let lastEtag = '';
|
||||
let lastJobs = null;
|
||||
let apiCalls = 0;
|
||||
let cachedCalls = 0;
|
||||
|
||||
async function fetchJobs() {
|
||||
const url = `GET /repos/{owner}/{repo}/actions/runs/{run_id}/jobs`;
|
||||
const params = {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
per_page: 100,
|
||||
headers: {},
|
||||
};
|
||||
if (lastEtag) {
|
||||
params.headers['if-none-match'] = lastEtag;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await github.request(url, params);
|
||||
apiCalls++;
|
||||
const rateRemaining = response.headers['x-ratelimit-remaining'] || '?';
|
||||
const rateLimit = response.headers['x-ratelimit-limit'] || '?';
|
||||
console.log(`[rate-limit] ${rateRemaining}/${rateLimit} remaining (ETag: ${lastEtag ? 'sent' : 'none'}) | this session: ${apiCalls} paid, ${cachedCalls} free`);
|
||||
lastEtag = response.headers.etag || '';
|
||||
const jobs = response.data.jobs;
|
||||
|
||||
// Handle pagination if >100 jobs
|
||||
// ETag only covers page 1, so invalidate it to avoid stale cache
|
||||
// when later pages change but page 1 doesn't.
|
||||
if (response.data.total_count > 100) {
|
||||
lastEtag = '';
|
||||
for (let page = 2; page <= Math.ceil(response.data.total_count / 100); page++) {
|
||||
const { data: pageData } = await github.request(url, {
|
||||
...params,
|
||||
page,
|
||||
headers: {},
|
||||
});
|
||||
jobs.push(...pageData.jobs);
|
||||
}
|
||||
}
|
||||
|
||||
lastJobs = jobs;
|
||||
return { jobs, cached: false };
|
||||
} catch (err) {
|
||||
if (err.status === 304 && lastJobs) {
|
||||
cachedCalls++;
|
||||
console.log(`[rate-limit] 304 Not Modified | this session: ${apiCalls} paid, ${cachedCalls} free`);
|
||||
return { jobs: lastJobs, cached: true };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const { jobs, cached } = await fetchJobs();
|
||||
|
||||
let allCompleted = true;
|
||||
let failedJobs = [];
|
||||
let completedCount = 0;
|
||||
let totalCount = 0;
|
||||
|
||||
for (const spec of normalizedSpecs) {
|
||||
const matchingJobs = jobs.filter(job => matchesSpec(job.name, spec));
|
||||
|
||||
for (const job of matchingJobs) {
|
||||
totalCount++;
|
||||
if (!cached) {
|
||||
console.log(`${job.name}: status=${job.status}, conclusion=${job.conclusion}`);
|
||||
}
|
||||
|
||||
if (job.status === 'completed') {
|
||||
completedCount++;
|
||||
if (job.conclusion !== 'success' && job.conclusion !== 'skipped') {
|
||||
failedJobs.push(job.name);
|
||||
}
|
||||
} else {
|
||||
allCompleted = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (matchingJobs.length < spec.expected_count) {
|
||||
// Job-level `if:` is evaluated before matrix expansion. When it
|
||||
// evaluates false, GitHub emits exactly one "skipped" entry
|
||||
// without a "(shard)" suffix instead of N matrix entries. Two
|
||||
// shapes are possible:
|
||||
// - inline matrix: `<prefix>`
|
||||
// - reusable workflow: `<prefix> / <called-job>`
|
||||
// Both lack the ` (` shard marker. Detect that precise shape so
|
||||
// we don't poll forever — and so we don't mistake a partially
|
||||
// materialized dynamic/reusable matrix for a skipped one.
|
||||
const unexpandedSkip = matchingJobs.length === 1 &&
|
||||
!matchingJobs[0].name.includes(' (') &&
|
||||
matchingJobs[0].status === 'completed' &&
|
||||
matchingJobs[0].conclusion === 'skipped';
|
||||
if (unexpandedSkip) {
|
||||
const missing = spec.expected_count - 1;
|
||||
totalCount += missing;
|
||||
completedCount += missing;
|
||||
if (!cached) {
|
||||
console.log(`${spec.prefix}: job-level skip (bare entry, conclusion=skipped); treating as all ${spec.expected_count} skipped`);
|
||||
}
|
||||
} else {
|
||||
console.log(`${spec.prefix}: found ${matchingJobs.length}/${spec.expected_count} jobs (waiting for more)`);
|
||||
allCompleted = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[${stageName}] Progress: ${completedCount}/${totalCount} jobs completed (expected ${totalExpectedJobs})${cached ? ' (cached, no rate limit cost)' : ''}`);
|
||||
|
||||
// Fail fast if any jobs failed
|
||||
if (failedJobs.length > 0) {
|
||||
core.setOutput('result', 'failure');
|
||||
core.setFailed(`${stageName} jobs failed: ${failedJobs.join(', ')}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (allCompleted && totalCount >= totalExpectedJobs) {
|
||||
core.setOutput('result', 'success');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Waiting ${pollIntervalSeconds}s... (attempt ${attempt + 1}/${maxAttempts})`);
|
||||
await new Promise(resolve => setTimeout(resolve, pollIntervalSeconds * 1000));
|
||||
}
|
||||
|
||||
core.setFailed(`Timeout waiting for ${stageName} jobs`);
|
||||
core.setOutput('result', 'timeout');
|
||||
Reference in New Issue
Block a user