chore: import upstream snapshot with attribution

.github/workflows/_docker-build-and-publish.yml

@@ -0,0 +1,331 @@
+name: Build and Publish Multi-Arch Docker Images
+
+# Reusable workflow: builds CUDA 12 + CUDA 13 images for amd64 and arm64,
+# then creates multi-arch manifests with caller-specified tags.
+
+on:
+  workflow_call:
+    inputs:
+      docker_target:
+        description: "Dockerfile target stage (framework or runtime)"
+        required: true
+        type: string
+      sgl_version:
+        description: "Version string passed as SGL_VERSION build arg (empty to skip)"
+        required: false
+        type: string
+        default: ""
+      extra_build_args:
+        description: "Additional --build-arg flags appended to docker buildx build"
+        required: false
+        type: string
+        default: ""
+      checkout_ref:
+        description: "Git ref to checkout (empty for default)"
+        required: false
+        type: string
+        default: ""
+      tag_config:
+        description: 'JSON array of {"cuda":"cu129|cu130","tags":["tag1","tag2"]}. Tags support {version} substitution.'
+        required: true
+        type: string
+      use_environment:
+        description: "GitHub environment name (e.g. prod) or empty for none"
+        required: false
+        type: string
+        default: ""
+      image_repo:
+        description: "Docker Hub repo to push to (e.g. lmsysorg/sglang-staging for testing)"
+        required: false
+        type: string
+        default: "lmsysorg/sglang"
+
+jobs:
+  build-x86:
+    if: github.repository == 'sgl-project/sglang'
+    environment: ${{ inputs.use_environment || null }}
+    runs-on: x64-docker-build-node
+    env:
+      TAG_CONFIG: ${{ inputs.tag_config }}
+      SGL_VERSION: ${{ inputs.sgl_version }}
+      IMAGE_REPO: ${{ inputs.image_repo }}
+    outputs:
+      digest-cu129: ${{ steps.build-cu129.outputs.digest }}
+      digest-cu130: ${{ steps.build-cu130.outputs.digest }}
+    steps:
+      - name: Delete huge unnecessary tools folder
+        run: rm -rf /opt/hostedtoolcache
+
+      - name: Cleanup workspace (remove root-owned files from prior runs)
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/* || true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.checkout_ref || github.ref }}
+
+      - name: Compute Docker build metadata args
+        run: |
+          set -euo pipefail
+          BUILD_COMMIT="$(git rev-parse HEAD)"
+          BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          for CUDA_VARIANT in cu129 cu130; do
+            python3 scripts/ci/utils/docker_build_metadata_args.py \
+              --cuda "${CUDA_VARIANT}" \
+              --tag-config "${TAG_CONFIG}" \
+              --image-repo "${IMAGE_REPO}" \
+              --sgl-version "${SGL_VERSION}" \
+              --build-commit "${BUILD_COMMIT}" \
+              --build-url "${BUILD_URL}" \
+              > "/tmp/docker-metadata-${CUDA_VARIANT}.args"
+          done
+
+      - name: Free disk space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          docker-images: true
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
+      - name: Prune Docker to reclaim disk space
+        run: |
+          docker buildx prune --filter "until=72h" -f
+          docker system prune -af --filter "until=72h"
+          docker volume prune -af
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push AMD64 image (CUDA 12)
+        id: build-cu129
+        run: |
+          VERSION_ARG=""
+          if [ -n "${SGL_VERSION}" ]; then
+            VERSION_ARG="--build-arg SGL_VERSION=${SGL_VERSION}"
+          fi
+          mapfile -t METADATA_ARGS < /tmp/docker-metadata-cu129.args
+
+          docker buildx build \
+            --target ${{ inputs.docker_target }} \
+            --platform linux/amd64 \
+            --output type=image,name=${{ inputs.image_repo }},push-by-digest=true,name-canonical=true,push=true \
+            -f docker/Dockerfile \
+            --build-arg CUDA_VERSION=12.9.1 \
+            --build-arg BUILD_TYPE=all \
+            --build-arg GRACE_BLACKWELL=0 \
+            --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
+            "${METADATA_ARGS[@]}" \
+            ${VERSION_ARG} \
+            ${{ inputs.extra_build_args }} \
+            --metadata-file /tmp/metadata-cu129.json \
+            --no-cache \
+            .
+
+          DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/metadata-cu129.json'))['containerimage.digest'])")
+          echo "Pushed digest: ${DIGEST}"
+          echo "digest=${DIGEST}" >> $GITHUB_OUTPUT
+
+      - name: Build and push AMD64 image (CUDA 13)
+        id: build-cu130
+        run: |
+          VERSION_ARG=""
+          if [ -n "${SGL_VERSION}" ]; then
+            VERSION_ARG="--build-arg SGL_VERSION=${SGL_VERSION}"
+          fi
+          mapfile -t METADATA_ARGS < /tmp/docker-metadata-cu130.args
+
+          docker buildx build \
+            --target ${{ inputs.docker_target }} \
+            --platform linux/amd64 \
+            --output type=image,name=${{ inputs.image_repo }},push-by-digest=true,name-canonical=true,push=true \
+            -f docker/Dockerfile \
+            --build-arg CUDA_VERSION=13.0.1 \
+            --build-arg BUILD_TYPE=all \
+            --build-arg GRACE_BLACKWELL=0 \
+            --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
+            "${METADATA_ARGS[@]}" \
+            ${VERSION_ARG} \
+            ${{ inputs.extra_build_args }} \
+            --metadata-file /tmp/metadata-cu130.json \
+            --no-cache \
+            .
+
+          DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/metadata-cu130.json'))['containerimage.digest'])")
+          echo "Pushed digest: ${DIGEST}"
+          echo "digest=${DIGEST}" >> $GITHUB_OUTPUT
+
+  build-arm64:
+    if: github.repository == 'sgl-project/sglang'
+    environment: ${{ inputs.use_environment || null }}
+    runs-on: arm-docker-build-node
+    env:
+      TAG_CONFIG: ${{ inputs.tag_config }}
+      SGL_VERSION: ${{ inputs.sgl_version }}
+      IMAGE_REPO: ${{ inputs.image_repo }}
+    outputs:
+      digest-cu129: ${{ steps.build-cu129.outputs.digest }}
+      digest-cu130: ${{ steps.build-cu130.outputs.digest }}
+    steps:
+      - name: Delete huge unnecessary tools folder
+        run: rm -rf /opt/hostedtoolcache
+
+      - name: Cleanup workspace (remove root-owned files from prior runs)
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/* || true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.checkout_ref || github.ref }}
+
+      - name: Compute Docker build metadata args
+        run: |
+          set -euo pipefail
+          BUILD_COMMIT="$(git rev-parse HEAD)"
+          BUILD_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          for CUDA_VARIANT in cu129 cu130; do
+            python3 scripts/ci/utils/docker_build_metadata_args.py \
+              --cuda "${CUDA_VARIANT}" \
+              --tag-config "${TAG_CONFIG}" \
+              --image-repo "${IMAGE_REPO}" \
+              --sgl-version "${SGL_VERSION}" \
+              --build-commit "${BUILD_COMMIT}" \
+              --build-url "${BUILD_URL}" \
+              > "/tmp/docker-metadata-${CUDA_VARIANT}.args"
+          done
+
+      - name: Prune Docker to reclaim disk space
+        run: |
+          docker buildx prune --filter "until=72h" -f
+          docker system prune -af --filter "until=72h"
+          docker volume prune -af
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push ARM64 image (CUDA 12)
+        id: build-cu129
+        run: |
+          VERSION_ARG=""
+          if [ -n "${SGL_VERSION}" ]; then
+            VERSION_ARG="--build-arg SGL_VERSION=${SGL_VERSION}"
+          fi
+          mapfile -t METADATA_ARGS < /tmp/docker-metadata-cu129.args
+
+          docker buildx build \
+            --target ${{ inputs.docker_target }} \
+            --platform linux/arm64 \
+            --output type=image,name=${{ inputs.image_repo }},push-by-digest=true,name-canonical=true,push=true \
+            -f docker/Dockerfile \
+            --build-arg CUDA_VERSION=12.9.1 \
+            --build-arg BUILD_TYPE=all \
+            --build-arg GRACE_BLACKWELL=1 \
+            --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
+            "${METADATA_ARGS[@]}" \
+            ${VERSION_ARG} \
+            ${{ inputs.extra_build_args }} \
+            --metadata-file /tmp/metadata-cu129.json \
+            --no-cache \
+            .
+
+          DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/metadata-cu129.json'))['containerimage.digest'])")
+          echo "Pushed digest: ${DIGEST}"
+          echo "digest=${DIGEST}" >> $GITHUB_OUTPUT
+
+      - name: Build and push ARM64 image (CUDA 13)
+        id: build-cu130
+        run: |
+          VERSION_ARG=""
+          if [ -n "${SGL_VERSION}" ]; then
+            VERSION_ARG="--build-arg SGL_VERSION=${SGL_VERSION}"
+          fi
+          mapfile -t METADATA_ARGS < /tmp/docker-metadata-cu130.args
+
+          docker buildx build \
+            --target ${{ inputs.docker_target }} \
+            --platform linux/arm64 \
+            --output type=image,name=${{ inputs.image_repo }},push-by-digest=true,name-canonical=true,push=true \
+            -f docker/Dockerfile \
+            --build-arg CUDA_VERSION=13.0.1 \
+            --build-arg BUILD_TYPE=all \
+            --build-arg GRACE_BLACKWELL=1 \
+            --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
+            "${METADATA_ARGS[@]}" \
+            ${VERSION_ARG} \
+            ${{ inputs.extra_build_args }} \
+            --metadata-file /tmp/metadata-cu130.json \
+            --no-cache \
+            .
+
+          DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/metadata-cu130.json'))['containerimage.digest'])")
+          echo "Pushed digest: ${DIGEST}"
+          echo "digest=${DIGEST}" >> $GITHUB_OUTPUT
+
+  create-manifests:
+    runs-on: ubuntu-latest
+    needs: [build-x86, build-arm64]
+    if: github.repository == 'sgl-project/sglang'
+    environment: ${{ inputs.use_environment || null }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Create multi-arch manifests
+        env:
+          TAG_CONFIG: ${{ inputs.tag_config }}
+          SGL_VERSION: ${{ inputs.sgl_version }}
+          IMAGE_REPO: ${{ inputs.image_repo }}
+          X86_CU129: ${{ needs.build-x86.outputs.digest-cu129 }}
+          X86_CU130: ${{ needs.build-x86.outputs.digest-cu130 }}
+          ARM64_CU129: ${{ needs.build-arm64.outputs.digest-cu129 }}
+          ARM64_CU130: ${{ needs.build-arm64.outputs.digest-cu130 }}
+          SHORT_SHA: ${{ github.sha }}
+        run: |
+          echo "${TAG_CONFIG}" | jq -c '.[]' | while read -r entry; do
+            CUDA=$(echo "${entry}" | jq -r '.cuda')
+
+            if [ "${CUDA}" = "cu129" ]; then
+              X86_DIGEST="${X86_CU129}"
+              ARM64_DIGEST="${ARM64_CU129}"
+            else
+              X86_DIGEST="${X86_CU130}"
+              ARM64_DIGEST="${ARM64_CU130}"
+            fi
+
+            TAG_ARGS=""
+            for tag in $(echo "${entry}" | jq -r '.tags[]'); do
+              # Substitute template variables
+              tag=$(echo "${tag}" | sed "s/{version}/${SGL_VERSION}/g")
+              tag=$(echo "${tag}" | sed "s/{date}/$(date +%Y%m%d)/g")
+              tag=$(echo "${tag}" | sed "s/{short_sha}/${SHORT_SHA:0:8}/g")
+              TAG_ARGS="${TAG_ARGS} -t ${IMAGE_REPO}:${tag}"
+            done
+
+            docker buildx imagetools create \
+              ${TAG_ARGS} \
+              ${IMAGE_REPO}@${X86_DIGEST} \
+              ${IMAGE_REPO}@${ARM64_DIGEST}
+
+            echo "Published:${TAG_ARGS}"
+          done

.github/workflows/_docker-cleanup-nightly.yml

@@ -0,0 +1,78 @@
+name: Cleanup Old Nightly Docker Tags
+
+# Reusable workflow: deletes old nightly Docker Hub tags, keeping the most recent ones.
+# Can also be triggered manually to clean up tags on demand.
+
+on:
+  workflow_call:
+    inputs:
+      tag_prefixes:
+        description: 'JSON array of tag prefixes to clean up (e.g. ["nightly-dev", "nightly-dev-cu13"])'
+        required: true
+        type: string
+      keep_count:
+        description: "Number of most recent tags to keep per prefix"
+        required: false
+        type: number
+        default: 14
+      image_repo:
+        description: "Docker Hub repo to clean up (e.g. lmsysorg/sglang-staging for testing)"
+        required: false
+        type: string
+        default: "lmsysorg/sglang"
+  workflow_dispatch:
+    inputs:
+      tag_prefixes:
+        description: 'JSON array of tag prefixes to clean up (e.g. ["nightly-dev", "nightly-dev-cu13"])'
+        required: true
+        type: string
+      keep_count:
+        description: "Number of most recent tags to keep per prefix"
+        required: false
+        type: number
+        default: 14
+      image_repo:
+        description: "Docker Hub repo to clean up (e.g. lmsysorg/sglang-staging for testing)"
+        required: false
+        type: string
+        default: "lmsysorg/sglang"
+
+jobs:
+  cleanup:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup old nightly tags
+        env:
+          TAG_PREFIXES: ${{ inputs.tag_prefixes }}
+          KEEP_COUNT: ${{ inputs.keep_count }}
+          IMAGE_REPO: ${{ inputs.image_repo }}
+        run: |
+          TOKEN=$(curl -s -H "Content-Type: application/json" \
+            -X POST -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' \
+            https://hub.docker.com/v2/users/login/ | jq -r .token)
+
+          TAGS_RESPONSE=$(curl -s -H "Authorization: JWT $TOKEN" \
+            "https://hub.docker.com/v2/repositories/${IMAGE_REPO}/tags/?page_size=100")
+
+          echo "${TAG_PREFIXES}" | jq -r '.[]' | while read -r PREFIX; do
+            echo "--- Checking prefix: ${PREFIX} ---"
+
+            TAGS=$(echo "$TAGS_RESPONSE" | jq -r \
+              --arg prefix "${PREFIX}" \
+              '.results[] | select(.name | test("^\($prefix)-[0-9]")) | "\(.last_updated)|\(.name)"' \
+              | sort -r | cut -d'|' -f2)
+
+            TAG_COUNT=$(echo "$TAGS" | grep -c . || true)
+            if [ "$TAG_COUNT" -gt "$KEEP_COUNT" ]; then
+              echo "Found $TAG_COUNT tags, keeping only the $KEEP_COUNT most recent"
+              TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +$((KEEP_COUNT + 1)))
+              for tag in $TAGS_TO_DELETE; do
+                echo "Deleting tag: $tag"
+                curl -X DELETE -H "Authorization: JWT $TOKEN" \
+                  "https://hub.docker.com/v2/repositories/${IMAGE_REPO}/tags/$tag/"
+              done
+            else
+              echo "Only $TAG_COUNT tags found, no cleanup needed"
+            fi
+          done

.github/workflows/_pr-test-check-changes.yml

@@ -0,0 +1,223 @@
+name: Check Changes
+
+on:
+  workflow_call:
+    inputs:
+      git_ref:
+        type: string
+        default: ''
+      run_all_tests:
+        type: boolean
+        default: false
+      force_continue_on_error:
+        type: boolean
+        default: false
+      pr_test_yml:
+        description: 'Workflow YAML whose stage `run_timeout_minutes` drives partition sizing.'
+        type: string
+        default: '.github/workflows/pr-test.yml'
+    outputs:
+      main_package:
+        value: ${{ jobs.run.outputs.main_package }}
+      sgl_kernel:
+        value: ${{ jobs.run.outputs.sgl_kernel }}
+      jit_kernel:
+        value: ${{ jobs.run.outputs.jit_kernel }}
+      multimodal_gen:
+        value: ${{ jobs.run.outputs.multimodal_gen }}
+      partitions:
+        value: ${{ jobs.run.outputs.partitions }}
+      partition_model_sha:
+        value: ${{ jobs.run.outputs.partition_model_sha }}
+      runs_on_map:
+        value: ${{ jobs.run.outputs.runs_on_map }}
+      enable_retry:
+        value: ${{ jobs.run.outputs.enable_retry }}
+      continue_on_error:
+        value: ${{ jobs.run.outputs.continue_on_error }}
+
+jobs:
+  run:
+    name: check-changes
+    runs-on: ubuntu-latest
+    outputs:
+      main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }}
+      sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }}
+      jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
+      multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
+      partitions: ${{ steps.partitions.outputs.partitions }}
+      partition_model_sha: ${{ steps.partition-model-sha.outputs.sha }}
+      runs_on_map: ${{ steps.runner-map.outputs.runs_on_map }}
+      enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
+      continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Determine run mode
+        id: run-mode
+        run: |
+          # Run all tests for scheduled runs and workflow_call (when ref input is provided)
+          # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.git_ref
+          if [[ "${{ github.event_name }}" == "schedule" || "${{ inputs.run_all_tests }}" == "true" ]]; then
+            echo "run_all_tests=true" >> $GITHUB_OUTPUT
+            echo "Run mode: ALL TESTS (schedule=${{ github.event_name == 'schedule' }}, run_all_tests=${{ inputs.run_all_tests }})"
+          else
+            echo "run_all_tests=false" >> $GITHUB_OUTPUT
+            echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
+          fi
+
+      - name: Detect file changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        if: steps.run-mode.outputs.run_all_tests != 'true'
+        with:
+          filters: |
+            main_package:
+              - ".github/workflows/pr-test.yml"
+              - ".github/workflows/pr-gate.yml"
+              - ".github/actions/**"
+              - "python/pyproject.toml"
+              - "python/sglang/!(multimodal_gen)/**/!(*.md)"
+              - "scripts/ci/cuda/*"
+              - "scripts/ci/utils/*"
+              - "test/**/!(*.md)"
+            multimodal_gen:
+              - ".github/workflows/pr-test.yml"
+              - ".github/workflows/pr-test-multimodal-gen.yml"
+              - "python/pyproject.toml"
+              - "python/sglang/multimodal_gen/**/!(*.md|*.ipynb)"
+              - "python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/**"
+              - "python/sglang/srt/observability/**"
+              - "python/sglang/jit_kernel/**"
+              - "test/registered/jit/diffusion/**"
+              - "test/registered/jit/benchmark/diffusion/**"
+              - "python/sglang/cli/**"
+            jit_kernel:
+              - ".github/workflows/pr-test.yml"
+              - ".github/workflows/pr-test-jit-kernel.yml"
+              - "python/pyproject.toml"
+              - "python/sglang/jit_kernel/**"
+              - "test/registered/jit/**"
+            sgl_kernel:
+              # Intentionally excludes ".github/workflows/pr-test-sgl-kernel.yml" —
+              # see API-side detector below for rationale.
+              - "sgl-kernel/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)"
+
+      - name: Determine full-parallel mode
+        id: parallel-mode
+        run: |
+          # `full=true` lifts the matrix-fanout throttle so each suite's
+          # max_parallel = size. Conditions:
+          #   1. Scheduled cron run.
+          #   2. run_all_tests run (manual full dispatch / release) -- a full
+          #      run should mirror the cron's parallelism, not just its test set.
+          #   3. pull_request event with the `high priority` label.
+          FULL=false
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            FULL=true
+            echo "Scheduled run -> full parallelism"
+          elif [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
+            FULL=true
+            echo "run_all_tests -> full parallelism"
+          elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then
+            FULL=true
+            echo "high priority PR -> full parallelism"
+          fi
+          echo "full=$FULL" >> "$GITHUB_OUTPUT"
+
+      - name: Resolve sglang-ci-stats SHA
+        id: partition-model-sha
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Pin all shards to one immutable commit so dispatch and every
+          # runtime LPT use the same model snapshot. Soft fail -> static.
+          SHA=$(gh api repos/sgl-project/sglang-ci-stats/commits/main --jq '.sha' 2>/dev/null || true)
+          if [[ -n "$SHA" ]]; then
+            echo "Pinned sglang-ci-stats@$SHA"
+            echo "sha=$SHA" >> "$GITHUB_OUTPUT"
+          else
+            echo "::warning::Could not resolve sglang-ci-stats SHA; using in-source est_time"
+            echo "sha=" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Fetch live partition model
+        if: steps.partition-model-sha.outputs.sha != ''
+        run: |
+          # SHA resolved -> require fetch (curl --retry). Soft fallback
+          # would risk cross-shard LPT divergence on transient curl flake.
+          rm -f /tmp/partition-model.json
+          URL="https://raw.githubusercontent.com/sgl-project/sglang-ci-stats/${{ steps.partition-model-sha.outputs.sha }}/model.json"
+          curl --fail --silent --show-error --max-time 15 --retry 3 --retry-delay 2 \
+            "$URL" -o /tmp/partition-model.json
+          echo "Fetched partition-model.json ($(wc -c < /tmp/partition-model.json) bytes)"
+
+      - name: Compute partitions
+        id: partitions
+        run: |
+          # Emit a single JSON output `partitions` keyed by suite name with
+          # {size, arr, max_parallel} fields per suite. Replaces the prior
+          # full/low max-parallel presets; `--full-parallel` keeps the
+          # `high priority` PR / scheduled cron escape hatch.
+          # See scripts/ci/utils/compute_partitions.py.
+          python3 scripts/ci/utils/compute_partitions.py \
+            --full-parallel ${{ steps.parallel-mode.outputs.full }} \
+            --partition-model-file /tmp/partition-model.json \
+            --pr-test-yml ${{ inputs.pr_test_yml }} \
+            >> "$GITHUB_OUTPUT"
+
+      - name: Set B200 runner tag
+        id: set-runner
+        run: |
+          # Use kernel-build runner only when sgl_kernel changes are detected.
+          sgl_kernel="${{ steps.filter.outputs.sgl_kernel }}"
+          if [[ "$sgl_kernel" == "true" ]]; then
+            echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT
+          else
+            echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build runs_on_map (resolves $b200_runner sentinel)
+        id: runner-map
+        run: |
+          python3 scripts/ci/runner_configs.py --map \
+            '${{ steps.set-runner.outputs.b200_runner }}' \
+            >> "$GITHUB_OUTPUT"
+
+      - name: Enable retry for CI
+        id: set-retry
+        run: |
+          echo "enable_retry=true" >> $GITHUB_OUTPUT
+          echo "Retry logic enabled for CI"
+
+      - name: Set continue-on-error for full test runs
+        id: set-continue-on-error
+        run: |
+          if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" || "${{ inputs.force_continue_on_error }}" == "true" ]]; then
+            echo "continue_on_error=true" >> $GITHUB_OUTPUT
+            echo "Full test run or force flag detected, enabling continue-on-error to run all tests"
+          else
+            echo "continue_on_error=false" >> $GITHUB_OUTPUT
+            echo "Filtered run, continue-on-error disabled"
+          fi
+
+      - name: Show filter results in summary (table)
+        run: |
+          {
+            echo "## Change Detection"
+            echo ""
+            echo "| Component         | Changed |"
+            echo "|-------------------|---------|"
+            echo "| main_package      | ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} |"
+            echo "| sgl_kernel        | ${{ steps.filter.outputs.sgl_kernel }} |"
+            echo "| jit_kernel        | ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} |"
+            echo "| multimodal_gen    | ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} |"
+            echo "| b200_runner       | ${{ steps.set-runner.outputs.b200_runner }} |"
+            echo "| enable_retry      | ${{ steps.set-retry.outputs.enable_retry }} |"
+            echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |"
+          } >> $GITHUB_STEP_SUMMARY

.github/workflows/_pr-test-sgl-kernel-build.yml

@@ -0,0 +1,118 @@
+name: PR Test - Build sgl-kernel Wheels
+
+# Reusable workflow that builds the sgl-kernel CUDA wheels for one
+# architecture / runner pool. Called twice from pr-test.yml: once for the
+# x86_64 runner pool, once for arm64. All architecture-specific values
+# (runner label, display name, artifact name suffix) are passed in via
+# inputs.
+
+on:
+  workflow_call:
+    inputs:
+      runs_on:
+        description: 'Runner label (e.g. x64-kernel-build-node, arm-kernel-build-node)'
+        type: string
+        required: true
+      job_display_name:
+        description: 'Display name shown in the Actions UI (e.g. "Build Wheel", "Build Wheel Arm")'
+        type: string
+        default: 'Build Wheel'
+      arch_suffix:
+        description: "Suffix appended to the upload-artifact name. '' for x86, '-aarch64' for arm."
+        type: string
+        default: ''
+      git_ref:
+        type: string
+        default: ''
+      skip_pr_test_health_check:
+        description: 'Forwarded from the caller workflow input of the same name. Not consumed inside this reusable workflow today, but mirrored so the resulting env context matches the inline-job baseline.'
+        type: boolean
+        default: false
+
+# Mirror the caller workflow's top-level env block. GitHub Actions does NOT
+# automatically inherit workflow-level env vars across the workflow_call
+# boundary, so anything pr-test.yml defines must be redeclared here for the
+# reusable jobs to see the same context. Most of these are not read by the
+# wheel-build path, but PR_TEST_BYPASS_MAINTENANCE_ON_MAIN is consumed by the
+# check-maintenance composite action and would otherwise be unset.
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
+  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check && 'true' || 'false' }}
+  FORCE_REBUILD_DEEPEP: '1'
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  USE_VENV: false
+
+jobs:
+  build:
+    runs-on: ${{ inputs.runs_on }}
+    timeout-minutes: 240
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.10"
+            cuda-version: "13.0"
+    name: ${{ inputs.job_display_name }}
+    steps:
+      - name: Cleanup
+        run: |
+          if [ -d "$GITHUB_WORKSPACE" ]; then
+            sudo rm -rf "$GITHUB_WORKSPACE"/* || true
+          else
+            echo "$GITHUB_WORKSPACE does not exist, nothing to clean"
+          fi
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Free Docker disk space
+        run: |
+          set -x
+          # `-f` only (no `-a`): drops dangling <none>:<none> layers from
+          # prior runs but keeps the tagged sgl-kernel-deps:* base images.
+          docker image prune -f
+          # Reap orphan buildx builders from interrupted prior runs. Must
+          # `buildx rm` (not `volume rm`) — the volume is held open by the
+          # buildkit helper container; removing the builder frees both.
+          # `sgl-kernel-builder` and `default` are skipped on purpose.
+          for b in $(docker buildx ls --format '{{.Name}}' | grep -E '^builder-' || true); do
+            echo "Removing orphan buildx builder: $b"
+            docker buildx rm "$b" || true
+          done
+          # Trim the persistent local buildx cache (--cache-to mode=max grows
+          # unbounded). Cache-hit imports rewrite the blob, so mtime tracks
+          # "last carried forward by a build" — 14d is well past the freshest
+          # CI cycle.
+          if [ -d "$HOME/.cache/sgl-kernel/buildx/blobs" ]; then
+            find "$HOME/.cache/sgl-kernel/buildx/blobs" -type f -mtime +14 -delete
+          fi
+          df -h /
+
+      - name: Build wheel for Python ${{ matrix.python-version }} and CUDA ${{ matrix.cuda-version }}
+        run: |
+          cd sgl-kernel
+          ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}"
+        env:
+          USE_CCACHE: 1
+
+      - name: Verify wheel artifacts
+        run: |
+          ls -alh sgl-kernel/dist
+          ls -alh sgl-kernel/dist/*.whl
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}${{ inputs.arch_suffix }}
+          path: sgl-kernel/dist/*
+          if-no-files-found: error

.github/workflows/_pr-test-stage.yml

@@ -0,0 +1,184 @@
+name: PR Test Stage
+# Reusable workflow for one CUDA test stage. Caller pr-test.yml forwards
+# `check_changes` = toJson(needs.check-changes.outputs) and
+# `caller_inputs` = toJson(inputs) as bundles. `partitions` is forwarded
+# separately to keep matrix expressions single-fromJson — its value is itself
+# a JSON string, so reading it via `check_changes` would need double fromJson,
+# and matrix can't use step-resolved values. base-a-test-cpu stays inline in
+# pr-test.yml (bespoke uv pip / protoc / rust-cache install).
+
+on:
+  workflow_call:
+    inputs:
+      self_name:
+        description: 'Caller job key; used for partitions[suite] lookup.'
+        type: string
+        required: true
+      runner_config:
+        description: 'Key in scripts/ci/runner_configs.yml. Resolves install script, artifact version, install timeout, runs-on label, and rdma_devices.'
+        type: string
+        required: true
+      check_changes:
+        description: 'toJson(needs.check-changes.outputs). Read via fromJson(...).main_package / sgl_kernel / continue_on_error / runs_on_map etc.'
+        type: string
+        required: true
+      caller_inputs:
+        description: 'toJson(inputs) from pr-test.yml. Read via fromJson(...).git_ref / skip_pr_test_health_check / test_parallel_dispatch.'
+        type: string
+        required: true
+      partitions:
+        description: 'check-changes.outputs.partitions raw — kept separate to avoid double-fromJson in matrix expressions.'
+        type: string
+        required: true
+      run_timeout_minutes:
+        description: 'timeout-minutes for the Run test step. Required so compute_partitions.py can read it from pr-test.yml without a duplicated default constant.'
+        type: string
+        required: true
+      timeout_per_file:
+        description: 'run_suite.py --timeout-per-file value (empty = unset).'
+        type: string
+        default: ''
+      warmup_deep_gemm_models:
+        description: 'Space-separated `model:gpus` list; empty skips DeepGEMM warmup.'
+        type: string
+        default: ''
+      warmup_server_models:
+        description: 'Space-separated `model:gpus` list; empty skips server-CUDA-graph warmup.'
+        type: string
+        default: ''
+      warmup_timeout_minutes:
+        description: 'timeout-minutes for each warmup step. Bump for stages with large cold-cache model lists.'
+        type: string
+        default: '25'
+      extra_pytest_path:
+        description: 'pytest path to run after the suite (b200 FA4 jit_kernel tests). Empty skips.'
+        type: string
+        default: ''
+
+# Mirror pr-test.yml top-level env. Reusable workflows do NOT inherit caller's
+# workflow-level env across the workflow_call boundary, so anything pr-test.yml
+# defines must be redeclared here for the called job to see the same context.
+env:
+  SGLANG_IS_IN_CI: true
+  # base-a runs the fwd-occupancy sanity kit; async-assert probes add GPU work
+  # and skew the occupancy measurement, so keep them off in that stage.
+  SGLANG_ENABLE_ASYNC_ASSERT: ${{ startsWith(inputs.self_name, 'base-a') && 'false' || 'true' }}
+  SGLANG_CUDA_COREDUMP: "1"
+  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }}
+  FORCE_REBUILD_DEEPEP: '1'
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  USE_VENV: false
+
+jobs:
+  run:
+    name: ${{ inputs.self_name }} (${{ matrix.partition }})
+    # Runs on schedule / parallel-dispatch / non-failed PR with main_package or sgl_kernel changes.
+    if: |
+      always() &&
+      ((github.event_name == 'schedule' || fromJson(inputs.caller_inputs).test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
+      (fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true')
+    # runs-on resolved from runs_on_map; check-changes already substituted
+    # $b200_runner (see runner_configs.py --map). rdma_devices is exported
+    # below in a setup step via $GITHUB_ENV.
+    runs-on: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }}
+    timeout-minutes: 240
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ fromJson(inputs.partitions)[inputs.self_name].max_parallel }}
+      matrix:
+        partition: ${{ fromJson(inputs.partitions)[inputs.self_name].arr }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ fromJson(inputs.caller_inputs).git_ref || github.sha }}
+
+      - name: Resolve runner_config
+        id: rc
+        run: python3 scripts/ci/runner_configs.py '${{ inputs.runner_config }}' >> "$GITHUB_OUTPUT"
+
+      - name: Export rdma_devices to job env
+        run: echo "SGLANG_CI_RDMA_ALL_DEVICES=${{ steps.rc.outputs.rdma_devices || '' }}" >> "$GITHUB_ENV"
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts (v4)
+        if: ${{ fromJson(inputs.check_changes).sgl_kernel == 'true' && steps.rc.outputs.artifact_version == 'v4' }}
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Download artifacts (v6)
+        if: ${{ fromJson(inputs.check_changes).sgl_kernel == 'true' && steps.rc.outputs.artifact_version == 'v6' }}
+        uses: actions/download-artifact@v6
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: ${{ fromJson(steps.rc.outputs.install_timeout) }}
+        env:
+          GRACE_BLACKWELL: ${{ steps.rc.outputs.grace_blackwell || '0' }}
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{ fromJson(inputs.check_changes).sgl_kernel }} bash ${{ steps.rc.outputs.install }}
+
+      - name: Warmup DeepGEMM JIT Compilation
+        if: inputs.warmup_deep_gemm_models != ''
+        timeout-minutes: ${{ fromJson(inputs.warmup_timeout_minutes) }}
+        run: |
+          # Activate venv if available (GITHUB_ENV may have failed to propagate)
+          [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
+          [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
+          python3 scripts/ci/cuda/warmup_deep_gemm.py ${{ inputs.warmup_deep_gemm_models }}
+
+      - name: Warmup Server CUDA Graphs
+        if: inputs.warmup_server_models != ''
+        timeout-minutes: ${{ fromJson(inputs.warmup_timeout_minutes) }}
+        run: |
+          [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate"
+          [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
+          python3 scripts/ci/cuda/warmup_server.py ${{ inputs.warmup_server_models }}
+
+      - name: Fetch live partition model
+        if: fromJson(inputs.check_changes).partition_model_sha != ''
+        run: |
+          # SHA resolved by check-changes -> require fetch (curl --retry)
+          # so all shards stay on the same snapshot.
+          rm -f /tmp/partition-model.json
+          URL="https://raw.githubusercontent.com/sgl-project/sglang-ci-stats/${{ fromJson(inputs.check_changes).partition_model_sha }}/model.json"
+          curl --fail --silent --show-error --max-time 15 --retry 3 --retry-delay 2 \
+            "$URL" -o /tmp/partition-model.json
+
+      - name: Run test
+        timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
+        env:
+          CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }}
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite ${{ inputs.self_name }} \
+            --auto-partition-id ${{ matrix.partition }} \
+            --auto-partition-size ${{ fromJson(inputs.partitions)[inputs.self_name].size }} \
+            --partition-model-file /tmp/partition-model.json \
+            ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
+            $CONTINUE_ON_ERROR_FLAG
+
+      - name: Run extra pytest
+        if: inputs.extra_pytest_path != ''
+        timeout-minutes: 10
+        run: python3 -m pytest -q ${{ inputs.extra_pytest_path }}
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+          tracker-issue: "26340"
+          bot-token: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+
+      - name: Cleanup venv
+        if: always()
+        run: bash scripts/ci/cuda/ci_cleanup_venv.sh

.github/workflows/amd-aiter-scout.yml

@@ -0,0 +1,170 @@
+name: AMD AITER Scout
+
+on:
+  schedule:
+    - cron: '0 20 * * 1'   # Monday 20:00 UTC
+    # - cron: '0 20 * * 4'   # Thursday 20:00 UTC (temporarily disabled)
+  workflow_dispatch:
+    inputs:
+      aiter_ref:
+        description: 'AITER git ref (branch, tag, or SHA). Default: main (latest commit)'
+        required: false
+        type: string
+        default: 'main'
+      job_filter:
+        description: 'Comma-separated workflows to run: nightly-amd, nightly-amd-rocm720, pr-test-amd, pr-test-amd-rocm720. Default: all'
+        required: false
+        type: string
+        default: 'all'
+      continue_on_error:
+        description: 'Continue running other workflows even if one fails'
+        required: false
+        type: boolean
+        default: true
+
+permissions:
+  actions: write
+  contents: read
+  issues: read
+  pull-requests: read
+
+concurrency:
+  group: amd-aiter-scout-${{ github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  resolve-aiter:
+    runs-on: ubuntu-latest
+    outputs:
+      aiter_sha: ${{ steps.resolve.outputs.sha }}
+      run_nightly_amd: ${{ steps.parse.outputs.run_nightly_amd }}
+      run_nightly_amd_rocm720: ${{ steps.parse.outputs.run_nightly_amd_rocm720 }}
+      run_pr_test_amd: ${{ steps.parse.outputs.run_pr_test_amd }}
+      run_pr_test_amd_rocm720: ${{ steps.parse.outputs.run_pr_test_amd_rocm720 }}
+    steps:
+      - name: Resolve AITER commit
+        id: resolve
+        run: |
+          REF="${{ inputs.aiter_ref || 'main' }}"
+          echo "Resolving AITER ref: ${REF}"
+
+          SHA=$(git ls-remote https://github.com/ROCm/aiter.git "refs/heads/${REF}" | head -1 | cut -f1)
+          if [ -z "$SHA" ]; then
+            SHA=$(git ls-remote https://github.com/ROCm/aiter.git "refs/tags/${REF}" | head -1 | cut -f1)
+          fi
+          if [ -z "$SHA" ]; then
+            SHA=$(git ls-remote https://github.com/ROCm/aiter.git "${REF}" | head -1 | cut -f1)
+          fi
+          if [ -z "$SHA" ]; then
+            SHA="${REF}"
+          fi
+
+          echo "sha=${SHA}" >> $GITHUB_OUTPUT
+          echo "### AITER Ref Resolution" >> $GITHUB_STEP_SUMMARY
+          echo "- **Requested ref:** \`${REF}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- **Resolved SHA:** \`${SHA}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- **AITER commit:** https://github.com/ROCm/aiter/commit/${SHA}" >> $GITHUB_STEP_SUMMARY
+
+      - name: Parse job filter
+        id: parse
+        run: |
+          FILTER="${{ inputs.job_filter || 'all' }}"
+          echo "Job filter: ${FILTER}"
+
+          if [[ "$FILTER" == "all" ]]; then
+            echo "run_nightly_amd=true" >> $GITHUB_OUTPUT
+            echo "run_nightly_amd_rocm720=true" >> $GITHUB_OUTPUT
+            echo "run_pr_test_amd=true" >> $GITHUB_OUTPUT
+            echo "run_pr_test_amd_rocm720=true" >> $GITHUB_OUTPUT
+          else
+            # Wrap with commas for exact substring matching (avoids "nightly-amd" matching "nightly-amd-rocm720")
+            PADDED=",${FILTER// /},"
+            echo "run_nightly_amd=$(echo "$PADDED" | grep -q ',nightly-amd,' && echo true || echo false)" >> $GITHUB_OUTPUT
+            echo "run_nightly_amd_rocm720=$(echo "$PADDED" | grep -q ',nightly-amd-rocm720,' && echo true || echo false)" >> $GITHUB_OUTPUT
+            echo "run_pr_test_amd=$(echo "$PADDED" | grep -q ',pr-test-amd,' && echo true || echo false)" >> $GITHUB_OUTPUT
+            echo "run_pr_test_amd_rocm720=$(echo "$PADDED" | grep -q ',pr-test-amd-rocm720,' && echo true || echo false)" >> $GITHUB_OUTPUT
+          fi
+
+          echo "### Job Filter" >> $GITHUB_STEP_SUMMARY
+          echo "- **Filter:** \`${FILTER}\`" >> $GITHUB_STEP_SUMMARY
+
+  call-nightly-amd:
+    if: needs.resolve-aiter.outputs.run_nightly_amd == 'true'
+    needs: resolve-aiter
+    uses: ./.github/workflows/nightly-test-amd.yml
+    secrets: inherit
+    with:
+      ref: amd/aiter-ci
+      aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
+      job_filter: 'all'
+      continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
+
+  call-nightly-amd-rocm720:
+    if: needs.resolve-aiter.outputs.run_nightly_amd_rocm720 == 'true'
+    needs: resolve-aiter
+    uses: ./.github/workflows/nightly-test-amd-rocm720.yml
+    secrets: inherit
+    with:
+      ref: amd/aiter-ci
+      aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
+      job_filter: 'all'
+      continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
+
+  call-pr-test-amd:
+    if: needs.resolve-aiter.outputs.run_pr_test_amd == 'true'
+    needs: resolve-aiter
+    uses: ./.github/workflows/pr-test-amd.yml
+    secrets: inherit
+    with:
+      ref: amd/aiter-ci
+      run_all_tests: true
+      aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
+      continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
+
+  call-pr-test-amd-rocm720:
+    if: needs.resolve-aiter.outputs.run_pr_test_amd_rocm720 == 'true'
+    needs: resolve-aiter
+    uses: ./.github/workflows/pr-test-amd-rocm720.yml
+    secrets: inherit
+    with:
+      ref: amd/aiter-ci
+      run_all_tests: true
+      aiter_ref: ${{ needs.resolve-aiter.outputs.aiter_sha }}
+      continue_on_error: ${{ inputs.continue_on_error == '' && true || inputs.continue_on_error }}
+
+  check-all-jobs:
+    if: always()
+    needs:
+      - resolve-aiter
+      - call-nightly-amd
+      - call-nightly-amd-rocm720
+      - call-pr-test-amd
+      - call-pr-test-amd-rocm720
+    runs-on: ubuntu-latest
+    steps:
+      - name: Summary
+        run: |
+          echo "## AMD AITER Scout Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- **SGLang ref:** \`amd/aiter-ci\`" >> $GITHUB_STEP_SUMMARY
+          echo "- **AITER SHA:** \`${{ needs.resolve-aiter.outputs.aiter_sha }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- **AITER commit:** https://github.com/ROCm/aiter/commit/${{ needs.resolve-aiter.outputs.aiter_sha }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Workflow | Result |" >> $GITHUB_STEP_SUMMARY
+          echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Nightly AMD (AITER Latest) | \`${{ needs.call-nightly-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| Nightly AMD ROCm 7.2 | \`${{ needs.call-nightly-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| PR Test AMD (AITER Latest) | \`${{ needs.call-pr-test-amd.result }}\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| PR Test AMD ROCm 7.2 | \`${{ needs.call-pr-test-amd-rocm720.result }}\` |" >> $GITHUB_STEP_SUMMARY
+
+      - name: Check if any job failed
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            echo "One or more workflows failed"
+            exit 1
+          fi
+          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more workflows were cancelled"
+            exit 1
+          fi
+          echo "All workflows passed"

.github/workflows/amd-ci-job-monitor.yml

@@ -0,0 +1,352 @@
+name: AMD CI Job Monitor
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily at midnight UTC
+  pull_request:
+    paths:
+      - '.github/workflows/amd-ci-job-monitor.yml'
+      - 'scripts/ci/utils/query_job_status.py'
+  workflow_dispatch:
+    inputs:
+      hours:
+        description: 'Time window in hours'
+        required: false
+        default: '24'
+        type: string
+      job_filter:
+        description: 'Job name filter (leave empty for all AMD jobs)'
+        required: false
+        type: string
+
+# Bound the API cost when the same ref pushes repeatedly. See note in
+# runner-utilization.yml for the original incident this guards against.
+concurrency:
+  group: amd-ci-job-monitor-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  fetch-actions-data:
+    name: Fetch Actions Snapshot
+    # Skip fork PRs entirely, and require the `run-ci` label on same-repo PRs.
+    # schedule and workflow_dispatch always run.
+    if: >-
+      github.event_name != 'pull_request' ||
+      (github.event.pull_request.head.repo.full_name == github.repository &&
+       contains(github.event.pull_request.labels.*.name, 'run-ci'))
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Select workflows for snapshot
+        id: select-workflows
+        run: |
+          if [[ -n "${{ inputs.job_filter }}" ]]; then
+            echo "workflows=pr-test-amd.yml" >> "$GITHUB_OUTPUT"
+          else
+            echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Fetch Actions data snapshot
+        timeout-minutes: 30
+        run: |
+          # PR-trigger is just a script smoke check; scan a 20-minute window
+          # (0.34h) to bound API cost. schedule / dispatch run the full report.
+          python scripts/ci/utils/query_job_status.py \
+            --repo ${{ github.repository }} \
+            --workflow "${{ steps.select-workflows.outputs.workflows }}" \
+            --hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \
+            --dump-data-file actions-job-snapshot.json
+
+      - name: Upload Actions data snapshot
+        uses: actions/upload-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: actions-job-snapshot.json
+          if-no-files-found: error
+
+  # Single job filter mode
+  custom-report:
+    name: Custom Job Report
+    if: ${{ inputs.job_filter }}
+    needs: fetch-actions-data
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Custom Job Report
+        timeout-minutes: 30
+        run: |
+          python scripts/ci/utils/query_job_status.py \
+            --repo ${{ github.repository }} \
+            --job "${{ inputs.job_filter }}" \
+            --workflow "pr-test-amd.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # Parse workflow files to get job names dynamically
+  parse-workflows:
+    name: Parse Workflow Jobs
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    outputs:
+      pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
+      nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }}
+      pr_rocm720_jobs: ${{ steps.parse.outputs.pr_rocm720_jobs }}
+      nightly_rocm720_jobs: ${{ steps.parse.outputs.nightly_rocm720_jobs }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Parse workflow files
+        id: parse
+        run: |
+          # Parse pr-test-amd.yml and extract job names (exclude utility jobs)
+          # Excluded: call-gate, check-changes, pr-test-amd-finish, cancel, check-all-jobs
+          pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
+            grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \
+            jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
+          echo "PR jobs: $pr_jobs"
+
+          # Parse nightly-test-amd.yml and extract job names (exclude utility jobs)
+          # Excluded: check-all-jobs
+          nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \
+            grep -v -E '^(check-all-jobs)$' | \
+            jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT
+          echo "Nightly jobs: $nightly_jobs"
+
+          # Parse pr-test-amd-rocm720.yml (exclude utility jobs)
+          # Excluded: call-gate, check-changes, pr-test-amd-finish, cancel, check-all-jobs
+          pr_rocm720_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd-rocm720.yml | \
+            grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \
+            jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "pr_rocm720_jobs=$pr_rocm720_jobs" >> $GITHUB_OUTPUT
+          echo "PR ROCm 7.2 jobs: $pr_rocm720_jobs"
+
+          # Parse nightly-test-amd-rocm720.yml (exclude utility jobs)
+          # Excluded: check-all-jobs
+          nightly_rocm720_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd-rocm720.yml | \
+            grep -v -E '^(check-all-jobs)$' | \
+            jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "nightly_rocm720_jobs=$nightly_rocm720_jobs" >> $GITHUB_OUTPUT
+          echo "Nightly ROCm 7.2 jobs: $nightly_rocm720_jobs"
+
+  # PR CI reports using dynamic matrix
+  pr-ci-reports:
+    name: PR - ${{ matrix.job_name }}
+    needs: [parse-workflows, fetch-actions-data]
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_jobs) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Report
+        timeout-minutes: 15
+        run: |
+          python scripts/ci/utils/query_job_status.py \
+            --repo ${{ github.repository }} \
+            --job "${{ matrix.job_name }}" \
+            --workflow "pr-test-amd.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # Nightly AMD test reports using dynamic matrix
+  nightly-reports:
+    name: Nightly - ${{ matrix.job_name }}
+    needs: [parse-workflows, fetch-actions-data]
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_jobs) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Nightly Report
+        timeout-minutes: 15
+        run: |
+          python scripts/ci/utils/query_job_status.py \
+            --repo ${{ github.repository }} \
+            --job "${{ matrix.job_name }}" \
+            --workflow "nightly-test-amd.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # PR ROCm 7.2 CI reports using dynamic matrix
+  pr-rocm720-ci-reports:
+    name: PR ROCm720 - ${{ matrix.job_name }}
+    needs: [parse-workflows, fetch-actions-data]
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_rocm720_jobs) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate PR ROCm 7.2 Report
+        timeout-minutes: 15
+        run: |
+          python scripts/ci/utils/query_job_status.py \
+            --repo ${{ github.repository }} \
+            --job "${{ matrix.job_name }}" \
+            --workflow "pr-test-amd-rocm720.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # Nightly ROCm 7.2 reports using dynamic matrix
+  nightly-rocm720-reports:
+    name: Nightly ROCm720 - ${{ matrix.job_name }}
+    needs: [parse-workflows, fetch-actions-data]
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_rocm720_jobs) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Nightly ROCm 7.2 Report
+        timeout-minutes: 15
+        run: |
+          python scripts/ci/utils/query_job_status.py \
+            --repo ${{ github.repository }} \
+            --job "${{ matrix.job_name }}" \
+            --workflow "nightly-test-amd-rocm720.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # Runner fleet report - cross-workflow runner analytics in a single pass
+  runner-fleet-report:
+    name: Runner Fleet Report
+    if: ${{ !inputs.job_filter }}
+    needs: fetch-actions-data
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Runner Fleet Report
+        timeout-minutes: 30
+        run: |
+          python scripts/ci/utils/query_job_status.py \
+            --repo ${{ github.repository }} \
+            --runner-report \
+            --workflow "pr-test-amd.yml,nightly-test-amd.yml,pr-test-amd-rocm720.yml,nightly-test-amd-rocm720.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary

.github/workflows/auto-tune.yml

@@ -0,0 +1,10 @@
+name: Auto tune
+
+on:
+  workflow_dispatch:
+
+jobs:
+  auto-tune-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4

.github/workflows/bot-bump-docs-version.yml

@@ -0,0 +1,88 @@
+name: Bot Bump Docs Version
+#
+# Bumps the "install from source" release-branch version pinned in the docs
+# (the `git clone -b v<version> ...sglang.git` line in
+# docs_new/docs/get-started/install.mdx "Method 2: From source" and
+# docs_new/docs/hardware-platforms/amd_gpu.mdx) whenever a release tag is
+# pushed, and opens a PR with the change.
+#
+# Triggers mirror release-docker.yml: a pushed `v*` tag (the release), or a
+# manual run with an explicit version.
+on:
+  push:
+    tags:
+      - "v[0-9]+.*"
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to set (without v prefix, e.g., 0.5.13)"
+        required: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  resolve-version:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - name: Get version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            VERSION="${GITHUB_REF_NAME#v}"
+          fi
+          if [ -z "$VERSION" ] || ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::Invalid version: $VERSION (expected: X.Y.Z)"
+            exit 1
+          fi
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+
+  bump-docs-version:
+    needs: resolve-version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # Branch off main so the PR targets the docs on the default branch,
+          # not the (detached) tag commit that triggered this run.
+          ref: main
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Python dependencies
+        run: |
+          pip install tomli
+
+      - name: Configure Git and branch
+        run: |
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+          # github.run_id is unique per run repo-wide; run_attempt disambiguates re-runs.
+          BRANCH_NAME="bot/bump-docs-version-${{ needs.resolve-version.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}"
+          git checkout -b "$BRANCH_NAME"
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+
+      - name: Run docs version bump script
+        run: |
+          python scripts/release/bump_docs_install_version.py "${{ needs.resolve-version.outputs.version }}"
+
+      - name: Commit and create PR
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+        run: |
+          if git diff --quiet; then
+            echo "Docs already pin this version; no PR needed."
+            exit 0
+          fi
+          bash scripts/release/commit_and_pr.sh "docs install" "${{ needs.resolve-version.outputs.version }}" "$BRANCH_NAME"

.github/workflows/bot-bump-flashinfer-version.yml

@@ -0,0 +1,50 @@
+name: Bot Bump Flashinfer Version
+
+on:
+  workflow_dispatch:
+    inputs:
+      new_version:
+        description: 'New flashinfer version (e.g., 0.6.4)'
+        required: true
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  bump-flashinfer-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Python dependencies
+        run: |
+            pip install tomli
+
+      - name: Configure Git and branch
+        run: |
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+          RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4)
+          BRANCH_NAME="bot/bump-flashinfer-version-${{ github.event.inputs.new_version }}-${RANDOM_SUFFIX}"
+          git checkout -b "$BRANCH_NAME"
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+
+      - name: Run flashinfer version bump script
+        run: |
+          python scripts/release/bump_flashinfer_version.py "${{ github.event.inputs.new_version }}"
+
+      - name: Commit and create PR
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+        run: |
+          bash scripts/release/commit_and_pr.sh "flashinfer" "${{ github.event.inputs.new_version }}" "$BRANCH_NAME"

.github/workflows/bot-bump-kernel-version-to-sglang.yml

@@ -0,0 +1,60 @@
+name: Bot Bump Kernel Version to SGLang
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  bump-kernel-version-to-sglang:
+    runs-on: ubuntu-latest
+    outputs:
+      branch_name: ${{ steps.set_output.outputs.branch_name }}
+      needs_sync: ${{ steps.check_sync.outputs.needs_sync }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Python dependencies
+        run: |
+          pip install tomli
+
+      - name: Check if sync is needed
+        id: check_sync
+        run: |
+          python scripts/release/check_kernel_version_to_sglang.py
+
+      - name: Configure Git and branch
+        if: steps.check_sync.outputs.needs_sync == 'true'
+        id: set_output
+        run: |
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+          RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4)
+          KERNEL_VERSION="${{ steps.check_sync.outputs.kernel_version }}"
+          BRANCH_NAME="bot/bump-kernel-version-to-sglang-${KERNEL_VERSION}-${RANDOM_SUFFIX}"
+          git checkout -b "$BRANCH_NAME"
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+          echo "KERNEL_VERSION=$KERNEL_VERSION" >> $GITHUB_ENV
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
+
+      - name: Run kernel version bump script
+        if: steps.check_sync.outputs.needs_sync == 'true'
+        run: |
+          python scripts/release/bump_kernel_version_to_sglang.py
+
+      - name: Commit and create PR
+        if: steps.check_sync.outputs.needs_sync == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+        run: |
+          bash scripts/release/commit_and_pr_kernel_to_sglang.sh "$KERNEL_VERSION" "$BRANCH_NAME"

.github/workflows/bot-bump-kernel-version.yml

@@ -0,0 +1,50 @@
+name: Bot Bump Kernel Version
+
+on:
+  workflow_dispatch:
+    inputs:
+      new_version:
+        description: 'New sgl-kernel version (e.g., 0.3.12)'
+        required: true
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  bump-kernel-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Python dependencies
+        run: |
+          pip install tomli
+
+      - name: Configure Git and branch
+        run: |
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+          RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4)
+          BRANCH_NAME="bot/bump-kernel-version-${{ github.event.inputs.new_version }}-${RANDOM_SUFFIX}"
+          git checkout -b "$BRANCH_NAME"
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+
+      - name: Run kernel version bump script
+        run: |
+          python scripts/release/bump_kernel_version.py "${{ github.event.inputs.new_version }}"
+
+      - name: Commit and create PR
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+        run: |
+          bash scripts/release/commit_and_pr.sh "sgl-kernel" "${{ github.event.inputs.new_version }}" "$BRANCH_NAME"

.github/workflows/bot-bump-sglang-version.yml

@@ -0,0 +1,89 @@
+name: Bot Bump SGLang Version
+
+on:
+  workflow_dispatch:
+    inputs:
+      new_version:
+        description: 'New SGLang version (e.g., 0.5.3 or 0.5.3rc0)'
+        required: true
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  bump-sglang-version:
+    runs-on: ubuntu-latest
+    outputs:
+      branch_name: ${{ steps.set_output.outputs.branch_name }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Python dependencies
+        run: |
+          pip install tomli
+
+      - name: Configure Git and branch
+        id: set_output
+        run: |
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+          RANDOM_SUFFIX=$(echo $RANDOM | md5sum | head -c 4)
+          BRANCH_NAME="bot/bump-sglang-version-${{ github.event.inputs.new_version }}-${RANDOM_SUFFIX}"
+          git checkout -b "$BRANCH_NAME"
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
+
+      - name: Run SGLang version bump script
+        run: |
+          python scripts/release/bump_sglang_version.py "${{ github.event.inputs.new_version }}"
+
+      - name: Commit and create PR
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+        run: |
+          bash scripts/release/commit_and_pr.sh "SGLang" "${{ github.event.inputs.new_version }}" "$BRANCH_NAME"
+
+  run-nightly-tests-nvidia:
+    needs: bump-sglang-version
+    uses: ./.github/workflows/nightly-test-nvidia.yml
+    with:
+      ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
+    secrets: inherit
+
+  run-nightly-tests-amd:
+    needs: bump-sglang-version
+    uses: ./.github/workflows/nightly-test-amd.yml
+    with:
+      ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
+    secrets: inherit
+
+  run-nightly-tests-npu:
+    needs: bump-sglang-version
+    uses: ./.github/workflows/nightly-test-npu.yml
+    with:
+      ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
+    secrets: inherit
+
+  run-pr-tests-xeon:
+    needs: bump-sglang-version
+    uses: ./.github/workflows/pr-test-xeon.yml
+    with:
+      ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
+    secrets: inherit
+
+  run-pr-tests-xpu:
+    needs: bump-sglang-version
+    uses: ./.github/workflows/pr-test-xpu.yml
+    with:
+      ref: ${{ needs.bump-sglang-version.outputs.branch_name }}
+    secrets: inherit

.github/workflows/bot-cherry-pick.yml

@@ -0,0 +1,273 @@
+name: Bot Cherry Pick to Release Branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'PR number to cherry-pick (preferred). Provide either pr_number OR commit_sha, not both.'
+        required: false
+        type: string
+      commit_sha:
+        description: 'Commit SHA to cherry-pick (alternative to pr_number; full or short hash).'
+        required: false
+        type: string
+      target_branch:
+        description: 'Target release branch (e.g., release/v0.5.7)'
+        required: true
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  cherry-pick:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Validate inputs
+        env:
+          TARGET_BRANCH: ${{ github.event.inputs.target_branch }}
+          PR_NUMBER: ${{ github.event.inputs.pr_number }}
+          COMMIT_SHA_INPUT: ${{ github.event.inputs.commit_sha }}
+        run: |
+          if [[ ! "$TARGET_BRANCH" =~ ^release/v[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then
+            echo "::error::Target branch must match pattern 'release/vX.Y' or 'release/vX.Y.Z' (e.g., release/v0.5.7)"
+            exit 1
+          fi
+
+          if [[ -z "$PR_NUMBER" && -z "$COMMIT_SHA_INPUT" ]]; then
+            echo "::error::Provide either 'pr_number' or 'commit_sha'."
+            exit 1
+          fi
+          if [[ -n "$PR_NUMBER" && -n "$COMMIT_SHA_INPUT" ]]; then
+            echo "::error::Provide exactly one of 'pr_number' or 'commit_sha', not both."
+            exit 1
+          fi
+          if [[ -n "$PR_NUMBER" && ! "$PR_NUMBER" =~ ^[0-9]+$ ]]; then
+            echo "::error::pr_number must be a positive integer."
+            exit 1
+          fi
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Configure Git
+        run: |
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+
+      - name: Validate target branch exists
+        env:
+          TARGET_BRANCH: ${{ github.event.inputs.target_branch }}
+        run: |
+          git fetch origin
+          if ! git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null 2>&1; then
+            echo "::error::Target branch '$TARGET_BRANCH' does not exist on remote"
+            exit 1
+          fi
+
+      - name: Resolve PR / commit
+        id: resolve
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.inputs.pr_number }}
+          COMMIT_SHA_INPUT: ${{ github.event.inputs.commit_sha }}
+        run: |
+          set -eo pipefail
+          shopt -s inherit_errexit
+
+          SOURCE_PR=""
+          PR_URL=""
+
+          if [[ -n "$PR_NUMBER" ]]; then
+            if ! PR_JSON=$(gh pr view "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --json state,mergeCommit,title,number,url); then
+              echo "::error::Failed to fetch PR #$PR_NUMBER from $GITHUB_REPOSITORY (see gh output above)."
+              exit 1
+            fi
+            STATE=$(jq -r '.state' <<<"$PR_JSON")
+            if [[ "$STATE" != "MERGED" ]]; then
+              echo "::error::PR #$PR_NUMBER is not merged (state=$STATE). Only merged PRs can be cherry-picked."
+              exit 1
+            fi
+            FULL_SHA=$(jq -r '.mergeCommit.oid // empty' <<<"$PR_JSON")
+            if [[ -z "$FULL_SHA" ]]; then
+              echo "::error::PR #$PR_NUMBER is marked merged but has no associated merge commit."
+              exit 1
+            fi
+            PR_TITLE=$(jq -r '.title // empty' <<<"$PR_JSON")
+            PR_URL=$(jq -r '.url // empty' <<<"$PR_JSON")
+            SOURCE_PR="#$PR_NUMBER"
+          else
+            if ! git cat-file -t "$COMMIT_SHA_INPUT" > /dev/null 2>&1; then
+              echo "::error::Commit SHA '$COMMIT_SHA_INPUT' does not exist in the fetched history"
+              exit 1
+            fi
+            FULL_SHA=$(git rev-parse --verify "${COMMIT_SHA_INPUT}^{commit}")
+            PR_TITLE=$(git log -1 --format="%s" "$FULL_SHA")
+
+            # Look up the merged PR associated with this commit. Distinguish "API call
+            # failed" (abort — could mask an unmerged commit) from "API returned no
+            # merged PR" (fall through to ancestry check).
+            if ! PR_LOOKUP_RAW=$(gh api -X GET "repos/${GITHUB_REPOSITORY}/commits/${FULL_SHA}/pulls"); then
+              echo "::error::Failed to query PRs associated with commit ${FULL_SHA}. Aborting to avoid cherry-picking unverified commits."
+              exit 1
+            fi
+            PR_LOOKUP=$(jq '[.[] | select(.merged_at != null)][0] // empty' <<<"$PR_LOOKUP_RAW")
+            if [[ -n "$PR_LOOKUP" ]]; then
+              ASSOC_PR_NUM=$(jq -r '.number' <<<"$PR_LOOKUP")
+              PR_URL=$(jq -r '.html_url // empty' <<<"$PR_LOOKUP")
+              PR_TITLE=$(jq -r '.title // empty' <<<"$PR_LOOKUP")
+              SOURCE_PR="#$ASSOC_PR_NUM"
+            else
+              # No merged PR found for this commit. Require it to at least be reachable
+              # from the default branch so we don't cherry-pick unmerged work.
+              if ! DEFAULT_BRANCH=$(gh repo view "$GITHUB_REPOSITORY" --json defaultBranchRef --jq '.defaultBranchRef.name'); then
+                echo "::error::Could not determine default branch for ${GITHUB_REPOSITORY}."
+                exit 1
+              fi
+              if [[ -z "$DEFAULT_BRANCH" ]]; then
+                echo "::error::GitHub API returned an empty default branch for ${GITHUB_REPOSITORY}."
+                exit 1
+              fi
+              if ! git rev-parse --verify "origin/${DEFAULT_BRANCH}" >/dev/null 2>&1; then
+                echo "::error::origin/${DEFAULT_BRANCH} is not fetched locally; cannot verify whether ${FULL_SHA} is merged."
+                exit 1
+              fi
+              if ! git merge-base --is-ancestor "$FULL_SHA" "origin/${DEFAULT_BRANCH}"; then
+                echo "::error::Commit '$FULL_SHA' has no associated merged PR and is not reachable from origin/${DEFAULT_BRANCH}. Refusing to cherry-pick unmerged commits."
+                exit 1
+              fi
+            fi
+          fi
+
+          SHORT_SHA=$(git rev-parse --short "$FULL_SHA")
+
+          # Detect merge commit by counting parents. `git show -s --format=%P` prints
+          # parent SHAs separated by spaces; >1 means it's a merge commit.
+          PARENTS_LINE=$(git show -s --format=%P "$FULL_SHA")
+          PARENT_COUNT=$(wc -w <<<"$PARENTS_LINE" | tr -d ' ')
+          IS_MERGE_COMMIT="false"
+          if [[ "$PARENT_COUNT" -gt 1 ]]; then
+            IS_MERGE_COMMIT="true"
+          fi
+
+          echo "full_sha=$FULL_SHA" >> $GITHUB_OUTPUT
+          echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT
+          echo "is_merge_commit=$IS_MERGE_COMMIT" >> $GITHUB_OUTPUT
+          echo "source_pr=$SOURCE_PR" >> $GITHUB_OUTPUT
+          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
+          {
+            echo "pr_title<<EOF"
+            echo "$PR_TITLE"
+            echo "EOF"
+          } >> $GITHUB_OUTPUT
+
+          echo "Cherry-picking ${SHORT_SHA} (${SOURCE_PR:-no associated PR}) - ${PR_TITLE}"
+
+      - name: Cherry-pick commit
+        id: cherry_pick
+        env:
+          TARGET_BRANCH: ${{ github.event.inputs.target_branch }}
+          FULL_SHA: ${{ steps.resolve.outputs.full_sha }}
+          SHORT_SHA: ${{ steps.resolve.outputs.short_sha }}
+          IS_MERGE_COMMIT: ${{ steps.resolve.outputs.is_merge_commit }}
+        run: |
+          RANDOM_SUFFIX=$(head -c 4 /dev/urandom | xxd -p)
+          NEW_BRANCH="cherry-pick/${SHORT_SHA}-to-${TARGET_BRANCH#release/}-${RANDOM_SUFFIX}"
+          git checkout -b "$NEW_BRANCH" "origin/$TARGET_BRANCH"
+          echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
+
+          CP_ARGS=()
+          if [[ "$IS_MERGE_COMMIT" == "true" ]]; then
+            CP_ARGS+=(-m 1)
+          fi
+          if git cherry-pick "${CP_ARGS[@]}" "$FULL_SHA"; then
+            echo "cherry_pick_success=true" >> $GITHUB_OUTPUT
+          else
+            echo "::error::Cherry-pick of ${FULL_SHA} onto ${TARGET_BRANCH} failed due to conflicts. Resolve locally: 'git checkout ${TARGET_BRANCH} && git cherry-pick ${FULL_SHA}', fix conflicts, push a branch, and open the PR by hand."
+            git cherry-pick --abort || echo "::warning::git cherry-pick --abort returned non-zero; ignoring since the job is exiting."
+            echo "cherry_pick_success=false" >> $GITHUB_OUTPUT
+            exit 1
+          fi
+
+      - name: Push branch
+        if: steps.cherry_pick.outputs.cherry_pick_success == 'true'
+        env:
+          NEW_BRANCH: ${{ steps.cherry_pick.outputs.new_branch }}
+        run: |
+          git push origin "$NEW_BRANCH"
+
+      - name: Create Pull Request
+        id: create_pr
+        if: steps.cherry_pick.outputs.cherry_pick_success == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_CHERRY_PICK }}
+          TARGET_BRANCH: ${{ github.event.inputs.target_branch }}
+          SHORT_SHA: ${{ steps.resolve.outputs.short_sha }}
+          FULL_SHA: ${{ steps.resolve.outputs.full_sha }}
+          PR_TITLE: ${{ steps.resolve.outputs.pr_title }}
+          SOURCE_PR: ${{ steps.resolve.outputs.source_pr }}
+          PR_URL: ${{ steps.resolve.outputs.pr_url }}
+          NEW_BRANCH: ${{ steps.cherry_pick.outputs.new_branch }}
+        run: |
+          if [[ -n "$SOURCE_PR" ]]; then
+            CHERRY_PICK_TITLE="[Cherry-pick to ${TARGET_BRANCH}] ${PR_TITLE} (${SOURCE_PR})"
+          else
+            CHERRY_PICK_TITLE="[Cherry-pick to ${TARGET_BRANCH}] ${PR_TITLE} (${SHORT_SHA})"
+          fi
+
+          BODY="Cherry-pick of commit \`${FULL_SHA}\` to \`${TARGET_BRANCH}\`."$'\n\n'
+          if [[ -n "$SOURCE_PR" ]]; then
+            BODY+="**Source PR:** ${SOURCE_PR} (${PR_URL})"$'\n'
+          fi
+          BODY+="**Source commit:** \`${FULL_SHA}\`"$'\n'
+          BODY+="**Original title:** ${PR_TITLE}"$'\n\n'
+          BODY+="---"$'\n'
+          BODY+="*This PR was automatically created by the cherry-pick workflow.*"
+
+          CREATED_PR_URL=$(printf '%s' "$BODY" | gh pr create \
+            --title "$CHERRY_PICK_TITLE" \
+            --base "$TARGET_BRANCH" \
+            --head "$NEW_BRANCH" \
+            --label "cherry-pick" \
+            --body-file -)
+          CREATED_PR_NUMBER="${CREATED_PR_URL##*/}"
+          echo "created_pr_url=$CREATED_PR_URL" >> $GITHUB_OUTPUT
+          echo "created_pr_number=$CREATED_PR_NUMBER" >> $GITHUB_OUTPUT
+          echo "Created cherry-pick PR #${CREATED_PR_NUMBER}: ${CREATED_PR_URL}"
+
+      - name: Summary
+        if: always()
+        env:
+          FULL_SHA: ${{ steps.resolve.outputs.full_sha }}
+          PR_TITLE: ${{ steps.resolve.outputs.pr_title }}
+          SOURCE_PR: ${{ steps.resolve.outputs.source_pr }}
+          TARGET_BRANCH: ${{ github.event.inputs.target_branch }}
+          CHERRY_PICK_SUCCESS: ${{ steps.cherry_pick.outputs.cherry_pick_success }}
+          NEW_BRANCH: ${{ steps.cherry_pick.outputs.new_branch }}
+          CREATED_PR_NUMBER: ${{ steps.create_pr.outputs.created_pr_number }}
+          CREATED_PR_URL: ${{ steps.create_pr.outputs.created_pr_url }}
+          ACTOR: ${{ github.actor }}
+        run: |
+          echo "## Cherry-Pick Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- **Triggered by:** @${ACTOR}" >> $GITHUB_STEP_SUMMARY
+          if [[ -n "$SOURCE_PR" ]]; then
+            echo "- **Source PR:** ${SOURCE_PR}" >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "- **Source commit:** ${FULL_SHA}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Title:** ${PR_TITLE}" >> $GITHUB_STEP_SUMMARY
+          echo "- **Target Branch:** ${TARGET_BRANCH}" >> $GITHUB_STEP_SUMMARY
+          if [[ "$CHERRY_PICK_SUCCESS" == "true" ]]; then
+            echo "- **Status:** Success" >> $GITHUB_STEP_SUMMARY
+            echo "- **PR Branch:** ${NEW_BRANCH}" >> $GITHUB_STEP_SUMMARY
+            if [[ -n "$CREATED_PR_NUMBER" ]]; then
+              echo "- **Cherry-pick PR:** [#${CREATED_PR_NUMBER}](${CREATED_PR_URL})" >> $GITHUB_STEP_SUMMARY
+            fi
+          else
+            echo "- **Status:** Failed" >> $GITHUB_STEP_SUMMARY
+          fi

.github/workflows/cancel-pr-workflows-on-close.yml

@@ -0,0 +1,104 @@
+name: Cancel PR Workflows on Close
+
+# Cancels every unfinished run on a PR's head branch when the PR closes
+# (merged or not). Runs are listed repo-wide by branch, not per workflow:
+# the previous styfle/cancel-workflow-action `workflow_id: all` fetched
+# only the first page (30) of repo workflows, so newer workflows (e.g.
+# pr-test-extra.yml) were never cancelled and kept occupying runners.
+
+on:
+  pull_request_target:
+    types:
+      - closed
+
+permissions:
+  actions: write
+
+jobs:
+  cancel:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel unfinished runs of this PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          HEAD_BRANCH: ${{ github.event.pull_request.head.ref }}
+          HEAD_REPO_ID: ${{ github.event.pull_request.head.repo.id }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [ -z "$HEAD_REPO_ID" ]; then
+            echo "Head repo no longer exists (fork deleted), nothing to match"
+            exit 0
+          fi
+
+          FAILURES=0
+
+          # `branch=` matches run.head_branch, shared with same-named
+          # branches of other forks -- filter by head repo id. Exclude this
+          # run itself: pull_request_target runs also carry the head branch.
+          # The status filter takes one value per request, so sweep every
+          # non-terminal state (pending/waiting/action_required runs would
+          # otherwise start after the PR is closed). Pass 2 catches runs
+          # still materializing during pass 1.
+          for pass in 1 2; do
+            run_ids=""
+            for status in queued in_progress waiting pending requested action_required; do
+              ids=""
+              for attempt in 1 2 3; do
+                if ids=$(gh api -X GET "repos/$REPO/actions/runs" \
+                  --paginate \
+                  -f branch="$HEAD_BRANCH" \
+                  -f status="$status" \
+                  -F per_page=100 \
+                  --jq ".workflow_runs[]
+                        | select(.head_repository.id == $HEAD_REPO_ID and .id != $GITHUB_RUN_ID)
+                        | .id"); then
+                  break
+                fi
+                ids=""
+                if [ "$attempt" -eq 3 ]; then
+                  echo "::error::Listing $status runs failed after 3 attempts"
+                  FAILURES=1
+                else
+                  sleep 5
+                fi
+              done
+              run_ids="$run_ids"$'\n'"$ids"
+            done
+            run_ids=$(echo "$run_ids" | sed '/^$/d' | sort -u)
+
+            if [ -z "$run_ids" ]; then
+              echo "Pass $pass: no unfinished runs found"
+              break
+            fi
+            echo "Pass $pass: cancelling $(echo "$run_ids" | wc -l | tr -d ' ') run(s)"
+
+            for run_id in $run_ids; do
+              echo "Cancelling https://github.com/$REPO/actions/runs/$run_id"
+              if gh run cancel "$run_id" --repo "$REPO" 2>/dev/null; then
+                continue
+              fi
+              # Plain cancel fails for runs stuck behind approval /
+              # deployment protection rules; force-cancel handles those.
+              if gh api -X POST "repos/$REPO/actions/runs/$run_id/force-cancel" >/dev/null 2>&1; then
+                echo "  force-cancelled"
+                continue
+              fi
+              # Finished between listing and cancelling is fine.
+              state=$(gh api "repos/$REPO/actions/runs/$run_id" --jq '.status' 2>/dev/null || echo "unknown")
+              if [ "$state" = "completed" ]; then
+                echo "  already finished, nothing to cancel"
+              else
+                echo "::error::Failed to cancel run $run_id (status: $state)"
+                FAILURES=1
+              fi
+            done
+
+            if [ "$pass" -eq 1 ]; then
+              sleep 20
+            fi
+          done
+
+          exit "$FAILURES"

.github/workflows/cancel-unfinished-pr-tests.yml

@@ -0,0 +1,234 @@
+name: Cancel Unfinished PR Runs
+
+on:
+  workflow_dispatch:
+    inputs:
+      workflows:
+        description: 'Space-separated list of workflow filenames to cancel'
+        required: true
+        type: string
+        default: 'pr-test.yml pr-test-extra.yml'
+      include_high_priority:
+        description: 'Also cancel runs from high-priority PRs'
+        required: false
+        type: boolean
+        default: false
+      include_rerun_test:
+        description: 'Also cancel /rerun-test dispatched runs (rerun-test.yml)'
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  actions: write   # Needed to cancel runs
+  contents: read   # Needed to read repo info
+  pull-requests: read  # needed for gh pr view (labels)
+
+jobs:
+  cancel-unfinished-pr-runs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install GitHub CLI
+        run: sudo apt-get install -y gh jq
+
+      - name: Cancel unfinished PR-associated runs
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          WORKFLOWS: ${{ github.event.inputs.workflows }}
+          INCLUDE_HIGH_PRIORITY: ${{ github.event.inputs.include_high_priority }}
+          INCLUDE_RERUN_TEST: ${{ github.event.inputs.include_rerun_test }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          # Read the space-separated string from the input into a bash array
+          read -r -a WORKFLOW_FILES <<< "${WORKFLOWS}"
+          if [ "$INCLUDE_RERUN_TEST" = "true" ]; then
+            WORKFLOW_FILES+=("rerun-test.yml")
+          fi
+
+          echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${WORKFLOW_FILES[*]}"
+          echo "include_high_priority=$INCLUDE_HIGH_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST"
+          echo ""
+
+          # Decide whether to cancel run_id given a PR-lookup endpoint.
+          #   $1 = run_id
+          #   $2 = gh api path returning a list of PR objects (head=... or commits/<sha>/pulls)
+          #   $3 = short label for log messages
+          maybe_cancel_for_pr() {
+            local run_id="$1"
+            local pr_query="$2"
+            local pr_label="$3"
+
+            local pr_info
+            pr_info=$(gh api -H "Accept: application/vnd.github+json" "$pr_query" \
+              --jq '.[0] | {number, state, merged_at}' 2>/dev/null || true)
+
+            if [ -z "$pr_info" ] || [ "$pr_info" = "null" ]; then
+              echo "  ⚠️  No PR found ($pr_label), skipping"
+              return
+            fi
+
+            local pr_number pr_state
+            pr_number=$(echo "$pr_info" | jq -r '.number // empty')
+            pr_state=$(echo "$pr_info" | jq -r '.state // empty')
+
+            if [ -z "$pr_number" ]; then
+              echo "  ⚠️  PR lookup returned empty number, skipping"
+              return
+            fi
+
+            local pr_url="https://github.com/$REPO/pull/$pr_number"
+            echo "  PR: $pr_url ($pr_state)"
+
+            # Closed PR (merged or not): always cancel, skip label checks.
+            if [ "$pr_state" = "closed" ]; then
+              echo "  🚫 Cancelling (PR closed)..."
+              gh run cancel "$run_id" --repo "$REPO" || echo "  ⚠️  Cancellation failed"
+              return
+            fi
+
+            # Open PR: apply label-based skip rules.
+            local labels
+            labels=$(gh pr view "$pr_number" --repo "$REPO" --json labels \
+              | jq -r '.labels[].name' 2>/dev/null || true)
+
+            if echo "$labels" | grep -Fxq "bypass-maintenance"; then
+              echo "  🛑 Skipping (bypass-maintenance label, never cancelled)"
+              return
+            fi
+
+            if echo "$labels" | grep -Fxq "high priority"; then
+              if [ "$INCLUDE_HIGH_PRIORITY" != "true" ]; then
+                echo "  🛑 Skipping (high priority label)"
+                return
+              fi
+              echo "  ⚠️  High priority PR, but include_high_priority is enabled"
+            fi
+
+            echo "  🚫 Cancelling..."
+            gh run cancel "$run_id" --repo "$REPO" || echo "  ⚠️  Cancellation failed"
+          }
+          export -f maybe_cancel_for_pr
+
+          for workflow_file in "${WORKFLOW_FILES[@]}"; do
+            echo "========================================="
+            echo "Workflow: $workflow_file"
+            echo "========================================="
+
+            # Get all unfinished runs.
+            # Use server-side --status filter: without it, `gh run list --limit 1000`
+            # only sees the most recent 1000 runs by createdAt, which on busy workflows
+            # like pr-test.yml is < 3 days. Old stuck runs would be missed.
+            all_runs=""
+            for status in queued in_progress waiting; do
+              batch=$(gh run list \
+                --repo "$REPO" \
+                --workflow "$workflow_file" \
+                --status "$status" \
+                --json databaseId,status,event,url,createdAt,displayTitle \
+                --limit 1000 \
+              | jq -c '.[]')
+              if [ -n "$batch" ]; then
+                if [ -n "$all_runs" ]; then
+                  all_runs="$all_runs"$'\n'"$batch"
+                else
+                  all_runs="$batch"
+                fi
+              fi
+            done
+
+            if [ -z "$all_runs" ]; then
+              echo "✅ No unfinished runs found"
+              echo ""
+              continue
+            fi
+
+            # Count runs by event type
+            total_runs=$(echo "$all_runs" | wc -l)
+            pr_runs=$(echo "$all_runs" | jq -s '[.[] | select(.event=="pull_request")] | length')
+            dispatch_runs=$(echo "$all_runs" | jq -s '[.[] | select(.event=="workflow_dispatch")] | length')
+            other_runs=$(echo "$all_runs" | jq -s '[.[] | select(.event!="pull_request" and .event!="workflow_dispatch")] | length')
+
+            echo "📊 Summary: $total_runs unfinished ($pr_runs pull_request, $dispatch_runs workflow_dispatch, $other_runs other)"
+            echo ""
+
+            # Other runs: list only, do not cancel.
+            if [ "$other_runs" -gt 0 ]; then
+              echo "--- Other Runs (listed only, not cancelled) ---"
+              echo "$all_runs" | jq -c 'select(.event!="pull_request" and .event!="workflow_dispatch")' | while read -r run; do
+                run_url=$(echo "$run" | jq -r '.url')
+                run_event=$(echo "$run" | jq -r '.event')
+                run_status=$(echo "$run" | jq -r '.status')
+                echo "  • $run_event ($run_status): $run_url"
+              done
+              echo ""
+            fi
+
+            # PR runs: resolve PR via head=owner:branch.
+            if [ "$pr_runs" -gt 0 ]; then
+              echo "--- PR Runs (resolving via head=owner:branch) ---"
+              echo "$all_runs" | jq -c 'select(.event=="pull_request")' | while read -r run; do
+                run_id=$(echo "$run" | jq -r '.databaseId')
+                run_url=$(echo "$run" | jq -r '.url')
+                run_status=$(echo "$run" | jq -r '.status')
+
+                echo ""
+                echo "Run ($run_status): $run_url"
+
+                run_details=$(gh api -H "Accept: application/vnd.github+json" \
+                  "repos/$REPO/actions/runs/$run_id" 2>/dev/null || true)
+                if [ -z "$run_details" ]; then
+                  echo "  ⚠️  Could not fetch run details, skipping"
+                  continue
+                fi
+
+                head_owner=$(echo "$run_details" | jq -r '.head_repository.owner.login // empty')
+                head_branch=$(echo "$run_details" | jq -r '.head_branch // empty')
+                if [ -z "$head_owner" ] || [ -z "$head_branch" ]; then
+                  echo "  ⚠️  Missing head info, skipping"
+                  continue
+                fi
+
+                echo "  Branch: ${head_owner}:${head_branch}"
+                maybe_cancel_for_pr "$run_id" \
+                  "repos/$REPO/pulls?state=all&head=${head_owner}:${head_branch}" \
+                  "head=${head_owner}:${head_branch}"
+              done
+              echo ""
+            fi
+
+            # workflow_dispatch runs (e.g. /rerun-test): resolve PR via pr_head_sha
+            # parsed from run-name (`[rerun-test] <cmd> <sha>`).
+            if [ "$dispatch_runs" -gt 0 ]; then
+              echo "--- Dispatch Runs (resolving via pr_head_sha in display_title) ---"
+              echo "$all_runs" | jq -c 'select(.event=="workflow_dispatch")' | while read -r run; do
+                run_id=$(echo "$run" | jq -r '.databaseId')
+                run_url=$(echo "$run" | jq -r '.url')
+                run_status=$(echo "$run" | jq -r '.status')
+                display_title=$(echo "$run" | jq -r '.displayTitle // empty')
+
+                echo ""
+                echo "Run ($run_status): $run_url"
+                echo "  Title: $display_title"
+
+                # Last whitespace-delimited token if it is a 40-hex SHA.
+                last_token=$(echo "$display_title" | awk '{print $NF}')
+                if ! [[ "$last_token" =~ ^[0-9a-f]{40}$ ]]; then
+                  echo "  ⚠️  No pr_head_sha in title, skipping"
+                  continue
+                fi
+
+                echo "  pr_head_sha: $last_token"
+                maybe_cancel_for_pr "$run_id" \
+                  "repos/$REPO/commits/$last_token/pulls" \
+                  "sha=$last_token"
+              done
+              echo ""
+            fi
+          done
+
+          echo "========================================="
+          echo "✅ Processing complete"
+          echo "========================================="

.github/workflows/ci-auto-bisect.yml

@@ -0,0 +1,66 @@
+name: CI Auto Bisect
+
+on:
+  workflow_run:
+    workflows: ["PR Test Base"]
+    types: [completed]
+    branches: [main]
+  workflow_dispatch: {}
+
+concurrency:
+  group: ci-auto-bisect
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  auto-bisect:
+    # Only run for scheduled pr-test completions (not PR-triggered), or manual dispatch
+    if: >
+      github.repository == 'sgl-project/sglang' && (
+        github.event_name == 'workflow_dispatch' ||
+        (github.event.workflow_run.event == 'schedule' &&
+         github.event.workflow_run.conclusion != 'cancelled')
+      )
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Full history needed for git log between SHAs
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.14'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests anthropic
+
+      - name: Run Auto Bisect
+        id: bisect
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          PYTHONUNBUFFERED: 1
+          PYTHONIOENCODING: utf-8
+        run: |
+          cd scripts/ci_monitor
+          python ci_auto_bisect.py \
+            --github-token $GITHUB_TOKEN \
+            --anthropic-api-key $ANTHROPIC_API_KEY \
+            --output bisect_results.json \
+            --max-failures 10
+
+      - name: Upload Bisect Results
+        if: always() && hashFiles('scripts/ci_monitor/bisect_results.json') != ''
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-auto-bisect-${{ github.run_number }}
+          path: scripts/ci_monitor/bisect_results.json
+          retention-days: 14

.github/workflows/ci-coverage-overview.yml

@@ -0,0 +1,164 @@
+name: CI Coverage Overview
+
+on:
+  schedule:
+    - cron: '0 6 * * *'  # Daily at 6 AM UTC
+  workflow_dispatch:
+    inputs:
+      output_format:
+        description: 'Output format'
+        required: false
+        default: 'markdown'
+        type: choice
+        options:
+          - markdown
+          - json
+
+jobs:
+  summary:
+    name: Summary
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Generate Summary Report
+        run: |
+          python scripts/ci/utils/ci_coverage_report.py --section summary
+
+  by-folder:
+    name: Tests by Folder
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Generate Tests by Folder Report
+        run: |
+          python scripts/ci/utils/ci_coverage_report.py --section by-folder
+
+  by-suite:
+    name: Tests by Suite
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Generate Tests by Suite Report
+        run: |
+          python scripts/ci/utils/ci_coverage_report.py --section by-suite
+
+  unit-test-coverage:
+    name: Unit Test Code Coverage
+    runs-on: 1-gpu-h100
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Rust toolchain + protoc
+        # Editable install of sglang triggers `build_rust` for the bundled
+        # native gRPC extension (rust/sglang-grpc). The 1-gpu-h100 runner
+        # image has no Rust toolchain by default, which is why this job has
+        # been failing on every scheduled run. Use the same helper that
+        # ci_install_dependency.sh uses so the install path matches the
+        # rest of CI.
+        run: |
+          bash scripts/ci/utils/install_rust_protoc.sh
+          echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        timeout-minutes: 10
+        run: |
+          pip install -e "python/[test]"
+
+      - name: Run unit tests with coverage
+        timeout-minutes: 10
+        run: |
+          pytest test/registered/unit/ \
+            --cov --cov-config=.coveragerc \
+            --cov-report=term-missing:skip-covered \
+            --continue-on-collection-errors \
+            -v | tee coverage_output.txt
+
+      - name: Write coverage to summary
+        if: always()
+        run: |
+          echo "## Unit Test Code Coverage" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Commit:** \`${GITHUB_SHA::8}\` | **Branch:** \`${GITHUB_REF_NAME}\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          # Test result line (e.g., "== 42 passed, 1 failed in 23.5s ==")
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          grep -E '^=+.*passed' coverage_output.txt >> $GITHUB_STEP_SUMMARY || true
+          echo "" >> $GITHUB_STEP_SUMMARY
+          # Coverage total, reformatted
+          awk '/^TOTAL / { for(i=1;i<=NF;i++) if($i~/^[0-9]+$/ || $i~/^[0-9]+%$/) a[++n]=$i; if(n>=3) printf "TOTAL   Stmts: %s   Miss: %s   Cover: %s\n", a[1], a[2], a[3] }' coverage_output.txt >> $GITHUB_STEP_SUMMARY || true
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+          # Core modules with coverage < 50%, sorted by uncovered lines (descending)
+          LOW_COV=$(awk '/^python\/.*%/ {
+            for (i=1; i<=NF; i++) {
+              if ($i ~ /^[0-9]+%$/) {
+                pct = $i + 0
+                if (pct >= 1 && pct < 50) {
+                  stmts = $(i-2) + 0
+                  miss = $(i-1) + 0
+                  printf "%d|%s|%d|%d|%d%%\n", miss, $1, stmts, miss, pct
+                }
+                break
+              }
+            }
+          }' coverage_output.txt \
+            | grep -E '/(mem_cache|managers|sampling|parser|observability|function_call|entrypoints|speculative|multimodal|utils)/' \
+            | sort -t'|' -k1 -nr | cut -d'|' -f2- | head -40 || true)
+          if [ -n "$LOW_COV" ]; then
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "<details><summary>Top uncovered core modules (coverage < 50%, sorted by uncovered lines)</summary>" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "| File | Stmts | Miss | Cover |" >> $GITHUB_STEP_SUMMARY
+            echo "|------|-------|------|-------|" >> $GITHUB_STEP_SUMMARY
+            echo "$LOW_COV" | while IFS='|' read -r file stmts miss pct; do
+              echo "| \`$file\` | $stmts | $miss | $pct |" >> $GITHUB_STEP_SUMMARY
+            done
+            echo "</details>" >> $GITHUB_STEP_SUMMARY
+          fi
+
+  json-export:
+    name: JSON Export
+    runs-on: ubuntu-latest
+    if: inputs.output_format == 'json'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Generate JSON Report
+        run: |
+          python scripts/ci/utils/ci_coverage_report.py --output-format json > ci_coverage.json
+
+      - name: Upload JSON artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-coverage-report
+          path: ci_coverage.json

.github/workflows/ci-failure-monitor.yml

@@ -0,0 +1,53 @@
+name: CI Failure Monitor
+
+on:
+  schedule:
+    - cron: '0 */12 * * *' # Every 12 hour
+  workflow_dispatch:
+
+concurrency:
+  group: ci-failure-monitor-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  failure-analysis:
+    if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.14'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+
+      - name: Run Failure Analysis
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GH_PAT_FOR_RUNNER_ADMIN: ${{ secrets.GH_PAT_FOR_RUNNER_ADMIN }}
+          PYTHONUNBUFFERED: 1
+          PYTHONIOENCODING: utf-8
+        run: |
+          cd scripts/ci_monitor
+          python ci_failures_analysis.py \
+            --token $GITHUB_TOKEN \
+            --limit 100 \
+            --output ci_failure_analysis_$(date +%Y%m%d_%H%M%S).json
+
+      - name: Upload Analysis Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-failure-analysis-${{ github.run_number }}
+          path: |
+            scripts/ci_monitor/ci_failure_analysis_*.json
+          retention-days: 7

.github/workflows/ci-model-inventory.yml

@@ -0,0 +1,68 @@
+name: CI Model Inventory
+
+# Builds models-per-stage.json: each NVIDIA (CUDA) CI suite -> the HuggingFace
+# model ids its tests reference, for runner cache pre-warming. Pure static
+# analysis (no GPU, no sglang install), so it stays fresh per push to main.
+# Grab the result with:
+#   gh run download -R sgl-project/sglang --name models-per-stage
+# or read the rendered table in the run's job summary.
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - "test/registered/**"
+      - "test/run_suite.py"
+      - "python/sglang/test/test_utils.py"
+      - "python/sglang/test/lora_utils.py"
+      - "python/sglang/test/ci/**"
+      - "scripts/ci/list_stage_models.py"
+      - "scripts/ci/stage_models_overrides.json"
+      - "scripts/ci/runner_configs.yml"
+      - ".github/workflows/ci-model-inventory.yml"
+  pull_request:
+    # Validate the tool itself end-to-end when it changes. Deliberately NOT
+    # filtered on test/registered/** -- that would run on most PRs; the
+    # published inventory is refreshed by the push-to-main trigger instead.
+    paths:
+      - "scripts/ci/list_stage_models.py"
+      - "scripts/ci/test_list_stage_models.py"
+      - "scripts/ci/stage_models_overrides.json"
+      - ".github/workflows/ci-model-inventory.yml"
+  schedule:
+    # Nightly safety net (09:00 UTC) so the artifact never goes stale even if a
+    # change lands outside the path filter above.
+    - cron: "0 9 * * *"
+
+jobs:
+  build-inventory:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Run extractor unit tests
+        run: python3 -m unittest discover -s scripts/ci -p 'test_list_stage_models.py'
+
+      - name: Generate model inventory
+        run: |
+          python3 scripts/ci/list_stage_models.py \
+            --backend cuda \
+            --commit "${{ github.sha }}" \
+            --output models-per-stage.json \
+            --markdown inventory.md
+
+      - name: Write job summary
+        run: cat inventory.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload inventory artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: models-per-stage
+          path: models-per-stage.json
+          retention-days: 90

.github/workflows/close-inactive-issues.yml

@@ -0,0 +1,96 @@
+name: Close Inactive Issues
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  contents: read
+
+jobs:
+  close-inactive-issues:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check and close inactive issues
+        uses: actions/github-script@v8
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const sixtyDaysAgo = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000);
+
+            const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/');
+            console.log(`Owner: ${owner}, Repo: ${repo}`);
+
+            async function fetchIssues(page = 1) {
+              console.log(`Fetching issues for ${owner}/${repo}, page ${page}`);
+              return await github.rest.issues.listForRepo({
+                owner,
+                repo,
+                state: 'open',
+                sort: 'updated',
+                direction: 'asc',
+                per_page: 100,
+                page: page
+              });
+            }
+
+            async function processIssues() {
+              console.log('Starting to process issues');
+              console.log(`Repository: ${owner}/${repo}`);
+
+              let page = 1;
+              let hasMoreIssues = true;
+              while (hasMoreIssues) {
+                try {
+                  const issues = await fetchIssues(page);
+                  console.log(`Fetched ${issues.data.length} issues on page ${page}`);
+
+                  if (issues.data.length === 0) {
+                    hasMoreIssues = false;
+                    break;
+                  }
+
+                  for (const issue of issues.data) {
+                    // Skip if the issue has 'good first issue' label
+                    if (issue.labels.some(label => label.name === 'good first issue')) {
+                      console.log(`Skipping issue #${issue.number} as it's marked as 'good first issue'`);
+                      continue;
+                    }
+                    if (new Date(issue.updated_at) < sixtyDaysAgo) {
+                      try {
+                        await github.rest.issues.update({
+                          owner,
+                          repo,
+                          issue_number: issue.number,
+                          state: 'closed',
+                          labels: [...issue.labels.map(l => l.name), 'inactive']
+                        });
+                        await github.rest.issues.createComment({
+                          owner,
+                          repo,
+                          issue_number: issue.number,
+                          body: 'This issue has been automatically closed due to inactivity. Please feel free to reopen it if needed.'
+                        });
+                        console.log(`Closed issue #${issue.number} due to inactivity.`);
+                      } catch (error) {
+                        console.error(`Failed to close issue #${issue.number}: ${error.message}`);
+                      }
+                    } else {
+                      console.log(`Issue #${issue.number} is still active. Stopping processing.`);
+                      hasMoreIssues = false;
+                      break;
+                    }
+                  }
+                  page += 1;
+                } catch (error) {
+                  console.error(`Error fetching issues on page ${page}: ${error.message}`);
+                  hasMoreIssues = false;
+                }
+              }
+              console.log('Finished processing issues');
+            }
+
+            await processIssues();

.github/workflows/diffusion-ci-gt-gen-npu.yml

@@ -0,0 +1,214 @@
+name: Diffusion CI Ground Truth Generation (NPU)
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Git ref to checkout'
+        required: false
+        default: ''
+        type: string
+      case_ids:
+        description: 'Specific case IDs to run (space-separated, optional)'
+        required: false
+        default: ''
+        type: string
+      output_name:
+        description: 'Custom local output/artifact folder name. Leave empty to use defaults.'
+        required: false
+        default: ''
+        type: string
+      publish_target_dir:
+        description: 'Remote target directory in sgl-project/ci-data. Leave empty to use diffusion-ci/consistency_gt/sglang_generated/ascend.'
+        required: false
+        default: ''
+        type: string
+
+
+concurrency:
+  group: diffusion-ci-gt-gen-${{ github.ref }}-${{ inputs.output_name || inputs.case_ids || 'default' }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  actions: read
+
+env:
+  SGLANG_IS_IN_CI: true
+  OUTPUT_NAME: ${{ inputs.output_name || 'diffusion-ci-outputs' }}
+  PUBLISH_TARGET_DIR: ${{ inputs.publish_target_dir || 'diffusion-ci/consistency_gt/sglang_generated/ascend' }}
+
+jobs:
+
+  compute-diffusion-partitions:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      matrix-1npu: ${{ steps.compute.outputs.matrix-1npu }}
+      matrix-2npu: ${{ steps.compute.outputs.matrix-2npu }}
+      partition-count-1npu: ${{ steps.compute.outputs['partition-count-1npu'] }}
+      partition-count-2npu: ${{ steps.compute.outputs['partition-count-2npu'] }}
+      plan-1npu: ${{ steps.compute.outputs.plan-1npu }}
+      plan-2npu: ${{ steps.compute.outputs.plan-2npu }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Compute partitions
+        env:
+          USE_NPU_CONFIGS: true
+        id: compute
+        run: |
+          python scripts/ci/utils/diffusion/compute_diffusion_partitions.py \
+            --min-time 1200 \
+            --target-time 1800 \
+            --max-time 2400 \
+            --max-partitions 10 \
+            --parametrized-only
+
+  multimodal-diffusion-gen-1npu:
+    needs: compute-diffusion-partitions
+    if: |
+      needs.compute-diffusion-partitions.result == 'success' &&
+      needs.compute-diffusion-partitions.outputs.matrix-1npu != '{"include":[]}'
+    runs-on: linux-aarch64-a3-2
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-1npu) }}
+    timeout-minutes: 150
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh a3 diffusion
+
+      - name: Generate outputs
+        env:
+          PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-1npu }}
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          cd python
+          python -m sglang.multimodal_gen.test.scripts.gen_diffusion_ci_outputs \
+            --suite 1-npu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions ${{ needs.compute-diffusion-partitions.outputs['partition-count-1npu'] }} \
+            --partition-plan-json "$PARTITION_PLAN_JSON" \
+            --out-dir ./${{ env.OUTPUT_NAME }} \
+            ${{ inputs.case_ids != '' && format('--case-ids {0}', inputs.case_ids) || '' }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_NAME }}-1npu-part${{ matrix.part }}
+          path: python/${{ env.OUTPUT_NAME }}
+          retention-days: 7
+
+      - name: Publish GT images to sgl-project/ci-data
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}"
+
+  multimodal-diffusion-gen-2npu:
+    needs: compute-diffusion-partitions
+    if: |
+      needs.compute-diffusion-partitions.result == 'success' &&
+      needs.compute-diffusion-partitions.outputs.matrix-2npu != '{"include":[]}'
+    runs-on: linux-aarch64-a3-16
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-2npu) }}
+    timeout-minutes: 150
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh a3 diffusion
+
+      - name: Generate outputs
+        env:
+          PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-2npu }}
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          cd python
+          python -m sglang.multimodal_gen.test.scripts.gen_diffusion_ci_outputs \
+            --suite 2-npu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions ${{ needs.compute-diffusion-partitions.outputs['partition-count-2npu'] }} \
+            --partition-plan-json "$PARTITION_PLAN_JSON" \
+            --out-dir ./${{ env.OUTPUT_NAME }} \
+            ${{ inputs.case_ids != '' && format('--case-ids {0}', inputs.case_ids) || '' }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_NAME }}-2npu-part${{ matrix.part }}
+          path: python/${{ env.OUTPUT_NAME }}
+          retention-days: 7
+
+      - name: Publish GT images to sgl-project/ci-data
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}"

.github/workflows/diffusion-ci-gt-gen.yml

@@ -0,0 +1,571 @@
+name: Diffusion CI Ground Truth Generation
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Git ref to checkout'
+        required: false
+        default: ''
+        type: string
+      case_ids:
+        description: 'Specific case IDs to run (space-separated, optional)'
+        required: false
+        default: ''
+        type: string
+      output_name:
+        description: 'Custom local output/artifact folder name. Leave empty to use defaults.'
+        required: false
+        default: ''
+        type: string
+      publish_target_dir:
+        description: 'Remote target directory in sgl-project/ci-data. Leave empty to use sglang_generated, or official_generated when run_official_cases is true.'
+        required: false
+        default: ''
+        type: string
+      kernel_artifact_run_id:
+        description: "Optional sgl-kernel wheel source: GitHub Actions run ID of a pr-test.yml run on the SAME commit as 'ref' that uploaded wheel-python3.10-cuda13.0. Use only when this branch's torch/ABI change makes the PyPI sglang-kernel wheel incompatible. Find it under Actions > pr-test; artifacts expire after 90 days."
+        required: false
+        default: ''
+        type: string
+      run_official_cases:
+        description: 'Run official comparable GT cases instead of native SGLang GT cases.'
+        required: false
+        default: false
+        type: boolean
+      official_case_ids:
+        description: 'Specific official case IDs to run (space-separated). Leave empty to run all official comparable cases.'
+        required: false
+        default: ''
+        type: string
+      official_source_group:
+        description: 'Official GT source group filter: all, diffusers, wan21, ltx, or ltx23. Used only when run_official_cases is true.'
+        required: false
+        default: ''
+        type: string
+      ci_data_ref:
+        description: 'ci-data ref to use for repro scripts when running official GT cases.'
+        required: false
+        default: 'main'
+        type: string
+
+concurrency:
+  group: diffusion-ci-gt-gen-${{ github.ref }}-${{ inputs.output_name || inputs.case_ids || inputs.official_case_ids || inputs.official_source_group || inputs.run_official_cases || 'default' }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  actions: read
+
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
+  OUTPUT_NAME: ${{ inputs.output_name || 'diffusion-ci-outputs' }}
+  PUBLISH_TARGET_DIR: ${{ inputs.publish_target_dir || (inputs.run_official_cases && 'diffusion-ci/consistency_gt/official_generated' || 'diffusion-ci/consistency_gt/sglang_generated') }}
+
+jobs:
+  compute-official-gt-matrix:
+    if: github.repository == 'sgl-project/sglang' && inputs.run_official_cases
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.compute.outputs.matrix }}
+      case-count: ${{ steps.compute.outputs.case-count }}
+    steps:
+      - name: Compute official case matrix
+        id: compute
+        env:
+          OFFICIAL_CASE_IDS: ${{ inputs.official_case_ids }}
+          OFFICIAL_SOURCE_GROUP: ${{ inputs.official_source_group || 'all' }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+
+          groups = {
+              "diffusers": [
+                  "flux_2_image_t2i",
+                  "flux_2_klein_image_t2i",
+                  "flux_2_klein_base_image_t2i",
+                  "flux_2_ti2i",
+                  "flux_image_t2i",
+                  "qwen_image_edit_2509_ti2i",
+                  "qwen_image_edit_2511_ti2i",
+                  "qwen_image_edit_ti2i",
+                  "qwen_image_layered_i2i",
+                  "qwen_image_t2i",
+                  "zimage_image_t2i",
+              ],
+              "wan21": ["wan2_1_t2v_1.3b"],
+              "ltx": [
+                  "ltx_2_two_stage_t2v",
+                  "ltx_2.3_two_stage_t2v_2gpus",
+                  "ltx_2_3_two_stage_ti2v_2gpus",
+                  "ltx_2.3_one_stage_ti2v",
+                  "ltx_2_3_hq_pipeline",
+              ],
+              "ltx23": [
+                  "ltx_2.3_two_stage_t2v_2gpus",
+                  "ltx_2.3_one_stage_ti2v",
+              ],
+          }
+          source_group = os.environ["OFFICIAL_SOURCE_GROUP"].strip() or "all"
+          if source_group == "all":
+              selected_groups = ["diffusers", "wan21", "ltx"]
+          elif source_group in groups:
+              selected_groups = [source_group]
+          else:
+              raise SystemExit(f"unknown official_source_group: {source_group}")
+
+          requested = os.environ["OFFICIAL_CASE_IDS"].split()
+          requested_set = set(requested)
+          h200_cases = {
+              "flux_2_image_t2i",
+              "flux_2_klein_image_t2i",
+              "flux_2_klein_base_image_t2i",
+              "flux_2_ti2i",
+          }
+          include = []
+          for group in selected_groups:
+              for case_id in groups[group]:
+                  if requested_set and case_id not in requested_set:
+                      continue
+                  include.append(
+                      {
+                          "source_group": group,
+                          "case_id": case_id,
+                          "runner": "8-gpu-h200" if case_id in h200_cases else "1-gpu-h100",
+                      }
+                  )
+
+          known_cases = {case for cases in groups.values() for case in cases}
+          unknown = sorted(requested_set - known_cases)
+          if unknown:
+              raise SystemExit(f"unknown official case id(s): {' '.join(unknown)}")
+          if not include:
+              raise SystemExit("official case matrix is empty")
+
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
+              f.write(f"matrix={json.dumps({'include': include}, separators=(',', ':'))}\n")
+              f.write(f"case-count={len(include)}\n")
+          PY
+
+  official-gt-hopper:
+    needs: compute-official-gt-matrix
+    if: needs.compute-official-gt-matrix.result == 'success'
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 240
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-official-gt-matrix.outputs.matrix) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Checkout ci-data repro scripts
+        uses: actions/checkout@v4
+        with:
+          repository: sgl-project/ci-data
+          ref: ${{ inputs.ci_data_ref || 'main' }}
+          path: ci-data
+          token: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          sparse-checkout: |
+            diffusion-ci/repro_scripts
+            diffusion-ci/consistency_gt/official_generated/case_map.json
+          sparse-checkout-cone-mode: false
+
+      - name: Prepare sgl-kernel/dist for prebuilt wheel
+        if: inputs.kernel_artifact_run_id != ''
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download prebuilt sgl-kernel wheel
+        if: inputs.kernel_artifact_run_id != ''
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          name: wheel-python3.10-cuda13.0
+          run-id: ${{ inputs.kernel_artifact_run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL="${{ inputs.kernel_artifact_run_id != '' && 'true' || 'false' }}" \
+            bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Install official LTX repro dependencies
+        if: matrix.source_group == 'ltx' || matrix.source_group == 'ltx23'
+        run: |
+          UV_SYSTEM_PYTHON=1 uv pip install \
+            "transformers==4.52.4" \
+            openimageio \
+            --index-strategy unsafe-best-match \
+            --prerelease allow
+          UV_SYSTEM_PYTHON=1 uv pip uninstall kernels
+
+      - name: Checkout official Wan2.1 repo
+        if: matrix.source_group == 'wan21'
+        run: |
+          mkdir -p /tmp/mmgen-official-code
+          if [ ! -d /tmp/mmgen-official-code/Wan2.1/.git ]; then
+            git clone https://github.com/Wan-Video/Wan2.1.git /tmp/mmgen-official-code/Wan2.1
+          fi
+          git -C /tmp/mmgen-official-code/Wan2.1 fetch --depth 1 origin 9737cba9c1c3c4d04b33fcad41c111989865d315
+          git -C /tmp/mmgen-official-code/Wan2.1 checkout 9737cba9c1c3c4d04b33fcad41c111989865d315
+          git -C /tmp/mmgen-official-code/Wan2.1 rev-parse HEAD
+
+      - name: Checkout official LTX-2 repo
+        if: matrix.source_group == 'ltx' || matrix.source_group == 'ltx23'
+        run: |
+          mkdir -p /tmp/mmgen-official-code
+          if [ ! -d /tmp/mmgen-official-code/LTX-2/.git ]; then
+            git clone https://github.com/Lightricks/LTX-2.git /tmp/mmgen-official-code/LTX-2
+          fi
+          git -C /tmp/mmgen-official-code/LTX-2 fetch --depth 1 origin 41d924371612b692c0fd1e4d9d94c3dfb3c02cb3
+          git -C /tmp/mmgen-official-code/LTX-2 checkout 41d924371612b692c0fd1e4d9d94c3dfb3c02cb3
+          git -C /tmp/mmgen-official-code/LTX-2 rev-parse HEAD
+
+      - name: Generate official output
+        env:
+          HF_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
+          HUGGING_FACE_HUB_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+          CUDA_VISIBLE_DEVICES: 0
+          CASE_ID: ${{ matrix.case_id }}
+          SOURCE_GROUP: ${{ matrix.source_group }}
+        run: |
+          git -C ci-data rev-parse HEAD
+          mkdir -p "python/${{ env.OUTPUT_NAME }}"
+          if [ "$SOURCE_GROUP" = "diffusers" ]; then
+            cd python
+            python ../ci-data/diffusion-ci/repro_scripts/gen_official_diffusion_gt.py \
+              --suite 1-gpu \
+              --out-dir ./${{ env.OUTPUT_NAME }} \
+              --case-ids "$CASE_ID" \
+              --dtype bf16 \
+              --device-map none \
+              --generator-device cuda
+          elif [ "$SOURCE_GROUP" = "wan21" ]; then
+            cd python
+            python ../ci-data/diffusion-ci/repro_scripts/gen_official_diffusion_gt.py \
+              --suite 1-gpu \
+              --out-dir ./${{ env.OUTPUT_NAME }} \
+              --case-ids "$CASE_ID" \
+              --wan-official-repo-dir /tmp/mmgen-official-code/Wan2.1 \
+              --dtype bf16 \
+              --device-map none \
+              --generator-device cuda
+          elif [ "$SOURCE_GROUP" = "ltx" ] || [ "$SOURCE_GROUP" = "ltx23" ]; then
+            cd python
+            extra_ltx_args=()
+            if [ "$CASE_ID" = "ltx_2_3_hq_pipeline" ]; then
+              extra_ltx_args+=(--num-frames 24)
+            fi
+            set +e
+            PYTHONPATH=/tmp/mmgen-official-code/LTX-2/packages/ltx-core/src:/tmp/mmgen-official-code/LTX-2/packages/ltx-pipelines/src:$PWD:$PYTHONPATH \
+              python ../ci-data/diffusion-ci/repro_scripts/gen_official_ltx23.py \
+                --out-dir ./${{ env.OUTPUT_NAME }} \
+                --case-ids "$CASE_ID" \
+                "${extra_ltx_args[@]}"
+            status=$?
+            set -e
+            if [ "$status" -ne 0 ]; then
+              find ./${{ env.OUTPUT_NAME }} \( -name 'official_ltx_manifest.json' -o -name 'official_ltx23_manifest.json' \) -print -exec cat {} \;
+              exit "$status"
+            fi
+          else
+            echo "Unknown SOURCE_GROUP=$SOURCE_GROUP"
+            exit 1
+          fi
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_NAME }}-${{ matrix.source_group }}-${{ matrix.case_id }}
+          path: |
+            python/${{ env.OUTPUT_NAME }}/*.jpg
+            python/${{ env.OUTPUT_NAME }}/*.png
+            python/${{ env.OUTPUT_NAME }}/official_gt_manifest_*.json
+            python/${{ env.OUTPUT_NAME }}/official_ltx_manifest.json
+            python/${{ env.OUTPUT_NAME }}/official_ltx23_manifest.json
+          retention-days: 7
+
+      - name: Validate generated GT images
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \
+            --check-only
+
+      - name: Publish official GT images to sgl-project/ci-data
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}"
+
+  compute-diffusion-partitions:
+    if: github.repository == 'sgl-project/sglang' && !inputs.run_official_cases
+    runs-on: ubuntu-latest
+    outputs:
+      matrix-1gpu: ${{ steps.compute.outputs.matrix-1gpu }}
+      matrix-2gpu: ${{ steps.compute.outputs.matrix-2gpu }}
+      matrix-b200: ${{ steps.compute.outputs.matrix-b200 }}
+      partition-count-1gpu: ${{ steps.compute.outputs['partition-count-1gpu'] }}
+      partition-count-2gpu: ${{ steps.compute.outputs['partition-count-2gpu'] }}
+      partition-count-b200: ${{ steps.compute.outputs['partition-count-b200'] }}
+      plan-1gpu: ${{ steps.compute.outputs.plan-1gpu }}
+      plan-2gpu: ${{ steps.compute.outputs.plan-2gpu }}
+      plan-b200: ${{ steps.compute.outputs.plan-b200 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Compute partitions
+        id: compute
+        run: |
+          python scripts/ci/utils/diffusion/compute_diffusion_partitions.py \
+            --min-time 1200 \
+            --target-time 1800 \
+            --max-time 2400 \
+            --max-partitions 10 \
+            --parametrized-only
+
+  multimodal-diffusion-gen-1gpu:
+    needs: compute-diffusion-partitions
+    if: |
+      needs.compute-diffusion-partitions.result == 'success' &&
+      needs.compute-diffusion-partitions.outputs.matrix-1gpu != '{"include":[]}'
+    runs-on: 1-gpu-h100
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-1gpu) }}
+    timeout-minutes: 150
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Prepare sgl-kernel/dist for prebuilt wheel
+        if: inputs.kernel_artifact_run_id != ''
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download prebuilt sgl-kernel wheel
+        if: inputs.kernel_artifact_run_id != ''
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          name: wheel-python3.10-cuda13.0
+          run-id: ${{ inputs.kernel_artifact_run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL="${{ inputs.kernel_artifact_run_id != '' && 'true' || 'false' }}" \
+            bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Generate outputs
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-1gpu }}
+        run: |
+          cd python
+          python -m sglang.multimodal_gen.test.scripts.gen_diffusion_ci_outputs \
+            --suite 1-gpu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions ${{ needs.compute-diffusion-partitions.outputs['partition-count-1gpu'] }} \
+            --partition-plan-json "$PARTITION_PLAN_JSON" \
+            --out-dir ./${{ env.OUTPUT_NAME }} \
+            ${{ inputs.case_ids != '' && format('--case-ids {0}', inputs.case_ids) || '' }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_NAME }}-1gpu-part${{ matrix.part }}
+          path: python/${{ env.OUTPUT_NAME }}
+          retention-days: 7
+
+      - name: Validate generated GT images
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \
+            --check-only
+
+      - name: Publish GT images to sgl-project/ci-data
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}"
+
+  multimodal-diffusion-gen-2gpu:
+    needs: compute-diffusion-partitions
+    if: |
+      needs.compute-diffusion-partitions.result == 'success' &&
+      needs.compute-diffusion-partitions.outputs.matrix-2gpu != '{"include":[]}'
+    runs-on: 2-gpu-h100
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-2gpu) }}
+    timeout-minutes: 150
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Prepare sgl-kernel/dist for prebuilt wheel
+        if: inputs.kernel_artifact_run_id != ''
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download prebuilt sgl-kernel wheel
+        if: inputs.kernel_artifact_run_id != ''
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          name: wheel-python3.10-cuda13.0
+          run-id: ${{ inputs.kernel_artifact_run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL="${{ inputs.kernel_artifact_run_id != '' && 'true' || 'false' }}" \
+            bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Generate outputs
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-2gpu }}
+        run: |
+          cd python
+          python -m sglang.multimodal_gen.test.scripts.gen_diffusion_ci_outputs \
+            --suite 2-gpu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions ${{ needs.compute-diffusion-partitions.outputs['partition-count-2gpu'] }} \
+            --partition-plan-json "$PARTITION_PLAN_JSON" \
+            --out-dir ./${{ env.OUTPUT_NAME }} \
+            ${{ inputs.case_ids != '' && format('--case-ids {0}', inputs.case_ids) || '' }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_NAME }}-2gpu-part${{ matrix.part }}
+          path: python/${{ env.OUTPUT_NAME }}
+          retention-days: 7
+
+      - name: Validate generated GT images
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \
+            --check-only
+
+      - name: Publish GT images to sgl-project/ci-data
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}"
+
+  multimodal-diffusion-gen-b200:
+    needs: compute-diffusion-partitions
+    if: |
+      needs.compute-diffusion-partitions.result == 'success' &&
+      needs.compute-diffusion-partitions.outputs.matrix-b200 != '{"include":[]}'
+    runs-on: 4-gpu-b200
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-b200) }}
+    timeout-minutes: 240
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Prepare sgl-kernel/dist for prebuilt wheel
+        if: inputs.kernel_artifact_run_id != ''
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download prebuilt sgl-kernel wheel
+        if: inputs.kernel_artifact_run_id != ''
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          name: wheel-python3.10-cuda13.0
+          run-id: ${{ inputs.kernel_artifact_run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL="${{ inputs.kernel_artifact_run_id != '' && 'true' || 'false' }}" \
+            bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Generate outputs
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-b200 }}
+        run: |
+          cd python
+          python -m sglang.multimodal_gen.test.scripts.gen_diffusion_ci_outputs \
+            --suite 1-gpu-b200 \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions ${{ needs.compute-diffusion-partitions.outputs['partition-count-b200'] }} \
+            --partition-plan-json "$PARTITION_PLAN_JSON" \
+            --out-dir ./${{ env.OUTPUT_NAME }} \
+            ${{ inputs.case_ids != '' && format('--case-ids {0}', inputs.case_ids) || '' }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.OUTPUT_NAME }}-b200-part${{ matrix.part }}
+          path: python/${{ env.OUTPUT_NAME }}
+          retention-days: 7
+
+      - name: Validate generated GT images
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}" \
+            --check-only
+
+      - name: Publish GT images to sgl-project/ci-data
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python scripts/ci/utils/diffusion/publish_diffusion_gt.py \
+            --source-dir python/${{ env.OUTPUT_NAME }} \
+            --target-dir "${{ env.PUBLISH_TARGET_DIR }}"

.github/workflows/full-test-npu.yml

@@ -0,0 +1,359 @@
+name: Full Test (NPU)
+
+on:
+#  pull_request:
+#    branches:
+#      - main
+#    paths:
+#      - ".github/workflows/full-test-npu.yml"
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      job_filter:
+        description: 'Select which job to run (leave empty or "all" to run all jobs)'
+        required: false
+        type: string
+        default: 'all'
+      image_a3:
+        description: 'The a3 running docker image of the test task.'
+        required: false
+        type: string
+        default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11'
+      skip_install_flag:
+        description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
+        required: false
+        type: string
+        default: 'false'
+
+concurrency:
+  group: full-test-npu-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+jobs:
+  set-image-config:
+    runs-on: ubuntu-latest
+    outputs:
+      ref: ${{ steps.set-vars.outputs.ref }}
+      job_filter: ${{ steps.set-vars.outputs.job_filter }}
+      image_a3: ${{ steps.set-vars.outputs.image_a3 }}
+      skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
+    steps:
+      # When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
+      - name: Set image config
+        id: set-vars
+        run: |
+          if [ -z "${{ inputs.ref }}" ]; then
+            echo "ref=" >> $GITHUB_OUTPUT
+          else
+            echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT
+          fi
+
+          if [ -z "${{ inputs.job_filter }}" ]; then
+            echo "job_filter=all" >> $GITHUB_OUTPUT
+          else
+            echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
+          fi
+
+          if [ -z "${{ inputs.image_a3 }}" ]; then
+            echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
+          else
+            echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
+          fi
+
+          if [ -z "${{ inputs.skip_install_flag }}" ]; then
+            echo "skip_install_flag=false" >> $GITHUB_OUTPUT
+          else
+            echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
+          fi
+
+  nighly-test-npu:
+    needs: [set-image-config]
+    name: nightly-test-npu
+    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
+    uses: ./.github/workflows/nightly-test-npu.yml
+    with:
+      ref:  ${{ needs.set-image-config.outputs.ref }}
+      job_filter: ${{ needs.set-image-config.outputs.job_filter }}
+      image_a3: ${{ needs.set-image-config.outputs.image_a3 }}
+      skip_install_flag: ${{ needs.set-image-config.outputs.skip_install_flag }}
+    secrets: inherit
+
+  full-1-npu-a3:
+    needs: [set-image-config]
+    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
+    runs-on: linux-aarch64-a3-2
+    container:
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
+            bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          fi
+
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Print Log Information
+        run: |
+          bash scripts/ci/npu/npu_log_print.sh
+
+      - name: Run test
+        timeout-minutes: 240
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          pip install sglang_router
+          hf download lmms-lab/MMMU --repo-type dataset
+          pip install sentence_transformers torchaudio==2.8.0
+          pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
+          pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
+          pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
+          git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+          cd ./lmms-eval
+          nohup pip install . > lmmslog.txt 2>&1 &
+          sleep 120
+          export PYTHONPATH=$PYTHONPATH:$(pwd)
+          cd ../
+          cd test
+          python3 run_suite.py --hw npu --suite full-1-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
+
+  full-2-npu-a3:
+    needs: [set-image-config]
+    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
+    runs-on: linux-aarch64-a3-2
+    container:
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
+            bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          fi
+
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Print Log Information
+        run: |
+          bash scripts/ci/npu/npu_log_print.sh
+
+      - name: Run test
+        timeout-minutes: 240
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          pip install sglang_router
+          hf download lmms-lab/MMMU --repo-type dataset
+          pip install sentence_transformers torchaudio==2.8.0
+          pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
+          pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
+          pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
+          git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+          cd ./lmms-eval
+          nohup pip install . > lmmslog.txt 2>&1 &
+          sleep 120
+          export PYTHONPATH=$PYTHONPATH:$(pwd)
+          cd ../
+          cd test
+          python3 run_suite.py --hw npu --suite full-2-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
+
+  full-4-npu-a3:
+    needs: [set-image-config]
+    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
+    runs-on: linux-aarch64-a3-4
+    container:
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
+            bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          fi
+
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Print Log Information
+        run: |
+          bash scripts/ci/npu/npu_log_print.sh
+
+      - name: Run test
+        timeout-minutes: 240
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          pip install sglang_router
+          hf download lmms-lab/MMMU --repo-type dataset
+          pip install sentence_transformers torchaudio==2.8.0
+          pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
+          pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
+          pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
+          git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+          cd ./lmms-eval
+          nohup pip install . > lmmslog.txt 2>&1 &
+          sleep 120
+          export PYTHONPATH=$PYTHONPATH:$(pwd)
+          cd ../
+          cd test
+          python3 run_suite.py --hw npu --suite full-4-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
+
+  full-16-npu-a3:
+    needs: [set-image-config]
+    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
+    runs-on: linux-aarch64-a3-16
+    container:
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.set-image-config.outputs.ref || github.ref }}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          if [ ${{ needs.set-image-config.outputs.skip_install_flag }} != "true" ];then
+            bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          fi
+
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Print Log Information
+        run: |
+          bash scripts/ci/npu/npu_log_print.sh
+
+      - name: Run test
+        timeout-minutes: 240
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          pip install sglang_router
+          hf download lmms-lab/MMMU --repo-type dataset
+          pip install sentence_transformers torchaudio==2.8.0
+          pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
+          pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
+          pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
+          git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+          cd ./lmms-eval
+          nohup pip install . > lmmslog.txt 2>&1 &
+          sleep 120
+          export PYTHONPATH=$PYTHONPATH:$(pwd)
+          cd ../
+          cd test
+          python3 run_suite.py --hw npu --suite full-16-npu-a3 --nightly --continue-on-error --timeout-per-file 3600
+
+  check-all-jobs:
+    if: github.repository == 'sgl-project/sglang' && always()
+    needs:
+      - nighly-test-npu
+      - full-1-npu-a3
+      - full-2-npu-a3
+      - full-4-npu-a3
+      - full-16-npu-a3
+    runs-on: ubuntu-latest
+    container:
+      image: docker.m.daocloud.io/ubuntu:22.04
+    steps:
+      - name: Check if any job failed
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            echo "One or more nightly test jobs failed"
+            exit 1
+          fi
+          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more nightly test jobs were cancelled"
+            exit 1
+          fi
+          echo "All nightly test jobs passed"

.github/workflows/labeler.yml

@@ -0,0 +1,20 @@
+name: Auto Label PRs
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  label:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Auto-label by file changes
+        uses: actions/labeler@v5
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          configuration-path: .github/labeler.yml
+          sync-labels: false

.github/workflows/lint.yml

@@ -0,0 +1,63 @@
+name: Lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Reject changes under legacy docs/
+        if: github.event_name == 'pull_request'
+        run: |
+          set -euo pipefail
+          BASE_REF="${{ github.base_ref }}"
+          # Refresh origin/<base_ref> with full history; --depth=1 would
+          # shallow the ref and break the merge-base used by `...`.
+          git fetch --no-tags origin "$BASE_REF"
+          # First, verify the diff itself succeeds and check whether any
+          # files changed. A silent failure here would let docs/ changes
+          # through, so the explicit `if !` guard is important.
+          if ! CHANGED=$(git diff --name-only --diff-filter=ACMRDTUXB "origin/${BASE_REF}...HEAD"); then
+            echo "git diff origin/${BASE_REF}...HEAD failed; aborting." >&2
+            exit 2
+          fi
+          if [ -z "$CHANGED" ]; then
+            echo "No changed files vs origin/${BASE_REF}; skipping."
+            exit 0
+          fi
+          # Re-emit with -z so xargs can safely handle whitespace in paths.
+          git diff -z --name-only --diff-filter=ACMRDTUXB "origin/${BASE_REF}...HEAD" \
+            | xargs -0 python3 scripts/ci/check_no_docs_changes.py
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install pre-commit hook
+        run: |
+          python -m pip install pre-commit
+          pre-commit install
+
+      - name: Run pre-commit checks
+        run: SKIP=no-commit-to-branch pre-commit run --all-files --show-diff-on-failure
+
+      - name: Run lychee docs checks (offline references)
+        uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2
+        with:
+          args: --config .github/linters/lychee.toml README.md "docs/**/*.md" "docs/**/*.rst" "docs/**/*.ipynb"
+
+      - name: Run sgl-kernel clang-format checks
+        uses: DoozyX/clang-format-lint-action@v0.20
+        with:
+          source: sgl-kernel
+          extensions: h,c,cpp,hpp,cu,cuh,cc
+          clangFormatVersion: 20
+          style: file

.github/workflows/list-active-pr-runs.yml

@@ -0,0 +1,317 @@
+name: List Active Runs
+
+on:
+  workflow_dispatch:
+    inputs:
+      workflows:
+        description: 'Space-separated list of workflow filenames to check'
+        required: false
+        type: string
+        default: 'pr-test.yml'
+
+permissions:
+  actions: read
+  contents: read
+  pull-requests: read
+
+jobs:
+  list-active-runs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install GitHub CLI
+        run: sudo apt-get install -y gh jq
+
+      - name: List active runs grouped by PR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          WORKFLOWS: ${{ github.event.inputs.workflows || 'pr-test.yml' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          echo "========================================="
+          echo "🔍 Active Workflow Runs Report"
+          echo "========================================="
+          echo ""
+
+          # Get all workflows or specific ones
+          read -r -a workflow_files <<< "${WORKFLOWS}"
+          echo "📋 Checking specified workflows: ${WORKFLOWS}"
+
+          echo ""
+
+          # Create a temporary file to store PR data
+          pr_data_file=$(mktemp)
+
+          # Process each workflow
+          for workflow_file in ${workflow_files[@]}; do
+            echo "Scanning workflow: $workflow_file"
+
+            # Get all active runs (queued, waiting, in_progress)
+            active_runs=$(gh run list \
+              --repo "$REPO" \
+              --workflow "$workflow_file" \
+              --json databaseId,status,event,headBranch,createdAt,updatedAt,headSha,number,attempt \
+              --limit 500 \
+              | jq -c '.[] | select(.status=="queued" or .status=="waiting" or .status=="in_progress")')
+
+            if [ -z "$active_runs" ]; then
+              continue
+            fi
+
+            # Process each run
+            echo "$active_runs" | while read -r run; do
+              run_id=$(echo "$run" | jq -r '.databaseId')
+              run_status=$(echo "$run" | jq -r '.status')
+              run_event=$(echo "$run" | jq -r '.event')
+              created_at=$(echo "$run" | jq -r '.createdAt')
+              head_sha=$(echo "$run" | jq -r '.headSha')
+              run_number=$(echo "$run" | jq -r '.number')
+              run_attempt=$(echo "$run" | jq -r '.attempt // 1')
+
+              # Get detailed run information including jobs
+              run_details=$(gh api "repos/$REPO/actions/runs/$run_id" 2>/dev/null || true)
+
+              if [ -z "$run_details" ]; then
+                continue
+              fi
+
+              head_owner=$(echo "$run_details" | jq -r '.head_repository.owner.login // empty')
+              head_branch=$(echo "$run_details" | jq -r '.head_branch // empty')
+
+              if [ -z "$head_owner" ] || [ -z "$head_branch" ]; then
+                continue
+              fi
+
+              # Find PR number (may be empty for non-PR runs)
+              pr_number=$(gh api "repos/$REPO/pulls?state=open&head=${head_owner}:${head_branch}" \
+                --jq '.[0].number // empty' 2>/dev/null || true)
+
+              if [ -z "$pr_number" ]; then
+                pr_number="NO_PR"
+              fi
+
+              # Get jobs for this run (with pagination to avoid missing jobs)
+              jobs=$(gh api "repos/$REPO/actions/runs/$run_id/jobs" --paginate --jq '.jobs[]' | jq -s '.')
+
+              running_jobs=$(echo "$jobs" | jq '[.[] | select(.status=="in_progress")] | length')
+              queued_jobs=$(echo "$jobs" | jq '[.[] | select(.status=="queued" or .status=="waiting")] | length')
+
+              # Get runner info for running jobs
+              runners=$(echo "$jobs" | jq -r '.[] | select(.status=="in_progress") | .runner_name // "N/A"' | paste -sd "," -)
+
+              # Calculate queue time
+              current_time=$(date -u +%s)
+              created_time=$(date -u -d "$created_at" +%s 2>/dev/null || echo "$current_time")
+              queue_time=$((current_time - created_time))
+              queue_minutes=$((queue_time / 60))
+
+              # Store data in temporary file (unified format with event and branch)
+              echo "$pr_number|$workflow_file|$run_id|$run_status|$running_jobs|$queued_jobs|$runners|$queue_minutes|$created_at|$head_sha|$run_attempt|$run_event|$head_branch" >> "$pr_data_file"
+            done
+          done
+
+          echo ""
+          echo "========================================="
+          echo "📊 Active Runs Summary"
+          echo "========================================="
+          echo ""
+
+          if [ ! -s "$pr_data_file" ]; then
+            echo "✅ No active runs found"
+            rm -f "$pr_data_file"
+            exit 0
+          fi
+
+          # Get unique PR numbers (exclude NO_PR entries)
+          pr_numbers=$(cut -d'|' -f1 < "$pr_data_file" | grep -v '^NO_PR$' | sort -u || true)
+
+          # Separate high priority and normal PRs
+          high_priority_prs=()
+          normal_prs=()
+
+          for pr_num in $pr_numbers; do
+            labels=$(gh pr view "$pr_num" --repo "$REPO" --json labels \
+              | jq -r '.labels[].name' 2>/dev/null || true)
+
+            if echo "$labels" | grep -Fxq "high priority"; then
+              high_priority_prs+=($pr_num)
+            else
+              normal_prs+=($pr_num)
+            fi
+          done
+
+          # Combine: high priority first, then normal
+          sorted_pr_numbers=("${high_priority_prs[@]}" "${normal_prs[@]}")
+
+          pr_count=0
+          total_running=0
+          total_queued=0
+
+          for pr_num in "${sorted_pr_numbers[@]}"; do
+            pr_count=$((pr_count + 1))
+
+            # Get PR details
+            pr_info=$(gh pr view "$pr_num" --repo "$REPO" --json title,author,labels,url 2>/dev/null || true)
+
+            if [ -z "$pr_info" ]; then
+              continue
+            fi
+
+            pr_title=$(echo "$pr_info" | jq -r '.title')
+            pr_author=$(echo "$pr_info" | jq -r '.author.login')
+            pr_url=$(echo "$pr_info" | jq -r '.url')
+            pr_labels=$(echo "$pr_info" | jq -r '.labels[].name' | paste -sd ", " -)
+
+            if [ -z "$pr_labels" ]; then
+              pr_labels="(no labels)"
+            fi
+
+            # Add priority indicator
+            priority_indicator=""
+            if echo "$pr_labels" | grep -q "high priority"; then
+              priority_indicator="🔴 [HIGH PRIORITY] "
+            fi
+
+            echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+            echo "🔗 ${priority_indicator}PR #$pr_num: $pr_title"
+            echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+            echo "👤 Author: $pr_author"
+            echo "🏷️  Labels: $pr_labels"
+            echo "🔗 URL: $pr_url"
+            echo ""
+
+            # Get all runs for this PR
+            pr_runs=$(grep "^$pr_num|" "$pr_data_file")
+
+            pr_running_total=0
+            pr_queued_total=0
+
+            echo "$pr_runs" | while read -r line; do
+              workflow=$(echo "$line" | cut -d'|' -f2)
+              run_id=$(echo "$line" | cut -d'|' -f3)
+              status=$(echo "$line" | cut -d'|' -f4)
+              running=$(echo "$line" | cut -d'|' -f5)
+              queued=$(echo "$line" | cut -d'|' -f6)
+              runners=$(echo "$line" | cut -d'|' -f7)
+              queue_min=$(echo "$line" | cut -d'|' -f8)
+              created=$(echo "$line" | cut -d'|' -f9)
+              attempt=$(echo "$line" | cut -d'|' -f11)
+
+              pr_running_total=$((pr_running_total + running))
+              pr_queued_total=$((pr_queued_total + queued))
+
+              run_url="https://github.com/$REPO/actions/runs/$run_id"
+
+              # Calculate retry count for this specific run
+              retry_count=$((attempt - 1))
+
+              # Show retry indicator
+              retry_indicator=""
+              if [ "$retry_count" -gt 0 ]; then
+                retry_indicator=" 🔄 Retry #$retry_count"
+              fi
+
+              echo "  📦 Workflow: $workflow (Run #$run_id)$retry_indicator"
+              echo "     Status: $status"
+              echo "     🟢 Running jobs: $running"
+              echo "     🟡 Queued jobs: $queued"
+
+              if [ "$running" -gt 0 ] && [ "$runners" != "" ]; then
+                echo "     🖥️  Runners: $runners"
+              fi
+
+              if [ "$queue_min" -gt 0 ]; then
+                echo "     ⏱️  Queue time: ${queue_min} minutes"
+              fi
+
+              echo "     🔗 Run URL: $run_url"
+              echo ""
+            done
+
+            # Summary for this PR
+            pr_running_total=$(grep "^$pr_num|" "$pr_data_file" | cut -d'|' -f5 | awk '{sum+=$1} END {print sum+0}')
+            pr_queued_total=$(grep "^$pr_num|" "$pr_data_file" | cut -d'|' -f6 | awk '{sum+=$1} END {print sum+0}')
+
+            total_running=$((total_running + pr_running_total))
+            total_queued=$((total_queued + pr_queued_total))
+
+            echo "  📊 PR Total: $pr_running_total running, $pr_queued_total queued"
+            echo ""
+          done
+
+          # --- Non-PR Runs Section ---
+          non_pr_runs=$(grep '^NO_PR|' "$pr_data_file" 2>/dev/null || true)
+          non_pr_running=0
+          non_pr_queued=0
+
+          if [ -n "$non_pr_runs" ]; then
+            echo "========================================="
+            echo "📦 Non-PR Runs (manual / scheduled / other)"
+            echo "========================================="
+            echo ""
+
+            echo "$non_pr_runs" | while read -r line; do
+              workflow=$(echo "$line" | cut -d'|' -f2)
+              run_id=$(echo "$line" | cut -d'|' -f3)
+              status=$(echo "$line" | cut -d'|' -f4)
+              running=$(echo "$line" | cut -d'|' -f5)
+              queued=$(echo "$line" | cut -d'|' -f6)
+              runners=$(echo "$line" | cut -d'|' -f7)
+              queue_min=$(echo "$line" | cut -d'|' -f8)
+              created=$(echo "$line" | cut -d'|' -f9)
+              attempt=$(echo "$line" | cut -d'|' -f11)
+              event=$(echo "$line" | cut -d'|' -f12)
+              branch=$(echo "$line" | cut -d'|' -f13)
+
+              run_url="https://github.com/$REPO/actions/runs/$run_id"
+
+              retry_count=$((attempt - 1))
+              retry_indicator=""
+              if [ "$retry_count" -gt 0 ]; then
+                retry_indicator=" 🔄 Retry #$retry_count"
+              fi
+
+              echo "  📦 Workflow: $workflow (Run #$run_id)$retry_indicator"
+              echo "     Event: $event"
+              echo "     Branch: $branch"
+              echo "     Status: $status"
+              echo "     🟢 Running jobs: $running"
+              echo "     🟡 Queued jobs: $queued"
+
+              if [ "$running" -gt 0 ] && [ "$runners" != "" ]; then
+                echo "     🖥️  Runners: $runners"
+              fi
+
+              if [ "$queue_min" -gt 0 ]; then
+                echo "     ⏱️  Queue time: ${queue_min} minutes"
+              fi
+
+              echo "     🔗 Run URL: $run_url"
+              echo ""
+            done
+
+            non_pr_running=$(echo "$non_pr_runs" | cut -d'|' -f5 | awk '{sum+=$1} END {print sum+0}')
+            non_pr_queued=$(echo "$non_pr_runs" | cut -d'|' -f6 | awk '{sum+=$1} END {print sum+0}')
+            non_pr_count=$(echo "$non_pr_runs" | wc -l | tr -d ' ')
+
+            total_running=$((total_running + non_pr_running))
+            total_queued=$((total_queued + non_pr_queued))
+
+            echo "  📊 Non-PR Total: $non_pr_running running, $non_pr_queued queued"
+            echo ""
+          fi
+
+          # Overall summary
+          echo "========================================="
+          echo "📈 Overall Summary"
+          echo "========================================="
+          echo "Total PRs with active runs: $pr_count"
+          echo "Total non-PR active runs: ${non_pr_count:-0}"
+          echo "Total running jobs: $total_running"
+          echo "Total queued jobs: $total_queued"
+          echo "========================================="
+
+          # Cleanup
+          rm -f "$pr_data_file"

.github/workflows/nightly-72-gpu-gb200.yml

@@ -0,0 +1,466 @@
+name: Nightly Test (GB200 72GPU)
+
+# NOTE: Nightly (schedule) runs require no approval.
+# Manual (workflow_dispatch) runs are gated by the gb200-ci environment
+# to prevent individuals from queuing arbitrary jobs on the shared GB200 cluster.
+on:
+  schedule:
+    - cron: '0 2 * * *'  # 2 AM UTC daily (offset from other nightly runs)
+  workflow_dispatch:  # allow manual trigger; gated by gb200-ci environment
+    inputs:
+      image:
+        description: 'Optional. SGLang Docker image to benchmark. Leave empty for the default nightly image. Mutually exclusive with pr_number and sglang_branch.'
+        required: false
+        default: ''
+      pr_number:
+        description: 'Optional. PR number to build from (works for PRs from forks too, via refs/pull/<N>/head). Preferred over sglang_branch when a PR exists. Mutually exclusive with image and sglang_branch.'
+        required: false
+        default: ''
+      sglang_branch:
+        description: 'Optional. Branch name on sgl-project/sglang to build from (use when no PR is open yet). For fork branches, open a PR and use pr_number instead. Mutually exclusive with image and pr_number.'
+        required: false
+        default: ''
+      configs:
+        description: 'Optional. Comma-separated names to run only a subset. Format: {model-prefix}-{precision}-{isl}{osl}-{recipe}. E.g. "dsr1-fp8-1k1k-max-tpt" or "dsr1-fp8-1k1k-max-tpt,dsr1-fp4-1k1k-mid-curve". Leave empty to run all. Available names are listed in the setup job log.'
+        required: false
+        default: ''
+
+concurrency:
+  group: nightly-test-gb200
+  cancel-in-progress: false
+
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
+  SRT_SLURM_BRANCH: sglang-nightly-regression
+  SLURM_PARTITION: batch
+  SLURM_ACCOUNT: sglang
+  # Docker Hub repo for ephemeral branch/PR build images (kept separate from
+  # the released `lmsysorg/sglang` repo). Cleaned up by `cleanup-image`.
+  CI_IMAGE_REPO: lmsysorg/sglang-staging
+  # How many most recent staging tags to retain after each run.
+  CI_IMAGE_KEEP_TAGS: 60
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Reject conflicting inputs early. At most one of `image`, `pr_number`,
+  # `sglang_branch` may be set — they select different image sources. Only runs
+  # on manual dispatch; all downstream jobs chain through this so invalid
+  # inputs halt the pipeline before cluster resources are reserved.
+  # ---------------------------------------------------------------------------
+  validate-inputs:
+    if: github.repository == 'sgl-project/sglang' && github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Reject conflicting inputs
+        run: |
+          IMAGE="${{ inputs.image }}"
+          PR="${{ inputs.pr_number }}"
+          BRANCH="${{ inputs.sglang_branch }}"
+          sources=0
+          [ -n "$IMAGE" ]  && sources=$((sources + 1))
+          [ -n "$PR" ]     && sources=$((sources + 1))
+          [ -n "$BRANCH" ] && sources=$((sources + 1))
+          if [ "$sources" -gt 1 ]; then
+            echo "::error::Specify at most one of 'image' ('$IMAGE'), 'pr_number' ('$PR'), or 'sglang_branch' ('$BRANCH')."
+            exit 1
+          fi
+          if [ -n "$PR" ] && ! echo "$PR" | grep -Eq '^[0-9]+$'; then
+            echo "::error::pr_number must be a positive integer, got '$PR'."
+            exit 1
+          fi
+
+  # ---------------------------------------------------------------------------
+  # Reads scripts/ci/slurm/nightly-configs.yaml and generates one matrix entry
+  # per recipe YAML. Each job runs the full concurrency sweep defined in the
+  # recipe as a single Slurm job.
+  # To add/remove configs, edit nightly-configs.yaml only.
+  # ---------------------------------------------------------------------------
+  setup:
+    needs: validate-inputs
+    # Run if validate-inputs succeeded (dispatch) or was skipped (cron).
+    if: |
+      always() && github.repository == 'sgl-project/sglang'
+      && (needs.validate-inputs.result == 'success' || needs.validate-inputs.result == 'skipped')
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Generate benchmark matrix
+        id: generate
+        env:
+          CONFIGS_FILTER: ${{ inputs.configs }}
+        run: |
+          pip install pyyaml -q
+
+          # List all available config names first so they're visible in logs
+          # even when a filter rejects an unknown name.
+          ALL_MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \
+            scripts/ci/slurm/nightly-configs.yaml --runner gb200)
+          echo "Available config names for runner gb200:"
+          echo "$ALL_MATRIX" | python3 -c "import json,sys; [print(f'  - {e[\"name\"]}') for e in json.load(sys.stdin)]"
+
+          FILTER_ARG=()
+          if [ -n "$CONFIGS_FILTER" ]; then
+            echo ""
+            echo "Filtering to: $CONFIGS_FILTER"
+            FILTER_ARG=(--filter "$CONFIGS_FILTER")
+          fi
+          MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \
+            scripts/ci/slurm/nightly-configs.yaml --runner gb200 "${FILTER_ARG[@]}")
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+
+  # ---------------------------------------------------------------------------
+  # When pr_number or sglang_branch is provided, build an ARM64 (GB200) image
+  # from that ref and push it to Docker Hub under lmsysorg/sglang-staging.
+  # Uses refs/pull/<N>/head for PRs so fork PRs work without cross-repo auth.
+  # Old staging tags are pruned by `cleanup-image` at the end of the run.
+  # Skipped on nightly (cron) runs and manual runs with neither pr_number nor
+  # sglang_branch.
+  # ---------------------------------------------------------------------------
+  build-image:
+    needs: [validate-inputs, setup]
+    if: |
+      github.repository == 'sgl-project/sglang' && github.event_name == 'workflow_dispatch'
+      && (inputs.pr_number != '' || inputs.sglang_branch != '')
+    runs-on: arm-docker-build-node
+    outputs:
+      image_ref: ${{ steps.build.outputs.image_ref }}
+      image_tag: ${{ steps.build.outputs.image_tag }}
+    steps:
+      # Self-hosted runners retain the workspace across jobs. Prior `docker buildx`
+      # runs on this node leave root-owned build artifacts (e.g. sgl-kernel/build/)
+      # that actions/checkout cannot remove, causing EACCES on rmdir. Wipe them
+      # via a throwaway root container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # PRs (including fork PRs) resolve via refs/pull/<N>/head on upstream.
+          # Otherwise fall back to the branch name on sgl-project/sglang.
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || inputs.sglang_branch }}
+
+      - name: Verify checkout
+        env:
+          PR_NUMBER: ${{ inputs.pr_number }}
+          BRANCH: ${{ inputs.sglang_branch }}
+        run: |
+          SHA=$(git rev-parse HEAD)
+          echo "Commit SHA: $SHA"
+          echo "Author:     $(git log -1 --format='%an <%ae>')"
+          echo "Date:       $(git log -1 --format='%aI')"
+          echo "Subject:    $(git log -1 --format='%s')"
+          echo ""
+          if [ -n "$PR_NUMBER" ]; then
+            echo "Cross-check: https://github.com/sgl-project/sglang/pull/${PR_NUMBER}/commits"
+          else
+            echo "Cross-check: https://github.com/sgl-project/sglang/commits/${BRANCH}"
+          fi
+          echo "Commit URL:  https://github.com/sgl-project/sglang/commit/${SHA}"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push ARM64 image
+        id: build
+        run: |
+          if [ -n "${{ inputs.pr_number }}" ]; then
+            TAG_STUB="pr-${{ inputs.pr_number }}"
+            SOURCE_DESC="PR #${{ inputs.pr_number }}"
+          else
+            TAG_STUB=$(echo "${{ inputs.sglang_branch }}" | tr '/' '-' | tr -cd '[:alnum:]._-')
+            SOURCE_DESC="branch ${{ inputs.sglang_branch }}"
+          fi
+          # run_attempt disambiguates "Re-run jobs" so the squash filename
+          # (derived from the image URL) doesn't collide with a stale one.
+          TAG="${TAG_STUB}-${{ github.run_id }}-${{ github.run_attempt }}"
+          IMAGE_REF="${CI_IMAGE_REPO}:${TAG}"
+          echo "Building ${IMAGE_REF} from ${SOURCE_DESC}"
+
+          docker buildx build \
+            --platform linux/arm64 \
+            --output type=image,name=${IMAGE_REF},push=true \
+            --target framework_final \
+            -f docker/Dockerfile \
+            --build-arg CUDA_VERSION=13.0.1 \
+            --build-arg BUILD_TYPE=all \
+            --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \
+            --build-arg GRACE_BLACKWELL=1 \
+            --build-arg BRANCH_TYPE=local \
+            --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
+            --no-cache \
+            .
+
+          echo "image_ref=${IMAGE_REF}" >> $GITHUB_OUTPUT
+          echo "image_tag=${TAG}" >> $GITHUB_OUTPUT
+
+  # ---------------------------------------------------------------------------
+  # Import Docker images to Lustre squash files once before all benchmark jobs.
+  # This avoids parallel jobs racing to enroot import the same image.
+  # When build-image ran, we import the freshly built Docker Hub staging image
+  # (lmsysorg/sglang-staging is public → no auth needed for enroot pull).
+  # Otherwise we use the `image` input (or its default public nightly image).
+  # ---------------------------------------------------------------------------
+  prepare-image:
+    needs: [setup, build-image]
+    if: |
+      always() && github.repository == 'sgl-project/sglang'
+      && needs.setup.result == 'success'
+      && (needs.build-image.result == 'success' || needs.build-image.result == 'skipped')
+    environment: ${{ github.event_name == 'workflow_dispatch' && 'gb200-ci' || '' }}
+    runs-on: 72-gpu-gb200
+    outputs:
+      squash_file: ${{ steps.import.outputs.squash_file }}
+      nginx_squash_file: ${{ steps.import.outputs.nginx_squash_file }}
+      image: ${{ steps.resolve.outputs.image }}
+    env:
+      NGINX_IMAGE: nginx:1.27.4
+    steps:
+      - name: Resolve image to import
+        id: resolve
+        run: |
+          BUILT_IMAGE="${{ needs.build-image.outputs.image_ref }}"
+          if [ -n "$BUILT_IMAGE" ]; then
+            echo "Using freshly built image: $BUILT_IMAGE"
+            echo "image=$BUILT_IMAGE" >> $GITHUB_OUTPUT
+          else
+            IMAGE="${{ inputs.image || 'lmsysorg/sglang:dev-cu13' }}"
+            echo "Using pre-existing image: $IMAGE"
+            echo "image=$IMAGE" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Import Docker images to Lustre
+        id: import
+        env:
+          IMAGE: ${{ steps.resolve.outputs.image }}
+        run: |
+          SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g')_$(date +%Y%m%d).sqsh"
+          NGINX_SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+
+          if [ -f "$SQUASH_FILE" ]; then
+            echo "Squash file already exists, skipping import: $SQUASH_FILE"
+          else
+            enroot import -o "$SQUASH_FILE" "docker://$IMAGE"
+          fi
+
+          if [ -f "$NGINX_SQUASH_FILE" ]; then
+            echo "Nginx squash file already exists, skipping import: $NGINX_SQUASH_FILE"
+          else
+            enroot import -o "$NGINX_SQUASH_FILE" "docker://$NGINX_IMAGE"
+          fi
+
+          echo "squash_file=$SQUASH_FILE" >> $GITHUB_OUTPUT
+          echo "nginx_squash_file=$NGINX_SQUASH_FILE" >> $GITHUB_OUTPUT
+
+  nightly-gb200-benchmark:
+    needs: [setup, prepare-image]
+    # Use always() + explicit success checks so a skipped transitive upstream
+    # (e.g. build-image when neither pr_number nor sglang_branch is set) does
+    # not propagate a skip to this job. Direct deps must still have succeeded.
+    if: |
+      always() && github.repository == 'sgl-project/sglang'
+      && needs.setup.result == 'success'
+      && needs.prepare-image.result == 'success'
+    runs-on: 72-gpu-gb200
+    strategy:
+      fail-fast: false
+      matrix:
+        config: ${{ fromJson(needs.setup.outputs.matrix) }}
+    env:
+      FRAMEWORK: dynamo-sglang
+      MODEL: ${{ matrix.config.model }}
+      MODEL_PREFIX: ${{ matrix.config.model_prefix }}
+      PRECISION: ${{ matrix.config.precision }}
+      ISL: ${{ matrix.config.isl }}
+      OSL: ${{ matrix.config.osl }}
+      CONFIG_FILE: ${{ matrix.config.config_file }}
+      RESULT_FILENAME: gb200-${{ matrix.config.name }}
+      MATRIX_CONFIG_NAME: ${{ matrix.config.name }}
+      SQUASH_FILE: ${{ needs.prepare-image.outputs.squash_file }}
+      NGINX_SQUASH_FILE: ${{ needs.prepare-image.outputs.nginx_squash_file }}
+      # S3 log-upload credentials — consumed by srt-slurm's postprocess stage
+      # to upload /logs after each Slurm job; prefix derived in launch_gb200.sh.
+      AWS_ACCESS_KEY_ID: ${{ secrets.NV_S3_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.NV_S3_SECRET_ACCESS_KEY }}
+      S3_BUCKET: ${{ secrets.NV_S3_BUCKET }}
+      S3_ENDPOINT_URL: ${{ secrets.NV_S3_ENDPOINT_URL }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Clean up prior Slurm jobs from this runner
+        continue-on-error: true
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: |
+          STALE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}')
+          if [ -n "$STALE_JOBS" ]; then
+            echo "Cancelling stale jobs: $STALE_JOBS"
+            scancel $STALE_JOBS
+          fi
+
+      - name: Launch GB200 benchmark via srt-slurm
+        timeout-minutes: 360
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: bash scripts/ci/slurm/launch_gb200.sh
+
+      - name: Process results
+        if: always()
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: |
+          pip install tabulate pyyaml -q
+          SRT_REPO_DIR="/mnt/lustre01/users-public/sglang-ci/workspace/${RUNNER_NAME}/srt-slurm"
+          for result_file in ${{ github.workspace }}/${RESULT_FILENAME}_*.json; do
+            [ -f "$result_file" ] || continue
+            basename_file=$(basename "$result_file")
+            ctx=$(echo "$basename_file" | sed -n 's/.*_ctx_\([0-9]*\)_gen.*/\1/p')
+            gen=$(echo "$basename_file" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
+            [ -n "$ctx" ] && [ -n "$gen" ] || continue
+            RESULT_FILENAME="${result_file%.json}" PREFILL_GPUS="$ctx" DECODE_GPUS="$gen" \
+              RECIPE_FILE="$SRT_REPO_DIR/$CONFIG_FILE" \
+              python3 scripts/ci/slurm/process_result.py
+          done
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gb200-${{ matrix.config.name }}-${{ github.run_id }}
+          path: |
+            ${{ github.workspace }}/*.json
+            ${{ github.workspace }}/multinode_server_logs.tar.gz
+          retention-days: 30
+          if-no-files-found: warn
+
+      - name: Analyze logs with AI on failure
+        if: failure()
+        continue-on-error: true
+        env:
+          MODAL_TOKEN_ID: ${{ secrets.NV_MODAL_TOKEN_ID }}
+          MODAL_TOKEN_SECRET: ${{ secrets.NV_MODAL_TOKEN_SECRET }}
+        run: |
+          TARBALL="${{ github.workspace }}/multinode_server_logs.tar.gz"
+          if [ -f "$TARBALL" ]; then
+            uv run --with modal python scripts/ci/slurm/analyze_logs_with_modal.py \
+              --tarball "$TARBALL" \
+              --job-id "${{ matrix.config.name }}-${{ github.run_id }}" \
+              --output "${{ github.workspace }}/ai_analysis.md"
+            if [ -f "${{ github.workspace }}/ai_analysis.md" ]; then
+              echo "## AI Log Analysis" >> $GITHUB_STEP_SUMMARY
+              cat "${{ github.workspace }}/ai_analysis.md" >> $GITHUB_STEP_SUMMARY
+            fi
+          else
+            echo "No log tarball found, skipping analysis"
+          fi
+
+      - name: Upload AI analysis to S3
+        if: failure()
+        continue-on-error: true
+        env:
+          ISL: ${{ matrix.config.isl }}
+          OSL: ${{ matrix.config.osl }}
+        run: |
+          ANALYSIS="${{ github.workspace }}/ai_analysis.md"
+          [ -f "$ANALYSIS" ] || { echo "no ai_analysis.md, skipping"; exit 0; }
+          case "${{ github.event_name }}" in
+            schedule)          TRIGGER=cron ;;
+            workflow_dispatch) TRIGGER=manual ;;
+            *)                 TRIGGER="${{ github.event_name }}" ;;
+          esac
+          fmt() { if [ $(( $1 % 1024 )) -eq 0 ]; then echo "$(( $1 / 1024 ))k"; else echo "$1"; fi; }
+          SEQ_LEN="$(fmt "$ISL")$(fmt "$OSL")"
+          KEY="${TRIGGER}/${{ github.run_id }}-${{ github.run_attempt }}/${SEQ_LEN}/${{ matrix.config.name }}/ai_analysis.md"
+          aws --endpoint-url "$S3_ENDPOINT_URL" s3 cp "$ANALYSIS" "s3://${S3_BUCKET}/${KEY}"
+          echo "uploaded to s3://${S3_BUCKET}/${KEY}"
+
+      - name: Clean up Slurm jobs on failure/cancel
+        if: failure() || cancelled()
+        continue-on-error: true
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: |
+          ACTIVE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}')
+          if [ -n "$ACTIVE_JOBS" ]; then
+            echo "Cancelling jobs: $ACTIVE_JOBS"
+            scancel $ACTIVE_JOBS
+          fi
+
+  collect-results:
+    needs: nightly-gb200-benchmark
+    if: github.repository == 'sgl-project/sglang' && always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: results/
+          pattern: gb200-*
+
+      - name: Print summary
+        run: |
+          pip install tabulate -q
+          python3 scripts/ci/slurm/summarize.py results/ >> $GITHUB_STEP_SUMMARY
+
+  # ---------------------------------------------------------------------------
+  # Prune old tags in the staging repo, keeping only the most recent N. Mirrors
+  # the pattern used by release-docker-dev.yml. Runs after benchmarks so the
+  # freshly built image (whose sqsh is already on Lustre) becomes a regular
+  # aged-out tag over time. No-op when the repo has ≤ CI_IMAGE_KEEP_TAGS tags.
+  # ---------------------------------------------------------------------------
+  cleanup-image:
+    needs: [build-image, nightly-gb200-benchmark]
+    if: always() && needs.build-image.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Prune old staging tags on Docker Hub
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        run: |
+          TOKEN=$(curl -s -H "Content-Type: application/json" \
+            -X POST -d "{\"username\": \"${DOCKERHUB_USERNAME}\", \"password\": \"${DOCKERHUB_TOKEN}\"}" \
+            https://hub.docker.com/v2/users/login/ | jq -r .token)
+          if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
+            echo "::error::Docker Hub login failed"
+            exit 1
+          fi
+
+          TAGS_RESPONSE=$(curl -s -H "Authorization: JWT $TOKEN" \
+            "https://hub.docker.com/v2/repositories/${CI_IMAGE_REPO}/tags/?page_size=100")
+
+          # Sort tags by last_updated (newest first), keep names only.
+          TAGS=$(echo "$TAGS_RESPONSE" | jq -r \
+            '.results[] | "\(.last_updated)|\(.name)"' \
+            | sort -r | cut -d'|' -f2)
+
+          TAG_COUNT=$(echo "$TAGS" | grep -c . || true)
+          if [ "$TAG_COUNT" -gt "$CI_IMAGE_KEEP_TAGS" ]; then
+            echo "Found $TAG_COUNT tags in ${CI_IMAGE_REPO}, keeping $CI_IMAGE_KEEP_TAGS most recent"
+            TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +$((CI_IMAGE_KEEP_TAGS + 1)))
+            for tag in $TAGS_TO_DELETE; do
+              echo "Deleting ${CI_IMAGE_REPO}:${tag}"
+              curl -s -X DELETE -H "Authorization: JWT $TOKEN" \
+                "https://hub.docker.com/v2/repositories/${CI_IMAGE_REPO}/tags/${tag}/"
+            done
+          else
+            echo "Only $TAG_COUNT tags in ${CI_IMAGE_REPO}, no cleanup needed"
+          fi

.github/workflows/nightly-amd-mi355x-disagg.yml

@@ -0,0 +1,250 @@
+name: Nightly Test (AMD MI355X 2N 1P1D Disagg)
+
+# A self-hosted runner on the amd-sglang login node salloc's two MI355X nodes
+# and runs the Docker harness (prefill + decode over MORI, a standalone load
+# balancer, then an sglang.bench_serving concurrency sweep) via
+# scripts/ci/slurm/launch_mi355x.sh.
+on:
+  schedule:
+    - cron: '17 3 * * *'  # 03:17 UTC
+  workflow_dispatch:
+    inputs:
+      image:
+        description: 'Optional. ROCm SGLang Docker image to benchmark. Leave empty for the recipe default.'
+        required: false
+        default: ''
+      configs:
+        description: 'Optional. Comma-separated names to run only a subset. Leave empty to run all. Available names are listed in the setup job log.'
+        required: false
+        default: ''
+
+concurrency:
+  group: nightly-test-mi355x
+  cancel-in-progress: false
+
+env:
+  SGLANG_IS_IN_CI: true
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Reads scripts/ci/slurm/nightly-configs.yaml and generates one matrix entry
+  # per recipe YAML for runner=mi355x. To add/remove configs, edit
+  # nightly-configs.yaml only.
+  # ---------------------------------------------------------------------------
+  setup:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+      image: ${{ steps.image.outputs.image }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Resolve latest MI35x nightly image
+        id: image
+        # Track the newest published image so the nightly actually catches
+        # regressions instead of re-testing one pinned build. We resolve on the
+        # GitHub-hosted runner (clean Docker Hub access, no rate limits) and pass
+        # the tag to the self-hosted runner. Scope is deliberately tight: only
+        # the clean main MI35x ROCm 7.2 line
+        # `lmsysorg/sglang-rocm:v*-rocm720-mi35x-YYYYMMDD` (excludes -test-,
+        # miles-, mori-, vllm-, rocm7_14 variants). On any failure we emit an
+        # empty string and the launcher falls back to the recipe's pinned
+        # `runtime.image`.
+        run: |
+          set -o pipefail
+          RESOLVED=$(curl -fsSL "https://hub.docker.com/v2/repositories/lmsysorg/sglang-rocm/tags?page_size=100&ordering=last_updated" \
+            | python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm720-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true)
+          if [ -n "$RESOLVED" ]; then
+            echo "Resolved latest MI35x image: lmsysorg/sglang-rocm:$RESOLVED"
+            echo "image=lmsysorg/sglang-rocm:$RESOLVED" >> $GITHUB_OUTPUT
+          else
+            echo "Could not resolve a latest image; launcher will use recipe default."
+            echo "image=" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Generate benchmark matrix
+        id: generate
+        env:
+          CONFIGS_FILTER: ${{ inputs.configs }}
+        run: |
+          pip install pyyaml -q
+
+          ALL_MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \
+            scripts/ci/slurm/nightly-configs.yaml --runner mi355x)
+          echo "Available config names for runner mi355x:"
+          echo "$ALL_MATRIX" | python3 -c "import json,sys; [print(f'  - {e[\"name\"]}') for e in json.load(sys.stdin)]"
+
+          FILTER_ARG=()
+          if [ -n "$CONFIGS_FILTER" ]; then
+            echo ""
+            echo "Filtering to: $CONFIGS_FILTER"
+            FILTER_ARG=(--filter "$CONFIGS_FILTER")
+          fi
+          MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \
+            scripts/ci/slurm/nightly-configs.yaml --runner mi355x "${FILTER_ARG[@]}")
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+
+  nightly-mi355x-benchmark:
+    needs: setup
+    if: |
+      always() && github.repository == 'sgl-project/sglang'
+      && needs.setup.result == 'success'
+    runs-on: linux-sglang-mi35x-di
+    strategy:
+      fail-fast: false
+      matrix:
+        config: ${{ fromJson(needs.setup.outputs.matrix) }}
+    env:
+      FRAMEWORK: sglang
+      HW: mi355x
+      MODEL: ${{ matrix.config.model }}
+      MODEL_PREFIX: ${{ matrix.config.model_prefix }}
+      PRECISION: ${{ matrix.config.precision }}
+      ISL: ${{ matrix.config.isl }}
+      OSL: ${{ matrix.config.osl }}
+      CONFIG_FILE: ${{ matrix.config.config_file }}
+      RESULT_FILENAME: mi355x-${{ matrix.config.name }}
+      MATRIX_CONFIG_NAME: ${{ matrix.config.name }}
+      # NOTE: RUNNER_NAME is a built-in default env var on every runner, so the
+      # launch + cleanup steps read it directly. Do NOT set it here from
+      # ${{ runner.name }} -- the `runner` context is not available in job-level
+      # env and makes the whole workflow file invalid.
+      # Shared-NFS HuggingFace cache dir; the launcher resolves the live snapshot
+      # via refs/main. Per-model value comes from nightly-configs.yaml model_path.
+      MODEL_PATH: ${{ matrix.config.model_path }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Clean up prior Slurm jobs from this runner
+        continue-on-error: true
+        run: |
+          # launch_mi355x.sh names the allocation
+          #   mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>
+          # Here we clear THIS runner's leftovers from a crashed previous run, so
+          # we match the RUNNER_NAME prefix (older runs have a different run id).
+          # Never a blanket `squeue --me`, which would kill a concurrent leg.
+          # %200j: squeue truncates the job name (%j) by default -- widen it so
+          # the grep sees the full name.
+          if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
+          STALE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true)
+          if [ -n "$STALE_JOBS" ]; then
+            echo "Cancelling stale jobs for ${RUNNER_NAME}: $STALE_JOBS"
+            scancel $STALE_JOBS
+          fi
+
+      - name: Launch MI355X 2N 1P1D benchmark
+        timeout-minutes: 180
+        env:
+          # Manual dispatch input wins; otherwise use the latest image resolved
+          # by the setup job; otherwise the launcher falls back to the recipe default.
+          IMAGE_OVERRIDE: ${{ inputs.image != '' && inputs.image || needs.setup.outputs.image }}
+          # Keep the scheduler off mia1-p01-g20: its ionic RDMA driver ABI
+          # mismatches the container, so MORI reports "no active RDMA device"
+          # and the disagg server dies on init. Remove once the node is fixed.
+          SLURM_EXCLUDE: mia1-p01-g20
+        run: bash scripts/ci/slurm/launch_mi355x.sh
+
+      - name: Pack logs
+        if: always()
+        run: |
+          LOG_DIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}"
+          LOG_BUNDLE="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_logs.tar.gz"
+          if [ ! -d "$LOG_DIR" ]; then
+            echo "WARN: log directory not found: $LOG_DIR"
+            exit 0
+          fi
+
+          mapfile -d '' LOG_FILES < <(
+            find "$LOG_DIR" -maxdepth 1 -type f \
+              \( -name '*.log' -o -name 'server_exit_*' -o -name 'bench_exit' \) \
+              -printf '%P\0' | sort -z
+          )
+          if [ "${#LOG_FILES[@]}" -eq 0 ]; then
+            echo "WARN: no log files found in $LOG_DIR"
+            exit 0
+          fi
+
+          STAGING_DIR="$(mktemp -d)"
+          trap 'rm -rf "$STAGING_DIR"' EXIT
+          for log_file in "${LOG_FILES[@]}"; do
+            cp -a "$LOG_DIR/$log_file" "$STAGING_DIR/$log_file" || true
+          done
+          tar czf "$LOG_BUNDLE" -C "$STAGING_DIR" .
+          echo "Packed ${#LOG_FILES[@]} log file(s) -> $LOG_BUNDLE"
+
+      - name: Process results
+        if: always()
+        run: |
+          pip install tabulate pyyaml -q
+          for result_file in ${{ github.workspace }}/${RESULT_FILENAME}_*.json; do
+            [ -f "$result_file" ] || continue
+            basename_file=$(basename "$result_file")
+            ctx=$(echo "$basename_file" | sed -n 's/.*_ctx_\([0-9]*\)_gen.*/\1/p')
+            gen=$(echo "$basename_file" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
+            [ -n "$ctx" ] && [ -n "$gen" ] || continue
+            RESULT_FILENAME="${result_file%.json}" PREFILL_GPUS="$ctx" DECODE_GPUS="$gen" \
+              RECIPE_FILE="${{ github.workspace }}/$CONFIG_FILE" \
+              python3 scripts/ci/slurm/process_result.py
+          done
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: mi355x-${{ matrix.config.name }}-${{ github.run_id }}
+          path: |
+            ${{ github.workspace }}/*.json
+            ${{ github.workspace }}/*_logs.tar.gz
+          retention-days: 30
+          if-no-files-found: warn
+
+      - name: Clean up Slurm jobs on failure/cancel
+        if: failure() || cancelled()
+        continue-on-error: true
+        run: |
+          # Cancel only THIS leg's allocation -- match the full job name
+          # mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>. GITHUB_RUN_ID +
+          # config are unique per matrix leg, so a concurrent leg is never hit.
+          if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
+          JOB_TAG="mi355x-ci-${RUNNER_NAME}-${GITHUB_RUN_ID}-${MATRIX_CONFIG_NAME}"
+          ACTIVE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true)
+          if [ -n "$ACTIVE_JOBS" ]; then
+            echo "Cancelling jobs for ${JOB_TAG}: $ACTIVE_JOBS"
+            scancel $ACTIVE_JOBS
+          fi
+
+      - name: Clean up MI355X scratch
+        if: always()
+        continue-on-error: true
+        run: |
+          LOG_DIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}"
+          if [ -d "$LOG_DIR" ]; then
+            echo "Removing MI355X scratch directory: $LOG_DIR"
+            rm -rf "$LOG_DIR"
+          fi
+
+  collect-results:
+    needs: nightly-mi355x-benchmark
+    if: github.repository == 'sgl-project/sglang' && always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: results/
+          pattern: mi355x-*
+
+      - name: Print summary
+        run: |
+          pip install tabulate -q
+          python3 scripts/ci/slurm/summarize.py results/ >> $GITHUB_STEP_SUMMARY

.github/workflows/nightly-experimental-sgl-router-docker.yml

@@ -0,0 +1,95 @@
+name: Nightly Experimental sgl-router Docker Image
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  IMAGE_REPO: lmsysorg/sglang-staging
+  IMAGE_TAG: experimental-router-dev
+
+jobs:
+  # Build each arch natively (no QEMU emulation) and push the image by digest.
+  # The manifest job then stitches the per-arch digests into one multi-arch
+  # tag, so `docker pull` resolves the right arch automatically.
+  build:
+    if: github.repository == 'sgl-project/sglang'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-24.04
+            arch: amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push by digest
+        run: |
+          docker buildx build . -f docker/sgl-router.Dockerfile \
+            --platform ${{ matrix.platform }} \
+            --output "type=image,name=${IMAGE_REPO},push-by-digest=true,name-canonical=true,push=true" \
+            --metadata-file /tmp/metadata.json
+          DIGEST=$(python3 -c "import json; print(json.load(open('/tmp/metadata.json'))['containerimage.digest'])")
+          echo "Pushed ${{ matrix.platform }} digest: ${DIGEST}"
+          mkdir -p /tmp/digests
+          echo "${DIGEST}" > "/tmp/digests/${{ matrix.arch }}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digest-${{ matrix.arch }}
+          path: /tmp/digests/${{ matrix.arch }}
+          retention-days: 1
+
+  create-manifest:
+    if: github.repository == 'sgl-project/sglang'
+    needs: build
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Create and push multi-arch manifest
+        run: |
+          AMD64_DIGEST=$(cat /tmp/digests/amd64)
+          ARM64_DIGEST=$(cat /tmp/digests/arm64)
+          docker buildx imagetools create \
+            -t "${IMAGE_REPO}:${IMAGE_TAG}" \
+            "${IMAGE_REPO}@${AMD64_DIGEST}" \
+            "${IMAGE_REPO}@${ARM64_DIGEST}"
+
+      - name: Inspect manifest
+        run: docker buildx imagetools inspect "${IMAGE_REPO}:${IMAGE_TAG}"

.github/workflows/nightly-link-check.yml

@@ -0,0 +1,32 @@
+name: Nightly Link Check
+
+on:
+  schedule:
+    - cron: "0 2 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: nightly-link-check-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lychee-online:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run lychee online link checks
+        uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2
+        with:
+          fail: true
+          args: >-
+            --config .github/linters/lychee-ci.toml
+            README.md
+            docs/**/*.md
+            docs/**/*.rst
+            docs/**/*.ipynb
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nightly-release-gateway.yml

@@ -0,0 +1,196 @@
+# Nightly release workflow for SGLang Model Gateway
+
+name: Nightly Release SGLang Model Gateway to PyPI
+
+on:
+  schedule:
+    # Run at 2 AM UTC every day
+    - cron: '0 2 * * *'
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  build:
+    name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos, windows]
+        target: [x86_64, aarch64]
+        manylinux: [auto]
+        include:
+          - os: ubuntu
+            platform: linux
+          - os: windows
+            ls: dir
+            target: x86_64
+            python-architecture: x64
+            interpreter: 3.9 3.10 3.11 3.12 3.13
+          - os: macos
+            target: aarch64
+            interpreter: 3.9 3.10 3.11 3.12 3.13
+          - os: ubuntu
+            platform: linux
+            target: aarch64
+          # musllinux
+          - os: ubuntu
+            platform: linux
+            target: x86_64
+            manylinux: musllinux_1_1
+          - os: ubuntu
+            platform: linux
+            target: aarch64
+            manylinux: musllinux_1_1
+        exclude:
+          - os: windows
+            target: aarch64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: sglang-repo
+
+      - name: Move sgl-model-gateway folder to root and delete sglang-repo
+        run: |
+          mv sglang-repo/sgl-model-gateway/* .
+          rm -rf sglang-repo
+          ls -alt
+        shell: bash
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          architecture: ${{ matrix.python-architecture || 'x64' }}
+
+      - name: Modify version for nightly release
+        run: |
+          # Get current version from pyproject.toml
+          CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])")
+          # Create nightly version with date: e.g., 0.2.1.dev20250128
+          NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)"
+          echo "Nightly version: $NIGHTLY_VERSION"
+
+          # Update pyproject.toml with nightly version (temporary, not committed)
+          sed -i.bak "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" bindings/python/pyproject.toml
+
+          # Verify the change
+          cat bindings/python/pyproject.toml | grep "^version"
+        shell: bash
+
+      - name: Install twine and tomli
+        run: pip install -U twine tomli
+
+      - name: Install protoc (macOS)
+        if: matrix.os == 'macos'
+        run: brew install protobuf
+
+      - name: Install protoc (Windows)
+        if: matrix.os == 'windows'
+        run: choco install protoc -y
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: bindings/python
+          target: ${{ matrix.target }}
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+          args: --release --out dist --features vendored-openssl --interpreter ${{ matrix.interpreter || '3.9 3.10 3.11 3.12 3.13 3.14' }}
+          rust-toolchain: stable
+          docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
+          before-script-linux: |
+            # Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC)
+            if command -v yum &> /dev/null; then
+              yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make
+              # Install cross-compilation toolchain for aarch64 if needed
+              if [ "${{ matrix.target }}" = "aarch64" ]; then
+                yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true
+              fi
+            elif command -v apt-get &> /dev/null; then
+              apt-get update && apt-get install -y wget unzip gcc g++ perl make
+              # Install cross-compilation toolchain for aarch64 if needed
+              if [ "${{ matrix.target }}" = "aarch64" ]; then
+                apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true
+              fi
+            fi
+            (cd /tmp && \
+             wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
+             unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
+             rm protoc-32.0-linux-x86_64.zip)
+            protoc --version
+
+      - name: List built packages
+        run: ${{ matrix.ls || 'ls -lh' }} bindings/python/dist/
+
+      - name: Check packages
+        run: twine check --strict bindings/python/dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }}
+          path: bindings/python/dist/
+
+  build-sdist:
+    name: Build SDist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: sglang-repo
+
+      - name: Move sgl-model-gateway folder to root and delete sglang-repo
+        run: |
+          mv sglang-repo/sgl-model-gateway/* .
+          rm -rf sglang-repo
+          ls -alt
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Modify version for nightly release
+        run: |
+          # Get current version from pyproject.toml
+          CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('bindings/python/pyproject.toml', 'rb'))['project']['version'])")
+          # Create nightly version with date: e.g., 0.2.1.dev20250128
+          NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)"
+          echo "Nightly version: $NIGHTLY_VERSION"
+
+          # Update pyproject.toml with nightly version (temporary, not committed)
+          sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" bindings/python/pyproject.toml
+
+          # Verify the change
+          cat bindings/python/pyproject.toml | grep "^version"
+
+      - name: Build SDist
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: bindings/python
+          command: sdist
+          args: --out dist
+          rust-toolchain: stable
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: bindings/python/dist/*.tar.gz
+
+  upload:
+    name: Upload to TestPyPI
+    if: github.repository == 'sgl-project/sglang'  # Ensure this job only runs for the sgl-project/sglang repository
+    needs: [build, build-sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: Upload to TestPyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN_ROUTER }}
+        run: |
+          pip install twine
+          twine upload --repository testpypi dist/* --verbose

.github/workflows/nightly-test-amd-miles-rocm700.yml

@@ -0,0 +1,93 @@
+name: Nightly Test Miles (AMD ROCm 7.0)
+
+on:
+  schedule:
+    # Runs after the 12:00 UTC miles image build has pushed the dated image.
+    - cron: "30 17 * * *"
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: "Override miles image tag (e.g. miles-rocm700-mi35x-20260621); empty = newest by date"
+        required: false
+        type: string
+        default: ""
+      continue_on_error:
+        description: "Continue on error (do not fail the workflow on test failures)"
+        required: false
+        type: boolean
+        default: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  test-miles-mi35x:
+    if: github.repository == 'sgl-project/sglang'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: stage-c-2-gpu-mi350
+            runner: linux-mi35x-gpu-2
+          # Use 8-GPU MI35X pool for 4-GPU suite.
+          - suite: stage-c-4-gpu-mi350
+            runner: linux-mi35x-gpu-8
+          - suite: stage-c-8-gpu-mi350
+            runner: linux-mi35x-gpu-8
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Resolve newest miles ROCm 7.0 image
+        run: |
+          repo="rocm/sgl-dev"
+          base="miles-rocm700-mi35x"
+          tag="${{ inputs.image_tag }}"
+          if [ -z "${tag}" ]; then
+            # Walk back from today; the build job pushes a dated tag at 12:00 UTC.
+            # Probe with `manifest inspect` so we don't pull a multi-GB image just
+            # to test existence; the real pull happens in amd_ci_start_container.sh.
+            for i in $(seq 0 6); do
+              d=$(date -u -d "-${i} day" +%Y%m%d)
+              candidate="${base}-${d}"
+              if docker manifest inspect "${repo}:${candidate}" >/dev/null 2>&1; then
+                tag="${candidate}"
+                break
+              fi
+            done
+          fi
+          if [ -z "${tag}" ]; then
+            echo "::error::No ${repo}:${base}-<date> image found on Docker Hub for the last 7 days"
+            exit 1
+          fi
+          echo "MILES_IMAGE=${repo}:${tag}" >> "$GITHUB_ENV"
+          echo "Using miles image: ${repo}:${tag}"
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
+      - name: Start container from the miles image
+        run: bash scripts/ci/amd/amd_ci_start_container.sh --custom-image "${MILES_IMAGE}"
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Run miles CI suite ${{ matrix.suite }}
+        timeout-minutes: 120
+        run: |
+          touch "${GITHUB_WORKSPACE}/github_summary.md"
+          TEST_EXIT_CODE=0
+          bash scripts/ci/amd/amd_ci_exec.sh -w /root/miles \
+            -e PYTHONPATH=/root/miles:/opt/tilelang \
+            -e GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            python3 tests/ci/run_suite.py --hw rocm --suite ${{ matrix.suite }} --match-all-labels \
+            ${{ (github.event_name != 'workflow_dispatch' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
+          cat "${GITHUB_WORKSPACE}/github_summary.md" >> "$GITHUB_STEP_SUMMARY" || true
+          exit ${TEST_EXIT_CODE}

.github/workflows/nightly-test-intel.yml

@@ -0,0 +1,181 @@
+name: Nightly Test (Intel)
+
+on:
+  schedule:
+    # 22:15 IST (UTC+5:30) → 16:45 UTC daily.
+    - cron: '45 16 * * *'
+  push:
+    branches:
+      - main
+    paths:
+      - "python/sglang/version.py"
+  workflow_dispatch:
+    inputs:
+      continue_on_error:
+        description: 'Continue on error (do not fail the workflow on test failures)'
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      ref:
+        description: "Branch, tag or SHA to checkout"
+        required: false
+        type: string
+        default: ""
+      continue_on_error:
+        description: 'Continue on error (do not fail the workflow on test failures)'
+        required: false
+        type: boolean
+        default: true
+
+concurrency:
+  group: nightly-test-intel-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || inputs.ref && format('caller-{0}', github.run_id) || github.ref }}
+  cancel-in-progress: ${{ !inputs.ref && github.event_name != 'workflow_call' && github.event_name != 'workflow_dispatch' }}
+
+jobs:
+  nightly-xpu-1-gpu:
+    # Placeholder: no models currently registered to nightly-xpu-1-gpu run on
+    # intel-bmg-nightly. Add a model test to this suite to re-enable.
+    if: false
+    runs-on: intel-bmg-nightly
+    steps:
+      - name: Placeholder
+        run: echo "nightly-xpu-1-gpu has no validated models; skipping."
+
+  nightly-xpu-2-gpu:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: intel-bmg-nightly
+    env:
+      DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
+      DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
+    steps:
+      - name: Reset workspace ownership
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Start CI container (pull intel/sglang-dev:latest)
+        run: |
+          export HF_TOKEN="$(cat ~/huggingface_token.txt)"
+          bash scripts/ci/xpu/xpu_ci_start_container.sh
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: HF login + install run_suite extras
+        timeout-minutes: 10
+        run: |
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
+          docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
+
+      - name: Nightly Test (2-GPU XPU)
+        timeout-minutes: 60
+        run: |
+          touch github_summary.md
+          docker exec ci_sglang_xpu bash -c "
+            source /opt/venv/bin/activate &&
+            cd /sglang-checkout/test &&
+            GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
+          " || TEST_EXIT_CODE=$?
+          echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
+          exit ${TEST_EXIT_CODE:-0}
+
+      - name: Cleanup container
+        if: always()
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+          rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true
+          find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
+          find . -type f -name "*.pyc" -delete || true
+          docker rm -f ci_sglang_xpu || true
+          if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
+            docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
+          fi
+
+  nightly-xpu-4-gpu:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: intel-bmg-nightly
+    env:
+      DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
+      DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
+    steps:
+      - name: Reset workspace ownership
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Start CI container (pull intel/sglang-dev:latest)
+        run: |
+          export HF_TOKEN="$(cat ~/huggingface_token.txt)"
+          bash scripts/ci/xpu/xpu_ci_start_container.sh
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: HF login + install run_suite extras
+        timeout-minutes: 10
+        run: |
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
+          docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
+
+      - name: Nightly Test (4-GPU XPU)
+        timeout-minutes: 120
+        run: |
+          touch github_summary.md
+          docker exec ci_sglang_xpu bash -c "
+            source /opt/venv/bin/activate &&
+            cd /sglang-checkout/test &&
+            GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
+            python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
+          " || TEST_EXIT_CODE=$?
+          echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
+          exit ${TEST_EXIT_CODE:-0}
+
+      - name: Cleanup container
+        if: always()
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+          rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true
+          find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
+          find . -type f -name "*.pyc" -delete || true
+          docker rm -f ci_sglang_xpu || true
+          if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
+            docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
+          fi
+
+  check-all-jobs:
+    if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'workflow_dispatch')
+    needs:
+      - nightly-xpu-2-gpu
+      - nightly-xpu-4-gpu
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if any job failed
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            echo "One or more nightly test jobs failed"
+            exit 1
+          fi
+          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more nightly test jobs were cancelled"
+            exit 1
+          fi
+          echo "All nightly test jobs passed"

.github/workflows/nightly-test-musa.yml

@@ -0,0 +1,301 @@
+name: Nightly Test (MUSA)
+
+on:
+  schedule:
+    - cron: '0 16 * * *'
+  workflow_dispatch:
+    inputs:
+      job_filter:
+        description: "Select which job to run (empty/all to run all jobs)"
+        required: false
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - 'nightly-test-musa-general-kernel'
+          - 'nightly-test-musa-general-multimodal-layer'
+          - 'nightly-test-llm-server-1-gpu-musa'
+          - 'nightly-test-multimodal-server-1-gpu-musa'
+          - 'nightly-test-multimodal-server-2-gpu-musa'
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      job_filter:
+        description: 'Select which job to run (empty or "all" to run all jobs)'
+        required: false
+        type: string
+        default: 'all'
+
+concurrency:
+  group: nightly-test-musa-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+env:
+  SGLANG_IS_IN_CI: true
+
+jobs:
+  # ==================== General: kernel ====================
+  nightly-test-musa-general-kernel:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HUB_CACHE: /hf-cache/hub
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-kernel" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        timeout-minutes: 10
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 15
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run sgl-kernel unit tests (MUSA)
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 30
+        run: |
+          pytest sgl-kernel/tests/test_dsv3_router_gemm.py
+          pytest sgl-kernel/tests/test_per_token_quant_fp8.py
+          pytest sgl-kernel/tests/speculative/test_eagle_utils.py
+          pytest sgl-kernel/tests/speculative/test_ngram_utils.py
+          pytest sgl-kernel/tests/speculative/test_speculative_sampling.py
+          pytest sgl-kernel/tests/test_torch_defaults_reset.py
+
+  # ==================== General: multimodal layer ====================
+  nightly-test-musa-general-multimodal-layer:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-multimodal-layer" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run multimodal MUSA layer unit tests
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 30
+        run: |
+          pytest python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_rmsnorm.py
+          pytest python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_silu_and_mul.py
+
+  # ==================== LLM server: 1-GPU ====================
+  nightly-test-llm-server-1-gpu-musa:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-llm-server-1-gpu-musa" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run DeepSeek-V2-Lite-Chat LLM server smoke test
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 120
+        run: |
+          cd test
+          python3 run_suite.py \
+            --hw musa \
+            --suite nightly-musa-1-gpu \
+            --nightly \
+            --timeout-per-file 7200
+
+  # ==================== Multimodal: 1-GPU (split) ====================
+  nightly-test-multimodal-server-1-gpu-musa:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        part: [0, 1]
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-1-gpu-musa" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run diffusion server tests (1-GPU)
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 60
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/server/musa/run_suite.py \
+            --suite 1-gpu-musa-nightly \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions 2 \
+            --continue-on-error
+
+  # ==================== Multimodal: 2-GPU ====================
+  nightly-test-multimodal-server-2-gpu-musa:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-2-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-2-gpu-musa" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run diffusion server tests (2-GPU)
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 60
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/server/musa/run_suite.py \
+            --suite 2-gpu-musa \
+            --continue-on-error
+
+  # ==================== finish ====================
+  nightly-test-musa-finish:
+    needs:
+      [
+        nightly-test-musa-general-kernel,
+        nightly-test-musa-general-multimodal-layer,
+        nightly-test-llm-server-1-gpu-musa,
+        nightly-test-multimodal-server-1-gpu-musa,
+        nightly-test-multimodal-server-2-gpu-musa,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          json_needs='${{ toJson(needs) }}'
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+
+          for job in $job_names; do
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+            echo "$job: $result"
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "Nightly failed."
+              exit 1
+            fi
+          done
+          echo "All jobs completed successfully."

.github/workflows/nightly-test-npu-e2e-multi-node.yml

@@ -0,0 +1,232 @@
+name: 'Nightly Test (NPU) Multi Node Template'
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        required: false
+        type: string
+        default: linux-amd64-cpu-8
+      test_type:
+        required: false
+        type: string
+        default: perf
+        description: perf or accuracy
+      test_config_name:
+        required: true
+        type: string
+        description: test config name
+      prefill_size:
+        required: false
+        type: number
+        default: 0
+        description: number of prefill nodes for pd-separation
+      decode_size:
+        required: false
+        type: number
+        default: 0
+        description: number of decode nodes for pd-separation
+      router_size:
+        required: false
+        type: number
+        default: 0
+        description: number of router nodes for pd-separation
+      node_size:
+        required: false
+        type: number
+        default: 0
+        description: number of nodes for pd-mix
+      test_case:
+        required: true
+        type: string
+        description: path of test case file
+      image:
+        required: false
+        type: string
+        description: image for pods
+        default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann8.5.0-a3"
+      install_sglang_from_source:
+        required: false
+        type: boolean
+        default: true
+        description: use sglang from source code or from docker image
+      prefill_decode_deployment:
+        required: true
+        type: string
+        description: "The deployment of prefill and decode nodes. ['separation', 'mix']"
+      transformers_version:
+        required: false
+        type: string
+        default: ""
+        description: "The transformers version number for running sglang. Use default version in image if keep empty."
+
+concurrency:
+  group: ascend-nightly-multi-node-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
+  cancel-in-progress: true
+
+jobs:
+  e2e:
+    name: ${{ inputs.test_config_name }}
+    runs-on: ${{ inputs.runner }}
+    container:
+      image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/sglang:main-x86
+      env:
+        KUBECONFIG: /root/.cache/.cache/kb.yaml
+        KUBECTL: /root/.cache/.cache/kubectl
+        NAMESPACE: sgl-project
+        KUBE_JOB_NAME: ascend-sglang-${{ inputs.test_type }}-test
+        SGLANG_IS_IN_CI: true
+        ASCEND_E2E_TEST_CONFIG_PATH: python/sglang/test/ascend/e2e
+        SGLANG_USE_MODELSCOPE: true
+        HF_ENDPOINT: https://hf-mirror.com
+        TRANSFORMERS_VERBOSITY: "error"
+        GDN_ATTN_BACKEND_TRITON: 1
+    steps:
+      - name: Install dependencies
+        run: |
+          pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple jinja2
+          pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple psutil
+          # Install kubernetes
+          pip3 install -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple kubernetes
+          cp $KUBECTL /usr/local/sbin/
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Prepare code for testing
+        run: |
+          # copy source code to shared-disk
+          current_path=$(pwd)
+          target_path=/root/.cache/tests/sglang
+          rm -rf ${target_path}
+          mkdir -p ${target_path}
+          cp -r ${current_path}/* ${target_path}/
+
+      - name: Clear resources
+        run: |
+          cd ${ASCEND_E2E_TEST_CONFIG_PATH}
+          kubectl delete -f ./k8s_multi_pd_*.yaml --ignore-not-found=true || true
+
+          pod_name_prefix="${KUBE_JOB_NAME}-"
+          echo "kube name space: $NAMESPACE, pod name prefix: ${pod_name_prefix}"
+          while true; do
+            if kubectl get po -A -n $NAMESPACE | grep -q "${pod_name_prefix}"; then
+              echo "Found exist sglang job, sleeping for 30 seconds..."
+              sleep 30
+              kubectl get pods | grep "${pod_name_prefix}" | awk '{print $1}' | xargs kubectl delete pod -n $NAMESPACE || true
+            else
+              echo "No sglang job exist, start test case..."
+              break
+            fi
+          done
+
+      - name: Run test
+        timeout-minutes: 300
+        run: |
+          # sglang_source_relative_path is shared-disk path
+          sglang_source_relative_path=tests/sglang
+          sglang_source_path=/root/.cache/${sglang_source_relative_path}
+          echo "Source code path: ${sglang_source_path}"
+
+          test_case="${{ inputs.test_case }}"
+          if [ ! -f "${sglang_source_path}/${test_case}" ]; then
+            echo "The testcase does not exit: ${sglang_source_path}/${test_case}"
+            exit 1
+          fi
+
+          # prepare for output data
+          current_date=$(date +%Y%m%d)
+          test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
+          mkdir -p ${test_data_output_path}
+          tc_name=${test_case##*/}
+          tc_name=${tc_name%.*}
+          metrics_data_path=${test_data_output_path}/${tc_name}
+          mkdir -p ${metrics_data_path}
+          echo "Metrics file path: ${metrics_data_path}"
+
+          prefill_decode_deployment="${{ inputs.prefill_decode_deployment }}"
+          kube_job_type=""
+          if [ "$prefill_decode_deployment" = "separation" ];then
+            kube_job_type=multi-pd-separation
+          elif [ "$prefill_decode_deployment" = "mix" ];then
+            kube_job_type=multi-pd-mix
+          else
+            echo "Unsupported deployment type: ${prefill_decode_deployment}"
+            exit 1
+          fi
+
+          transformers_version="${{ inputs.transformers_version }}"
+          cd ${ASCEND_E2E_TEST_CONFIG_PATH}
+          CMD="python3 -u run_npu_e2e_test.py \
+            --env ci \
+            --image ${{ inputs.image }} \
+            --sglang-source-relative-path ${sglang_source_relative_path} \
+            --metrics-data-file ${metrics_data_path} \
+            --test-case ${test_case} \
+            --kube-name-space ${NAMESPACE} \
+            --kube-job-type ${kube_job_type} \
+            --kube-job-name-prefix ${KUBE_JOB_NAME}"
+
+          if [ "$prefill_decode_deployment" = "separation" ];then
+            CMD="${CMD} \
+              --prefill-size ${{ inputs.prefill_size }} \
+              --decode-size ${{ inputs.decode_size }} \
+              --router-size ${{ inputs.router_size }}"
+          elif [ "$prefill_decode_deployment" = "mix" ];then
+            CMD="${CMD} --node-size ${{ inputs.node_size }}"
+          else
+            echo "Unsupported deployment type: ${prefill_decode_deployment}"
+            exit 1
+          fi
+
+          if [ "$SGLANG_IS_IN_CI" = "true" ] || [ "$SGLANG_IS_IN_CI" = "True" ];then
+            echo "Run test in ci."
+            CMD="${CMD} --sglang-is-in-ci"
+          fi
+
+          install_sglang_from_source="${{ inputs.install_sglang_from_source }}"
+          if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then
+            echo "Install sglang from source."
+            CMD="${CMD} --install-sglang-from-source"
+            commit_id=${{ github.sha }}
+            echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
+          else
+            echo "Use sglang from image: ${{ inputs.image }}"
+          fi
+
+          if [ "$transformers_version" != "" ];then
+            CMD="${CMD} --transformers-version ${transformers_version}"
+          else
+            echo "Use default transformers version in image."
+          fi
+
+          echo "Run command: ${CMD}"
+          eval "${CMD}"
+
+      - name: Upload metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-${{ inputs.test_config_name }}
+          path: /tmp/metrics.json
+          retention-days: 7
+
+      - name: Post process
+        if: always()
+        run: |
+          cd ${ASCEND_E2E_TEST_CONFIG_PATH}
+          kubectl get pods -n $NAMESPACE | grep $KUBE_JOB_NAME
+          kubectl delete -f ./k8s_multi_pd_*.yaml --ignore-not-found=true || true
+
+          pod_name_prefix="${KUBE_JOB_NAME}-"
+          echo "kube name space: $NAMESPACE, pod name prefix: ${pod_name_prefix}"
+          while true; do
+            if kubectl get po -A -n $NAMESPACE | grep -q "${pod_name_prefix}"; then
+              echo "Found exist sglang job, sleeping for 30 seconds..."
+              sleep 30
+              kubectl get pods | grep "${pod_name_prefix}" | awk '{print $1}' | xargs kubectl delete pod -n $NAMESPACE || true
+            else
+              echo "No sglang job exist, start test case..."
+              break
+            fi
+          done

.github/workflows/nightly-test-npu-e2e-single-node.yml

@@ -0,0 +1,255 @@
+name: 'Nightly Test (NPU) Single Node Template'
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        required: true
+        type: string
+        default: linux-aarch64-a3-16
+      test_type:
+        required: false
+        type: string
+        default: perf
+        description: perf or accuracy
+      test_config_name:
+        required: true
+        type: string
+        description: test config name
+      test_case:
+        required: true
+        type: string
+        description: path of test case file
+      image:
+        required: false
+        type: string
+        description: image for pods
+        default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3"
+      install_sglang_from_source:
+        required: false
+        type: boolean
+        default: false
+        description: use sglang from source code or from docker image
+      install_sglang_deps:
+        required: false
+        type: boolean
+        default: false
+        description: install sglang dependencies (e.g. PyTorch, CANN packages) when using source installation
+      device_type_for_deps:
+        required: false
+        type: string
+        default: 'a3'
+        description: device type for dependency installation (a3 or 910b)
+      transformers_version:
+        required: false
+        type: string
+        default: ""
+        description: "The transformers version number for running sglang. Use default version in image if keep empty."
+
+concurrency:
+  group: ascend-nightly-e2e-singlenode-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.test_config_name }}
+  cancel-in-progress: true
+
+jobs:
+  e2e:
+    name: ${{ inputs.test_config_name }}
+    runs-on: ${{ inputs.runner }}
+    container:
+      image: ${{ inputs.image }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check npu info
+        run: |
+          npu-smi info
+
+      - name: Install sglang dependencies
+        if: ${{ inputs.install_sglang_deps == true }}
+        shell: bash
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+          bash scripts/ci/npu/npu_ci_install_dependency.sh ${{ inputs.device_type_for_deps }}
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+
+      - name: Run test
+        timeout-minutes: 300
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          HF_ENDPOINT: https://hf-mirror.com
+          SGLANG_IS_IN_CI: true
+          TRANSFORMERS_VERBOSITY: "error"
+          GDN_ATTN_BACKEND_TRITON: 1
+          SGLANG_TEST_METRICS_OUTPUT: /root/.cache/tests/output/metrics/metrics
+        shell: bash
+        run: |
+          sglang_source_path=$(pwd)
+          echo "Source code path: ${sglang_source_path}"
+          ln -sf ${sglang_source_path} /root/sglang
+
+          test_case=${{ inputs.test_case }}
+          if [ ! -f "${sglang_source_path}/${test_case}" ]; then
+            echo "The testcase does not exit: ${sglang_source_path}/${test_case}"
+            exit 1
+          fi
+
+          # prepare for output data
+          current_date=$(date +%Y%m%d)
+          test_data_output_path=/root/.cache/tests/output/${{ inputs.test_type }}/${current_date}
+          mkdir -p ${test_data_output_path}
+          tc_name=${test_case##*/}
+          tc_name=${tc_name%.*}
+          export METRICS_DATA_FILE=${test_data_output_path}/${tc_name}
+          mkdir -p ${METRICS_DATA_FILE}
+          echo "Metrics file path: ${METRICS_DATA_FILE}"
+
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
+
+          export TRANSFORMERS_VERSION_FOR_SGLANG="${{ inputs.transformers_version }}"
+          PYTHON_FOR_SGLANG="python"
+          PIP_FOR_SGLANG="pip"
+          if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
+            echo "===== Install transformers for sglang - Begin ====="
+            TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
+            if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
+              echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
+              echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
+              pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+            else
+              echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
+              TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
+              mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
+              cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
+              pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
+            fi
+            echo "===== Install transformers for sglang in virtual env - End ====="
+          fi
+          echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
+
+          echo "scaling_governor performance num: \
+            $(cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | grep performance | wc -l)"
+          echo "swappiness: $(cat /proc/sys/vm/swappiness)"
+          echo "numa_balancing: $(cat /proc/sys/kernel/numa_balancing)"
+          echo "sched_migration_cost_ns: $(cat /proc/sys/kernel/sched_migration_cost_ns)"
+
+          export SGLANG_TEST_MAX_RETRY=0
+          export SGLANG_SET_CPU_AFFINITY=1
+          echo "SGLANG_SET_CPU_AFFINITY: $SGLANG_SET_CPU_AFFINITY"
+
+          install_sglang_from_source=${{ inputs.install_sglang_from_source }}
+          if [ "$install_sglang_from_source" = "true" ] || [ "$install_sglang_from_source" = "True" ];then
+            echo "Install sglang from source"
+            commit_id=${{ github.sha }}
+            echo "commit id: ${commit_id}" > ${test_data_output_path}/commit_id
+            export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH
+          else
+            echo "Use sglang from image: ${{ inputs.image }}"
+            sglang_pkg_path=/sgl-workspace/sglang/python
+            ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
+            mkdir -p ${ascend_test_util_path}
+            mv ${ascend_test_util_path} ${ascend_test_util_path}_bak
+            cp -r ${sglang_source_path}/python/sglang/test/ascend ${ascend_test_util_path}
+          fi
+
+          source /usr/local/Ascend/cann/set_env.sh || true
+          source /usr/local/Ascend/nnal/atb/set_env.sh || true
+
+          # Set environment of cann
+          log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+          rm -rf ${log_path}
+          mkdir -p ${log_path}
+          echo "Log path: ${log_path}"
+
+          echo "Running test case ${test_case}"
+          test_exit_code=0
+          ${PYTHON_FOR_SGLANG} -u ${test_case} 2>&1 | tee /tmp/test_output.log || test_exit_code=$?
+          echo "Finished test case ${test_case}"
+
+          if [ "${test_exit_code}" = "0" ]; then
+            test_status="pass"
+            status_icon="✅"
+          else
+            test_status="fail"
+            status_icon="❌"
+          fi
+
+          echo "test_status=${test_status}" >> $GITHUB_ENV
+          echo "tc_name=${tc_name}" >> $GITHUB_ENV
+          export test_status tc_name
+
+          echo "## ${tc_name}  ${status_icon} ${test_status}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          metric_count=$(grep -c '\[METRIC\]' /tmp/test_output.log 2>/dev/null || echo 0)
+          if [ "${metric_count}" -gt 0 ]; then
+            echo "| Metric | Value | Pass |" >> $GITHUB_STEP_SUMMARY
+            echo "|--------|-------|------|" >> $GITHUB_STEP_SUMMARY
+            grep '\[METRIC\]' /tmp/test_output.log | while IFS= read -r line; do
+              metric_name=$(echo "$line" | sed -E 's/.*\[METRIC\] ([^=]+)=.*/\1/')
+              metric_value=$(echo "$line" | sed -E 's/.*\[METRIC\] [^=]+=([^ ]+).*/\1/')
+              echo "| ${metric_name} | ${metric_value} | ${status_icon} |" >> $GITHUB_STEP_SUMMARY
+            done
+          else
+            echo "No metrics collected (test may have failed before producing results)." >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "import json, re, os" > /tmp/dump_metrics.py
+          echo "tc = os.environ.get('tc_name', 'unknown')" >> /tmp/dump_metrics.py
+          echo "st = os.environ.get('test_status', 'unknown')" >> /tmp/dump_metrics.py
+          echo "tp = '${{ inputs.test_type }}'" >> /tmp/dump_metrics.py
+          echo "metrics = {}" >> /tmp/dump_metrics.py
+          echo "with open('/tmp/test_output.log') as f:" >> /tmp/dump_metrics.py
+          echo "    for line in f:" >> /tmp/dump_metrics.py
+          echo "        m = re.match(r'\[METRIC\] (\S+)=(\S+)', line)" >> /tmp/dump_metrics.py
+          echo "        if m:" >> /tmp/dump_metrics.py
+          echo "            v = m.group(2)" >> /tmp/dump_metrics.py
+          echo "            try:" >> /tmp/dump_metrics.py
+          echo "                v = float(v)" >> /tmp/dump_metrics.py
+          echo "            except ValueError:" >> /tmp/dump_metrics.py
+          echo "                pass" >> /tmp/dump_metrics.py
+          echo "            metrics[m.group(1)] = v" >> /tmp/dump_metrics.py
+          echo "baselines = {}" >> /tmp/dump_metrics.py
+          echo "for k, v in list(metrics.items()):" >> /tmp/dump_metrics.py
+          echo "    if k.endswith('_baseline'):" >> /tmp/dump_metrics.py
+          echo "        baselines[k[:-9]] = v" >> /tmp/dump_metrics.py
+          echo "        del metrics[k]" >> /tmp/dump_metrics.py
+          echo "with open('/tmp/metrics.json', 'w') as f:" >> /tmp/dump_metrics.py
+          echo "    json.dump({'test_case': tc, 'test_type': tp, 'status': st, 'metrics': metrics, 'baselines': baselines}, f)" >> /tmp/dump_metrics.py
+          python3 /tmp/dump_metrics.py
+          exit ${test_exit_code}
+
+      - name: Upload metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-${{ inputs.test_config_name }}
+          path: /tmp/metrics.json
+          retention-days: 7
+
+      - name: Backup plog
+        if: always()
+        shell: bash
+        run: |
+          plog_path="/root/ascend/log/debug/plog"
+          if [ -d "$plog_path" ];then
+            echo "Plog files found. Begin to backup them."
+            tc_name=${{ inputs.test_case }}
+            tc_name=${tc_name##*/}
+            tc_name=${tc_name%.*}
+            target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
+            echo "Save path: ${target_plog_path}"
+            rm -rf ${target_plog_path}
+            mkdir -p ${target_plog_path}
+            cp ${plog_path}/* ${target_plog_path}
+          fi

.github/workflows/nightly-test-npu.yml

@@ -0,0 +1,406 @@
+name: Nightly Test (NPU)
+on:
+  schedule:
+    - cron: '0 18 * * *'  # Execute at 2:00 a.m. Beijing Time every day
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/nightly-test-npu.yml"
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      job_filter:
+        description: 'Select which job to run (leave empty or "all" to run all jobs)'
+        required: false
+        type: string
+        default: 'all'
+      image_a2:
+        description: 'The a2 running docker image of the test task.'
+        required: false
+        type: string
+        default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
+      image_a3:
+        description: 'The a3 running docker image of the test task.'
+        required: false
+        type: string
+        default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
+      skip_install_flag:
+        description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
+        required: false
+        type: string
+        default: 'true'
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      job_filter:
+        description: 'Select which job to run (leave empty or "all" to run all jobs)'
+        required: false
+        type: string
+        default: 'all'
+      image_a2:
+        description: 'The a2 running docker image of the test task.'
+        required: false
+        type: string
+        default: ''
+      image_a3:
+        description: 'The a3 running docker image of the test task.'
+        required: false
+        type: string
+        default: ''
+      skip_install_flag:
+        description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
+        required: false
+        type: string
+        default: ''
+concurrency:
+  group: nightly-test-npu-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+jobs:
+  set-image-config:
+    runs-on: ubuntu-latest
+    outputs:
+      ref: ${{ steps.set-vars.outputs.ref }}
+      job_filter: ${{ steps.set-vars.outputs.job_filter }}
+      image_a2: ${{ steps.set-vars.outputs.image_a2 }}
+      image_a3: ${{ steps.set-vars.outputs.image_a3 }}
+      skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
+    steps:
+      # When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
+      - name: Set image config
+        id: set-vars
+        run: |
+          if [ -z "${{ inputs.ref }}" ]; then
+            echo "ref=" >> $GITHUB_OUTPUT
+          else
+            echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT
+          fi
+          if [ -z "${{ inputs.job_filter }}" ]; then
+            echo "job_filter=all" >> $GITHUB_OUTPUT
+          else
+            echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
+          fi
+          if [ -z "${{ inputs.image_a2 }}" ]; then
+            echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
+          else
+            echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
+          fi
+          if [ -z "${{ inputs.image_a3 }}" ]; then
+            echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
+          else
+            echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
+          fi
+          if [ -z "${{ inputs.skip_install_flag }}" ]; then
+            echo "skip_install_flag=false" >> $GITHUB_OUTPUT
+          else
+            echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
+          fi
+  nightly-poc-single-node-a2-tests:
+    name: single-node-poc-a2
+    if: ${{ !cancelled() }}
+    needs: [ set-image-config ]
+    strategy:
+      fail-fast: false
+      max-parallel: 6
+      matrix:
+        test_config:
+          - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2
+            runner: linux-aarch64-a2-4
+            test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
+            test_type: 'perf'
+    uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
+    with:
+      runner: ${{ matrix.test_config.runner }}
+      test_type: ${{ matrix.test_config.test_type }}
+      test_config_name: ${{ matrix.test_config.name }}
+      test_case: ${{ matrix.test_config.test_case }}
+      image: ${{ needs.set-image-config.outputs.image_a2 }}
+      install_sglang_from_source: false
+      transformers_version: ''
+
+  nightly-poc-single-node-tests:
+    name: single-node-poc
+    if: ${{ !cancelled() }}
+    needs: [set-image-config]
+    strategy:
+      fail-fast: false
+      max-parallel: 6
+      matrix:
+        test_config:
+          # qwen3_6_35b_a3b performance tests
+          - name: qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_35b_a3b_1p_aime26
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py
+          - name: qwen3_6_35b_a3b_1p_in64k_out1k_50ms
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_35b_a3b_1p_in128k_out1k_50ms
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py
+            test_type: 'perf'
+          - name: qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py
+            test_type: 'perf'
+          # qwen3_6_27b performance tests
+          - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+            test_type: 'perf'
+          - name: qwen3_6_27b_w8a8_2p_in16k_out1k_50ms
+            runner: linux-aarch64-a3-4
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
+            runner: linux-aarch64-a3-4
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms
+            runner: linux-aarch64-a3-4
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_27b_2p_in64k_out1k_prefix90_50ms
+            runner: linux-aarch64-a3-4
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_27b_1p_in1080p_30_out256_50ms
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
+            test_type: 'perf'
+          - name: qwen3_6_27b_1p_gpqa
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py
+          # qwen3_32b performance tests
+          - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa
+            runner: linux-aarch64-a3-4
+            test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
+            test_type: 'perf'
+          - name: qwen3_32b_bf16_8p_in18k_out4k_6ms
+            runner: linux-aarch64-a3-16
+            test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
+            test_type: 'perf'
+          - name: qwen3_32b_bf16_8p_gpqa
+            runner: linux-aarch64-a3-16
+            test_case: test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
+          # qwen3_30b_a3b performance tests
+          - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
+            test_type: 'perf'
+          # qwen3-8b performance tests
+          - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+            test_type: 'perf'
+          - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
+            test_type: 'perf'
+          # qwen3_next_80b_a3b_instruct performance tests
+          - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25
+            runner: linux-aarch64-a3-4
+            test_case: test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
+            test_type: 'perf'
+          # minimax_m2_5 performance tests
+          - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa
+            runner: linux-aarch64-a3-16
+            test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
+            test_type: 'perf'
+          - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa
+            runner: linux-aarch64-a3-16
+            test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
+            test_type: 'perf'
+          # deepseek_v3_2 accuracy tests
+          - name: deepseek_v3_2_8p_aime25
+            runner: linux-aarch64-a3-16
+            test_case: test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
+          # glm4_7_flash accuracy tests
+          - name: glm4_7_flash_1p_aime25
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
+          # glm4_6v_flash accuracy tests
+          - name: glm4_6v_flash_1p_mmmu
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py
+          # qwen3_vl_8b_thinking accuracy tests
+          - name: qwen3_vl_8b_thinking_1p_mmmu
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
+          # qwen3_vl_30b_a3b_thinking accuracy tests
+          - name: qwen3_vl_30b_a3b_thinking_1p_mmmu
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
+    uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
+    with:
+      runner: ${{ matrix.test_config.runner }}
+      test_type: ${{ matrix.test_config.test_type }}
+      test_config_name: ${{ matrix.test_config.name }}
+      test_case: ${{ matrix.test_config.test_case }}
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_from_source: false
+      transformers_version: ''
+
+  nightly-poc-multi-node-tests:
+    name: multi-node-poc
+    if: ${{ !cancelled() }}
+    needs: [set-image-config, nightly-poc-single-node-tests]
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        test_config:
+          # glm5_1 performance tests
+          - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26
+            prefill_size: 2
+            decode_size: 2
+            router_size: 1
+            test_case: test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
+            test_type: 'perf'
+            prefill_decode_deployment: 'separation'
+    uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
+    with:
+      runner: linux-amd64-cpu-8
+      test_type: ${{ matrix.test_config.test_type }}
+      test_config_name: ${{ matrix.test_config.name }}
+      prefill_size: ${{ matrix.test_config.prefill_size }}
+      decode_size: ${{ matrix.test_config.decode_size }}
+      router_size: ${{ matrix.test_config.router_size }}
+      test_case: ${{ matrix.test_config.test_case }}
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_from_source: false
+      prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
+      transformers_version: ''
+
+  nightly-poc-multi-node-mix-tests:
+    name: multi-node-mix-poc
+    if: ${{ !cancelled() }}
+    needs: [set-image-config, nightly-poc-single-node-tests, nightly-poc-multi-node-tests]
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        test_config:
+          # kimi_k2_6 performance tests
+          - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25
+            node_size: 2
+            test_case: test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
+            test_type: 'perf'
+    uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
+    with:
+      runner: linux-amd64-cpu-8
+      test_type: ${{ matrix.test_config.test_type }}
+      test_config_name: ${{ matrix.test_config.name }}
+      node_size: ${{ matrix.test_config.node_size }}
+      test_case: ${{ matrix.test_config.test_case }}
+      image: ${{ needs.set-image-config.outputs.image_a3 }}
+      install_sglang_from_source: false
+      prefill_decode_deployment: 'mix'
+      transformers_version: ''
+  check-all-jobs:
+    if: ${{ !cancelled() }}
+    needs:
+      - nightly-poc-single-node-a2-tests
+      - nightly-poc-single-node-tests
+      - nightly-poc-multi-node-tests
+      - nightly-poc-multi-node-mix-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all metrics
+        uses: actions/download-artifact@v4
+        with:
+          pattern: metrics-*
+          path: /tmp/metrics
+          merge-multiple: false
+
+      - name: Generate results table
+        run: |
+          status_emoji() {
+            case "$1" in
+              pass)   echo "✅" ;;
+              fail)   echo "❌" ;;
+              *)      echo "❓" ;;
+            esac
+          }
+
+          single_result_a2="${{ needs.nightly-poc-single-node-a2-tests.result }}"
+          single_result="${{ needs.nightly-poc-single-node-tests.result }}"
+          multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
+          mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"
+
+          group_icon() {
+            case "$1" in
+              success)   echo "✅" ;;
+              failure)   echo "❌" ;;
+              cancelled) echo "⏭️" ;;
+              skipped)   echo "⏭️" ;;
+              *)         echo "❓" ;;
+            esac
+          }
+
+          echo "## Nightly Test Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY
+          echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
+          echo "| single-node-poc-a2 | $(group_icon ${single_result_a2}) ${single_result_a2} |" >> $GITHUB_STEP_SUMMARY
+          echo "| single-node-poc | $(group_icon ${single_result}) ${single_result} |" >> $GITHUB_STEP_SUMMARY
+          echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
+          echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Per-Test Metrics" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          has_metrics=false
+          for dir in /tmp/metrics/metrics-*/; do
+            file="${dir}metrics.json"
+            if [ -f "$file" ]; then
+              has_metrics=true
+              echo "import json" > /tmp/parse_metrics.py
+              echo "with open('$file') as f:" >> /tmp/parse_metrics.py
+              echo "    d = json.load(f)" >> /tmp/parse_metrics.py
+              echo "tc = d.get('test_case', '-')" >> /tmp/parse_metrics.py
+              echo "tp = d.get('test_type', '-')" >> /tmp/parse_metrics.py
+              echo "st = d.get('status', 'unknown')" >> /tmp/parse_metrics.py
+              echo "icon = '✅' if st == 'pass' else '❌'" >> /tmp/parse_metrics.py
+              echo "metrics = d.get('metrics', {})" >> /tmp/parse_metrics.py
+              echo "baselines = d.get('baselines', {})" >> /tmp/parse_metrics.py
+              echo "mstr = ', '.join(f'{k}={v}' for k,v in metrics.items()) if metrics else '-'" >> /tmp/parse_metrics.py
+              echo "bstr = ', '.join(f'{k}={v}' for k,v in baselines.items()) if baselines else '-'" >> /tmp/parse_metrics.py
+              echo "print(f'| {tc} | {tp} | {icon} {st} | {mstr} | {bstr} |')" >> /tmp/parse_metrics.py
+              python3 /tmp/parse_metrics.py
+            fi
+          done > /tmp/metrics_table.txt
+
+          if [ "${has_metrics}" = "true" ]; then
+            echo "| Test Case | Type | Status | Metrics | Baseline |" >> $GITHUB_STEP_SUMMARY
+            echo "|-----------|------|--------|---------|----------|" >> $GITHUB_STEP_SUMMARY
+            cat /tmp/metrics_table.txt >> $GITHUB_STEP_SUMMARY
+          else
+            echo "No per-test metrics available (artifacts not found)." >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          FAIL=0
+          if [ "${single_result}" != "success" ] && [ "${single_result}" != "skipped" ]; then FAIL=1; fi
+          exit $FAIL

.github/workflows/nightly-test-nvidia.yml

@@ -0,0 +1,808 @@
+name: Nightly Test (Nvidia)
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+    inputs:
+      job_filter:
+        description: 'Select which job to run (leave empty or "all" to run all jobs)'
+        required: false
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - 'nightly-test-general-1-gpu-h100'
+          - 'nightly-test-general-4-gpu-h100'
+          - 'nightly-test-general-8-gpu-h200'
+          - 'nightly-test-general-8-gpu-h20'
+          - 'nightly-test-general-8-gpu-b200'
+          - 'nightly-test-text-accuracy-2-gpu-h100'
+          - 'nightly-test-text-perf-2-gpu-h100'
+          - 'nightly-test-vlm-accuracy-2-gpu-h100'
+          - 'nightly-test-vlm-perf-2-gpu-h100'
+          - 'nightly-test-perf-4-gpu-b200'
+          - 'nightly-test-perf-8-gpu-b200'
+          - 'nightly-test-specialized-8-gpu-b200'
+          - 'nightly-test-perf-4-gpu-gb300'
+          - 'nightly-test-kernel-1-gpu-h100'
+          - 'nightly-test-diffusion'
+          - 'nightly-test-kernel-8-gpu-h200'
+          - 'nightly-test-precision-8-gpu-h200'
+      force_baseline_update:
+        description: 'precision job only: refresh the rolling baseline instead of comparing (sets SGLANG_PRECISION_FORCE_UPDATE=1). Dispatch once after an intentional forward-path dtype/precision change stales the baseline; later scheduled runs compare against it and go green.'
+        required: false
+        type: boolean
+        default: false
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      job_filter:
+        description: 'Select which job to run (leave empty or "all" to run all jobs)'
+        required: false
+        type: string
+        default: 'all'
+
+concurrency:
+  group: nightly-test-nvidia-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
+  SGLANG_CUDA_COREDUMP: "1"
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 300
+
+jobs:
+  # General tests - 1 GPU
+  nightly-test-general-1-gpu-h100:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-1-gpu-h100')
+    runs-on: 1-gpu-h100
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-1-gpu --nightly --continue-on-error
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # JIT kernel full unit tests (expanded parameter ranges via SGLANG_JIT_KERNEL_RUN_FULL_TESTS)
+  nightly-test-kernel-1-gpu-h100:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-kernel-1-gpu-h100')
+    runs-on: 1-gpu-h100
+    timeout-minutes: 60
+    env:
+      # Full jit_kernel test grids (see sglang.jit_kernel.utils.should_run_full_tests)
+      SGLANG_JIT_KERNEL_RUN_FULL_TESTS: "1"
+      # Match pr-test-jit-kernel workflow for consistent JIT warmup behavior
+      SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+      # Allow maintenance bypass on default branch (same semantics as PR JIT workflow)
+      PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run jit kernel nightly suite
+        timeout-minutes: 60
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-kernel-1-gpu --nightly --continue-on-error
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  nightly-test-kernel-8-gpu-h200:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-kernel-8-gpu-h200')
+    runs-on: 8-gpu-h200
+    timeout-minutes: 240
+    env:
+      SGLANG_JIT_KERNEL_RUN_FULL_TESTS: "1"
+      SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+      PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run multi-GPU jit kernel nightly suite
+        timeout-minutes: 90
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-kernel-8-gpu-h200 --nightly --continue-on-error
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # General tests - 4 GPU H100
+  nightly-test-general-4-gpu-h100:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100')
+    runs-on: 4-gpu-h100
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # General tests - 8 GPU H200
+  nightly-test-general-8-gpu-h200:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200')
+    runs-on: 8-gpu-h200
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        partition: [0, 1, 2, 3]
+    env:
+      RUNNER_LABELS: 8-gpu-h200
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run common 8-GPU model tests
+        if: always()
+        timeout-minutes: 300
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "8-gpu-h200"
+          IS_H200: "1"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=18000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=4
+
+      - name: Publish traces to storage repo
+        if: always()
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          TRACE_ARGS=""
+          for dir in test/performance_profiles_*/; do
+            [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
+          done
+          if [ -n "$TRACE_ARGS" ]; then
+            python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
+            find test/performance_profiles_*/ -name '*.json.gz' -delete
+          else
+            echo "No trace directories found, skipping publish"
+          fi
+
+      - name: Run test
+        timeout-minutes: 30
+        env:
+          GPU_CONFIG: "8-gpu-h200"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=4
+
+      - name: Collect performance metrics
+        if: always()
+        run: |
+          python3 scripts/ci/utils/save_metrics.py \
+            --gpu-config 8-gpu-h200 \
+            --partition ${{ matrix.partition }} \
+            --run-id ${{ github.run_id }} \
+            --output test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json \
+            --search-dir test/performance_profiles_8_gpu \
+            --search-dir test
+
+      - name: Upload partition metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-8gpu-h200-partition-${{ matrix.partition }}
+          path: test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json
+          retention-days: 5
+          if-no-files-found: ignore
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
+  # General tests - 8 GPU H20
+  nightly-test-general-8-gpu-h20:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
+    runs-on: 8-gpu-h20
+    env:
+      SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 30
+        env:
+          GPU_CONFIG: "8-gpu-h20"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # General tests - 8 GPU B200
+  nightly-test-general-8-gpu-b200:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200')
+    runs-on: 8-gpu-b200
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        partition: [0, 1, 2, 3]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run common 8-GPU model tests
+        if: always()
+        timeout-minutes: 200
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "8-gpu-b200"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=4
+
+      - name: Publish traces to storage repo
+        if: always()
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          TRACE_ARGS=""
+          for dir in test/performance_profiles_*/; do
+            [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
+          done
+          if [ -n "$TRACE_ARGS" ]; then
+            python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
+            find test/performance_profiles_*/ -name '*.json.gz' -delete
+          else
+            echo "No trace directories found, skipping publish"
+          fi
+
+      - name: Collect performance metrics
+        if: always()
+        run: |
+          python3 scripts/ci/utils/save_metrics.py \
+            --gpu-config 8-gpu-b200 \
+            --partition ${{ matrix.partition }} \
+            --run-id ${{ github.run_id }} \
+            --output test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json \
+            --search-dir test/performance_profiles_8_gpu \
+            --search-dir test
+
+      - name: Upload partition metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-8gpu-b200-partition-${{ matrix.partition }}
+          path: test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json
+          retention-days: 5
+          if-no-files-found: ignore
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+        with:
+          artifact-suffix: ${{ matrix.partition }}
+
+  # Text model accuracy tests
+  nightly-test-text-accuracy-2-gpu-h100:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-h100')
+    runs-on: 2-gpu-h100
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run eval test for text models
+        timeout-minutes: 120
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-eval-text-2-gpu --nightly --continue-on-error --timeout-per-file 4500
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # Text model performance tests
+  nightly-test-text-perf-2-gpu-h100:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-h100')
+    runs-on: 2-gpu-h100
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run performance test for text models
+        timeout-minutes: 30
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "2-gpu-h100"
+        run: |
+          cd test
+          rm -rf performance_profiles_text_models/
+          python3 run_suite.py --hw cuda --suite nightly-perf-text-2-gpu --nightly --continue-on-error --timeout-per-file 3600
+
+      - name: Publish traces to storage repo
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_text_models
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # VLM accuracy tests
+  nightly-test-vlm-accuracy-2-gpu-h100:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-h100')
+    runs-on: 2-gpu-h100
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run eval test for VLM models (fixed MMMU-100)
+        timeout-minutes: 120
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # VLM performance tests
+  nightly-test-vlm-perf-2-gpu-h100:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-h100')
+    runs-on: 2-gpu-h100
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run perf test for VLM models (MMMU)
+        timeout-minutes: 30
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "2-gpu-h100"
+        run: |
+          cd test
+          rm -rf performance_profiles_vlms/
+          python3 run_suite.py --hw cuda --suite nightly-perf-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 3600
+
+      - name: Publish traces to storage repo
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          python3 scripts/ci/utils/publish_traces.py --traces-dir test/performance_profiles_vlms
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # B200 Performance tests - 4 GPU
+  nightly-test-perf-4-gpu-b200:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200')
+    runs-on: 4-gpu-b200
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 200
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "4-gpu-b200"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000
+
+      - name: Publish traces to storage repo
+        if: always()
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          TRACE_ARGS=""
+          for dir in test/performance_profiles_*/; do
+            [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
+          done
+          if [ -n "$TRACE_ARGS" ]; then
+            python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
+            find test/performance_profiles_*/ -name '*.json.gz' -delete
+          else
+            echo "No trace directories found, skipping publish"
+          fi
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64)
+  nightly-test-perf-4-gpu-gb300:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
+    name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
+    runs-on: 4-gpu-gb300-nightly
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - model: glm5-nvfp4
+            suite: nightly-4-gpu-gb300-glm5-nvfp4
+          - model: qwen35-fp8
+            suite: nightly-4-gpu-gb300-qwen35-fp8
+          - model: deepseek-v4-pro-fp4
+            suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
+          - model: kimi-k25-nvfp4
+            suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        env:
+          GRACE_BLACKWELL: "1"
+        run: |
+          bash scripts/ci/cuda/ci_install_deepep.sh
+
+      - name: Run test
+        timeout-minutes: 600
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "4-gpu-gb300"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
+
+      - name: Publish traces to storage repo
+        if: always()
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          TRACE_ARGS=""
+          for dir in test/performance_profiles_*/; do
+            [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
+          done
+          if [ -n "$TRACE_ARGS" ]; then
+            python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
+            find test/performance_profiles_*/ -name '*.json.gz' -delete
+          else
+            echo "No trace directories found, skipping publish"
+          fi
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # Specialized B200 tests - 8 GPU, for specific backends and configs
+  nightly-test-specialized-8-gpu-b200:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200')
+    runs-on: 8-gpu-b200
+    env:
+      RUNNER_LABELS: 8-gpu-b200
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          GPU_CONFIG: "8-gpu-b200"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # SGLang-Diffusion nightly benchmark
+  nightly-test-diffusion:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion')
+    runs-on: 4-gpu-h100
+    timeout-minutes: 300
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run diffusion benchmark
+        env:
+          GITHUB_SHA: ${{ github.sha }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          PYTHONUNBUFFERED: "1"
+        timeout-minutes: 210
+        run: |
+          python3 -u scripts/ci/utils/diffusion/run_comparison.py \
+            --output comparison-results.json
+
+      - name: Generate dashboard
+        if: always()
+        env:
+          GH_PAT_FOR_NIGHTLY_CI_DATA: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python3 scripts/ci/utils/diffusion/generate_diffusion_dashboard.py \
+            --results comparison-results.json \
+            --output dashboard.md \
+            --charts-dir comparison-charts \
+            --fetch-history \
+            --step-summary
+
+      - name: Publish to sglang-ci-data
+        if: always()
+        env:
+          GH_PAT_FOR_NIGHTLY_CI_DATA: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python3 scripts/ci/utils/diffusion/publish_comparison_results.py \
+            --results comparison-results.json \
+            --dashboard dashboard.md \
+            --charts-dir comparison-charts
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-benchmark-${{ github.run_id }}
+          path: |
+            comparison-results.json
+            dashboard.md
+            comparison-charts/
+            comparison-logs/
+          retention-days: 90
+          if-no-files-found: ignore
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # Nightly precision regression - per-layer hidden state comparison
+  nightly-test-precision-8-gpu-h200:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-precision-8-gpu-h200')
+    runs-on: 8-gpu-h200
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run precision regression test
+        timeout-minutes: 120
+        env:
+          SGLANG_PRECISION_BASELINE_DIR: /tmp/sglang_precision_baselines
+          # Required: the test errors if SGLANG_PRECISION_HF_REPO is unset (no
+          # local-only mode). Set the var + the HF_TOKEN_PRECISION_STORE secret.
+          SGLANG_PRECISION_HF_REPO: ${{ vars.SGLANG_PRECISION_HF_REPO }}
+          SGLANG_PRECISION_HF_REVISION: ${{ vars.SGLANG_PRECISION_HF_REVISION || 'main' }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN_PRECISION_STORE }}
+          SGLANG_PRECISION_COMMIT: ${{ github.sha }}
+          # '0' on scheduled runs (input unset); '1' only on a manual baseline-refresh dispatch.
+          SGLANG_PRECISION_FORCE_UPDATE: ${{ inputs.force_baseline_update && '1' || '0' }}
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-precision-8-gpu-h200 --nightly --continue-on-error --timeout-per-file 3600
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  # Consolidate performance metrics from all jobs
+  consolidate-metrics:
+    if: github.repository == 'sgl-project/sglang' && always()
+    needs:
+      - nightly-test-general-8-gpu-h200
+      - nightly-test-general-8-gpu-b200
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Download all partition metrics
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "*metrics-*"
+          path: metrics/
+          merge-multiple: true
+
+      - name: List downloaded metrics
+        run: |
+          echo "Downloaded metrics files:"
+          find metrics/ -name "*.json" -type f 2>/dev/null || echo "No metrics files found"
+
+      - name: Merge metrics
+        run: |
+          python3 scripts/ci/utils/merge_metrics.py \
+            --input-dir metrics/ \
+            --output consolidated-metrics-${{ github.run_id }}.json \
+            --run-id ${{ github.run_id }} \
+            --commit-sha ${{ github.sha }} \
+            --branch ${{ github.ref_name }}
+
+      - name: Upload consolidated metrics
+        uses: actions/upload-artifact@v4
+        with:
+          name: consolidated-metrics-${{ github.run_id }}
+          path: consolidated-metrics-${{ github.run_id }}.json
+          retention-days: 90
+          if-no-files-found: warn
+
+  # Final check job
+  check-all-jobs:
+    if: github.repository == 'sgl-project/sglang' && always()
+    needs:
+      - nightly-test-general-1-gpu-h100
+      - nightly-test-general-4-gpu-h100
+      - nightly-test-general-8-gpu-h200
+      - nightly-test-general-8-gpu-h20
+      - nightly-test-general-8-gpu-b200
+      - nightly-test-text-accuracy-2-gpu-h100
+      - nightly-test-text-perf-2-gpu-h100
+      - nightly-test-vlm-accuracy-2-gpu-h100
+      - nightly-test-vlm-perf-2-gpu-h100
+      - nightly-test-perf-4-gpu-b200
+      - nightly-test-specialized-8-gpu-b200
+      - nightly-test-perf-4-gpu-gb300
+      - nightly-test-diffusion
+      - nightly-test-precision-8-gpu-h200
+      - consolidate-metrics
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if any job failed
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            echo "One or more nightly test jobs failed"
+            exit 1
+          fi
+          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more nightly test jobs were cancelled"
+            exit 1
+          fi
+          echo "All nightly test jobs passed"

.github/workflows/open-pr-copy-from-oss.yml

@@ -0,0 +1,28 @@
+name: Open A PR to Copy Code From OSS
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   - cron: '0 10 * * *'
+
+permissions:
+  contents: write
+
+jobs:
+  copy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: 'main'
+
+      - name: Install GitHub CLI (if not present)
+        run: |
+          bash scripts/code_sync/install_github_cli.sh
+
+      - name: Copy from OSS code
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_OPEN_PR_TO_PRIVATE }}
+        run: |
+          python3 scripts/code_sync/copy_from_oss.py

.github/workflows/open-pr-copy-to-oss.yml

@@ -0,0 +1,31 @@
+name: Open A PR to Copy Diff To OSS
+
+on:
+  workflow_dispatch:
+    inputs:
+      commit_sha:
+        description: 'The commit SHA to copy. Defaults to LAST to copy the latest commit.'
+        required: false
+        default: 'LAST'
+
+permissions:
+  contents: write
+
+jobs:
+  copy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install GitHub CLI (if not present)
+        run: |
+          bash scripts/code_sync/install_github_cli.sh
+
+      - name: Copy to OSS code
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_OPEN_PR_TO_OSS }}
+        run: |
+          python3 scripts/code_sync/copy_to_oss.py --commit ${{ github.event.inputs.commit_sha }}

.github/workflows/patch-docker-dev.yml

@@ -0,0 +1,118 @@
+name: Patch Docker Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_numbers:
+        description: "Comma-separated PR numbers to apply (e.g. 18962,19010)"
+        required: false
+        default: ""
+      image_tag:
+        description: "Base image tag to patch (e.g. dev-x86, dev-x86-cu13)"
+        required: true
+
+concurrency:
+  group: patch-docker-${{ inputs.image_tag }}
+  cancel-in-progress: true
+
+jobs:
+  patch:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: x64-docker-build-node
+    steps:
+      - name: Cleanup workspace (remove root-owned files from prior runs)
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/* || true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Pull base image and extract commit
+        run: |
+          IMAGE="lmsysorg/sglang:${{ inputs.image_tag }}"
+          docker pull "${IMAGE}"
+          if BASE_SHA=$(docker run --rm "${IMAGE}" git -C /sgl-workspace/sglang rev-parse HEAD 2>/dev/null); then
+            echo "Image built from commit: ${BASE_SHA}"
+          else
+            BASE_SHA=""
+            echo "::warning::Image has no .git directory — cannot extract base commit"
+          fi
+          echo "BASE_SHA=${BASE_SHA}" >> "$GITHUB_ENV"
+
+      - name: Generate patches
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git fetch origin main
+          mkdir -p /tmp/patch-ctx
+
+          if [ -n "${{ inputs.pr_numbers }}" ]; then
+            IFS=',' read -ra PRS <<< "${{ inputs.pr_numbers }}"
+            for pr in "${PRS[@]}"; do
+              pr=$(echo "${pr}" | xargs)
+              echo "Fetching PR #${pr}"
+              git fetch origin "pull/${pr}/head:pr-${pr}"
+              MERGE_BASE=$(git merge-base origin/main "pr-${pr}")
+              echo "  PR #${pr}: merge-base=${MERGE_BASE}"
+              git diff "${MERGE_BASE}..pr-${pr}" > "/tmp/patch-ctx/${pr}.patch"
+              echo "  PR #${pr}: $(wc -l < /tmp/patch-ctx/${pr}.patch) lines"
+            done
+          elif [ -n "${BASE_SHA}" ]; then
+            echo "Generating diff: image ${BASE_SHA} → latest main"
+            git fetch origin "${BASE_SHA}"
+            git diff "${BASE_SHA}..origin/main" > /tmp/patch-ctx/main.patch
+            echo "  main: $(wc -l < /tmp/patch-ctx/main.patch) lines"
+          else
+            echo "::error::No PR numbers specified and image has no .git — cannot generate diff against main"
+            exit 1
+          fi
+
+          TOTAL=$(cat /tmp/patch-ctx/*.patch | wc -l)
+          if [ "${TOTAL}" -eq 0 ]; then
+            echo "::warning::All patches are empty — image is already up to date"
+            echo "SKIP_BUILD=true" >> "$GITHUB_ENV"
+          fi
+
+      - name: Build patched image
+        if: env.SKIP_BUILD != 'true'
+        run: |
+          IMAGE="lmsysorg/sglang:${{ inputs.image_tag }}"
+
+          cat <<'DOCKERFILE' > /tmp/patch-ctx/Dockerfile
+          ARG BASE_IMAGE
+          FROM ${BASE_IMAGE}
+          COPY *.patch /tmp/patches/
+          RUN cd /sgl-workspace/sglang \
+              && for p in /tmp/patches/*.patch; do \
+                   if [ ! -s "${p}" ]; then \
+                     echo "Skipping ${p} (empty)"; \
+                   else \
+                     echo "Applying ${p}..." \
+                     && patch -p1 --fuzz=2 --no-backup-if-mismatch -f < "${p}" \
+                     || { echo "ERROR: Failed to apply ${p}"; exit 1; }; \
+                   fi; \
+                 done \
+              && rm -rf /tmp/patches
+          DOCKERFILE
+
+          docker build \
+            --no-cache \
+            --build-arg BASE_IMAGE="${IMAGE}" \
+            -t "${IMAGE}" \
+            /tmp/patch-ctx/
+
+      - name: Push patched image
+        if: env.SKIP_BUILD != 'true'
+        run: |
+          IMAGE="lmsysorg/sglang:${{ inputs.image_tag }}"
+          docker push "${IMAGE}"
+
+          echo "### Patched \`${IMAGE}\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "- **Base commit:** \`${BASE_SHA:-unknown (no .git)}\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "- **Source:** ${{ inputs.pr_numbers && format('PRs: {0}', inputs.pr_numbers) || 'latest main' }}" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/pr-benchmark-rust.yml

@@ -0,0 +1,198 @@
+name: PR Benchmark (SMG Components)
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - "sgl-model-gateway/**"
+  pull_request:
+    branches: [ main ]
+    paths:
+      - "sgl-model-gateway/**"
+  workflow_dispatch:
+
+concurrency:
+  group: pr-benchmark-rust-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUSTC_WRAPPER: sccache
+  SCCACHE_GHA_ENABLED: "true"
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+
+jobs:
+  benchmark-compile-check:
+    name: Benchmark Compilation Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_gateway_dependencies.sh
+
+      - name: Configure sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+        with:
+          version: "v0.12.0"
+          disable_annotations: true
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: sgl-model-gateway
+          shared-key: "rust-cache"
+          save-if: true
+          cache-all-crates: true
+          cache-on-failure: true
+
+      - name: Check benchmarks compile
+        run: |
+          source "$HOME/.cargo/env"
+          cd sgl-model-gateway/
+          cargo check --benches
+
+      - name: Show sccache stats
+        if: always()
+        run: sccache --show-stats
+
+  benchmark:
+    name: Benchmark - ${{ matrix.name }}
+    if: |
+      github.repository == 'sgl-project/sglang' &&
+      (github.event_name == 'push' ||
+       github.event_name == 'workflow_dispatch' ||
+       (contains(github.event.pull_request.labels.*.name, 'router-benchmark') &&
+        contains(github.event.pull_request.labels.*.name, 'run-ci')))
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Request Processing
+            bench_name: request_processing
+            bench_args: "benchmark_summary --exact"
+            runner: ubuntu-latest
+            sccache_version: "v0.12.0"
+            artifact_name: request-processing-results
+            artifact_path: criterion/benchmark_summary/
+          - name: Manual Policy
+            bench_name: manual_policy_benchmark
+            bench_args: ""
+            runner: ubuntu-latest
+            sccache_version: "v0.12.0"
+            artifact_name: manual-policy-results
+            artifact_path: criterion/manual_policy*/
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_gateway_dependencies.sh
+
+      - name: Configure sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+        with:
+          version: ${{ matrix.sccache_version }}
+          disable_annotations: true
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: sgl-model-gateway
+          shared-key: "rust-cache"
+          cache-all-crates: true
+          cache-on-failure: true
+          save-if: true
+
+      - name: Run benchmark
+        timeout-minutes: 30
+        run: |
+          source "$HOME/.cargo/env"
+          cd sgl-model-gateway/
+          if command -v sccache &> /dev/null; then
+            echo "Testing sccache availability..."
+            export RUSTC_WRAPPER=sccache
+            export SCCACHE_GHA_ENABLED="true"
+            if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then
+              echo "sccache is working, using it for compilation"
+            else
+              echo "sccache failed to start, falling back to regular cargo"
+              unset RUSTC_WRAPPER
+              unset SCCACHE_GHA_ENABLED
+            fi
+          else
+            echo "sccache not available, using regular cargo"
+          fi
+          cargo bench --bench ${{ matrix.bench_name }} -- ${{ matrix.bench_args }} 2>&1 | tee benchmark_output.txt
+
+      - name: Upload benchmark results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}-${{ github.sha }}
+          path: |
+            sgl-model-gateway/target/${{ matrix.artifact_path }}
+            sgl-model-gateway/benchmark_output.txt
+          retention-days: 30
+
+      - name: Show sccache stats
+        if: always()
+        run: sccache --show-stats
+
+  benchmark-summary:
+    name: Benchmark Summary
+    needs: [benchmark]
+    if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: '*-results-${{ github.sha }}'
+          path: benchmark-results
+
+      - name: Generate summary
+        run: |
+          generate_section() {
+            local title="$1" dir_name="$2" lines="${3:-100}"
+            local dir="benchmark-results/${dir_name}-${{ github.sha }}"
+            echo "### $title" >> summary.md
+            if [ -d "$dir" ]; then
+              echo "✅ **Completed**" >> summary.md
+              if [ -f "$dir/benchmark_output.txt" ]; then
+                echo -e "\n<details>\n<summary>View Results</summary>\n\n\`\`\`" >> summary.md
+                tail -"$lines" "$dir/benchmark_output.txt" >> summary.md
+                echo -e "\`\`\`\n</details>" >> summary.md
+              fi
+            else
+              echo "❌ Failed or skipped" >> summary.md
+            fi
+            echo "" >> summary.md
+          }
+
+          echo "## 🚀 Benchmark Results Summary" > summary.md
+          echo "" >> summary.md
+
+          generate_section "Request Processing" "request-processing-results" 60
+          generate_section "Manual Policy (Sticky Sessions)" "manual-policy-results" 100
+
+          echo -e "---\n_Generated at $(date -u '+%Y-%m-%d %H:%M:%S UTC')_" >> summary.md
+
+          cat summary.md
+          cat summary.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload summary
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-summary-${{ github.sha }}
+          path: summary.md
+          retention-days: 30

.github/workflows/pr-gate.yml

@@ -0,0 +1,272 @@
+on:
+  workflow_call:
+    inputs:
+      require-run-ci:
+        description: "Whether the PR must have the run-ci label"
+        type: boolean
+        default: true
+      require-run-ci-extra:
+        description: "Whether the PR must also have the run-ci-extra label (in addition to run-ci)"
+        type: boolean
+        default: false
+      cool-down-minutes:
+        description: "Cooldown period in minutes for low-permission users; 0 disables rate limiting"
+        type: number
+        default: 120
+
+jobs:
+  pr-gate:
+    # 1. for commits on main: no gating needed
+    # 2. for workflow_dispatch: this can only be triggered by users with write access
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch latest PR info
+        if: github.event_name == 'pull_request'
+        id: pr
+        uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+            core.setOutput("labels", JSON.stringify(pr.data.labels.map(l => l.name)));
+            core.setOutput("draft", pr.data.draft);
+            core.setOutput("user", pr.data.user.login);
+
+      - name: Log PR info
+        if: github.event_name == 'pull_request'
+        run: |
+          echo "===== PR Info ====="
+          echo "PR Event: ${{ github.event_name }}"
+          echo "PR Labels: ${{ steps.pr.outputs.labels }}"
+          echo "PR Draft: ${{ steps.pr.outputs.draft }}"
+          echo "PR User: ${{ steps.pr.outputs.user }}"
+          echo "Require run-ci: ${{ inputs.require-run-ci }}"
+          echo "Require run-ci-extra: ${{ inputs.require-run-ci-extra }}"
+          echo "Cool down minutes: ${{ inputs.cool-down-minutes }}"
+          echo "==================="
+
+      - name: Block draft PR
+        if: github.event_name == 'pull_request' && fromJson(steps.pr.outputs.draft)
+        run: |
+          echo "PR is draft. Blocking CI."
+          exit 1
+
+      - name: Require run-ci label (optional)
+        if:  github.event_name == 'pull_request' && inputs.require-run-ci == true
+        run: |
+          labels='${{ steps.pr.outputs.labels }}'
+          if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci') }}" == "false" ]]; then
+            echo "Missing required label 'run-ci'. See https://docs.sglang.io/developer_guide/contribution_guide.html#how-to-trigger-ci-tests for more details."
+            exit 1
+          fi
+
+      # Live-fetch label gate for pr-test-extra.yml. Workflow-level `if`
+      # checks would read the frozen event payload, which doesn't update on
+      # rerun — so /tag-and-rerun-ci extra (slash handler adds the label
+      # then reruns) could never un-skip. A runtime step using gh-api-fetched
+      # labels lets reruns pick up the new label set.
+      - name: Require run-ci-extra label (optional)
+        if: github.event_name == 'pull_request' && inputs.require-run-ci-extra == true
+        run: |
+          if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci-extra') }}" == "false" ]]; then
+            echo "Missing required label 'run-ci-extra'. Add the label (e.g. via /tag-and-rerun-ci extra) to opt this PR into the extra test workflow."
+            exit 1
+          fi
+
+      - name: Enforce rate limit for low-permission actors (optional)
+        if: github.event_name == 'pull_request' && inputs.cool-down-minutes > 0
+        uses: actions/github-script@v8
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const DEFAULT_MINUTES = Number("${{ inputs.cool-down-minutes }}");
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const eventName = context.eventName;
+            const curRun = await github.rest.actions.getWorkflowRun({
+              owner, repo, run_id: context.runId
+            });
+            let triggeringActor = curRun.data.triggering_actor?.login || context.actor;
+            if (triggeringActor === "github-actions[bot]") {
+              triggeringActor = `${{ steps.pr.outputs.user }}`;
+              core.info(
+                `triggering_actor is github-actions[bot]; substituting PR author '${triggeringActor}'.`
+              );
+            }
+
+            async function hasHighPermission(username) {
+              try {
+                const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ owner, repo, username });
+                const perm = data.permission || 'none';
+                return perm === 'write' || perm === 'maintain' || perm === 'admin';
+              } catch (e) {
+                if (e.status === 404 || e.status === 403) return false;
+                throw e;
+              }
+            }
+
+            if (await hasHighPermission(triggeringActor)) {
+              core.info(`Triggering user '${triggeringActor}' has high permission. No rate limit applied.`);
+              return;
+            }
+
+            let effectiveCooldownMinutes = DEFAULT_MINUTES;
+            let perUserCooldownMinutes = null;
+
+            try {
+              const contentResp = await github.rest.repos.getContent({
+                owner,
+                repo,
+                path: ".github/CI_PERMISSIONS.json",
+                ref: "main",
+              });
+
+              if (!Array.isArray(contentResp.data) && contentResp.data && "content" in contentResp.data) {
+                const raw = Buffer.from(
+                  contentResp.data.content,
+                  contentResp.data.encoding || "base64"
+                ).toString();
+                const ciPermissions = JSON.parse(raw);
+
+                const userPerm = ciPermissions[triggeringActor];
+                if (userPerm && typeof userPerm.cooldown_interval_minutes === "number") {
+                  perUserCooldownMinutes = userPerm.cooldown_interval_minutes;
+                  core.info(
+                    `Per-user cooldown for '${triggeringActor}' from CI_PERMISSIONS.json: ${perUserCooldownMinutes} minutes.`
+                  );
+                } else {
+                  core.info(`No per-user cooldown found for '${triggeringActor}' in CI_PERMISSIONS.json.`);
+                }
+              } else {
+                core.info("CI_PERMISSIONS.json content response is not a file; skipping per-user cooldown.");
+              }
+            } catch (e) {
+              core.info(`CI_PERMISSIONS.json not found or unreadable: ${e.message}. Using default rate limit only.`);
+            }
+
+            if (perUserCooldownMinutes !== null) {
+              effectiveCooldownMinutes = Math.min(effectiveCooldownMinutes, perUserCooldownMinutes);
+            }
+
+            if (effectiveCooldownMinutes <= 0) {
+              core.info(
+                `Effective cooldown for '${triggeringActor}' is 0 minutes; no rate limit enforced for this user.`
+              );
+              return;
+            }
+
+            const cutoff = new Date(Date.now() - effectiveCooldownMinutes * 60 * 1000);
+            core.info(
+              `Checking for workflow runs since ${cutoff.toISOString()} (last ${effectiveCooldownMinutes} minutes) for event '${eventName}'.`
+            );
+
+            const { data } = await github.rest.actions.listWorkflowRuns({
+              owner,
+              repo,
+              workflow_id: 'pr-test.yml',
+              event: eventName,
+              per_page: 100,
+            });
+
+            const runs = data.workflow_runs || [];
+
+            // Rate Limiting Logic:
+            // We only count workflow runs that actually consumed CI resources (i.e., passed the gate).
+            // A run "passes the gate" if any jobs beyond the gate jobs (check-changes, pr-gate, call-gate)
+            // actually executed (not skipped/cancelled). This prevents scenarios where:
+            // - User has PR A with missing 'run-ci' label (fails at gate)
+            // - User opens PR B with 'run-ci' label
+            // - PR B should be able to run even though PR A triggered a run recently
+
+            // Helper function to check if a run passed the gate (i.e., actually consumed CI resources)
+            async function didRunPassGate(run) {
+              try {
+                // Note: Fetching up to 100 jobs (API maximum). If a workflow has >100 jobs,
+                // we may miss some, but this is unlikely in practice.
+                const { data: jobsData } = await github.rest.actions.listJobsForWorkflowRun({
+                  owner, repo, run_id: run.id, per_page: 100
+                });
+                const jobs = jobsData.jobs || [];
+
+                // If no jobs exist yet, the run hasn't started consuming resources
+                if (jobs.length === 0) {
+                  core.info(`Run ${run.id} has no jobs yet; not counting against rate limit.`);
+                  return false;
+                }
+
+                // Gate jobs that don't consume significant CI resources
+                const gateJobs = ['check-changes', 'pr-gate', 'call-gate', 'pr-test-finish'];
+                const jobsBeyondGate = jobs.filter(j => !gateJobs.some(g => j.name === g || j.name.startsWith(g + ' ')));
+
+                // A job "ran" if it reached a terminal conclusion state that indicates actual execution
+                const ranStates = ['success', 'failure', 'timed_out', 'action_required'];
+                const hasJobsThatRan = jobsBeyondGate.some(j => j.conclusion && ranStates.includes(j.conclusion));
+                return hasJobsThatRan;
+              } catch (e) {
+                core.warning(`Could not check jobs for run ${run.id}: ${e.message}`);
+
+                // If it's a rate limit error, count it conservatively to prevent abuse
+                if (e.status === 429) {
+                  core.warning(`Hit rate limit checking run ${run.id}; counting it to be safe.`);
+                  return true;
+                }
+
+                // For cancelled/skipped runs, they likely didn't consume resources
+                if (run.conclusion === 'cancelled' || run.conclusion === 'skipped') {
+                  return false;
+                }
+
+                // Default to counting it to prevent abuse
+                return true;
+              }
+            }
+
+            // Limit the number of runs we'll check in detail to avoid API rate limits
+            const MAX_RUNS_TO_CHECK = 5;
+            let runsChecked = 0;
+            let runsSkippedAtGate = 0;
+            let recentFound = null;
+
+            for (const run of runs) {
+              if (String(run.id) === String(context.runId)) continue;
+              if (new Date(run.created_at) < cutoff) continue;
+              const isUserRun = (run.actor?.login === triggeringActor) || (run.triggering_actor?.login === triggeringActor);
+              if (!isUserRun) continue;
+
+              runsChecked++;
+              core.info(`Checking run ${run.id} (created: ${run.created_at}, conclusion: ${run.conclusion})`);
+
+              // Safety limit: if we've checked too many runs, assume the next one passed to be conservative
+              if (runsChecked > MAX_RUNS_TO_CHECK) {
+                core.warning(`Checked ${MAX_RUNS_TO_CHECK} runs; assuming this one passed gate to avoid API limits.`);
+                recentFound = run;
+                break;
+              }
+
+              // Only count runs that actually passed the gate and consumed CI resources
+              if (await didRunPassGate(run)) {
+                recentFound = run;
+                core.info(`Found recent run ${run.id} that passed gate.`);
+                break;
+              } else {
+                runsSkippedAtGate++;
+                core.info(`Run ${run.id} failed at gate; not counting against rate limit.`);
+              }
+            }
+
+            core.info(`Rate limit check summary: checked ${runsChecked} runs, ${runsSkippedAtGate} failed at gate.`);
+
+            if (recentFound) {
+              core.setFailed(
+                `User '${triggeringActor}' already triggered '${context.workflow}' via '${eventName}' at ${recentFound.created_at}. ` +
+                `Please wait ${effectiveCooldownMinutes} minutes before triggering again.`
+              );
+            } else {
+              core.info(
+                `No recent runs detected for '${triggeringActor}' within the last ${effectiveCooldownMinutes} minutes; proceeding.`
+              );
+            }

.github/workflows/pr-states.yml

@@ -0,0 +1,219 @@
+name: PR States
+# Maintains the CI-states block at the bottom of the PR body. Uses
+# pull_request_target (not pull_request) for fork-PR write access; safe
+# because we never check out PR head code, only API-read and PATCH the body.
+#
+# Three triggers, each covering a gap:
+#   - pull_request_target: push / open / label change (all PRs incl forks).
+#   - workflow_run: initial pr-test* completion (incl fork PRs; the
+#     notify-job below can't fire on forks — their pull_request
+#     GITHUB_TOKEN is forced read-only). Does NOT re-fire for rerun
+#     attempts, which is why workflow_dispatch is needed too.
+#   - workflow_dispatch: rerun completions for non-fork PRs, dispatched by
+#     pr-test*'s notify-pr-states job. workflow_dispatch is the only event
+#     GITHUB_TOKEN can cascade-create.
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+  workflow_run:
+    workflows: ["PR Test Base", "PR Test Extra"]
+    types: [requested, completed]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'PR number whose CI-states block should be refreshed.'
+        required: true
+        type: string
+
+permissions:
+  pull-requests: write
+  actions: read
+
+concurrency:
+  # Per-SHA dedupe (PR number for workflow_dispatch). Queues concurrent
+  # fires across the three triggers so they don't race on the body PATCH.
+  group: pr-states-${{ github.event.pull_request.head.sha || github.event.workflow_run.head_sha || inputs.pr_number }}
+  cancel-in-progress: false
+
+jobs:
+  update-pr-body:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Update CI states block in PR body
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const RETRY_ATTEMPTS = 3;
+            const RETRY_DELAY_MS = 6000;
+            const LABEL_ON_PRESLEEP_MS = 3000;
+            const LABEL_OFF_PRESLEEP_MS = 6000;
+            const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+            // Event-agnostic PR lookup.
+            // - pull_request_target: PR is in the payload.
+            // - workflow_run: pull_requests[] is populated for same-repo
+            //   PRs; fork PRs need a head-ref reverse lookup
+            //   ("forkOwner:branch") because
+            //   listPullRequestsAssociatedWithCommit requires the commit to
+            //   be in the default branch. Bail when no open PR matches.
+            // - workflow_dispatch: pr_number is an explicit input.
+            let prNumber;
+            if (context.payload.pull_request) {
+              prNumber = context.payload.pull_request.number;
+            } else if (context.eventName === 'workflow_run') {
+              const wr = context.payload.workflow_run;
+              if (wr.pull_requests && wr.pull_requests.length > 0) {
+                prNumber = wr.pull_requests[0].number;
+              } else if (wr.head_repository && wr.head_branch) {
+                const headSpec = `${wr.head_repository.owner.login}:${wr.head_branch}`;
+                const { data: prs } = await github.rest.pulls.list({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  state: 'open',
+                  head: headSpec,
+                });
+                // Multiple PRs could share a head ref if a branch was reused;
+                // match by head_sha to pick the right one.
+                const match = prs.find(p => p.head.sha === wr.head_sha) || prs[0];
+                if (!match) {
+                  core.info(`No open PR for head=${headSpec}; skipping.`);
+                  return;
+                }
+                prNumber = match.number;
+              } else {
+                core.info(`workflow_run has no PR linkage; skipping.`);
+                return;
+              }
+            } else if (context.eventName === 'workflow_dispatch') {
+              // Reject non-digit pr_number; parseInt('123abc', 10) returns
+              // 123 and would silently truncate trailing garbage.
+              const raw = context.payload.inputs.pr_number;
+              if (!/^\d+$/.test(raw)) {
+                core.setFailed(`workflow_dispatch: invalid pr_number input '${raw}' (must be digits only)`);
+                return;
+              }
+              prNumber = parseInt(raw, 10);
+            } else {
+              core.info(`Unsupported event '${context.eventName}'; skipping.`);
+              return;
+            }
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber,
+            });
+            const sha = pr.head.sha;
+            const labels = pr.labels.map(l => l.name);
+            const hasCI = labels.includes('run-ci');
+            const hasExtra = labels.includes('run-ci-extra');
+
+            // Pre-sleep below only matters when a label was just removed;
+            // other no-label fires don't race against fresh indexing.
+            const justRemovedLabel = context.eventName === 'pull_request_target'
+                                     && context.payload.action === 'unlabeled';
+
+            async function findRunBySha(workflowFile) {
+              const { data } = await github.rest.actions.listWorkflowRuns({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                workflow_id: workflowFile,
+                head_sha: sha,
+                per_page: 1,
+              });
+              return data.workflow_runs[0] || null;
+            }
+            async function findRun(workflowFile, { preSleepMs = 0, attempts = 1, delayMs = 0 }) {
+              if (preSleepMs > 0) await sleep(preSleepMs);
+              for (let i = 0; i < attempts; i++) {
+                const run = await findRunBySha(workflowFile);
+                if (run) return run;
+                if (i < attempts - 1) await sleep(delayMs);
+              }
+              return null;
+            }
+
+            // Query unconditionally: removing a label does NOT cancel an
+            // in-flight workflow, so a run from before removal can still be
+            // live. Label state only picks the fallback text. labelOnOpts
+            // covers fresh-push indexing race; labelOffOpts only pre-sleeps
+            // on just-removed-label to catch removal-after-push.
+            const labelOnOpts  = { preSleepMs: LABEL_ON_PRESLEEP_MS, attempts: RETRY_ATTEMPTS, delayMs: RETRY_DELAY_MS };
+            const labelOffOpts = { preSleepMs: justRemovedLabel ? LABEL_OFF_PRESLEEP_MS : 0, attempts: 1 };
+            const ptRun = await findRun('pr-test.yml', hasCI ? labelOnOpts : labelOffOpts);
+            // pr-test-extra gates on BOTH labels (see check-changes there).
+            const peRun = await findRun('pr-test-extra.yml', (hasCI && hasExtra) ? labelOnOpts : labelOffOpts);
+
+            // Skipped run = "no real run" -- happens when a label is added
+            // after a commit, because GHA doesn't retrigger on `labeled`.
+            const isReal = (run) => run && run.conclusion !== 'skipped';
+
+            const missingCIText = ':x: **Missing `run-ci` label** -- add it to run CI tests.';
+            const peBlockedByCIText = ':x: **Blocked** -- `run-ci` is required first.';
+            const notExtraEnabledText = ':warning: **Not enabled** -- add `run-ci-extra` label to opt in.';
+            const stalePushText = ':warning: **Not run on latest push** -- push again to dispatch.';
+
+            // Status icon: body otherwise looks uniformly "dispatched" even
+            // for gate-failed runs (call-gate exits on missing label).
+            function runIcon(run) {
+              if (run.status !== 'completed') return ':hourglass_flowing_sand:';
+              switch (run.conclusion) {
+                case 'success': return ':white_check_mark:';
+                case 'failure': return ':x:';
+                case 'cancelled': return ':no_entry_sign:';
+                case 'timed_out': return ':alarm_clock:';
+                case 'action_required': return ':warning:';
+                case 'neutral': return ':grey_question:';
+                default: return ':grey_question:';
+              }
+            }
+            const runLink = (run) => `${runIcon(run)} [Run #${run.id}](${run.html_url})`;
+            const ptText = isReal(ptRun)
+              ? runLink(ptRun)
+              : (!hasCI ? missingCIText : '_Not run yet_');
+            const peText = isReal(peRun)
+              ? runLink(peRun)
+              : !hasCI
+                ? peBlockedByCIText
+                : !hasExtra
+                  ? notExtraEnabledText
+                  : stalePushText;
+
+            const outerStart = '<!-- pr-states:start -->';
+            const outerEnd   = '<!-- pr-states:end -->';
+            const ptStart    = '<!-- slot:pr-test:start -->';
+            const ptEnd      = '<!-- slot:pr-test:end -->';
+            const peStart    = '<!-- slot:pr-test-extra:start -->';
+            const peEnd      = '<!-- slot:pr-test-extra:end -->';
+
+            const newBlock = [
+              outerStart,
+              '---',
+              '### CI States',
+              '',
+              `Latest PR Test (Base): ${ptStart}${ptText}${ptEnd}`,
+              `Latest PR Test (Extra): ${peStart}${peText}${peEnd}`,
+              outerEnd,
+            ].join('\n');
+
+            const body = pr.body || '';
+
+            const blockRe = new RegExp(`(?:\\n+---\\n+)?${outerStart}[\\s\\S]*?${outerEnd}`);
+            let newBody;
+            if (blockRe.test(body)) {
+              newBody = body.replace(blockRe, `\n\n${newBlock}`);
+            } else {
+              const sep = body.length === 0
+                ? ''
+                : (body.endsWith('\n') ? '\n' : '\n\n');
+              newBody = `${body}${sep}${newBlock}`;
+            }
+
+            if (newBody !== body) {
+              await github.rest.pulls.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: prNumber,
+                body: newBody,
+              });
+            }

.github/workflows/pr-test-amd-extra.yml

@@ -0,0 +1,244 @@
+name: PR Test Extra (AMD)
+# Label-gated AMD extra CI workflow — the AMD mirror of pr-test-extra.yml.
+#
+# Adds AMD runtime to a PR only when the author opts in: the PR must carry
+# BOTH `run-ci` (basic-CI prerequisite) and `run-ci-extra` (explicit opt-in).
+# The label check happens at runtime in pr-gate.yml via a live
+# `gh pr view`-style fetch, so reruns after adding the labels (e.g. via a
+# slash command) pick up the new label set — a workflow-level `if` would read
+# the frozen event payload, which never updates on rerun. The job graph also
+# runs unconditionally on workflow_dispatch / workflow_call so it can be
+# triggered manually or chained from the AMD scheduler.
+#
+# Stage: extra-a. Each job mirrors the container bring-up of pr-test-amd.yml
+# and dispatches `run_suite.py --hw amd --suite extra-a-test-{config}-amd`:
+#   - 1-gpu-small: mock-model / kv_canary unit + single-GPU canary e2e tests
+#   - 1-gpu-large: single-GPU model e2e tests (quant fp8kv-triton,
+#                  sessions streaming-session, spec standalone triton,
+#                  kv_canary self-bench)
+#   - 2-gpu-large: multi-GPU (TP/PP/PD) mock-model + kv_canary e2e tests
+# kv_canary e2e is registered to the same extra-a stage as its CUDA siblings
+# (it now exercises the ROCm canary kernels end-to-end).
+
+on:
+  pull_request:
+    # `labeled` lets the workflow re-fire when `run-ci-extra` (or `run-ci`)
+    # is added after the latest push. See call-gate.if for the matching guard
+    # that prevents unrelated label additions from dispatching a full run.
+    types: [opened, synchronize, reopened, labeled]
+  workflow_dispatch:
+    inputs:
+      runner_arch:
+        description: 'AMD runner pool to dispatch GPU jobs to'
+        required: false
+        type: choice
+        default: mi325
+        options:
+          - mi300
+          - mi325
+      rocm_version:
+        description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)'
+        required: false
+        type: choice
+        default: ''
+        options:
+          - ''
+          - rocm720
+      aiter_ref:
+        description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
+        required: false
+        type: string
+        default: ''
+      continue_on_error:
+        description: 'Continue on error (do not fail the workflow on test failures)'
+        required: false
+        type: boolean
+        default: false
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      rocm_version:
+        description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)'
+        required: false
+        type: string
+        default: ''
+      aiter_ref:
+        description: 'Override AITER commit (optional, leave empty to use Dockerfile default)'
+        required: false
+        type: string
+        default: ''
+      continue_on_error:
+        description: 'Continue on error (do not fail the workflow on test failures)'
+        required: false
+        type: boolean
+        default: false
+
+env:
+  AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
+  DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+  DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+concurrency:
+  # Mirror pr-test-amd.yml: schedule/dispatch runs key on run_id (unique group,
+  # never cancel each other); PR runs share a per-branch group so pushes cancel
+  # stale runs. (In a reusable workflow github.event_name is the originating
+  # event, never workflow_call, so the guard keys off the real events.)
+  group: pr-test-amd-extra-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && format('full-{0}', github.run_id) || github.head_ref || github.ref_name || inputs.ref || 'default' }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  actions: write
+  contents: read
+  issues: read
+  pull-requests: read
+
+jobs:
+  # =============================================== PR Gate ====================================================
+  # Runtime live-fetch label gate (mirrors pr-test-extra.yml's call-gate):
+  # requires both `run-ci` and `run-ci-extra`. A failure here cascades to
+  # every test job via `needs`, so a PR without the labels ends in one red
+  # ~30s gate job plus a row of skipped jobs instead of consuming AMD runners.
+  #
+  # The job-level `if` only filters the `labeled` event type so that adding an
+  # unrelated label doesn't dispatch a full run; the actual label-presence
+  # gate is enforced at runtime inside pr-gate.yml.
+  call-gate:
+    if: |
+      github.event_name != 'pull_request' ||
+      github.event.action != 'labeled' ||
+      github.event.label.name == 'run-ci' ||
+      github.event.label.name == 'run-ci-extra'
+    uses: ./.github/workflows/pr-gate.yml
+    with:
+      require-run-ci: true
+      require-run-ci-extra: true
+    secrets: inherit
+
+  # =============================================== extra-a (1-gpu-small) ===============================================
+  # Single unpartitioned job: the 21 onboarded unit tests total ~233s, so the
+  # expensive per-job setup (container bring-up + sgl-kernel ROCm build + dep
+  # install, several minutes) dominates. Partitioning would multiply that
+  # setup across scarce AMD GPUs to shave only a couple minutes of test time,
+  # so one GPU running the whole suite sequentially is the better trade.
+  extra-a-test-1-gpu-small-amd:
+    name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }}
+    needs: [call-gate]
+    if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
+      - name: Start CI container
+        # `rocm_version` (e.g. rocm720) selects an alternate ROCm container; empty uses the Dockerfile default.
+        run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }}
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 45
+        run: |
+          bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite extra-a-test-1-gpu-small-amd --timeout-per-file 1800 ${{ inputs.continue_on_error == true && '--continue-on-error' || '' }}
+
+  # =============================================== extra-a (1-gpu-large) ===============================================
+  # Single-GPU lora / spec / quant model e2e tests. Runs on the same 1-gpu
+  # pool as small (AMD GPUs carry enough VRAM that "large" here is a CUDA
+  # memory-tier label, not a separate AMD runner pool).
+  extra-a-test-1-gpu-large-amd:
+    name: ${{ format('extra-a-test-1-gpu-large-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }}
+    needs: [call-gate]
+    if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
+      - name: Start CI container
+        run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }}
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite extra-a-test-1-gpu-large-amd --timeout-per-file 1800 ${{ inputs.continue_on_error == true && '--continue-on-error' || '' }}
+
+  # =============================================== extra-a (2-gpu-large) ===============================================
+  # Multi-GPU TP / PP / PD mock-model + kv_canary e2e tests. Mirrors CUDA's
+  # extra-a 2-gpu-large stage; runs on the 2-GPU AMD pool.
+  extra-a-test-2-gpu-large-amd:
+    name: ${{ format('extra-a-test-2-gpu-large-amd{0} (linux-{1}-2gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }}
+    needs: [call-gate]
+    if: ${{ !cancelled() && needs.call-gate.result == 'success' }}
+    runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || 'mi325') }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
+      - name: Start CI container
+        run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }}
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install dependencies
+        run: bash scripts/ci/amd/amd_ci_install_dependency.sh
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite extra-a-test-2-gpu-large-amd --timeout-per-file 1800 ${{ inputs.continue_on_error == true && '--continue-on-error' || '' }}
+
+  # =============================================== aggregator ====================================================
+  # Single fan-in job so branch protection / notifications depend on one job
+  # rather than every matrix leg. Fails if any dependent failed or was
+  # cancelled; `skipped` (e.g. PR without the opt-in labels) is not a failure.
+  pr-test-amd-extra-finish:
+    needs:
+      [
+        call-gate,
+        extra-a-test-1-gpu-small-amd,
+        extra-a-test-1-gpu-large-amd,
+        extra-a-test-2-gpu-large-amd,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          json_needs='${{ toJson(needs) }}'
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+          for job in $job_names; do
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+            echo "$job: $result"
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "The above jobs failed."
+              exit 1
+            fi
+          done
+          echo "All jobs completed successfully"
+          exit 0

.github/workflows/pr-test-arm64.yml

@@ -0,0 +1,118 @@
+name: PR Test (Arm64)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      run_all_tests:
+        description: "Run all tests (for releasing or testing purpose)"
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: pr-test-arm64-${{ inputs.ref || github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Determine run mode
+        id: run-mode
+        run: |
+          if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
+            echo "run_all_tests=true" >> $GITHUB_OUTPUT
+            echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})"
+          else
+            echo "run_all_tests=false" >> $GITHUB_OUTPUT
+            echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
+          fi
+
+      - name: Detect file changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        if: steps.run-mode.outputs.run_all_tests != 'true'
+        with:
+          filters: |
+            main_package:
+              - "python/sglang/!(multimodal_gen)/**/!(*.md)"
+              - "python/pyproject_cpu.toml"
+              - "test/**/!(*.md)"
+              - "sgl-kernel/**/*.!(md|txt)"
+              - ".github/workflows/pr-test-arm64.yml"
+              - "docker/arm64.Dockerfile"
+
+  pr-gate:
+    needs: check-changes
+    if: needs.check-changes.outputs.main_package == 'true'
+    uses: ./.github/workflows/pr-gate.yml
+    secrets: inherit
+
+  build-test:
+    needs: [check-changes, pr-gate]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Build container
+        run: |
+          PR_REPO=${{ github.event.pull_request.head.repo.clone_url }}
+          PR_HEAD_REF=${{ github.head_ref }}
+
+          docker build \
+            ${PR_REPO:+--build-arg SGLANG_REPO=$PR_REPO} \
+            ${PR_HEAD_REF:+--build-arg VER_SGLANG=$PR_HEAD_REF} \
+            . -f docker/arm64.Dockerfile -t sglang_arm64 --no-cache
+
+      - name: Run container
+        run: |
+          docker run -dt \
+            -v ${{ github.workspace }}:/sglang-checkout/ --ipc=host \
+            --name ci_sglang_arm64 \
+            sglang_arm64
+
+      - name: Arm sanity check
+        timeout-minutes: 5
+        run: |
+          docker exec -w /sglang-checkout/ ci_sglang_arm64 \
+            bash -c "source /opt/.venv/bin/activate && python3 -c 'import platform; import torch; import sgl_kernel; from sglang.srt.utils.common import is_host_cpu_arm64; assert platform.machine() in (\"aarch64\", \"arm64\"); assert is_host_cpu_arm64(); assert hasattr(torch.ops.sgl_kernel, \"decode_attention_cpu\"); assert hasattr(torch.ops.sgl_kernel, \"initialize\");'"
+
+      - name: Run unit tests
+        timeout-minutes: 36
+        run: |
+          docker exec -w /sglang-checkout/ ci_sglang_arm64 \
+            bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite base-b-test-cpu-arm64 --timeout-per-file 1500"
+
+      - name: Change permission
+        timeout-minutes: 2
+        run: |
+          docker exec -u root ci_sglang_arm64 bash -c "
+            rm -rf /tmp/ci-home  &&
+            chown -R  $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true
+          "
+
+      - name: Cleanup container
+        if: always()
+        run: |
+          docker rm -f ci_sglang_arm64 || true

.github/workflows/pr-test-extra.yml

@@ -0,0 +1,320 @@
+name: PR Test Extra
+# Label-gated extra CI workflow opted into per-PR via labels.
+#
+# Adds runtime to a PR only when the author asks for it: the PR must carry
+# BOTH `run-ci` (basic-CI prerequisite) and `run-ci-extra` (explicit opt-in
+# to this workflow). The label check happens at runtime in pr-gate.yml via
+# live `gh pr view --json labels`, so reruns after adding the labels via
+# slash command pick up the new label set (workflow-level `if` checks would
+# read the frozen event payload, which never updates on rerun). The same
+# job graph runs unconditionally on workflow_dispatch / workflow_call so it
+# can be triggered manually or chained from another workflow.
+#
+# Stages: extra-a (1-/2-gpu) and extra-b (4-/8-gpu) caller stubs reuse
+# `_pr-test-stage.yml` and `_pr-test-check-changes.yml` from pr-test.yml.
+
+on:
+  pull_request:
+    # `labeled` lets the workflow re-fire when `run-ci-extra` (or `run-ci`)
+    # is added after the latest push — otherwise GHA leaves the stale
+    # skipped run in place and there's no way to enable extra without
+    # another push. See check-changes.if below for the matching guard.
+    types: [opened, synchronize, reopened, labeled]
+  workflow_dispatch:
+    inputs:
+      force_continue_on_error:
+        description: "Force continue-on-error (test scheduled CI behavior)"
+        required: false
+        type: boolean
+        default: false
+      test_parallel_dispatch:
+        description: "Test parallel dispatch behavior (simulates scheduled run)"
+        required: false
+        type: boolean
+        default: false
+      run_all_tests:
+        description: "Run all tests — bypasses paths-filter (needed when dispatching against long-lived branches whose diff against main can't resolve a merge-base in a shallow clone)"
+        required: false
+        type: boolean
+        default: false
+  workflow_call:
+    inputs:
+      git_ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      run_all_tests:
+        description: "Run all tests (for releasing or testing purpose)"
+        required: false
+        type: boolean
+        default: false
+      skip_pr_test_health_check:
+        description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: pr-test-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
+  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.run_all_tests == true) && 'true' || 'false' }}
+  FORCE_REBUILD_DEEPEP: '1'
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  USE_VENV: false
+
+permissions:
+  actions: write
+  contents: read
+  issues: read
+  pull-requests: read
+
+jobs:
+  # =============================================== check changes ====================================================
+  # The actual `run-ci` + `run-ci-extra` label gate now lives in the
+  # `call-gate` job below, which calls pr-gate.yml. pr-gate.yml live-fetches
+  # the PR's current labels at runtime, so a rerun (e.g. via
+  # /tag-and-rerun-ci extra, which adds labels via GITHUB_TOKEN — these
+  # don't cascade-trigger labeled events) picks up the new label set and
+  # passes. The previous workflow-level `if` checked the frozen event
+  # payload, which never updates on rerun and made the slash-add-label flow
+  # impossible to recover.
+  #
+  # We only keep one filter here: for `labeled` events, only proceed if the
+  # just-added label is one of the gating labels. This prevents every
+  # unrelated label addition from dispatching a full CI run.
+  check-changes:
+    if: |
+      github.event_name != 'pull_request' ||
+      github.event.action != 'labeled' ||
+      github.event.label.name == 'run-ci' ||
+      github.event.label.name == 'run-ci-extra'
+    uses: ./.github/workflows/_pr-test-check-changes.yml
+    with:
+      git_ref: ${{ inputs.git_ref || '' }}
+      run_all_tests: ${{ inputs.run_all_tests == true }}
+      force_continue_on_error: ${{ inputs.force_continue_on_error == true }}
+      pr_test_yml: '.github/workflows/pr-test-extra.yml'
+    secrets: inherit
+
+  # =============================================== PR Gate ====================================================
+  # Runtime live-fetch gate. Mirrors pr-test.yml's call-gate, but additionally
+  # requires `run-ci-extra`. Failure here cascades down to every test job
+  # via `needs`, so the workflow ends with one red gate job (~30s) plus a
+  # row of skipped test jobs instead of consuming a runner.
+  call-gate:
+    needs: check-changes
+    if: github.event_name != 'schedule' && needs.check-changes.result == 'success'
+    uses: ./.github/workflows/pr-gate.yml
+    with:
+      require-run-ci: true
+      require-run-ci-extra: true
+    secrets: inherit
+
+  # =============================================== sgl-kernel ====================================================
+  sgl-kernel-build-wheels:
+    needs: [check-changes, call-gate]
+    if: |
+      needs.check-changes.result == 'success' &&
+      (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') &&
+      needs.check-changes.outputs.sgl_kernel == 'true'
+    uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
+    with:
+      runs_on: x64-kernel-build-node
+      job_display_name: Build Wheel
+      git_ref: ${{ inputs.git_ref || '' }}
+      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
+    secrets: inherit
+
+  # =============================================== extra-a (1-/2-gpu) ===============================================
+  extra-a-test-1-gpu-small:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-a-test-1-gpu-small
+      runner_config: 1-gpu-small
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+    secrets: inherit
+
+  extra-a-test-1-gpu-large:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-a-test-1-gpu-large
+      runner_config: 1-gpu-large
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  extra-a-test-2-gpu-large:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-a-test-2-gpu-large
+      runner_config: 2-gpu-large
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+    secrets: inherit
+
+  # =============================================== extra-b (4-/8-gpu) ===============================================
+  extra-b-test-4-gpu-h100:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-b-test-4-gpu-h100
+      runner_config: 4-gpu-h100
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+    secrets: inherit
+
+  extra-b-test-4-gpu-b200:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-b-test-4-gpu-b200
+      runner_config: 4-gpu-b200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  extra-b-test-8-gpu-h200:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-b-test-8-gpu-h200
+      runner_config: 8-gpu-h200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+    secrets: inherit
+
+  extra-b-test-deepep-4-gpu-h100:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-b-test-deepep-4-gpu-h100
+      runner_config: deepep-4-gpu-h100
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+    secrets: inherit
+
+  extra-b-test-deepep-4-gpu-b200:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-b-test-deepep-4-gpu-b200
+      runner_config: deepep-4-gpu-b200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  extra-b-test-deepep-8-gpu-h200:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: extra-b-test-deepep-8-gpu-h200
+      runner_config: deepep-8-gpu-h200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '60'
+    secrets: inherit
+
+  # =============================================== aggregator ====================================================
+  # Mirrors pr-test.yml's `pr-test-finish` so notify-pr-states below only
+  # depends on one job rather than re-listing every stage. Fails if any
+  # dependent failed/cancelled.
+  pr-test-extra-finish:
+    needs:
+      [
+        check-changes,
+        call-gate,
+        sgl-kernel-build-wheels,
+        extra-a-test-1-gpu-small,
+        extra-a-test-1-gpu-large,
+        extra-a-test-2-gpu-large,
+        extra-b-test-4-gpu-h100,
+        extra-b-test-4-gpu-b200,
+        extra-b-test-8-gpu-h200,
+        extra-b-test-deepep-4-gpu-h100,
+        extra-b-test-deepep-4-gpu-b200,
+        extra-b-test-deepep-8-gpu-h200,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          json_needs='${{ toJson(needs) }}'
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+          for job in $job_names; do
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+            echo "$job: $result"
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "The above jobs failed."
+              exit 1
+            fi
+          done
+          echo "All jobs completed successfully"
+          exit 0
+
+  # =============================================== notify pr-states ====================================================
+  # Dispatches pr-states.yml after every attempt (initial + every rerun).
+  # workflow_run.completed only fires on the initial completion (one
+  # workflow_run record across attempts), so we explicitly dispatch —
+  # workflow_dispatch is the only event GITHUB_TOKEN can cascade-create.
+  #
+  # Fork PRs are excluded: their pull_request GITHUB_TOKEN is forced
+  # read-only regardless of declared `actions: write`, so dispatch 403s.
+  # Fork PRs get pr-states updates via the workflow_run subscription
+  # there (initial completion) and pull_request_target (push / label).
+  notify-pr-states:
+    needs: [pr-test-extra-finish]
+    if: |
+      always() &&
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    steps:
+      - name: Dispatch pr-states refresh
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          gh workflow run pr-states.yml \
+            --repo ${{ github.repository }} \
+            --ref main \
+            -f pr_number="$PR_NUMBER"

.github/workflows/pr-test-jit-kernel.yml

@@ -0,0 +1,197 @@
+name: PR Test - JIT Kernel
+
+on:
+  workflow_call:
+    inputs:
+      jit_kernel:
+        required: true
+        type: string
+      sgl_kernel:
+        required: true
+        type: string
+      runner_config:
+        required: true
+        type: string
+      runs_on_map:
+        required: true
+        type: string
+      git_ref:
+        required: false
+        type: string
+        default: ''
+      test_parallel_dispatch:
+        required: false
+        type: string
+        default: 'false'
+      skip_pr_test_health_check:
+        required: false
+        type: boolean
+        default: false
+
+# Workflow-level env is NOT inherited from the caller in reusable workflows (verified by CI test).
+# The github context (including github.event_name) IS inherited from the caller.
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
+  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
+
+jobs:
+  jit-kernel-unit-test:
+    # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
+    # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
+    # the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
+    runs-on: 1-gpu-h100
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Cleanup
+        if: inputs.sgl_kernel == 'true'
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda13.0
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run test
+        timeout-minutes: 30
+        run: |
+          cd test/
+          python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-1-gpu-large
+
+  jit-kernel-multigpu-unit-test:
+    # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
+    # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
+    # the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
+    runs-on: 8-gpu-h200
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Cleanup
+        if: inputs.sgl_kernel == 'true'
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda13.0
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run multi-GPU test
+        timeout-minutes: 45
+        run: |
+          cd test/
+          python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-8-gpu-h200
+
+  jit-kernel-benchmark-test:
+    # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
+    # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
+    # the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
+    runs-on: 1-gpu-h100
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Cleanup
+        if: inputs.sgl_kernel == 'true'
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda13.0
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run benchmark tests
+        timeout-minutes: 45
+        run: |
+          cd test/
+          python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1-gpu-large
+
+  jit-kernel-b200-test:
+    # Runs whenever call-jit-kernel-tests dispatches this workflow. That caller is the
+    # single gate (PR jit_kernel changes, or scheduled/parallel-dispatch full runs), so
+    # the sub-jobs no longer re-exclude schedule/parallel-dispatch here.
+    runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }}
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Cleanup
+        if: inputs.sgl_kernel == 'true'
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda13.0
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run B200 diffusion test
+        timeout-minutes: 30
+        run: |
+          cd test/
+          python3 run_suite.py --hw cuda --suite base-b-kernel-unit-test-4-gpu-b200

.github/workflows/pr-test-mlx.yml

@@ -0,0 +1,184 @@
+name: PR Test (MLX)
+
+on:
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+    inputs:
+      # target_stage is the job id (e.g. stage-b-e2e-test-mlx), distinct from
+      # the run_suite.py suite name it dispatches (e.g. stage-b-e2e-mlx).
+      target_stage:
+        description: "Select a stage to run from dropdown (leave empty for auto-detect)"
+        required: false
+        type: choice
+        default: ''
+        options:
+          - ''
+          - stage-a-unit-test-mlx
+          - stage-b-e2e-test-mlx
+
+concurrency:
+  group: pr-test-mlx-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # ==================== Check Changes ==================== #
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changes_exist: ${{ steps.filter.outputs.main_package == 'true'
+        || steps.filter.outputs.sgl_kernel == 'true'
+        || inputs.target_stage != '' }}
+      main_package: ${{ steps.filter.outputs.main_package }}
+      sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Detect file changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          # Ignored on pull_request; workflow_dispatch has no PR base without it.
+          base: main
+          filters: |
+            main_package:
+              - "python/sglang/!(multimodal_gen)/**/!(*.md)"
+              - "python/pyproject_other.toml"
+              - "test/**/!(*.md)"
+              - ".github/workflows/pr-test-mlx.yml"
+            sgl_kernel:
+              - "sgl-kernel/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)"
+              - ".github/workflows/pr-test-mlx.yml"
+
+  # ==================== PR Gate ==================== #
+  pr-gate:
+    needs: check-changes
+    if: needs.check-changes.outputs.changes_exist == 'true'
+    uses: ./.github/workflows/pr-gate.yml
+    secrets: inherit
+
+  stage-a-unit-test-mlx:
+    needs: [check-changes, pr-gate]
+    # !cancelled() lets dispatch past a skipped pr-gate; the pull_request
+    # path still requires pr-gate success, keeping the run-ci gate intact.
+    if: |
+      !cancelled() && (
+        (needs.pr-gate.result == 'success' &&
+          (needs.check-changes.outputs.main_package == 'true' ||
+           needs.check-changes.outputs.sgl_kernel == 'true')) ||
+        (github.event_name == 'workflow_dispatch' &&
+         inputs.target_stage == 'stage-a-unit-test-mlx')
+      )
+    runs-on: macos-26
+    timeout-minutes: 60
+    env:
+      SGLANG_IS_IN_CI: true
+      # use_mlx() needs this; without it the profiler takes the MPS branch.
+      SGLANG_USE_MLX: 1
+      # Forbid HF downloads to enforce the model-free guarantee.
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: 'python/pyproject_other.toml'
+
+      - name: Verify Apple Silicon runner
+        run: |
+          echo "uname -m: $(uname -m)"
+          python3 -c "import platform; assert platform.machine()=='arm64', platform.machine(); print('machine:', platform.machine(), 'system:', platform.system())"
+
+      - name: Install dependencies (MLX / srt_mps extra)
+        timeout-minutes: 30
+        run: |
+          test -f python/pyproject_other.toml || { echo "alt pyproject_other.toml missing"; exit 1; }
+          # Swap in the Apple Silicon pyproject; srt_mps skips the all_mps diffusion chain.
+          rm -f python/pyproject.toml
+          mv python/pyproject_other.toml python/pyproject.toml
+          uv venv
+          uv pip install -e "python[srt_mps,test]"
+
+      - name: Report MLX / torch versions
+        run: |
+          uv run python -c "import mlx.core as mx; print('mlx', mx.__version__)"
+          uv run python -c "import torch; print('torch', torch.__version__)"
+
+      - name: Run model-free MLX unit tests
+        timeout-minutes: 15
+        run: |
+          uv run python test/run_suite.py --hw mlx --suite stage-a-unit-test-mlx
+
+  stage-b-e2e-test-mlx:
+    needs: [check-changes, pr-gate]
+    # Manual-only: no self hosted Apple Silicon runner is generally available; hosted runners are too small for e2e.
+    if: |
+      !cancelled() && inputs.target_stage == 'stage-b-e2e-test-mlx'
+    runs-on: [self-hosted, macOS, ARM64]
+    timeout-minutes: 120
+    env:
+      SGLANG_IS_IN_CI: true
+      SGLANG_USE_MLX: 1
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: 'python/pyproject_other.toml'
+
+      - name: Verify Apple Silicon runner
+        run: |
+          echo "uname -m: $(uname -m)"
+          python3 -c "import platform; assert platform.machine()=='arm64', platform.machine(); print('machine:', platform.machine(), 'system:', platform.system())"
+
+      - name: Install dependencies (MLX / srt_mps extra)
+        timeout-minutes: 30
+        run: |
+          test -f python/pyproject_other.toml || { echo "alt pyproject_other.toml missing"; exit 1; }
+          rm -f python/pyproject.toml
+          mv python/pyproject_other.toml python/pyproject.toml
+          uv venv --python 3.11
+          uv pip install -e "python[srt_mps,test]"
+
+      # Needs a warm HF cache or network and roughly 24 GB of unified memory.
+      - name: Run MLX e2e smoke tests
+        timeout-minutes: 90
+        run: |
+          uv run python test/run_suite.py --hw mlx --suite stage-b-e2e-mlx
+
+  pr-test-mlx-finish:
+    needs:
+      [
+        pr-gate,
+        check-changes,
+        stage-a-unit-test-mlx,
+        stage-b-e2e-test-mlx,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          json_needs='${{ toJson(needs) }}'
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+          for job in $job_names; do
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+            echo "$job: $result"
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "The above jobs failed."
+              exit 1
+            fi
+          done
+          echo "All jobs completed successfully"

.github/workflows/pr-test-multimodal-gen.yml

@@ -0,0 +1,505 @@
+name: PR Test - Multimodal Gen
+
+on:
+  workflow_call:
+    inputs:
+      multimodal_gen:
+        required: true
+        type: string
+      sgl_kernel:
+        required: true
+        type: string
+      runner_config:
+        required: true
+        type: string
+      runs_on_map:
+        required: true
+        type: string
+      continue_on_error:
+        required: false
+        type: string
+        default: 'false'
+      git_ref:
+        required: false
+        type: string
+        default: ''
+      test_parallel_dispatch:
+        required: false
+        type: string
+        default: 'false'
+      caller_needs_failure:
+        required: false
+        type: string
+        default: 'false'
+      skip_pr_test_health_check:
+        required: false
+        type: string
+        default: 'false'
+
+# Workflow-level env is NOT inherited from the caller in reusable workflows.
+# The github context (including github.event_name) IS inherited from the caller.
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == 'true' }}
+
+jobs:
+  compute-diffusion-partitions:
+    if: inputs.multimodal_gen == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+      matrix-1gpu: ${{ steps.compute.outputs.matrix-1gpu }}
+      matrix-2gpu: ${{ steps.compute.outputs.matrix-2gpu }}
+      partition-count-1gpu: ${{ steps.compute.outputs['partition-count-1gpu'] }}
+      partition-count-2gpu: ${{ steps.compute.outputs['partition-count-2gpu'] }}
+      plan-1gpu: ${{ steps.compute.outputs.plan-1gpu }}
+      plan-2gpu: ${{ steps.compute.outputs.plan-2gpu }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Compute partitions
+        id: compute
+        run: |
+          python scripts/ci/utils/diffusion/compute_diffusion_partitions.py --min-time 1200 --target-time 1800 --max-time 2400 --max-partitions 10
+
+  multimodal-gen-test-1-gpu:
+    needs: compute-diffusion-partitions
+    if: |
+      always() &&
+      needs.compute-diffusion-partitions.result == 'success' &&
+      needs.compute-diffusion-partitions.outputs.matrix-1gpu != '{"include":[]}' &&
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: 1-gpu-h100
+    timeout-minutes: 240
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-1gpu) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+      - name: Run diffusion server tests
+        timeout-minutes: 240
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-1gpu }}
+          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite 1-gpu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions ${{ needs.compute-diffusion-partitions.outputs['partition-count-1gpu'] }} \
+            --partition-plan-json "$PARTITION_PLAN_JSON" \
+            $CONTINUE_ON_ERROR_FLAG
+
+      - name: Upload execution report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-report-1gpu-${{ matrix.part }}
+          path: python/sglang/multimodal_gen/test/execution_report_*.json
+          retention-days: 1
+
+      - name: Upload diffusion failure artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-failures-1gpu-${{ matrix.part }}-${{ github.run_attempt }}
+          path: diffusion-failures/
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
+  multimodal-gen-test-1-5090:
+    if: |
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: 1-gpu-5090
+    timeout-minutes: 180
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run 5090 diffusion canary tests
+        timeout-minutes: 120
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite 1-gpu-5090 \
+            $CONTINUE_ON_ERROR_FLAG
+
+      - name: Upload execution report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-5090-report-${{ github.run_attempt }}
+          path: python/sglang/multimodal_gen/test/execution_report_*.json
+          retention-days: 7
+
+      - name: Upload diffusion failure artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-failures-5090-${{ github.run_attempt }}
+          path: diffusion-failures/
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+        with:
+          artifact-suffix: 1-gpu-5090
+
+  bcg-diffusion:
+    if: |
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: 1-gpu-h100
+    timeout-minutes: 90
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run BCG diffusion tests
+        timeout-minutes: 60
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-bcg-artifacts
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite bcg-diffusion \
+            $CONTINUE_ON_ERROR_FLAG
+
+      - name: Upload BCG diffusion artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-bcg-artifacts-${{ github.run_attempt }}
+          path: diffusion-bcg-artifacts/
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+        with:
+          artifact-suffix: bcg-diffusion
+
+  multimodal-gen-test-2-gpu:
+    needs: compute-diffusion-partitions
+    if: |
+      always() &&
+      needs.compute-diffusion-partitions.result == 'success' &&
+      needs.compute-diffusion-partitions.outputs.matrix-2gpu != '{"include":[]}' &&
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: 2-gpu-h100
+    timeout-minutes: 240
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-2gpu) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run diffusion server tests
+        timeout-minutes: 240
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-2gpu }}
+          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite 2-gpu \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions ${{ needs.compute-diffusion-partitions.outputs['partition-count-2gpu'] }} \
+            --partition-plan-json "$PARTITION_PLAN_JSON" \
+            $CONTINUE_ON_ERROR_FLAG
+
+      - name: Upload execution report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-report-2gpu-${{ matrix.part }}
+          path: python/sglang/multimodal_gen/test/execution_report_*.json
+          retention-days: 1
+
+      - name: Upload diffusion failure artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-failures-2gpu-${{ matrix.part }}-${{ github.run_attempt }}
+          path: diffusion-failures/
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+        with:
+          artifact-suffix: ${{ matrix.part }}
+
+  multimodal-gen-component-accuracy:
+    if: |
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: 2-gpu-h100
+    timeout-minutes: 240
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run diffusion component accuracy tests
+        timeout-minutes: 240
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite component-accuracy \
+            $CONTINUE_ON_ERROR_FLAG
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+        with:
+          artifact-suffix: component-accuracy
+
+  multimodal-gen-test-1-b200:
+    if: |
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }}
+    timeout-minutes: 240
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        env:
+          SGLANG_CI_EARLY_LD_LIBRARY_PATH: "1"
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run diffusion server tests
+        timeout-minutes: 240
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+          CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py \
+            --suite 1-gpu-b200 \
+            $CONTINUE_ON_ERROR_FLAG
+
+      - name: Upload diffusion failure artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-failures-${{ github.job }}-${{ github.run_attempt }}
+          path: diffusion-failures/
+          if-no-files-found: ignore
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  multimodal-gen-unit-test:
+    if: |
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
+      inputs.multimodal_gen == 'true'
+    runs-on: 1-gpu-h100
+    timeout-minutes: 120
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Download artifacts
+        if: inputs.sgl_kernel == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run diffusion unit tests
+        timeout-minutes: 60
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py --suite unit
+
+  diffusion-coverage-check:
+    needs: [multimodal-gen-test-1-gpu, multimodal-gen-test-2-gpu]
+    if: |
+      always() &&
+      inputs.multimodal_gen == 'true' &&
+      (
+        needs.multimodal-gen-test-1-gpu.result == 'success' ||
+        needs.multimodal-gen-test-1-gpu.result == 'failure' ||
+        needs.multimodal-gen-test-2-gpu.result == 'success' ||
+        needs.multimodal-gen-test-2-gpu.result == 'failure'
+      )
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Download all execution reports
+        uses: actions/download-artifact@v4
+        with:
+          path: reports/
+          pattern: diffusion-report-*
+          merge-multiple: true
+
+      - name: Verify coverage
+        run: |
+          python scripts/ci/utils/diffusion/verify_diffusion_coverage.py --reports-dir reports/

.github/workflows/pr-test-musa.yml

@@ -0,0 +1,264 @@
+name: PR Test (MUSA)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+    input:
+      target_stage:
+        description: "Specific test stage to run (Optional)"
+        required: false
+        type: string
+        default: ""
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      run_all_tests:
+        description: "Run all tests (for releasing or testing purpose)"
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: pr-test-musa-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+jobs:
+  # ==================== Check Changes ==================== #
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changes_exist: ${{ steps.filter.outputs.main_package == 'true'
+        || steps.filter.outputs.multimodal_gen == 'true'
+        || steps.filter.outputs.sgl_kernel == 'true'
+        || steps.run-mode.outputs.run_all_tests == 'true'
+        || inputs.target_stage != '' }}
+      main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
+      multimodal_gen: ${{ steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
+      sgl_kernel: ${{ steps.filter.outputs.sgl_kernel == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Determine run mode
+        id: run-mode
+        run: |
+          # Run all tests for workflow_call (when ref input is provided)
+          # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
+          if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
+            echo "run_all_tests=true" >> $GITHUB_OUTPUT
+            echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})"
+          else
+            echo "run_all_tests=false" >> $GITHUB_OUTPUT
+            echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
+          fi
+
+      - name: Detect file changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        if: steps.run-mode.outputs.run_all_tests != 'true'
+        with:
+          filters: |
+            main_package:
+              - ".github/workflows/pr-test-musa.yml"
+              - "python/pyproject_other.toml"
+              - "python/sglang/!(multimodal_gen)/**"
+              - "python/sglang/srt/hardware_backend/musa/**"
+              - "scripts/ci/musa/*"
+              - "scripts/ci/utils/*"
+            multimodal_gen:
+              - "python/pyproject_other.toml"
+              - "python/sglang/multimodal_gen/runtime/platforms/musa.py"
+              - "python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_rmsnorm.py"
+              - "python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_silu_and_mul.py"
+              - "python/sglang/multimodal_gen/test/server/musa/run_suite.py"
+              - "python/sglang/multimodal_gen/test/server/musa/**"
+            sgl_kernel:
+              - ".github/workflows/pr-test-musa.yml"
+              - "sgl-kernel/csrc/musa/**"
+
+  # ==================== PR Gate ==================== #
+  pr-gate:
+    needs: check-changes
+    if: needs.check-changes.outputs.changes_exist == 'true'
+    uses: ./.github/workflows/pr-gate.yml
+    secrets: inherit
+
+  # ==================== Multimodal Gen Tests ==================== #
+  multimodal-gen-test-1-gpu-musa:
+    needs: [check-changes, pr-gate]
+    if: needs.check-changes.outputs.multimodal_gen == 'true' || inputs.target_stage == 'multimodal-gen-test-1-gpu-musa'
+    strategy:
+      fail-fast: false
+      matrix:
+        part: [0, 1]
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Checkout code
+        timeout-minutes: 10
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run diffusion server tests (1-GPU)
+        timeout-minutes: 30
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/server/musa/run_suite.py \
+            --suite 1-gpu-musa \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions 2
+
+  multimodal-gen-test-2-gpu-musa:
+    needs: [check-changes, pr-gate]
+    if: needs.check-changes.outputs.multimodal_gen == 'true' || inputs.target_stage == 'multimodal-gen-test-2-gpu-musa'
+    runs-on: s5000-2-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Checkout code
+        timeout-minutes: 10
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run diffusion server tests (2-GPU)
+        timeout-minutes: 30
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/server/musa/run_suite.py \
+            --suite 2-gpu-musa
+
+  multimodal-gen-layer-unit-test-musa:
+    needs: [check-changes, pr-gate]
+    if: needs.check-changes.outputs.multimodal_gen == 'true' || inputs.target_stage == 'multimodal-gen-layer-unit-test-musa'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Checkout code
+        timeout-minutes: 10
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run multimodal gen layer unit test
+        timeout-minutes: 30
+        run: |
+          pytest python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_rmsnorm.py
+          pytest python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_silu_and_mul.py
+
+# =============================================== sgl-kernel ====================================================
+  sgl-kernel-unit-test-musa:
+    needs: [check-changes, pr-gate]
+    if: needs.check-changes.outputs.sgl_kernel == 'true' || inputs.target_stage == 'sgl-kernel-unit-test-musa'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HUB_CACHE: /hf-cache/hub
+    steps:
+      - name: Checkout code
+        timeout-minutes: 10
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run sgl-kernel test
+        timeout-minutes: 20
+        run: |
+          pytest sgl-kernel/tests/test_dsv3_router_gemm.py
+          pytest sgl-kernel/tests/test_per_token_quant_fp8.py
+          pytest sgl-kernel/tests/speculative/test_eagle_utils.py
+          pytest sgl-kernel/tests/speculative/test_ngram_utils.py
+          pytest sgl-kernel/tests/speculative/test_speculative_sampling.py
+          pytest sgl-kernel/tests/test_torch_defaults_reset.py
+
+
+  pr-test-musa-finish:
+    needs:
+      [
+        pr-gate,
+        check-changes,
+        multimodal-gen-test-1-gpu-musa,
+        multimodal-gen-test-2-gpu-musa,
+        multimodal-gen-layer-unit-test-musa,
+        sgl-kernel-unit-test-musa,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          # Convert the 'needs' context to a JSON string
+          json_needs='${{ toJson(needs) }}'
+
+          # Get a list of all job names from the JSON keys
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+
+          for job in $job_names; do
+            # For each job, extract its result
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+
+            # Print the job name and its result
+            echo "$job: $result"
+
+            # Check for failure or cancellation and exit if found
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "The above jobs failed."
+              exit 1
+            fi
+          done
+          # If the loop completes, all jobs were successful
+          echo "All jobs completed successfully"
+          exit

.github/workflows/pr-test-npu.yml

@@ -0,0 +1,478 @@
+name: PR Test (NPU)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      run_all_tests:
+        description: "Run all tests (for releasing or testing purpose)"
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: pr-test-npu-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+jobs:
+  # ==================== Check Changes ==================== #
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changes_exist: ${{ steps.filter.outputs.main_package == 'true' || steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true'}}
+      main_package: ${{ steps.filter.outputs.main_package == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
+      multimodal_gen: ${{ steps.filter.outputs.multimodal_gen == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Determine run mode
+        id: run-mode
+        run: |
+          # Run all tests for workflow_call (when ref input is provided)
+          # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
+          if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
+            echo "run_all_tests=true" >> $GITHUB_OUTPUT
+            echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})"
+          else
+            echo "run_all_tests=false" >> $GITHUB_OUTPUT
+            echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
+          fi
+
+      - name: Detect file changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        if: steps.run-mode.outputs.run_all_tests != 'true'
+        with:
+          filters: |
+            main_package:
+              - "python/sglang/!(multimodal_gen)/**/!(*.md)"
+              - "python/pyproject_npu.toml"
+              - "scripts/ci/npu/npu_ci_install_dependency.sh"
+              - "test/registered/ascend/**"
+              - ".github/workflows/pr-test-npu.yml"
+            multimodal_gen:
+              - "python/sglang/multimodal_gen/**/!(*.md|*.ipynb)"
+              - "python/sglang/jit_kernel/diffusion/triton/npu_fallback.py"
+              - "python/sglang/srt/**"
+              - "python/pyproject_npu.toml"
+              - "scripts/ci/npu/npu_ci_install_dependency.sh"
+              - ".github/workflows/pr-test-npu.yml"
+
+  # ==================== PR Gate ==================== #
+  pr-gate:
+    needs: check-changes
+    if: needs.check-changes.outputs.changes_exist == 'true'
+    uses: ./.github/workflows/pr-gate.yml
+    secrets: inherit
+
+  set-image-config:
+    runs-on: ubuntu-latest
+    outputs:
+      CANN_image_a3: ${{ steps.set-vars.outputs.CANN_image_a3 }}
+      CANN_image_910b: ${{ steps.set-vars.outputs.CANN_image_910b }}
+    steps:
+      # When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
+      - name: Set image config
+        id: set-vars
+        run: |
+          echo "CANN_image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
+          echo "CANN_image_910b=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-910b-ubuntu22.04-py3.11" >> $GITHUB_OUTPUT
+
+  stage-b-test-1-npu-a2:
+    needs: [check-changes, pr-gate, set-image-config]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: linux-aarch64-a2-1
+    strategy:
+      fail-fast: false
+      matrix:
+        part: [ 0, 1 ]
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh 910b
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          cd test
+          python3 run_suite.py --hw npu --suite stage-b-test-1-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
+
+  stage-b-test-2-npu-a2:
+    needs: [check-changes, pr-gate, set-image-config]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: linux-aarch64-a2-2
+    strategy:
+      fail-fast: true
+      matrix:
+        part: [0, 1]
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_910b }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh 910b
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          cd test
+          python3 run_suite.py --hw npu --suite stage-b-test-2-npu-a2 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
+
+  stage-b-test-4-npu-a3:
+    needs: [check-changes, pr-gate, set-image-config]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: linux-aarch64-a3-4
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          cd test
+          python3 run_suite.py --hw npu --suite stage-b-test-4-npu-a3 --timeout-per-file 3600
+
+
+  stage-b-test-16-npu-a3:
+    needs: [check-changes, pr-gate, set-image-config]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: linux-aarch64-a3-16
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+        run: |
+          cd test
+          python3 run_suite.py --hw npu --suite stage-b-test-16-npu-a3 --timeout-per-file 3600
+
+  multimodal-gen-test-1-npu-a3:
+    needs: [check-changes, pr-gate, set-image-config]
+    if: needs.check-changes.outputs.multimodal_gen == 'true'
+    runs-on: linux-aarch64-a3-2
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py --suite 1-npu
+
+      - name: Upload diffusion failure artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-failures-npu-1-${{ github.run_attempt }}
+          path: diffusion-failures/
+          if-no-files-found: ignore
+          retention-days: 7
+
+  multimodal-gen-test-2-npu-a3:
+    needs: [check-changes, pr-gate, set-image-config]
+    if: needs.check-changes.outputs.multimodal_gen == 'true'
+    runs-on: linux-aarch64-a3-16
+    container:
+      image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Mark repository safe
+        run: |
+          git config --system --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install dependencies
+        env:
+          TORCH_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu"
+          PYPI_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
+          GITHUB_PROXY_URL: "https://gh-proxy.test.osinfra.cn/"
+          RUSTUP_CACHE_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local:8082"
+        run: |
+          # speed up by using infra cache services
+          CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local"
+          sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list
+          pip config set global.index-url http://${CACHING_URL}/pypi/simple
+          pip config set global.trusted-host "${CACHING_URL}"
+
+          bash scripts/ci/npu/npu_ci_install_dependency.sh a3
+          # copy required file from our daily cache
+          cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+          # copy gsm8k dataset
+          cp ~/.cache/modelscope/hub/datasets/tmp/test.jsonl /tmp
+
+      - name: Run test
+        timeout-minutes: 60
+        env:
+          SGLANG_USE_MODELSCOPE: true
+          SGLANG_IS_IN_CI: true
+          HF_ENDPOINT: https://hf-mirror.com
+          TORCH_EXTENSIONS_DIR: /tmp/torch_extensions
+          PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+          STREAMS_PER_DEVICE: 32
+          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite.py --suite 2-npu
+
+      - name: Upload diffusion failure artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-failures-npu-2-${{ github.run_attempt }}
+          path: diffusion-failures/
+          if-no-files-found: ignore
+          retention-days: 7
+
+  pr-single-node-tests:
+    name: single-node-poc
+    needs: [check-changes, pr-gate, set-image-config]
+    if: needs.check-changes.outputs.main_package == 'true'
+    strategy:
+      fail-fast: false
+      max-parallel: 6
+      matrix:
+        test_config:
+          # qwen3_6_27b performance tests
+          - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms
+            runner: linux-aarch64-a3-2
+            test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py
+            test_type: 'perf'
+    uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml
+    with:
+      runner: ${{ matrix.test_config.runner }}
+      test_type: ${{ matrix.test_config.test_type }}
+      test_config_name: ${{ matrix.test_config.name }}
+      test_case: ${{ matrix.test_config.test_case }}
+      image: ${{ needs.set-image-config.outputs.CANN_image_a3 }}
+      install_sglang_from_source: false
+      install_sglang_deps: true
+      device_type_for_deps: 'a3'
+      transformers_version: ''
+
+  pr-test-npu-finish:
+    needs:
+      [
+        check-changes,
+
+        stage-b-test-1-npu-a2,
+        stage-b-test-2-npu-a2,
+        stage-b-test-4-npu-a3,
+        stage-b-test-16-npu-a3,
+
+        multimodal-gen-test-1-npu-a3,
+        multimodal-gen-test-2-npu-a3,
+
+        pr-single-node-tests,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          # Convert the 'needs' context to a JSON string
+          json_needs='${{ toJson(needs) }}'
+
+          # Get a list of all job names from the JSON keys
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+
+          for job in $job_names; do
+            # For each job, extract its result
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+
+            # Print the job name and its result
+            echo "$job: $result"
+
+            # Check for failure or cancellation and exit if found
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "The above jobs failed."
+              exit 1
+            fi
+          done
+          # If the loop completes, all jobs were successful
+          echo "All jobs completed successfully"
+          exit 0

.github/workflows/pr-test-rust.yml

@@ -0,0 +1,456 @@
+name: PR Test (SMG)
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - "sgl-model-gateway/**"
+      - ".github/workflows/pr-test-rust.yml"
+      - "scripts/ci/cuda/ci_install_dependency.sh"
+      - "scripts/ci/cuda/ci_install_gateway_dependencies.sh"
+  pull_request:
+    branches: [ main ]
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - "sgl-model-gateway/**"
+      - ".github/workflows/pr-test-rust.yml"
+      - "scripts/ci/cuda/ci_install_dependency.sh"
+      - "scripts/ci/cuda/ci_install_gateway_dependencies.sh"
+  workflow_dispatch:
+
+concurrency:
+  group: gateway-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  SGLANG_IS_IN_CI: true
+
+jobs:
+  build-wheel:
+    if: |
+      github.event_name != 'pull_request' ||
+      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
+    # Pin to 22.04 so the wheel auditwheel-tags as manylinux_2_35; the
+    # self-hosted GPU runners are Ubuntu 22.04 (glibc 2.35) and reject
+    # manylinux_2_39 wheels produced on ubuntu-latest (Ubuntu 24.04).
+    runs-on: ubuntu-22.04
+    # sccache is only installed on the GitHub-hosted runners that run this
+    # job and `unit-tests`; setting RUSTC_WRAPPER workflow-wide leaks it to
+    # gateway-e2e on the self-hosted GPU runners (which don't have sccache),
+    # so any pip-install that compiles a Rust extension would fail with
+    # `could not execute process \`sccache rustc\``.
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install rust dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_gateway_dependencies.sh
+
+      - name: Configure sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+        with:
+          version: "v0.12.0"
+          disable_annotations: true
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: sgl-model-gateway
+          shared-key: "rust-cache"
+          cache-all-crates: true
+          cache-on-failure: true
+          save-if: true
+
+      - name: Build python binding
+        run: |
+          source "$HOME/.cargo/env"
+          export RUSTC_WRAPPER=sccache
+          cd sgl-model-gateway/bindings/python
+          python3 -m pip install --upgrade pip maturin
+          maturin build --profile ci --features vendored-openssl --out dist
+
+      - name: List built wheel
+        run: ls -lh sgl-model-gateway/bindings/python/dist/
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: smg-wheel
+          path: sgl-model-gateway/bindings/python/dist/*.whl
+          retention-days: 1
+
+      - name: Test wheel install
+        run: |
+          pip install sgl-model-gateway/bindings/python/dist/*.whl
+          python3 -c "import sglang_router; print('Python package: OK')"
+          python3 -c "from sglang_router.sglang_router_rs import Router; print('Rust extension: OK')"
+          python3 -m sglang_router.launch_router --help > /dev/null && echo "Entry point: OK"
+
+  python-unit-tests:
+    needs: build-wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: sglang-repo
+
+      - name: Move sgl-model-gateway folder to root
+        run: |
+          mv sglang-repo/sgl-model-gateway/* .
+          rm -rf sglang-repo
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Download wheel artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: smg-wheel
+          path: dist/
+
+      - name: Install wheel
+        run: pip install dist/*.whl
+
+      - name: Run Python unit tests
+        run: |
+          cd bindings/python
+          python3 -m pip install pytest pytest-cov pytest-xdist
+          pytest -q tests --cov=sglang_router --cov-config=.coveragerc --cov-report=term-missing --cov-fail-under=80
+
+  unit-tests:
+    if: |
+      github.event_name != 'pull_request' ||
+      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
+    runs-on: ubuntu-latest
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_gateway_dependencies.sh
+
+      - name: Configure sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+        with:
+          version: "v0.12.0"
+          disable_annotations: true
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: sgl-model-gateway
+          shared-key: "rust-cache"
+          cache-all-crates: true
+          cache-on-failure: true
+          save-if: true
+
+      - name: Run lint
+        run: |
+          source "$HOME/.cargo/env"
+          cd sgl-model-gateway/
+          rustup component add clippy
+          cargo clippy --all-targets --all-features -- -D warnings
+
+      - name: Run fmt
+        run: |
+          source "$HOME/.cargo/env"
+          cd sgl-model-gateway/
+          rustup component add --toolchain nightly-x86_64-unknown-linux-gnu rustfmt
+          rustup toolchain install nightly --profile minimal
+          cargo +nightly fmt -- --check
+
+      - name: Generate vision golden fixtures
+        run: |
+          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+
+          pip install transformers pillow numpy scipy
+          pip install transformers pillow numpy
+          cd sgl-model-gateway/
+          python scripts/generate_vision_golden.py
+
+      - name: Run Rust tests
+        timeout-minutes: 20
+        run: |
+          source "$HOME/.cargo/env"
+          cd sgl-model-gateway/
+          cargo test
+
+      - name: Show sccache stats
+        if: always()
+        run: sccache --show-stats
+
+  gateway-e2e:
+    name: ${{ matrix.name }}
+    needs: build-wheel
+    if: |
+      github.event_name != 'pull_request' ||
+      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
+    # The `responses` matrix entry is intentionally omitted. It needs
+    # `docker run gvenzl/oracle-xe` + `docker run shoofio/brave-search-mcp-sse`
+    # on the runner host, but the 2-/4-gpu-h100 runners are themselves
+    # containers without a Docker daemon. Re-enable by adding back:
+    #   - name: responses
+    #     runner: 2-gpu-h100
+    #     timeout: 45
+    #     test_dirs: "e2e_test/responses"
+    #     extra_deps: ""
+    #     env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
+    #     reruns: "--reruns 2 --reruns-delay 5"
+    #     setup_oracle: true
+    #     setup_brave: true
+    #     parallel_opts: ""
+    # plus the Oracle Instant Client / `gvenzl/oracle-xe` /
+    # `shoofio/brave-search-mcp-sse` setup + cleanup steps (see commit
+    # cf346bb15 for the exact step bodies) once a runner with
+    # `docker.sock` (or binary-installed deps) is available.
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: benchmarks
+            # 4 GPUs: test_pd_perf.py uses workers(prefill=2, decode=2) and
+            # test_regular_perf.py uses workers(count=4) — both need tp*workers=4.
+            runner: 4-gpu-h100
+            timeout: 32
+            test_dirs: "e2e_test/benchmarks"
+            extra_deps: "genai-bench==0.0.3"
+            env_vars: ""
+            reruns: ""
+            upload_benchmarks: true
+            parallel_opts: ""  # No parallel for benchmarks (performance measurement)
+          - name: e2e
+            runner: 2-gpu-h100
+            timeout: 45
+            test_dirs: "e2e_test/router e2e_test/embeddings"
+            extra_deps: ""
+            env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
+            reruns: "--reruns 2 --reruns-delay 5"
+            # Run tests serially. pytest-parallel (unmaintained since 2019)
+            # has buggy fixture-finalize handling under thread dispatch:
+            # both class- and function-scoped fixture references leaked
+            # between tests, leaving model_pool instances pinned at
+            # _ref_count > 0 and deadlocking later tests that needed
+            # eviction (50+ min hangs). On a 2-GPU runner with 5 distinct
+            # model:mode combos in router+embeddings, the suite is
+            # eviction-bound anyway, so the parallel speedup was illusory.
+            parallel_opts: ""
+          - name: chat-completions
+            runner: 2-gpu-h100
+            timeout: 45
+            test_dirs: "e2e_test/chat_completions"
+            extra_deps: ""
+            env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
+            reruns: "--reruns 2 --reruns-delay 5"
+            parallel_opts: ""
+          - name: chat-completions-4gpu
+            runner: 4-gpu-h100
+            timeout: 45
+            # qwen-30b (tp=4) tests can't fit on the 2-gpu-h100 matrix entries —
+            # they get skipped there by hooks.py. Run them here so coverage holds.
+            test_dirs: "e2e_test/chat_completions/test_enable_thinking.py"
+            extra_deps: ""
+            env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
+            reruns: "--reruns 2 --reruns-delay 5"
+            parallel_opts: ""
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: ${{ matrix.timeout }}
+    # Self-hosted GPU runners are scarce; serialize per hardware type so
+    # 2-gpu-h100 and 4-gpu-h100 each run one job at a time across all
+    # in-flight PRs. Queue rather than cancel — different refs shouldn't
+    # interrupt each other. Include matrix.name in the group so siblings
+    # within the same run don't collide: GitHub keeps only one pending job
+    # per concurrency group, so matrix entries sharing a group (e.g.
+    # benchmarks + chat-completions-4gpu on 4-gpu-h100) would evict each
+    # other regardless of cancel-in-progress: false.
+    concurrency:
+      group: pr-test-rust-${{ matrix.runner }}-${{ matrix.name }}
+      cancel-in-progress: false
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install SGLang dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Download wheel artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: smg-wheel
+          path: wheel/
+
+      - name: Install wheel
+        run: |
+          pip uninstall -y sglang-router || true
+          pip install wheel/*.whl
+
+      - name: Install e2e test dependencies
+        run: |
+          python3 -m pip install pytest pytest-rerunfailures httpx openai grpcio grpcio-health-checking numpy
+          if [ -n "${{ matrix.extra_deps }}" ]; then
+            if echo "${{ matrix.extra_deps }}" | grep -q "genai-bench"; then
+              # genai-bench's transitive deps (oci/locust) pull
+              # transformers<5 which requires huggingface_hub<1.0 — that
+              # downgrades the 1.11.0 ci_install_dependency.sh settled on
+              # and breaks `kernels` (requires huggingface_hub>=1.3.0,<2.0).
+              # Install --no-deps and supply the runtime deps explicitly.
+              python3 -m pip --no-cache-dir install --no-deps ${{ matrix.extra_deps }}
+              python3 -m pip --no-cache-dir install \
+                locust click rich tenacity oci openpyxl gevent matplotlib
+            else
+              # Other extras with well-behaved transitive deps —
+              # normal --upgrade install path.
+              python3 -m pip --no-cache-dir install --upgrade ${{ matrix.extra_deps }}
+            fi
+          fi
+
+      - name: Run E2E tests
+        run: |
+          python3 python/sglang/cli/killall.py
+          cd sgl-model-gateway
+          ${{ matrix.env_vars }} pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.test_dirs }} -s -vv -o log_cli=true --log-cli-level=INFO
+
+      - name: Upload benchmark results
+        if: matrix.upload_benchmarks && success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: genai-bench-results-all-policies
+          path: sgl-model-gateway/benchmark_**/
+
+  docker-build-test:
+    if: |
+      github.event_name != 'pull_request' ||
+      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image (no push)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/gateway.Dockerfile
+          push: false
+          tags: sgl-model-gateway:test
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  k8s-integration:
+    # Runs SMG against a kind cluster with fake worker pods to exercise the
+    # K8s service discovery / reconciliation path. No GPU required (workers
+    # are python:3.12-slim mocks); the h100 matrix runners are unsuitable
+    # because they're containers without a Docker daemon.
+    if: |
+      github.event_name != 'pull_request' ||
+      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install kind and kubectl
+        run: |
+          curl -fsSLo /tmp/kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
+          chmod +x /tmp/kind && sudo mv /tmp/kind /usr/local/bin/kind
+          KUBECTL_VERSION=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
+          curl -fsSLo /tmp/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
+          chmod +x /tmp/kubectl && sudo mv /tmp/kubectl /usr/local/bin/kubectl
+          kind --version
+          kubectl version --client
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build smg-gateway:test image
+        uses: docker/build-push-action@v5
+        with:
+          context: sgl-model-gateway
+          file: sgl-model-gateway/e2e_test/k8s_integration/Dockerfile.gateway
+          tags: smg-gateway:test
+          load: true
+          cache-from: type=gha,scope=k8s-integration
+          cache-to: type=gha,scope=k8s-integration,mode=max
+
+      - name: Install Python test dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install pytest httpx
+
+      - name: Set up kind cluster and deploy
+        env:
+          SKIP_DOCKER_BUILD: "1"
+        run: |
+          cd sgl-model-gateway
+          bash e2e_test/k8s_integration/setup.sh
+
+      - name: Run K8s integration tests
+        run: |
+          cd sgl-model-gateway
+          # confcutdir avoids loading the parent e2e_test/conftest.py, which
+          # pulls in heavy infra deps (requests, sglang_router) that this job
+          # intentionally doesn't install.
+          pytest e2e_test/k8s_integration/ \
+            --confcutdir=e2e_test/k8s_integration \
+            -v -s -o log_cli=true --log-cli-level=INFO
+
+      - name: Dump cluster state on failure
+        if: failure()
+        run: |
+          kubectl --context kind-smg-test get all -A || true
+          kubectl --context kind-smg-test -n smg-test describe pods || true
+          kubectl --context kind-smg-test -n smg-test logs deploy/smg-gateway --tail=200 || true
+
+      - name: Tear down kind cluster
+        if: always()
+        run: |
+          cd sgl-model-gateway
+          bash e2e_test/k8s_integration/setup.sh teardown || true
+
+  finish:
+    needs: [build-wheel, python-unit-tests, unit-tests, gateway-e2e, docker-build-test, k8s-integration]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finish
+        run: echo "This is an empty step to ensure that all jobs are completed."
+
+  summarize-benchmarks:
+    needs: gateway-e2e
+    runs-on: ubuntu-latest
+    if: success()
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Download benchmark results
+      uses: actions/download-artifact@v4
+      with:
+        name: genai-bench-results-all-policies
+
+    - name: Create benchmark summary
+      run: python3 sgl-model-gateway/e2e_test/benchmarks/summarize.py .

.github/workflows/pr-test-sgl-kernel.yml

@@ -0,0 +1,178 @@
+name: PR Test - SGL Kernel
+
+on:
+  workflow_call:
+    inputs:
+      sgl_kernel:
+        required: true
+        type: string
+      runner_config:
+        required: true
+        type: string
+      runs_on_map:
+        required: true
+        type: string
+      git_ref:
+        required: false
+        type: string
+        default: ''
+      skip_pr_test_health_check:
+        required: false
+        type: boolean
+        default: false
+
+# Workflow-level env is NOT inherited from the caller in reusable workflows.
+# The github context (including github.event_name) IS inherited from the caller.
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
+
+jobs:
+  sgl-kernel-unit-test:
+    runs-on: 1-gpu-h100
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Cleanup
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run test
+        timeout-minutes: 30
+        run: |
+          cd sgl-kernel
+          pytest tests/
+
+  sgl-kernel-benchmark-test:
+    runs-on: 1-gpu-h100
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Cleanup
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run benchmark tests
+        timeout-minutes: 45
+        run: |
+          cd sgl-kernel/benchmark
+          echo "Running sgl-kernel benchmark tests in CI mode..."
+
+          echo "CI environment variable: $CI"
+          echo "GITHUB_ACTIONS environment variable: $GITHUB_ACTIONS"
+
+          for bench_file in bench_*.py; do
+            echo "Testing $bench_file..."
+            timeout 60 python3 "$bench_file" || echo "Warning: $bench_file timed out or failed, continuing..."
+            echo "Completed $bench_file"
+            echo "---"
+          done
+
+          echo "All benchmark tests completed!"
+
+  sgl-kernel-b200-test:
+    runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }}
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Cleanup
+        run: |
+          ls -alh sgl-kernel/dist || true
+          rm -rf sgl-kernel/dist/* || true
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-python3.10-cuda*
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run sgl-kernel unit tests on B200
+        timeout-minutes: 30
+        run: |
+          cd sgl-kernel
+          pytest tests/
+
+  # Adding a single CUDA13 build-and-run check for the kernel
+  # TODO: Add back this test when it can pass on CI
+  # cuda13-kernel-build-check:
+  #   if: inputs.sgl_kernel == 'true'
+  #   runs-on: x64-cu13-kernel-tests
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: Cleanup
+  #       run: |
+  #         ls -alh sgl-kernel/dist || true
+  #         rm -rf sgl-kernel/dist/* || true
+
+  #     - name: Download CUDA 13.0 artifacts
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         path: sgl-kernel/dist/
+  #         merge-multiple: true
+  #         pattern: wheel-python3.10-cuda*
+
+  #     - name: Install dependencies
+  #       run: |
+  #         CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh
+
+  #     - name: Run kernel unit tests
+  #       timeout-minutes: 30
+  #       run: |
+  #         cd sgl-kernel
+  #         pytest tests/

.github/workflows/pr-test-sgl-router.yml

@@ -0,0 +1,434 @@
+name: PR Test (sgl-router)
+
+# Trigger contract — modeled on `pr-test.yml`:
+#
+#   * No `paths:` filter at the trigger level. The workflow ALWAYS
+#     fires on every PR synchronize / push, so the run record always
+#     appears as a check on the PR (no more "workflow silently didn't
+#     fire" mystery debugging). Path-based skip decisions are made
+#     inside the `sgl-router-gate` job below, where we can log the
+#     reason. Same for the `run-ci` label gate — moved off the
+#     workflow trigger and into the gate job so its outcome is
+#     observable.
+#
+#   * `push` on `main` keeps firing so post-merge runs still record
+#     against the default branch.
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened, labeled]
+  workflow_dispatch:
+
+concurrency:
+  group: sgl-router-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}
+  cancel-in-progress: true
+
+env:
+  SGLANG_IS_IN_CI: true
+
+jobs:
+  # Stage 0 — decide whether to run the heavy tiers. Always runs
+  # (cheap, ubuntu-latest), and emits a single `should_run` output
+  # consumed by every tier below. Reasons it might evict downstream
+  # work:
+  #   * `pull_request` event without the `run-ci` label (budget gate)
+  #   * No files matching the sgl-router paths changed in this PR
+  # Either outcome is logged on the gate job's page, so operators can
+  # see *why* the tiers were skipped instead of guessing.
+  sgl-router-gate:
+    name: gate
+    runs-on: ubuntu-latest
+    outputs:
+      should_run: ${{ steps.decide.outputs.should_run }}
+      paths_changed: ${{ steps.paths.outputs.sgl_router }}
+      has_run_ci_label: ${{ steps.label.outputs.has_run_ci }}
+    steps:
+      - uses: actions/checkout@v4
+      # `dorny/paths-filter` computes the diff against the PR base
+      # (for pull_request) or the push's `before` SHA (for push) and
+      # exposes `sgl_router=true|false` on whether any tracked path
+      # changed. Replaces the old workflow-level `paths:` filter so
+      # the gate is observable.
+      - name: Detect sgl-router path changes
+        id: paths
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            sgl_router:
+              - 'experimental/sgl-router/**'
+              - '.github/workflows/pr-test-sgl-router.yml'
+              - 'scripts/ci/cuda/ci_install_dependency.sh'
+      # `run-ci` label gate. Cheap workflow runs (e.g. docs-only PRs
+      # that happen to touch the sgl-router dir) still need the
+      # opt-in label before consuming the H100 / kind tiers below.
+      # `push` events on main and `workflow_dispatch` skip this gate.
+      - name: Check run-ci label
+        id: label
+        run: |
+          if [[ "${{ github.event_name }}" != "pull_request" ]]; then
+            echo "has_run_ci=true" >> "$GITHUB_OUTPUT"
+            echo "Non-PR event (${{ github.event_name }}); label gate bypassed."
+            exit 0
+          fi
+          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci') }}" == "true" ]]; then
+            echo "has_run_ci=true" >> "$GITHUB_OUTPUT"
+            echo "PR has run-ci label; downstream tiers will run."
+          else
+            echo "has_run_ci=false" >> "$GITHUB_OUTPUT"
+            echo "::warning::PR is missing the 'run-ci' label; skipping sgl-router tiers. Apply the label to opt in."
+          fi
+      - name: Decide
+        id: decide
+        run: |
+          paths='${{ steps.paths.outputs.sgl_router }}'
+          label='${{ steps.label.outputs.has_run_ci }}'
+          if [[ "$paths" == "true" && "$label" == "true" ]]; then
+            echo "should_run=true" >> "$GITHUB_OUTPUT"
+            echo "Both path filter and run-ci label match; running tiers."
+          else
+            echo "should_run=false" >> "$GITHUB_OUTPUT"
+            echo "Skipping tiers (paths_changed=$paths, has_run_ci=$label)."
+          fi
+
+  sgl-router-lint:
+    name: tier-1 — lint
+    needs: sgl-router-gate
+    if: needs.sgl-router-gate.outputs.should_run == 'true'
+    runs-on: ubuntu-latest
+    env:
+      # Scoped per-job (not workflow-wide) to avoid SMG's documented breakage
+      # of self-hosted-runner pip installs that compile Rust extensions.
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      # Supply-chain guard: deny.toml allowlists the dynamo git source but
+      # cannot pin a SHA. Bumps to this constant must be a deliberate PR.
+      DYNAMO_TOKENIZERS_EXPECTED_REV: "1efdd4dcb901caeae636131321094090d252c8d6"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Verify dynamo-* SHA pin
+        run: |
+          set -euo pipefail
+          revs=$(grep -E '^dynamo-[a-z]+[[:space:]]*=.*\brev[[:space:]]*=' experimental/sgl-router/Cargo.toml \
+            | sed -E 's/.*rev[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/')
+          if [ -z "${revs:-}" ]; then
+            echo "::error::no dynamo-* git deps with rev= found in experimental/sgl-router/Cargo.toml"
+            exit 1
+          fi
+          count=0
+          while IFS= read -r r; do
+            count=$((count + 1))
+            if [ "$r" != "$DYNAMO_TOKENIZERS_EXPECTED_REV" ]; then
+              echo "::error::dynamo-* SHA pin mismatch in experimental/sgl-router/Cargo.toml"
+              echo "  expected: $DYNAMO_TOKENIZERS_EXPECTED_REV"
+              echo "  actual:   $r"
+              echo "All revs found:"
+              echo "$revs" | sed 's/^/  /'
+              exit 1
+            fi
+          done <<< "$revs"
+          echo "dynamo-* SHA pin OK: $count entries all at $DYNAMO_TOKENIZERS_EXPECTED_REV"
+
+      # sccache install is best-effort: the GitHub release CDN intermittently
+      # returns 5xx and the action's built-in retry is shallow. If it fails,
+      # we unset RUSTC_WRAPPER below and proceed without compile caching —
+      # rust-cache still covers the bulk of warm-cache wins.
+      - name: Configure sccache
+        id: sccache
+        continue-on-error: true
+        uses: mozilla-actions/sccache-action@v0.0.9
+        with:
+          version: "v0.12.0"
+          disable_annotations: true
+
+      - name: Disable sccache if install failed
+        if: steps.sccache.outcome != 'success'
+        run: |
+          echo "::warning::sccache install failed (${{ steps.sccache.outcome }}); proceeding without compile caching"
+          echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV"
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: experimental/sgl-router
+          shared-key: "sgl-router-cache"
+          cache-all-crates: true
+          cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: cargo check
+        working-directory: experimental/sgl-router
+        run: cargo check --all-targets
+
+      - name: cargo clippy
+        working-directory: experimental/sgl-router
+        run: cargo clippy --all-targets -- -D warnings
+
+      - name: cargo fmt
+        working-directory: experimental/sgl-router
+        run: |
+          rustup toolchain install nightly --profile minimal --component rustfmt
+          cargo +nightly fmt -- --check
+
+      # cargo-deny 0.19.6+ required for CVSS 4.0 parsing.
+      - name: Install cargo-deny
+        run: cargo install --locked cargo-deny@0.19.6
+
+      - name: cargo deny check
+        working-directory: experimental/sgl-router
+        run: cargo deny check licenses bans sources advisories
+
+      - name: Show sccache stats
+        if: always() && steps.sccache.outcome == 'success'
+        run: sccache --show-stats
+
+  sgl-router-build-and-test:
+    name: tier-2 — build + test
+    needs: [sgl-router-gate, sgl-router-lint]
+    if: needs.sgl-router-gate.outputs.should_run == 'true'
+    runs-on: ubuntu-latest
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install OS deps (libssl, pkg-config)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libssl-dev pkg-config
+
+      # sccache install is best-effort: the GitHub release CDN intermittently
+      # returns 5xx and the action's built-in retry is shallow. If it fails,
+      # we unset RUSTC_WRAPPER below and proceed without compile caching —
+      # rust-cache still covers the bulk of warm-cache wins.
+      - name: Configure sccache
+        id: sccache
+        continue-on-error: true
+        uses: mozilla-actions/sccache-action@v0.0.9
+        with:
+          version: "v0.12.0"
+          disable_annotations: true
+
+      - name: Disable sccache if install failed
+        if: steps.sccache.outcome != 'success'
+        run: |
+          echo "::warning::sccache install failed (${{ steps.sccache.outcome }}); proceeding without compile caching"
+          echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV"
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: experimental/sgl-router
+          shared-key: "sgl-router-cache"
+          cache-all-crates: true
+          cache-on-failure: true
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: cargo build (record wall time)
+        working-directory: experimental/sgl-router
+        run: |
+          START=$(date +%s)
+          cargo build --release --all-targets
+          END=$(date +%s)
+          BUILD_SECONDS=$((END - START))
+          echo "tier_2_build_duration_seconds=${BUILD_SECONDS}" >> "$GITHUB_ENV"
+          echo "::notice title=Tier-2 build duration::${BUILD_SECONDS}s"
+
+      - name: cargo test
+        working-directory: experimental/sgl-router
+        # Skip the tokenizer parity_matrix test here: ubuntu-latest has
+        # no HuggingFace cache, so the matrix would hard-fail (see the
+        # test docstring). The e2e job runs it after pytest populates
+        # the Qwen3-0.6B tokenizer.json. Filter is matched against the
+        # full test path `tokenizer::parity::parity_matrix`; substring
+        # `parity_matrix` is unique to that test.
+        run: cargo test --release -- --skip parity_matrix
+
+      # Regenerate the cross-impl block-hash parity fixture and fail if it
+      # differs from the committed file.  The Python script replicates
+      # SGLang's `radix_cache.hash_page` algorithm verbatim (no
+      # `import sglang`); a diff here means EITHER the Rust port's algorithm
+      # drifted from the script, OR the script drifted from SGLang's actual
+      # code — in both cases the operator must investigate and re-commit
+      # the fixture in the same PR as the algorithm change.
+      #
+      # `set -euo pipefail` makes a Python-side failure (syntax error,
+      # missing import) cause this step to fail instead of falling
+      # through to a no-diff "pass" when no fixture was actually
+      # regenerated.
+      - name: kv_events hash parity fixture (drift check)
+        working-directory: experimental/sgl-router
+        run: |
+          set -euo pipefail
+          python3 tests/scripts/generate_kv_events_hash_parity.py
+          if ! git diff --exit-code -- tests/fixtures/kv_events_hash_parity.json; then
+            echo "::error::kv_events hash parity fixture drifted." \
+              "Re-run experimental/sgl-router/tests/scripts/generate_kv_events_hash_parity.py" \
+              "and commit the updated tests/fixtures/kv_events_hash_parity.json."
+            exit 1
+          fi
+
+      - name: Show sccache stats
+        if: always() && steps.sccache.outcome == 'success'
+        run: sccache --show-stats
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "## Tier-2 results" >> "$GITHUB_STEP_SUMMARY"
+          echo "- Build duration: \`${tier_2_build_duration_seconds:-(not recorded)}\` s" >> "$GITHUB_STEP_SUMMARY"
+
+  sgl-router-docker-build-test:
+    name: tier-3 — docker (placeholder)
+    needs: [sgl-router-gate, sgl-router-build-and-test]
+    if: needs.sgl-router-gate.outputs.should_run == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - name: Placeholder
+        run: |
+          echo "docker-build-test not implemented yet."
+          exit 0
+
+  # tier-3 k8s integration is decoupled from tier-3 e2e: each runs on a
+  # different runner type (ubuntu kind cluster vs H100), each manages its
+  # own test scope (k8s_integration/ vs everything-else-in-e2e), and
+  # each is allowed to fail without blocking the other. They both still
+  # gate on tier-2's build+test passing — a Rust compile failure blocks
+  # the GPU/cluster runners from spinning up.
+  sgl-router-k8s-integration:
+    name: tier-3 — k8s integration
+    needs: [sgl-router-gate, sgl-router-build-and-test]
+    if: needs.sgl-router-gate.outputs.should_run == 'true'
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Install kind
+        run: |
+          curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
+          chmod +x /tmp/kind && sudo mv /tmp/kind /usr/local/bin/kind
+      - name: Install kubectl
+        run: |
+          curl -Lo /tmp/kubectl https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl
+          chmod +x /tmp/kubectl && sudo mv /tmp/kubectl /usr/local/bin/kubectl
+      - name: Build router image
+        run: docker build -t sgl-router:e2e -f experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.router .
+      - name: Build fake_worker image
+        run: |
+          docker build -t sgl-router-fake-worker:e2e \
+            -f experimental/sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker \
+            experimental/sgl-router/tests/e2e/k8s_integration/
+      - name: Bootstrap kind + deploy
+        run: bash experimental/sgl-router/tests/e2e/k8s_integration/setup.sh
+      - name: Set up Python
+        run: |
+          python3 -m venv /tmp/e2e-venv
+          /tmp/e2e-venv/bin/pip install -r experimental/sgl-router/tests/e2e/k8s_integration/requirements.txt
+      - name: Run E2E
+        run: /tmp/e2e-venv/bin/pytest experimental/sgl-router/tests/e2e/k8s_integration/ -v --tb=short
+      - name: Dump cluster + router diagnostics on failure
+        if: failure()
+        run: |
+          set +e
+          kubectl -n sgl-router-test get pods -o wide
+          kubectl -n sgl-router-test describe pod -l app=sgl-router
+          kubectl -n sgl-router-test describe pod -l app=sglang
+          kubectl -n sgl-router-test logs deploy/sgl-router --tail=300
+          kubectl -n sgl-router-test logs deploy/sgl-router --tail=300 --previous
+          kubectl -n sgl-router-test get events --sort-by=.lastTimestamp
+          kubectl -n sgl-router-test get endpointslices -o wide
+          kubectl -n sgl-router-test get svc
+          true
+
+  sgl-router-e2e:
+    name: tier-3 — e2e
+    needs: [sgl-router-gate, sgl-router-build-and-test]
+    if: needs.sgl-router-gate.outputs.should_run == 'true'
+    runs-on: 2-gpu-h100
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust toolchain
+        run: bash scripts/ci/utils/install_rust_protoc.sh
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: experimental/sgl-router
+          shared-key: "sgl-router-cache"
+      - name: cargo build (release)
+        working-directory: experimental/sgl-router
+        run: |
+          source "$HOME/.cargo/env"
+          cargo build --release
+
+      # Install SGLang from the local checkout in editable mode (no PyPI
+      # version pin) — mirrors `pr-test-rust.yml` so the router e2e runs
+      # against whatever SGLang version is on the branch. Replaces the
+      # previous `pip install sglang[srt]==X.Y.Z` pin, which forced the
+      # e2e to test against a stale wheel and would silently miss any
+      # new ServerArgs field (e.g. `disaggregation_mode` /
+      # `disaggregation_bootstrap_port`) added in the branch.
+      - name: Install SGLang dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Install e2e test dependencies
+        run: |
+          python3 -m pip install -r experimental/sgl-router/tests/e2e/requirements.txt
+
+      # IMPORTANT: --ignore=tests/e2e/k8s_integration. The k8s integration
+      # suite is owned by the `sgl-router-k8s-integration` job which has
+      # kind + kubectl installed and a real cluster running. The e2e GPU
+      # runner has neither, so pytest's recursive collection of
+      # tests/e2e/ would ERROR every k8s test at setup. Both tiers must
+      # be allowed to fail independently — k8s flakes (cluster boot,
+      # image pull) must not surface as e2e failures on H100, and e2e
+      # flakes (model load, HF auth) must not surface as k8s failures.
+      - name: Run e2e
+        working-directory: experimental/sgl-router
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          python3 -m pytest tests/e2e/ -v -s --tb=short \
+            --ignore=tests/e2e/k8s_integration
+
+      - name: Tokenizer parity matrix (uses HF cache populated by e2e)
+        working-directory: experimental/sgl-router
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          source "$HOME/.cargo/env"
+          cargo test --release --test component tokenizer::parity
+
+  sgl-router-finish:
+    name: finish
+    needs:
+      - sgl-router-gate
+      - sgl-router-lint
+      - sgl-router-build-and-test
+      - sgl-router-docker-build-test
+      - sgl-router-k8s-integration
+      - sgl-router-e2e
+    # `always()` lets `finish` run after upstream `skipped` outcomes
+    # (when the gate decided to skip). The downstream-skipped path is
+    # an explicit success — the workflow was correctly bypassed for a
+    # PR that didn't need full CI. The downstream-failure /
+    # downstream-cancelled path is a real fail. We also accept the
+    # gate's `skipped` (impossible today, kept for symmetry) and
+    # require the gate itself to have succeeded.
+    if: |
+      always() &&
+      needs.sgl-router-gate.result == 'success' &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled')
+    runs-on: ubuntu-latest
+    steps:
+      - name: All required checks completed
+        run: echo "OK"

.github/workflows/pr-test-xeon.yml

@@ -0,0 +1,148 @@
+name: PR Test (Xeon)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      run_all_tests:
+        description: "Run all tests (for releasing or testing purpose)"
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: pr-test-xeon-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+jobs:
+  # ==================== Check Changes ==================== #
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests}}
+    env:
+      PYTHONDONTWRITEBYTECODE: 1
+    steps:
+      - name: clean up
+        run: |
+          sudo rm -rf ${{ github.workspace }} || true
+          sudo mkdir -p ${{ github.workspace }}
+          sudo chown -R $(id -u):$(id -g) ${{ github.workspace }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          clean: false
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Determine run mode
+        id: run-mode
+        run: |
+          # Run all tests for workflow_call (when ref input is provided)
+          # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
+          if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
+            echo "run_all_tests=true" >> $GITHUB_OUTPUT
+            echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})"
+          else
+            echo "run_all_tests=false" >> $GITHUB_OUTPUT
+            echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
+          fi
+
+      - name: Detect file changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        if: steps.run-mode.outputs.run_all_tests != 'true'
+        with:
+          filters: |
+            main_package:
+              - "python/sglang/!(multimodal_gen)/**/!(*.md)"
+              - "python/pyproject_cpu.toml"
+              - "test/**/!(*.md)"
+              - "sgl-kernel/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)"
+              - ".github/workflows/pr-test-xeon.yml"
+              - "docker/xeon.Dockerfile"
+
+  # ==================== PR Gate ==================== #
+  pr-gate:
+    needs: check-changes
+    if: needs.check-changes.outputs.main_package == 'true'
+    uses: ./.github/workflows/pr-gate.yml
+    secrets: inherit
+
+  build-test:
+    needs: [check-changes, pr-gate]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: ${{ matrix.runner }}
+
+    strategy:
+      matrix:
+        include:
+          - runner: xeon-gnr
+            suite: base-b-test-cpu
+            partition_args: ""
+    steps:
+      - name: Cleanup workspace
+        run: |
+          sudo rm -rf ${{ github.workspace }} || true
+          sudo mkdir -p ${{ github.workspace }}
+          sudo chown $(id -u):$(id -g) ${{ github.workspace }}
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          clean: false
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Build and Push
+        run: |
+          version=$(cat python/sglang/version.py | cut -d'"' -f2)
+          tag=v${version}-xeon
+          PR_REPO=${{ github.event.pull_request.head.repo.clone_url }}
+          PR_HEAD_REF=${{ github.head_ref }}
+
+          docker build \
+            ${PR_REPO:+--build-arg SGLANG_REPO=$PR_REPO} \
+            ${PR_HEAD_REF:+--build-arg VER_SGLANG=$PR_HEAD_REF} \
+            . -f docker/xeon.Dockerfile  -t sglang_xeon --no-cache
+
+      - name: Run container
+        run: |
+          docker run -dt \
+            -v ${{ github.workspace }}:/sglang-checkout/ --ipc=host \
+            -v $HOME/.cache/huggingface:/root/.cache/huggingface \
+            -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \
+            --name ci_sglang_${{ matrix.runner }} \
+            sglang_xeon
+
+      - name: Check AMX support
+        id: check_amx
+        timeout-minutes: 5
+        run: |
+          docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.runner }} \
+            bash -c "source /opt/.venv/bin/activate && python3 -c 'import torch; import sgl_kernel; assert torch.cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\"); '"
+
+      - name: Run unit tests
+        timeout-minutes: 120
+        run: |
+          docker exec -w /sglang-checkout/ ci_sglang_${{ matrix.runner }} \
+            bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite ${{ matrix.suite }} ${{ matrix.partition_args }}"
+
+      - name: Change permission
+        timeout-minutes: 2
+        run: |
+          docker exec -u root ci_sglang_${{ matrix.runner }} bash -c "
+            rm -rf /tmp/ci-home  &&
+            chown -R  $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true
+          "
+
+      - name: Cleanup container
+        if: always()
+        run: |
+          docker rm -f ci_sglang_${{ matrix.runner }} || true

.github/workflows/pr-test-xpu.yml

@@ -0,0 +1,238 @@
+name: PR Test (XPU)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      run_all_tests:
+        description: "Run all tests (for releasing or testing purpose)"
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  group: pr-test-xpu-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+jobs:
+  # ==================== Check Changes ==================== #
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Determine run mode
+        id: run-mode
+        run: |
+          # Run all tests for workflow_call (when ref input is provided)
+          # Note: github.event_name is inherited from caller, so we detect workflow_call by checking inputs.ref
+          if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
+            echo "run_all_tests=true" >> $GITHUB_OUTPUT
+            echo "Run mode: ALL TESTS (run_all_tests=${{ inputs.run_all_tests }})"
+          else
+            echo "run_all_tests=false" >> $GITHUB_OUTPUT
+            echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
+          fi
+      - name: Detect file changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        if: steps.run-mode.outputs.run_all_tests != 'true'
+        with:
+          filters: |
+            main_package:
+              - "python/sglang/!(multimodal_gen)/**/!(*.md)"
+              - "python/pyproject_xpu.toml"
+              - "test/**/!(*.md)"
+              - "sgl-kernel/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)"
+              - ".github/workflows/pr-test-xpu.yml"
+              - "docker/xpu.Dockerfile"
+
+  # ==================== PR Gate ==================== #
+  pr-gate:
+    needs: check-changes
+    if: needs.check-changes.outputs.main_package == 'true'
+    uses: ./.github/workflows/pr-gate.yml
+    secrets: inherit
+
+  # ==================== Stage A ==================== #
+  stage-a-test-1-gpu-xpu:
+    needs: [check-changes, pr-gate]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: intel-bmg
+    env:
+      DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
+      DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
+    steps:
+      - name: Reset workspace ownership
+        run: |
+          # Prior runs install sglang as root inside the CI container against
+          # the bind-mounted workspace, leaving root-owned egg-info on the host.
+          # actions/checkout's pre-clean step can't unlink those as the runner
+          # user, so reset ownership before checkout runs.
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Start CI container
+        run: |
+          export HF_TOKEN="$(cat ~/huggingface_token.txt)"
+          bash scripts/ci/xpu/xpu_ci_start_container.sh
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install Dependency
+        timeout-minutes: 60
+        run: |
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --upgrade pip
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python sgl-kernel sglang
+          docker exec ci_sglang_xpu cp /sglang-checkout/python/pyproject_xpu.toml /sglang-checkout/python/pyproject.toml
+          docker exec -w /sglang-checkout/python ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir . --extra-index-url https://download.pytorch.org/whl/xpu
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
+          docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
+
+      - name: Run stage-a tests
+        timeout-minutes: 30
+        run: |
+          docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu"
+
+      - name: Cleanup container
+        if: always()
+        run: |
+          # pip install ran as root inside the container against the
+          # bind-mounted workspace, so build artifacts are root-owned on
+          # the host. Chown them back before rm to avoid Permission denied.
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+          # Wipe everything the run wrote into the workspace so the next
+          # job starts from a clean tree.
+          rm -rf \
+            python/build \
+            python/dist \
+            python/sglang.egg-info \
+            python/sglang/*.egg-info \
+            test/result.jsonl \
+            test/results \
+            test/.pytest_cache \
+            .pytest_cache || true
+          find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
+          find . -type f -name "*.pyc" -delete || true
+          docker rm -f ci_sglang_xpu || true
+          if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
+            docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
+          fi
+
+  # ==================== Wait for Stage A ==================== #
+  wait-for-stage-a:
+    needs: [stage-a-test-1-gpu-xpu]
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "stage-a passed"
+
+  # ==================== Stage B ==================== #
+  stage-b-test-1-gpu-xpu:
+    needs: [check-changes, pr-gate, wait-for-stage-a]
+    if: needs.check-changes.outputs.main_package == 'true'
+    runs-on: intel-bmg
+    env:
+      DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
+      DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
+    steps:
+      - name: Reset workspace ownership
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Start CI container
+        run: |
+          export HF_TOKEN="$(cat ~/huggingface_token.txt)"
+          bash scripts/ci/xpu/xpu_ci_start_container.sh
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+
+      - name: Install Dependency
+        timeout-minutes: 60
+        run: |
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --upgrade pip
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python sgl-kernel sglang
+          docker exec ci_sglang_xpu cp /sglang-checkout/python/pyproject_xpu.toml /sglang-checkout/python/pyproject.toml
+          docker exec -w /sglang-checkout/python ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir . --extra-index-url https://download.pytorch.org/whl/xpu
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
+          docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
+
+      - name: Run stage-b tests
+        timeout-minutes: 60
+        run: |
+          docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-b-test-1-gpu-xpu"
+
+      - name: Cleanup container
+        if: always()
+        run: |
+          # pip install ran as root inside the container against the
+          # bind-mounted workspace, so build artifacts are root-owned on
+          # the host. Chown them back before rm to avoid Permission denied.
+          docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
+            chown -R "$(id -u):$(id -g)" /w || true
+          # Wipe everything the run wrote into the workspace so the next
+          # job starts from a clean tree.
+          rm -rf \
+            python/build \
+            python/dist \
+            python/sglang.egg-info \
+            python/sglang/*.egg-info \
+            test/result.jsonl \
+            test/results \
+            test/.pytest_cache \
+            .pytest_cache || true
+          find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
+          find . -type f -name "*.pyc" -delete || true
+          docker rm -f ci_sglang_xpu || true
+          if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
+            docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
+          fi
+
+  finish:
+    if: always()
+    needs: [stage-a-test-1-gpu-xpu, stage-b-test-1-gpu-xpu, pr-gate]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check job status
+        run: |
+          stage_a="${{ needs.stage-a-test-1-gpu-xpu.result }}"
+          stage_b="${{ needs.stage-b-test-1-gpu-xpu.result }}"
+          if [ "$stage_a" != "success" ] && [ "$stage_a" != "skipped" ]; then
+            echo "stage-a failed with result: $stage_a"
+            exit 1
+          fi
+          if [ "$stage_b" != "success" ] && [ "$stage_b" != "skipped" ]; then
+            echo "stage-b failed with result: $stage_b"
+            exit 1
+          fi
+          echo "All jobs completed successfully"
+          exit 0

.github/workflows/pr-test.yml

@@ -0,0 +1,620 @@
+name: PR Test Base
+run-name: >-
+  ${{
+    github.event_name == 'schedule' && 'Scheduled Full Run'
+    || github.event_name == 'workflow_dispatch' && format('Manual run by {0}', github.actor)
+    || github.event_name == 'pull_request' && format('PR #{0} - {1}', github.event.pull_request.number, github.event.pull_request.title)
+    || 'PR Test Base'
+  }}
+
+on:
+  schedule:
+    - cron: '0 11,23 * * *'  # Run 2x daily, 12h apart, off PR-push peaks
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      force_continue_on_error:
+        description: "Force continue-on-error (test scheduled CI behavior)"
+        required: false
+        type: boolean
+        default: false
+      test_parallel_dispatch:
+        description: "Test parallel dispatch behavior (simulates scheduled run)"
+        required: false
+        type: boolean
+        default: false
+      run_all_tests:
+        description: "Run all tests (mirrors the scheduled cron's behavior — combine with test_parallel_dispatch for full sim)"
+        required: false
+        type: boolean
+        default: false
+  workflow_call:
+    inputs:
+      git_ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      run_all_tests:
+        description: "Run all tests (for releasing or testing purpose)"
+        required: false
+        type: boolean
+        default: false
+      skip_pr_test_health_check:
+        description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
+        required: false
+        type: boolean
+        default: false
+
+concurrency:
+  # Concurrency group structure: pr-test-{event}-{branch}-{git_ref}
+  # - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main'
+  #   (without it, both resolve the branch segment to 'main' and block each other)
+  # - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name
+  group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_CUDA_COREDUMP: "1"
+  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
+  # Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  USE_VENV: false
+
+permissions:
+  actions: write
+  contents: read
+  issues: read
+  pull-requests: read
+
+jobs:
+  # =============================================== check changes ====================================================
+  check-changes:
+    uses: ./.github/workflows/_pr-test-check-changes.yml
+    with:
+      git_ref: ${{ inputs.git_ref || '' }}
+      run_all_tests: ${{ inputs.run_all_tests == true }}
+      force_continue_on_error: ${{ inputs.force_continue_on_error == true }}
+    secrets: inherit
+
+  call-pr-test-extra:
+    if: github.event_name == 'schedule' || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true
+    uses: ./.github/workflows/pr-test-extra.yml
+    with:
+      git_ref: ${{ inputs.git_ref || '' }}
+      run_all_tests: true
+    secrets: inherit
+
+  # =============================================== Wait Jobs for Sequential PR Execution ====================================================
+  # These jobs poll GitHub API to wait for previous stages to complete.
+  # For PR runs: wait jobs run and enforce sequential execution via polling.
+  # For scheduled runs: wait jobs are skipped, enabling parallel execution for easier retry.
+  # For PRs with the `bypass-fastfail` label: wait jobs run but return success immediately
+  # (handled inside the wait-for-jobs action), so downstream stages dispatch in parallel.
+
+  wait-for-base-a:
+    needs: [check-changes, call-gate]
+    if: |
+      always() &&
+      !cancelled() &&
+      github.event_name == 'pull_request' &&
+      inputs.test_parallel_dispatch != true &&
+      (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
+      (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
+    runs-on: ubuntu-latest
+    outputs:
+      base_a_result: ${{ steps.wait.outputs.result }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/check-maintenance
+
+      - uses: ./.github/actions/wait-for-jobs
+        id: wait
+        with:
+          stage-name: base-a
+          jobs: |
+            [
+              {"prefix": "base-a-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-1-gpu-small'].size }}},
+              {"prefix": "base-a-test-cpu", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].size }}}
+            ]
+          max-wait-minutes: '240'
+
+  wait-for-base-b:
+    needs: [check-changes, call-gate, wait-for-base-a]
+    if: |
+      always() &&
+      !cancelled() &&
+      github.event_name == 'pull_request' &&
+      inputs.test_parallel_dispatch != true &&
+      (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
+      (needs.wait-for-base-a.result == 'success' || needs.wait-for-base-a.result == 'skipped') &&
+      (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
+    runs-on: ubuntu-latest
+    outputs:
+      base_b_result: ${{ steps.wait.outputs.result }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/check-maintenance
+
+      - uses: ./.github/actions/wait-for-jobs
+        id: wait
+        with:
+          stage-name: base-b
+          jobs: |
+            [
+              {"prefix": "base-b-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-1-gpu-small'].size }}},
+              {"prefix": "base-b-test-1-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-1-gpu-large'].size }}},
+              {"prefix": "base-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-2-gpu-large'].size }}},
+              {"prefix": "base-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-4-gpu-b200'].size }}}
+            ]
+          max-wait-minutes: '480'
+
+  # =============================================== PR Gate ====================================================
+  call-gate:
+    needs: check-changes
+    # Skip for scheduled runs (they run all tests)
+    if: |
+      github.event_name != 'schedule' &&
+      inputs.test_parallel_dispatch != true &&
+      (
+        needs.check-changes.outputs.main_package == 'true' ||
+        needs.check-changes.outputs.sgl_kernel == 'true' ||
+        needs.check-changes.outputs.jit_kernel == 'true' ||
+        needs.check-changes.outputs.multimodal_gen == 'true'
+      )
+    uses: ./.github/workflows/pr-gate.yml
+    secrets: inherit
+
+  # =============================================== sgl-kernel ====================================================
+
+  sgl-kernel-build-wheels:
+    needs: [check-changes, call-gate]
+    # Skip for scheduled runs (they run stages independently).
+    if: |
+      github.event_name != 'schedule' &&
+      inputs.test_parallel_dispatch != true &&
+      needs.check-changes.result == 'success' &&
+      needs.check-changes.outputs.sgl_kernel == 'true' &&
+      needs.call-gate.result == 'success'
+    uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
+    with:
+      runs_on: x64-kernel-build-node
+      job_display_name: Build Wheel
+      git_ref: ${{ inputs.git_ref || '' }}
+      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
+    secrets: inherit
+
+  sgl-kernel-build-wheels-arm:
+    needs: [check-changes, call-gate]
+    # Skip for scheduled runs (they run stages independently).
+    if: |
+      github.event_name != 'schedule' &&
+      inputs.test_parallel_dispatch != true &&
+      needs.check-changes.result == 'success' &&
+      needs.check-changes.outputs.sgl_kernel == 'true' &&
+      needs.call-gate.result == 'success'
+    uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
+    with:
+      runs_on: arm-kernel-build-node
+      job_display_name: Build Wheel Arm
+      arch_suffix: '-aarch64'
+      git_ref: ${{ inputs.git_ref || '' }}
+      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
+    secrets: inherit
+
+  call-sgl-kernel-tests:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: |
+      github.event_name != 'schedule' &&
+      inputs.test_parallel_dispatch != true &&
+      needs.check-changes.outputs.sgl_kernel == 'true'
+    uses: ./.github/workflows/pr-test-sgl-kernel.yml
+    with:
+      runner_config: 4-gpu-b200
+      runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
+      sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
+      git_ref: ${{ inputs.git_ref || '' }}
+      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
+    secrets: inherit
+
+  # =============================================== jit-kernel ====================================================
+
+  call-jit-kernel-tests:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    # Run on scheduled/parallel-dispatch runs (same pattern as the base-* stages) so the
+    # jit_kernel suite is exercised on main 3x daily, not only on PRs that touch jit_kernel/**.
+    # check-changes already forces jit_kernel='true' on scheduled runs (run_all_tests).
+    if: |
+      always() &&
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
+      needs.check-changes.outputs.jit_kernel == 'true'
+    uses: ./.github/workflows/pr-test-jit-kernel.yml
+    with:
+      runner_config: 4-gpu-b200
+      runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
+      jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }}
+      # On scheduled/parallel-dispatch runs sgl-kernel-build-wheels is skipped, so the wheel
+      # artifact does not exist. Force sgl_kernel='false' there so the jit-kernel jobs skip the
+      # wheel download and run against the released/pinned sgl-kernel instead of a fresh build.
+      sgl_kernel: ${{ (github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) && 'false' || needs.check-changes.outputs.sgl_kernel }}
+      git_ref: ${{ inputs.git_ref || '' }}
+      test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
+      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
+    secrets: inherit
+
+  # =============================================== primary ====================================================
+
+  # Runs on 5090 (32GB, SM120)
+  base-a-test-1-gpu-small:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-a-test-1-gpu-small
+      runner_config: 1-gpu-small
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '10'
+    secrets: inherit
+
+  base-a-test-cpu:
+    needs: [check-changes, call-gate]
+    if: |
+      always() &&
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
+      (needs.check-changes.outputs.main_package == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 240
+    env:
+      HF_HOME: ${{ github.workspace }}/.hf-cache
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].max_parallel }}
+      matrix:
+        partition: ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].arr }}
+    steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+          df -h
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      # Needed by setuptools-rust to build the bundled native gRPC extension
+      # (rust/sglang-grpc) when installing the main `sglang` wheel from source.
+      - name: Install protoc + Rust toolchain
+        timeout-minutes: 10
+        run: bash scripts/ci/utils/install_rust_protoc.sh
+
+      - name: Rust cache (sglang-grpc)
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust/sglang-grpc
+          shared-key: "sglang-grpc-cpu"
+          save-if: ${{ matrix.partition == 0 }}
+
+      # uv pip targets a venv by default; setup-python has no venv — install into that interpreter (see UV_SYSTEM_PYTHON in https://docs.astral.sh/uv/guides/integration/github/)
+      - name: Install dependencies
+        timeout-minutes: 20
+        env:
+          UV_SYSTEM_PYTHON: "1"
+        run: |
+          uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow
+
+      # Hosted runners are ephemeral, so models are re-fetched every run and the
+      # Hub occasionally returns 429s. Persist the HF cache in GitHub's cache
+      # storage (rolling key + restore-keys) so warm runs never hit the network.
+      - name: Cache HF hub
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.hf-cache
+          key: hf-cpu-${{ matrix.partition }}-${{ github.run_id }}
+          restore-keys: hf-cpu-${{ matrix.partition }}-
+
+      - name: Run test
+        timeout-minutes: 10
+        env:
+          CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+        run: |
+          cd test/
+          python3 run_suite.py --hw cpu --suite base-a-test-cpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].size }} $CONTINUE_ON_ERROR_FLAG
+
+  # Runs on 5090 (32GB, SM120)
+  base-b-test-1-gpu-small:
+    needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-b-test-1-gpu-small
+      runner_config: 1-gpu-small
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+    secrets: inherit
+
+  # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
+  base-b-test-1-gpu-large:
+    needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-b-test-1-gpu-large
+      runner_config: 1-gpu-large
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  base-b-test-2-gpu-large:
+    needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-b-test-2-gpu-large
+      runner_config: 2-gpu-large
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+    secrets: inherit
+
+  base-b-test-4-gpu-b200:
+    needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-b-test-4-gpu-b200
+      runner_config: 4-gpu-b200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '40'
+    secrets: inherit
+
+  call-multimodal-gen-tests:
+    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
+    if: |
+      always() &&
+      !cancelled() &&
+      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
+      needs.check-changes.outputs.multimodal_gen == 'true'
+    uses: ./.github/workflows/pr-test-multimodal-gen.yml
+    with:
+      runner_config: 4-gpu-b200
+      runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
+      multimodal_gen: ${{ needs.check-changes.outputs.multimodal_gen }}
+      sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
+      continue_on_error: ${{ needs.check-changes.outputs.continue_on_error }}
+      git_ref: ${{ inputs.git_ref || '' }}
+      test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
+      caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }}
+      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
+    secrets: inherit
+
+  base-c-test-4-gpu-h100:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-4-gpu-h100
+      runner_config: 4-gpu-h100
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+    secrets: inherit
+
+  base-c-test-8-gpu-h200:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-8-gpu-h200
+      runner_config: 8-gpu-h200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      # Per-model TP must match the test's launch in test/registered/ -- see
+      # FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep
+      # flags. Only models that actually invoke DeepGEMM kernels at runtime
+      # are listed. Cold-cache ~13 min; warm-cache <=30 s via marker file.
+      # Server CUDA Graph warmup is combined into this step (warmup_server_models unset).
+      warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8'
+      warmup_timeout_minutes: '60'
+    secrets: inherit
+
+  base-c-test-8-gpu-h20:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-8-gpu-h20
+      runner_config: 8-gpu-h20
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+    secrets: inherit
+
+  base-c-test-deepep-4-gpu-h100:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-deepep-4-gpu-h100
+      runner_config: deepep-4-gpu-h100
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4'
+      warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4'
+    secrets: inherit
+
+  base-c-test-deepep-4-gpu-b200:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-deepep-4-gpu-b200
+      runner_config: deepep-4-gpu-b200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  base-c-test-deepep-8-gpu-h200:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-deepep-8-gpu-h200
+      runner_config: deepep-8-gpu-h200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  base-c-test-4-gpu-b200:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-4-gpu-b200
+      runner_config: 4-gpu-b200
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  base-c-test-4-gpu-gb300:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-4-gpu-gb300
+      runner_config: 4-gpu-gb300
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      timeout_per_file: '1800'
+    secrets: inherit
+
+  pr-test-finish:
+    needs:
+      [
+        call-gate,
+        check-changes,
+
+        sgl-kernel-build-wheels,
+        sgl-kernel-build-wheels-arm,
+        call-sgl-kernel-tests,
+
+        wait-for-base-a,
+        wait-for-base-b,
+
+        call-jit-kernel-tests,
+
+        call-multimodal-gen-tests,
+
+        base-a-test-1-gpu-small,
+        base-a-test-cpu,
+        base-b-test-1-gpu-small,
+        base-b-test-1-gpu-large,
+        base-b-test-2-gpu-large,
+        base-b-test-4-gpu-b200,
+        base-c-test-4-gpu-h100,
+        base-c-test-8-gpu-h20,
+        base-c-test-8-gpu-h200,
+        base-c-test-deepep-4-gpu-h100,
+        base-c-test-deepep-4-gpu-b200,
+        base-c-test-deepep-8-gpu-h200,
+        base-c-test-4-gpu-b200,
+        base-c-test-4-gpu-gb300,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          # Convert the 'needs' context to a JSON string
+          json_needs='${{ toJson(needs) }}'
+
+          # Get a list of all job names from the JSON keys
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+
+          for job in $job_names; do
+            # For each job, extract its result
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+
+            # Print the job name and its result
+            echo "$job: $result"
+
+            # Check for failure or cancellation and exit if found
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "The above jobs failed."
+              exit 1
+            fi
+          done
+          # If the loop completes, all jobs were successful
+          echo "All jobs completed successfully"
+          exit 0
+
+  # =============================================== notify pr-states ====================================================
+  # Dispatches pr-states.yml after every attempt (initial + every rerun).
+  # workflow_run.completed only fires on the initial completion (one
+  # workflow_run record across attempts), so we explicitly dispatch —
+  # workflow_dispatch is the only event GITHUB_TOKEN can cascade-create.
+  #
+  # Fork PRs are excluded: their pull_request GITHUB_TOKEN is forced
+  # read-only regardless of declared `actions: write`, so dispatch 403s.
+  # Fork PRs get pr-states updates via the workflow_run subscription
+  # there (initial completion) and pull_request_target (push / label).
+  notify-pr-states:
+    needs: [pr-test-finish]
+    if: |
+      always() &&
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    steps:
+      - name: Dispatch pr-states refresh
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          gh workflow run pr-states.yml \
+            --repo ${{ github.repository }} \
+            --ref main \
+            -f pr_number="$PR_NUMBER"

.github/workflows/release-branch-cut.yml

@@ -0,0 +1,233 @@
+name: Release Branch Cut
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch_name:
+        description: 'Branch name to create (e.g., release/v0.5.7)'
+        required: true
+        type: string
+      commit_sha:
+        description: 'Commit SHA from main to cut the release branch from (defaults to latest main)'
+        required: false
+        type: string
+        default: ''
+
+permissions:
+  actions: write
+  contents: write
+  issues: read
+  pull-requests: read
+
+jobs:
+  cut-release-branch:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    environment: 'prod'
+    outputs:
+      branch_name: ${{ steps.set_output.outputs.branch_name }}
+      branch_exists: ${{ steps.check_exists.outputs.branch_exists }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Validate branch name
+        run: |
+          BRANCH_NAME="${{ github.event.inputs.branch_name }}"
+
+          if [ -z "$BRANCH_NAME" ]; then
+            echo "::error::Branch name is required"
+            exit 1
+          fi
+
+          # Validate branch name format (should start with release/)
+          if [[ ! "$BRANCH_NAME" =~ ^release/ ]]; then
+            echo "::warning::Branch name '$BRANCH_NAME' does not follow convention 'release/vX.Y.Z'"
+          fi
+
+          echo "Branch name: $BRANCH_NAME"
+
+      - name: Validate commit SHA
+        id: validate
+        run: |
+          COMMIT_SHA="${{ github.event.inputs.commit_sha }}"
+
+          # If no commit SHA provided, use latest main
+          if [ -z "$COMMIT_SHA" ]; then
+            COMMIT_SHA=$(git rev-parse HEAD)
+            echo "No commit SHA provided, using latest main: $COMMIT_SHA"
+          fi
+
+          # Verify the commit exists and is on main
+          if ! git cat-file -t "$COMMIT_SHA" > /dev/null 2>&1; then
+            echo "::error::Commit SHA '$COMMIT_SHA' does not exist"
+            exit 1
+          fi
+
+          # Check if commit is an ancestor of main (i.e., is on main branch)
+          if ! git merge-base --is-ancestor "$COMMIT_SHA" main; then
+            echo "::error::Commit SHA '$COMMIT_SHA' is not on the main branch"
+            exit 1
+          fi
+
+          echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_OUTPUT
+          echo "Validated commit SHA: $COMMIT_SHA"
+
+      - name: Check if branch already exists
+        id: check_exists
+        run: |
+          BRANCH_NAME="${{ github.event.inputs.branch_name }}"
+
+          if git ls-remote --heads origin "$BRANCH_NAME" | grep -q "$BRANCH_NAME"; then
+            echo "Branch '$BRANCH_NAME' already exists, skipping creation and proceeding to downstream tests"
+            echo "branch_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "Branch '$BRANCH_NAME' does not exist, proceeding with creation"
+            echo "branch_exists=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create release branch
+        if: steps.check_exists.outputs.branch_exists != 'true'
+        run: |
+          COMMIT_SHA="${{ steps.validate.outputs.COMMIT_SHA }}"
+          BRANCH_NAME="${{ github.event.inputs.branch_name }}"
+
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+
+          # Create branch from the specified commit
+          git checkout -b "$BRANCH_NAME" "$COMMIT_SHA"
+
+          echo "Successfully created branch '$BRANCH_NAME' from commit '$COMMIT_SHA'"
+
+      - name: Update version references in documentation
+        if: steps.check_exists.outputs.branch_exists != 'true'
+        run: |
+          BRANCH_NAME="${{ github.event.inputs.branch_name }}"
+          # Extract version from branch name (e.g., release/v0.5.8 -> v0.5.8)
+          VERSION=$(echo "$BRANCH_NAME" | sed 's/release\///')
+
+          # Update git clone version references in docs
+          sed -i "s/git clone -b v[0-9]\+\.[0-9]\+\.[0-9]\+\.\?post\?[0-9]*/git clone -b $VERSION/" docs/get_started/install.md
+          sed -i "s/git clone -b v[0-9]\+\.[0-9]\+\.[0-9]\+\.\?post\?[0-9]*/git clone -b $VERSION/" docs/platforms/amd_gpu.md
+
+          # Check if any changes were made
+          if git diff --quiet; then
+            echo "No version references needed updating"
+          else
+            git add docs/get_started/install.md docs/platforms/amd_gpu.md
+            git commit -m "docs: update version references to $VERSION"
+            echo "Updated version references to $VERSION"
+          fi
+
+      - name: Push release branch
+        if: steps.check_exists.outputs.branch_exists != 'true'
+        run: |
+          BRANCH_NAME="${{ github.event.inputs.branch_name }}"
+          git push origin "$BRANCH_NAME"
+          echo "Successfully pushed branch '$BRANCH_NAME'"
+
+      - name: Emit branch_name output
+        id: set_output
+        run: |
+          BRANCH_NAME="${{ github.event.inputs.branch_name }}"
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
+
+      - name: Summary
+        run: |
+          COMMIT_SHA="${{ steps.validate.outputs.COMMIT_SHA }}"
+          BRANCH_NAME="${{ github.event.inputs.branch_name }}"
+          BRANCH_EXISTS="${{ steps.check_exists.outputs.branch_exists }}"
+
+          if [ "$BRANCH_EXISTS" = "true" ]; then
+            STATUS="Branch already existed — creation skipped, downstream tests will run"
+          else
+            STATUS="Newly created"
+          fi
+
+          echo "## Release Branch Cut Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
+          echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Branch | \`$BRANCH_NAME\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| Commit | \`$COMMIT_SHA\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| Status | $STATUS |" >> $GITHUB_STEP_SUMMARY
+          echo "| Triggered by | @${{ github.actor }} |" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
+          echo "1. Tests are automatically triggered on the release branch" >> $GITHUB_STEP_SUMMARY
+          echo "2. Apply any hotfixes if needed" >> $GITHUB_STEP_SUMMARY
+          echo "3. Create a tag to trigger release: \`gh workflow run release-tag.yml -f version=X.Y.Z -f ref=$BRANCH_NAME\`" >> $GITHUB_STEP_SUMMARY
+
+  run-pr-tests-nvidia:
+    needs: cut-release-branch
+    uses: ./.github/workflows/pr-test.yml
+    with:
+      git_ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+      run_all_tests: true
+      skip_pr_test_health_check: true
+    secrets: inherit
+
+  run-pr-tests-amd:
+    needs: cut-release-branch
+    uses: ./.github/workflows/pr-test-amd.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+      run_all_tests: true
+    secrets: inherit
+
+  run-pr-test-npu:
+    needs: cut-release-branch
+    uses: ./.github/workflows/pr-test-npu.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+      run_all_tests: true
+    secrets: inherit
+
+  run-pr-tests-xeon:
+    needs: cut-release-branch
+    uses: ./.github/workflows/pr-test-xeon.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+      run_all_tests: true
+    secrets: inherit
+
+  run-pr-tests-xpu:
+    needs: cut-release-branch
+    uses: ./.github/workflows/pr-test-xpu.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+      run_all_tests: true
+    secrets: inherit
+
+  run-nightly-tests-nvidia:
+    needs: cut-release-branch
+    uses: ./.github/workflows/nightly-test-nvidia.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+    secrets: inherit
+
+  run-nightly-tests-amd:
+    needs: cut-release-branch
+    uses: ./.github/workflows/nightly-test-amd.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+    secrets: inherit
+
+  run-nightly-tests-npu:
+    needs: cut-release-branch
+    uses: ./.github/workflows/nightly-test-npu.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+    secrets: inherit
+
+  run-nightly-tests-intel:
+    needs: cut-release-branch
+    uses: ./.github/workflows/nightly-test-intel.yml
+    with:
+      ref: ${{ needs.cut-release-branch.outputs.branch_name }}
+    secrets: inherit

.github/workflows/release-docker-amd-miles-rocm700-nightly.yml

@@ -0,0 +1,63 @@
+name: Release Docker Images Nightly Miles ROCm7.0 (AMD)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: amd-docker-scale
+    environment: "prod"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - gpu_arch: gfx950
+            rocm_tag: rocm700-mi35x
+            sglang_image_repo: rocm/sgl-dev
+            sglang_image_tag: v0.5.14-rocm700-mi35x-20260627
+          - gpu_arch: gfx942
+            rocm_tag: rocm700-mi30x
+            sglang_image_tag: v0.5.10-rocm700-mi30x
+    steps:
+      - name: Checkout miles repository
+        uses: actions/checkout@v4
+        with:
+          repository: radixark/miles
+          ref: main
+          fetch-depth: 0
+
+      - name: Set date
+        run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+
+      - name: Record miles commit
+        id: miles
+        run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Build and push miles image to rocm/sgl-dev
+        run: |
+          base="miles-${{ matrix.rocm_tag }}"
+          dated="${base}-${{ env.DATE }}"
+          echo "IMAGE_TAG=${dated}" >> $GITHUB_ENV
+          echo "Building rocm/sgl-dev:${dated} from miles main @ ${{ steps.miles.outputs.sha }}"
+          docker build . \
+            -f docker/Dockerfile.rocm \
+            --build-arg GPU_ARCH=${{ matrix.gpu_arch }} \
+            --build-arg SGLANG_IMAGE_REPO=${{ matrix.sglang_image_repo || 'lmsysorg/sglang' }} \
+            --build-arg SGLANG_IMAGE_TAG=${{ matrix.sglang_image_tag }} \
+            --build-arg MILES_COMMIT=${{ steps.miles.outputs.sha }} \
+            -t rocm/sgl-dev:${dated} \
+            --no-cache
+          docker push rocm/sgl-dev:${dated}

.github/workflows/release-docker-amd-miles-rocm720-nightly.yml

@@ -0,0 +1,58 @@
+name: Release Docker Images Nightly Miles ROCm7.2 (AMD)
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: amd-docker-scale
+    environment: "prod"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - gpu_arch: gfx950
+            rocm_tag: rocm720-mi35x
+            sglang_image_tag: v0.5.10-rocm720-mi35x
+    steps:
+      - name: Checkout miles repository
+        uses: actions/checkout@v4
+        with:
+          repository: radixark/miles
+          ref: main
+          fetch-depth: 0
+
+      - name: Set date
+        run: echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+
+      - name: Record miles commit
+        id: miles
+        run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Build and push miles image to rocm/sgl-dev
+        run: |
+          base="miles-${{ matrix.rocm_tag }}"
+          dated="${base}-${{ env.DATE }}"
+          echo "IMAGE_TAG=${dated}" >> $GITHUB_ENV
+          echo "Building rocm/sgl-dev:${dated} from miles main @ ${{ steps.miles.outputs.sha }}"
+          docker build . \
+            -f docker/Dockerfile.rocm \
+            --build-arg GPU_ARCH=${{ matrix.gpu_arch }} \
+            --build-arg SGLANG_IMAGE_TAG=${{ matrix.sglang_image_tag }} \
+            --build-arg MILES_COMMIT=${{ steps.miles.outputs.sha }} \
+            -t rocm/sgl-dev:${dated} \
+            --no-cache
+          docker push rocm/sgl-dev:${dated}

.github/workflows/release-docker-amd-nightly.yml

@@ -0,0 +1,252 @@
+name: Release Docker Images Nightly ROCm7.0 (AMD)
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 12 * * *'
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: amd-docker-scale
+    environment: 'prod'
+    strategy:
+      fail-fast: false
+      matrix:
+        gpu_arch: ['gfx942', 'gfx950']
+        build_type: ['all']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Required for git describe to find tags
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: "Set Date"
+        run: |
+          echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+
+      - name: Get version from latest tag
+        id: version
+        run: |
+          # Use the shared helper so stable/post releases sort above rc tags.
+          VERSION=$(python3 python/tools/get_version_tag.py --tag-only | sed 's/^v//')
+
+          if [ -z "$VERSION" ]; then
+            echo "::error::Could not determine version from git tags"
+            exit 1
+          fi
+
+          # Get short commit hash of current HEAD
+          COMMIT_HASH=$(git rev-parse --short HEAD)
+
+          # Compose pretend version for setuptools_scm: e.g., 0.5.8.dev20260129+g1a2b3c4
+          PRETEND_VERSION="${VERSION}.dev${{ env.DATE }}+g${COMMIT_HASH}"
+
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "pretend_version=${PRETEND_VERSION}" >> $GITHUB_OUTPUT
+          echo "Detected version: ${VERSION}"
+          echo "Pretend version for pip: ${PRETEND_VERSION}"
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Build and Push to rocm/sgl-dev
+        run: |
+          version=${{ steps.version.outputs.version }}
+          pretend_version=${{ steps.version.outputs.pretend_version }}
+          echo "Version: ${version}"
+          echo "Pretend version: ${pretend_version}"
+
+          if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
+            rocm_tag="rocm700-mi30x"
+          elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then
+            rocm_tag="rocm700-mi35x"
+          else
+            echo "Unsupported gfx arch"
+            exit 1
+          fi
+
+          tag=v${version}-${rocm_tag}
+          echo "IMAGE_TAG=${tag}-${{ env.DATE }}" >> $GITHUB_ENV
+
+          # remove --build-arg NIC_BACKEND=ainic for auto detection nic support in mori
+          # UBUNTU_MIRROR forces apt over HTTPS to dodge port-80 reachability flakes
+          # to Canonical's archive.ubuntu.com mirror IPs from the amd-docker-scale runner.
+          docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg ENABLE_NIXL=1 --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
+          docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
+
+      # Persist the tag right after rocm/sgl-dev push succeeds so the local
+      # registry mirror can run even if a later step in this job (lmsys push)
+      # fails. By default this step only runs when the previous step succeeded,
+      # so the artifact only exists when an image actually landed on Docker Hub.
+      - name: Save published image tag
+        run: |
+          mkdir -p image-tag
+          echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
+
+      - name: Upload image tag artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-tag-${{ matrix.gpu_arch }}
+          path: image-tag/${{ matrix.gpu_arch }}.txt
+          retention-days: 1
+
+      - name: Login to Docker Hub (lmsys)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push to lmsysorg/sglang-rocm
+        run: |
+          docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
+          docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
+
+  # Mirror the freshly published rocm/sgl-dev image to the in-network Docker
+  # registry so AMD CI runners can pull without hitting Docker Hub rate limits.
+  # The tag is read verbatim from the publish job's artifact so this job uses
+  # exactly the same tag that publish pushed (only the registry prefix differs).
+  # `!cancelled()` lets us still mirror successful matrix legs when other legs
+  # of publish failed; legs without an artifact will fail at download and be
+  # the only ones marked red.
+  push_local_registry:
+    if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' }}
+    runs-on: linux-mi300-1gpu-sglang
+    environment: 'prod'
+    needs: publish
+    strategy:
+      fail-fast: false
+      matrix:
+        gpu_arch: ['gfx942', 'gfx950']
+    steps:
+      - name: Download image tag artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: image-tag-${{ matrix.gpu_arch }}
+
+      - name: Read image tag
+        run: |
+          image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt")
+          if [ -z "${image_tag}" ]; then
+            echo "::error::Image tag artifact is empty"
+            exit 1
+          fi
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
+          echo "Resolved IMAGE_TAG=${image_tag}"
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Mirror rocm/sgl-dev to local registry
+        run: |
+          src="rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          dst="10.44.14.109:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          docker pull "${src}"
+          docker tag "${src}" "${dst}"
+          docker push "${dst}"
+
+  # Temporarily disable docker cache seeding until performant storage is in place
+  # cache:
+  #   if: false
+  #   # if: always() && github.repository == 'sgl-project/sglang'
+  #   runs-on: linux-mi300-gpu-1
+  #   environment: 'prod'
+  #   needs: publish
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       gpu_arch: ['gfx942']
+  #       build_type: ['all']
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0  # Required for git describe to find tags
+
+  #     - name: "Set Date"
+  #       run: |
+  #         echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+
+  #     - name: Get version from latest tag
+  #       id: version
+  #       run: |
+  #         # Use the shared helper so stable/post releases sort above rc tags.
+  #         VERSION=$(python3 python/tools/get_version_tag.py --tag-only | sed 's/^v//')
+
+  #         if [ -z "$VERSION" ]; then
+  #           echo "::error::Could not determine version from git tags"
+  #           exit 1
+  #         fi
+
+  #         echo "version=${VERSION}" >> $GITHUB_OUTPUT
+  #         echo "Detected version: ${VERSION}"
+
+  #     - name: Login to Docker Hub
+  #       uses: docker/login-action@v2
+  #       with:
+  #         username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+  #         password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+  #     - name: Pull and Save Docker Image to Cache
+  #       run: |
+  #         set -euxo pipefail
+
+  #         version=${{ steps.version.outputs.version }}
+  #         echo "Version: ${version}"
+
+  #         if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
+  #           rocm_tag="rocm700-mi30x"
+  #         else
+  #           echo "Unsupported gfx arch"
+  #           exit 1
+  #         fi
+
+  #         tag=v${version}-${rocm_tag}
+
+  #         if [ "${{ matrix.build_type }}" = "all" ]; then
+  #           tag_suffix=""
+  #         else
+  #           echo "Unsupported build type"
+  #           exit 1
+  #         fi
+
+  #         image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}"
+
+  #         # Determine target cache file name based on ROCm variant
+  #         if [[ "${rocm_tag}" == rocm700* ]]; then
+  #           final_path="/home/runner/sgl-data/docker/image-700.tar"
+  #         else
+  #           echo "Unexpected ROCm tag: ${rocm_tag}"
+  #           exit 1
+  #         fi
+
+  #         tmp_path="${final_path}.tmp"
+
+  #         echo "Pulling image: ${image}"
+  #         docker pull "${image}"
+
+  #         echo "Saving to temp file: ${tmp_path}"
+  #         docker save "${image}" -o "${tmp_path}"
+
+  #         echo "Moving to final path: ${final_path}"
+  #         mv -f "${tmp_path}" "${final_path}"
+
+  #         echo "Cache populated successfully at ${final_path}"

.github/workflows/release-docker-amd-rocm720-nightly.yml

@@ -0,0 +1,173 @@
+name: Release Docker Images Nightly ROCm7.2 (AMD)
+on:
+  workflow_dispatch:
+    inputs:
+      job_select:
+        description: 'Select which release job to run'
+        required: false
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - publish
+  schedule:
+    - cron: '0 12 * * *'
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: True
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish')
+    runs-on: amd-docker-scale
+    environment: 'prod'
+    strategy:
+      fail-fast: false
+      matrix:
+        gpu_arch: ['gfx942-rocm720', 'gfx950-rocm720']
+        build_type: ['all']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Required for git describe to find tags
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: "Set Date"
+        run: |
+          echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
+
+      - name: Get version from latest tag
+        id: version
+        run: |
+          # Use the shared helper so stable/post releases sort above rc tags.
+          VERSION=$(python3 python/tools/get_version_tag.py --tag-only | sed 's/^v//')
+
+          if [ -z "$VERSION" ]; then
+            echo "::error::Could not determine version from git tags"
+            exit 1
+          fi
+
+          # Get short commit hash of current HEAD
+          COMMIT_HASH=$(git rev-parse --short HEAD)
+
+          # Compose pretend version for setuptools_scm: e.g., 0.5.8.post1.dev20260211+g1a2b3c4
+          PRETEND_VERSION="${VERSION}.dev${{ env.DATE }}+g${COMMIT_HASH}"
+
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "pretend_version=${PRETEND_VERSION}" >> $GITHUB_OUTPUT
+          echo "Detected version: ${VERSION}"
+          echo "Pretend version for pip: ${PRETEND_VERSION}"
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Build and Push to rocm/sgl-dev
+        run: |
+          version=${{ steps.version.outputs.version }}
+          pretend_version=${{ steps.version.outputs.pretend_version }}
+          echo "Version: ${version}"
+          echo "Pretend version: ${pretend_version}"
+
+          if [ "${{ matrix.gpu_arch }}" = "gfx942-rocm720" ]; then
+            rocm_tag="rocm720-mi30x"
+          elif [ "${{ matrix.gpu_arch }}" = "gfx950-rocm720" ]; then
+            rocm_tag="rocm720-mi35x"
+          else
+            echo "Unsupported gfx arch"
+            exit 1
+          fi
+
+          tag=v${version}-${rocm_tag}
+          echo "IMAGE_TAG=${tag}-${{ env.DATE }}" >> $GITHUB_ENV
+
+          # remove --build-arg NIC_BACKEND=ainic for auto detection nic support in mori
+          # UBUNTU_MIRROR forces apt over HTTPS to dodge port-80 reachability flakes
+          # to Canonical's archive.ubuntu.com mirror IPs from the amd-docker-scale runner.
+          docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg ENABLE_NIXL=1 --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
+          docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
+
+      # Persist the tag right after rocm/sgl-dev push succeeds so the local
+      # registry mirror can run even if a later step in this job (lmsys push)
+      # fails. By default this step only runs when the previous step succeeded,
+      # so the artifact only exists when an image actually landed on Docker Hub.
+      - name: Save published image tag
+        run: |
+          mkdir -p image-tag
+          echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
+
+      - name: Upload image tag artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-tag-${{ matrix.gpu_arch }}
+          path: image-tag/${{ matrix.gpu_arch }}.txt
+          retention-days: 1
+
+      - name: Login to Docker Hub (lmsys)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push to lmsysorg/sglang-rocm
+        run: |
+          docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
+          docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
+
+  # Mirror the freshly published rocm/sgl-dev image to the in-network Docker
+  # registry so AMD CI runners can pull without hitting Docker Hub rate limits.
+  # The tag is read verbatim from the publish job's artifact so this job uses
+  # exactly the same tag that publish pushed (only the registry prefix differs).
+  # `!cancelled()` lets us still mirror successful matrix legs when other legs
+  # of publish failed; legs without an artifact will fail at download and be
+  # the only ones marked red.
+  push_local_registry:
+    if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' && (github.event_name != 'workflow_dispatch' || inputs.job_select == 'all' || inputs.job_select == 'publish') }}
+    runs-on: linux-mi300-1gpu-sglang
+    environment: 'prod'
+    needs: publish
+    strategy:
+      fail-fast: false
+      matrix:
+        gpu_arch: ['gfx942-rocm720', 'gfx950-rocm720']
+    steps:
+      - name: Download image tag artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: image-tag-${{ matrix.gpu_arch }}
+
+      - name: Read image tag
+        run: |
+          image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt")
+          if [ -z "${image_tag}" ]; then
+            echo "::error::Image tag artifact is empty"
+            exit 1
+          fi
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
+          echo "Resolved IMAGE_TAG=${image_tag}"
+
+      - name: Login to Docker Hub (AMD)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
+
+      - name: Mirror rocm/sgl-dev to local registry
+        run: |
+          src="rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          dst="10.44.14.109:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}"
+          docker pull "${src}"
+          docker tag "${src}" "${dst}"
+          docker push "${dst}"

.github/workflows/release-docker-amd.yml

@@ -0,0 +1,91 @@
+name: Release Docker Images (AMD)
+on:
+  push:
+    tags:
+      - 'v[0-9]+.*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to build (without v prefix, e.g., 0.5.7)'
+        required: true
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: amd-docker-scale
+    environment: 'prod'
+    strategy:
+      matrix:
+        rocm_version: ['rocm700', 'rocm720']
+        gpu_arch: ['gfx942', 'gfx950']
+        build_type: ['all']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version from tag
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            # Extract version from tag (e.g., v0.5.7 -> 0.5.7)
+            VERSION="${GITHUB_REF_NAME#v}"
+          fi
+
+          # Validate version format
+          if [ -z "$VERSION" ]; then
+            echo "::error::Version is empty"
+            exit 1
+          fi
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::Invalid version format: $VERSION (expected: X.Y.Z)"
+            exit 1
+          fi
+
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+
+      - name: Build and Push
+        run: |
+          version=${{ steps.version.outputs.version }}
+          echo "Version: ${version}"
+
+          gpu_arch_suffix=""
+          if [ "${{ matrix.rocm_version }}" = "rocm700" ]; then
+            if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
+              rocm_tag="rocm700-mi30x"
+            elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then
+              rocm_tag="rocm700-mi35x"
+            else
+              echo "Unsupported gfx arch"
+              exit 1
+            fi
+          elif [ "${{ matrix.rocm_version }}" = "rocm720" ]; then
+            gpu_arch_suffix="-${{ matrix.rocm_version }}"
+            if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
+              rocm_tag="rocm720-mi30x"
+            elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then
+              rocm_tag="rocm720-mi35x"
+            else
+              echo "Unsupported gfx arch"
+              exit 1
+            fi
+          else
+            echo "Unsupported rocm version"
+            exit 1
+          fi
+
+          tag=v${version}-${rocm_tag}
+
+          # rocm.Dockerfile expects SGL_BRANCH with 'v' prefix for git tag checkout
+          # remove --build-arg NIC_BACKEND=ainic for auto detection nic support in mori
+          # UBUNTU_MIRROR forces apt over HTTPS to dodge port-80 reachability flakes
+          # to Canonical's archive.ubuntu.com mirror IPs from the amd-docker-scale runner.
+          docker build . -f docker/rocm.Dockerfile --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }}${gpu_arch_suffix} --build-arg SGL_BRANCH=v${version} --build-arg ENABLE_MORI=1 --build-arg UBUNTU_MIRROR=https://archive.ubuntu.com -t lmsysorg/sglang:${tag} --no-cache
+          docker push lmsysorg/sglang:${tag}

.github/workflows/release-docker-dev.yml

@@ -0,0 +1,200 @@
+name: Build and Push Development Docker Images
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: "PR number to build from (leave empty to use current branch)"
+        required: false
+        default: ""
+      tag:
+        description: "Custom tag suffix (overrides pr_number in tag). E.g. 'my-test' → dev-my-test, dev-cu13-my-test, etc."
+        required: false
+        default: ""
+      image_repo:
+        description: "Docker Hub repo to push to. Use lmsysorg/sglang-staging for testing."
+        required: false
+        default: "lmsysorg/sglang"
+      overlay_dockerfile:
+        description: "Optional extra Dockerfile lines appended after FROM <base> to build a layered image (e.g. 'RUN pip install ...'). Note: this job has no repo checkout, so only FROM + RUN work — you cannot COPY files from this repo. Leave empty to skip overlay."
+        required: false
+        default: ""
+      overlay_cudas:
+        description: "Comma-separated cuda variants to overlay onto (cu12, cu13). Default 'cu13'."
+        required: false
+        default: "cu13"
+      overlay_tag_suffix:
+        description: "Overlay output suffix appended to the base tag. EMPTY = overwrite the base tag(s); non-empty = append (e.g. 'msa')."
+        required: false
+        default: ""
+  schedule:
+    - cron: "0 0 * * *"
+
+concurrency:
+  group: release-docker-dev-${{ inputs.tag || inputs.pr_number || 'nightly' }}
+  cancel-in-progress: true
+
+jobs:
+  prepare:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      checkout_ref: ${{ steps.config.outputs.checkout_ref }}
+      extra_build_args: ${{ steps.config.outputs.extra_build_args }}
+      tag_config: ${{ steps.config.outputs.tag_config }}
+      overlay_cudas_json: ${{ steps.config.outputs.overlay_cudas_json }}
+      overlay_tags_json: ${{ steps.config.outputs.overlay_tags_json }}
+    steps:
+      - name: Compute build configuration
+        id: config
+        env:
+          # Inject multi-line/free-text inputs via env to avoid ${{ }} injection
+          # breaking the bash script on quotes or $ in the Dockerfile snippet.
+          OVERLAY_DOCKERFILE_INPUT: ${{ inputs.overlay_dockerfile }}
+          OVERLAY_CUDAS_INPUT: ${{ inputs.overlay_cudas }}
+          OVERLAY_SUFFIX_INPUT: ${{ inputs.overlay_tag_suffix }}
+        run: |
+          # Determine checkout ref
+          if [ -n "${{ inputs.pr_number }}" ]; then
+            echo "checkout_ref=refs/pull/${{ inputs.pr_number }}/head" >> $GITHUB_OUTPUT
+          else
+            echo "checkout_ref=" >> $GITHUB_OUTPUT
+          fi
+
+          # Determine extra build args. INSTALL_DYNAMO=1 installs the ai-dynamo
+          # nightly here only; release-docker.yml / release-docker-runtime.yml
+          # leave it at the Dockerfile default (0), keeping it out of releases.
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            echo "extra_build_args=--build-arg USE_LATEST_SGLANG=1 --build-arg INSTALL_DYNAMO=1 --build-arg CMAKE_BUILD_PARALLEL_LEVEL=\$(nproc)" >> $GITHUB_OUTPUT
+          else
+            echo "extra_build_args=--build-arg BRANCH_TYPE=local --build-arg INSTALL_DYNAMO=1 --build-arg CMAKE_BUILD_PARALLEL_LEVEL=\$(nproc)" >> $GITHUB_OUTPUT
+          fi
+
+          # Determine tag suffix
+          SUFFIX=""
+          if [ -n "${{ inputs.tag }}" ]; then
+            SUFFIX="-${{ inputs.tag }}"
+          elif [ -n "${{ inputs.pr_number }}" ]; then
+            SUFFIX="-pr-${{ inputs.pr_number }}"
+          fi
+
+          # Build tag config. dev-cu13 / nightly-dev-cu13 are published as
+          # aliases on the cu130 image for backwards compatibility with
+          # consumers pinned to the pre-flip names.
+          if [ -z "${SUFFIX}" ]; then
+            # Nightly: include dated tags
+            TAG_CONFIG='[{"cuda":"cu129","tags":["dev-cu12","nightly-dev-cu12-{date}-{short_sha}"]},{"cuda":"cu130","tags":["dev","dev-cu13","nightly-dev-{date}-{short_sha}","nightly-dev-cu13-{date}-{short_sha}"]}]'
+          else
+            TAG_CONFIG="[{\"cuda\":\"cu129\",\"tags\":[\"dev-cu12${SUFFIX}\"]},{\"cuda\":\"cu130\",\"tags\":[\"dev${SUFFIX}\",\"dev-cu13${SUFFIX}\"]}]"
+          fi
+          echo "tag_config=${TAG_CONFIG}" >> $GITHUB_OUTPUT
+
+          # Overlay configuration (optional). When overlay_dockerfile is set,
+          # build one or more layered images on top of the just-built base.
+          # SUFFIX was computed above from inputs.tag / inputs.pr_number.
+          if [ -n "${OVERLAY_DOCKERFILE_INPUT}" ]; then
+            # Refuse to overlay without a tag/pr_number: SUFFIX would be empty,
+            # so base would point at the moving 'dev'/'dev-cu12'/'dev-cu13'
+            # latest tags and overwrite mode would clobber them.
+            if [ -z "${SUFFIX}" ]; then
+              echo "overlay_dockerfile is set but neither 'tag' nor 'pr_number' was given." >&2
+              echo "Set one so the overlay targets a specific tag, not the moving 'dev' latest." >&2
+              exit 1
+            fi
+            CUDAS_INPUT="${OVERLAY_CUDAS_INPUT}"
+            [ -z "${CUDAS_INPUT}" ] && CUDAS_INPUT="cu13"
+            OVERLAY_TAGS='{}'
+            IFS=',' read -ra CUDAS <<< "${CUDAS_INPUT}"
+            for C in "${CUDAS[@]}"; do
+              C="$(echo "${C}" | xargs)"
+              if [ "${C}" = "cu13" ]; then
+                BASES='["dev'"${SUFFIX}"'","dev-cu13'"${SUFFIX}"'"]'
+              elif [ "${C}" = "cu12" ]; then
+                BASES='["dev-cu12'"${SUFFIX}"'"]'
+              else
+                echo "Unknown overlay cuda variant: ${C} (expected cu12 or cu13)" >&2
+                exit 1
+              fi
+              if [ -n "${OVERLAY_SUFFIX_INPUT}" ]; then
+                # Append mode: primary base tag + suffix.
+                PRIMARY="$(echo "${BASES}" | jq -r '.[0]')"
+                OUTS='["'"${PRIMARY}-${OVERLAY_SUFFIX_INPUT}"'"]'
+              else
+                # Overwrite mode: replace every base alias.
+                OUTS="${BASES}"
+              fi
+              OVERLAY_TAGS="$(echo "${OVERLAY_TAGS}" | jq -c \
+                --arg c "${C}" --argjson b "${BASES}" --argjson o "${OUTS}" \
+                '.[$c] = {bases:$b, outs:$o}')"
+            done
+            CUDAS_JSON="$(echo "${CUDAS_INPUT}" | jq -c -R 'split(",") | map(gsub("^\\s+|\\s+$";""))')"
+            echo "overlay_cudas_json=${CUDAS_JSON}" >> $GITHUB_OUTPUT
+            echo "overlay_tags_json=${OVERLAY_TAGS}" >> $GITHUB_OUTPUT
+          else
+            echo "overlay_cudas_json=[]" >> $GITHUB_OUTPUT
+            echo "overlay_tags_json={}" >> $GITHUB_OUTPUT
+          fi
+
+  build-and-publish:
+    needs: prepare
+    uses: ./.github/workflows/_docker-build-and-publish.yml
+    with:
+      docker_target: framework_final
+      checkout_ref: ${{ needs.prepare.outputs.checkout_ref }}
+      extra_build_args: ${{ needs.prepare.outputs.extra_build_args }}
+      tag_config: ${{ needs.prepare.outputs.tag_config }}
+      image_repo: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
+    secrets: inherit
+
+  cleanup-nightly:
+    needs: build-and-publish
+    if: ${{ !inputs.tag && !inputs.pr_number }}
+    uses: ./.github/workflows/_docker-cleanup-nightly.yml
+    with:
+      tag_prefixes: '["nightly-dev", "nightly-dev-cu12", "nightly-dev-cu13"]'
+      image_repo: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
+    secrets: inherit
+
+  build-overlay:
+    needs: [prepare, build-and-publish]
+    if: ${{ inputs.overlay_dockerfile != '' && github.repository == 'sgl-project/sglang' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cuda: ${{ fromJson(needs.prepare.outputs.overlay_cudas_json) }}
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push overlay image (amd64 + arm64)
+        env:
+          REPO: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
+          TAGS: ${{ needs.prepare.outputs.overlay_tags_json }}
+          CUDA: ${{ matrix.cuda }}
+          # Injected via env to keep Dockerfile quotes/$ verbatim.
+          SNIPPET: ${{ inputs.overlay_dockerfile }}
+        run: |
+          set -euo pipefail
+          BASE="$(printf '%s' "$TAGS" | jq -r --arg c "$CUDA" '.[$c].bases[0]')"
+          mapfile -t OUTS < <(printf '%s' "$TAGS" | jq -r --arg c "$CUDA" '.[$c].outs[]')
+          {
+            printf 'FROM %s:%s\n' "$REPO" "$BASE"
+            printf '%s\n' "$SNIPPET"
+          } > Dockerfile.overlay
+          echo "::group::overlay Dockerfile ($CUDA)"
+          cat Dockerfile.overlay
+          echo "::endgroup::"
+          OUT_ARGS=""
+          for o in "${OUTS[@]}"; do OUT_ARGS="$OUT_ARGS -t $REPO:$o"; done
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            -f Dockerfile.overlay \
+            $OUT_ARGS \
+            --push .

.github/workflows/release-docker-gateway.yml

@@ -0,0 +1,39 @@
+name: Release SGLang Model Gateway Docker Image
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - sgl-model-gateway/bindings/python/pyproject.toml
+  workflow_dispatch:
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and Push
+        run: |
+          version=$(cat sgl-model-gateway/bindings/python/src/sglang_router/version.py | cut -d'"' -f2)
+          tag=v${version}
+
+          docker buildx build . -f docker/gateway.Dockerfile \
+            --platform linux/amd64,linux/arm64 \
+            -t lmsysorg/sgl-model-gateway:${tag} \
+            -t lmsysorg/sgl-model-gateway:latest \
+            --push

.github/workflows/release-docker-intel-xpu-nightly.yml

@@ -0,0 +1,59 @@
+name: Release Docker Images Nightly (Intel XPU)
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 12 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: intel-bmg
+    environment: 'prod'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          path: sglang-nightly-build
+          fetch-depth: 0
+
+      - name: Set date and commit hash
+        id: image_meta
+        run: |
+          cd sglang-nightly-build
+          date_tag=$(date +%Y%m%d)
+          commit_hash=$(git rev-parse --short=7 HEAD)
+          if [ -z "${commit_hash}" ]; then
+            echo "::error::Could not resolve commit hash"
+            exit 1
+          fi
+          image_tag="nightly-dev-xpu-bmg-${date_tag}-${commit_hash}"
+          echo "DATE=${date_tag}" >> $GITHUB_ENV
+          echo "commit_hash=${commit_hash}" >> $GITHUB_OUTPUT
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
+          echo "Image tag: ${image_tag}"
+
+      - name: Login to Docker Hub (Intel)
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
+
+      - name: Build intel/sglang-dev
+        run: |
+          echo "Building intel/sglang-dev:${{ env.IMAGE_TAG }}"
+          docker build sglang-nightly-build \
+            -f sglang-nightly-build/docker/xpu.Dockerfile \
+            --build-arg SG_LANG_REPO=https://github.com/sgl-project/sglang.git \
+            --build-arg SG_LANG_BRANCH=main \
+            --no-cache --progress=plain \
+            -t "intel/sglang-dev:${{ env.IMAGE_TAG }}" \
+            -t "intel/sglang-dev:latest"
+
+      - name: Push intel/sglang-dev
+        run: |
+          docker push "intel/sglang-dev:${{ env.IMAGE_TAG }}"
+          docker push "intel/sglang-dev:latest"

.github/workflows/release-docker-npu-nightly.yml

@@ -0,0 +1,85 @@
+name: Release Docker Images Nightly (NPU)
+on:
+  pull_request:
+    branches:
+      - 'main'
+    paths:
+      - '.github/workflows/release-docker-npu-nightly.yml'
+      - 'docker/npu.Dockerfile'
+  workflow_dispatch:
+  schedule:
+    - cron: "0 16 * * *" # Execute at 0:00 a.m. Beijing Time every day
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04-arm
+    strategy:
+      matrix:
+        cann_version: ["9.0.0"]
+        device_type: ["910b", "a3"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Free up disk space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          tool-cache: true
+          docker-images: false
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            lmsysorg/sglang
+          # push with schedule event
+          # push with workflow_dispatch event
+          tags: |
+            type=ref,event=pr
+            type=ref,event=branch
+            type=schedule,pattern=main
+          flavor: |
+            latest=false
+            suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }},onlatest=true
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into docker hub
+        uses: docker/login-action@v3
+        if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Enable Docker multi-architecture build environment
+      # Emulate non-native architectures
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # Required for building and pushing multi-arch Docker images
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v6
+        with:
+          context: docker
+          file: docker/npu.Dockerfile
+          platforms: linux/arm64,linux/amd64
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
+          provenance: false
+          build-args: |
+            SGLANG_KERNEL_NPU_TAG=2026.6.2
+            CANN_VERSION=${{ matrix.cann_version }}
+            DEVICE_TYPE=${{ matrix.device_type }}

.github/workflows/release-docker-npu.yml

@@ -0,0 +1,93 @@
+name: Release Docker Images (NPU)
+on:
+  push:
+    tags:
+      - 'v[0-9]+.*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to build (without v prefix, e.g., 0.5.7)'
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04-arm
+    strategy:
+      matrix:
+        cann_version: ["9.0.0"]
+        device_type: ["910b", "a3"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Free up disk space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          tool-cache: true
+          docker-images: false
+
+        # push with tag
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            lmsysorg/sglang
+          tags: |
+            type=ref,event=pr
+          flavor: |
+            latest=false
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version from tag
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            # Extract version from tag (e.g., v0.5.7 -> 0.5.7)
+            VERSION="${GITHUB_REF_NAME#v}"
+          fi
+          # Validate version format
+          if [ -z "$VERSION" ]; then
+            echo "::error::Version is empty"
+            exit 1
+          fi
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::Invalid version format: $VERSION (expected: X.Y.Z)"
+            exit 1
+          fi
+          echo "version=v${VERSION}" >> $GITHUB_OUTPUT
+          echo "TAG=lmsysorg/sglang:v${VERSION}-cann${{ matrix.cann_version }}-${{ matrix.device_type }}" >> $GITHUB_OUTPUT
+      # Enable Docker multi-architecture build environment
+      # Emulate non-native architectures
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # Required for building and pushing multi-arch Docker images
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v6
+        with:
+          context: docker
+          file: docker/npu.Dockerfile
+          platforms: linux/arm64,linux/amd64
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags || steps.version.outputs.TAG }}
+          push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }}
+          provenance: false
+          build-args: |
+            SGLANG_KERNEL_NPU_TAG=2026.6.2
+            CANN_VERSION=${{ matrix.cann_version }}
+            DEVICE_TYPE=${{ matrix.device_type }}
+            SGLANG_TAG=${{ steps.version.outputs.version }}

.github/workflows/release-docker-runtime.yml

@@ -0,0 +1,55 @@
+name: Release Docker Runtime Images
+#
+# Builds and publishes runtime Docker images (production-optimized, ~50% smaller):
+#   - lmsysorg/sglang:v{version}-runtime, lmsysorg/sglang:latest-runtime
+#   - lmsysorg/sglang:v{version}-cu129-runtime, lmsysorg/sglang:latest-cu129-runtime
+#
+on:
+  push:
+    tags:
+      - "v[0-9]+.*"
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to build (without v prefix, e.g., 0.5.7)"
+        required: true
+      image_repo:
+        description: "Docker Hub repo to push to. Use lmsysorg/sglang-staging for testing."
+        required: false
+        default: "lmsysorg/sglang"
+
+jobs:
+  resolve-version:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - name: Get version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            VERSION="${GITHUB_REF_NAME#v}"
+          fi
+          if [ -z "$VERSION" ] || ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::Invalid version: $VERSION (expected: X.Y.Z)"
+            exit 1
+          fi
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+
+  build-and-publish:
+    needs: resolve-version
+    uses: ./.github/workflows/_docker-build-and-publish.yml
+    with:
+      docker_target: runtime
+      sgl_version: ${{ needs.resolve-version.outputs.version }}
+      use_environment: prod
+      image_repo: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
+      tag_config: |
+        [
+          {"cuda": "cu130", "tags": ["v${{ needs.resolve-version.outputs.version }}-runtime", "latest-runtime", "v${{ needs.resolve-version.outputs.version }}-cu130-runtime", "latest-cu130-runtime"]},
+          {"cuda": "cu129", "tags": ["v${{ needs.resolve-version.outputs.version }}-cu129-runtime", "latest-cu129-runtime"]}
+        ]
+    secrets: inherit

.github/workflows/release-docker-xeon.yml

@@ -0,0 +1,62 @@
+name: Release Docker Xeon Images
+on:
+  push:
+    tags:
+      - 'v[0-9]+.*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to build (without v prefix, e.g., 0.5.7)'
+        required: true
+
+jobs:
+  publish:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-24.04
+    environment: 'prod'
+    strategy:
+      matrix:
+        build_type: ['all']
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version from tag
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            # Extract version from tag (e.g., v0.5.7 -> 0.5.7)
+            VERSION="${GITHUB_REF_NAME#v}"
+          fi
+
+          # Validate version format
+          if [ -z "$VERSION" ]; then
+            echo "::error::Version is empty"
+            exit 1
+          fi
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::Invalid version format: $VERSION (expected: X.Y.Z)"
+            exit 1
+          fi
+
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+
+      - name: Build and Push
+        run: |
+          version=${{ steps.version.outputs.version }}
+          tag=v${version}-xeon
+
+          docker build . -f docker/xeon.Dockerfile \
+            --build-arg VER_SGLANG=v${version} \
+            -t lmsysorg/sglang:${tag} \
+            --no-cache
+          docker push lmsysorg/sglang:${tag}

.github/workflows/release-docker.yml

@@ -0,0 +1,55 @@
+name: Release Docker Images
+#
+# Builds and publishes framework Docker images (full development environment):
+#   - lmsysorg/sglang:v{version}, lmsysorg/sglang:latest (cuda 13)
+#   - lmsysorg/sglang:v{version}-cu129, lmsysorg/sglang:latest-cu129
+#
+on:
+  push:
+    tags:
+      - "v[0-9]+.*"
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to build (without v prefix, e.g., 0.5.7)"
+        required: true
+      image_repo:
+        description: "Docker Hub repo to push to. Use lmsysorg/sglang-staging for testing."
+        required: false
+        default: "lmsysorg/sglang"
+
+jobs:
+  resolve-version:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - name: Get version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            VERSION="${GITHUB_REF_NAME#v}"
+          fi
+          if [ -z "$VERSION" ] || ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::Invalid version: $VERSION (expected: X.Y.Z)"
+            exit 1
+          fi
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+
+  build-and-publish:
+    needs: resolve-version
+    uses: ./.github/workflows/_docker-build-and-publish.yml
+    with:
+      docker_target: framework_final
+      sgl_version: ${{ needs.resolve-version.outputs.version }}
+      use_environment: prod
+      image_repo: ${{ inputs.image_repo || 'lmsysorg/sglang' }}
+      tag_config: |
+        [
+          {"cuda": "cu130", "tags": ["v${{ needs.resolve-version.outputs.version }}", "latest", "v${{ needs.resolve-version.outputs.version }}-cu130", "latest-cu130"]},
+          {"cuda": "cu129", "tags": ["v${{ needs.resolve-version.outputs.version }}-cu129", "latest-cu129"]}
+        ]
+    secrets: inherit

.github/workflows/release-pypi-gateway.yml

@@ -0,0 +1,167 @@
+name: Release SGLang Model Gateway to PyPI
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - sgl-model-gateway/bindings/python/pyproject.toml
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos, windows]
+        target: [x86_64, aarch64]
+        manylinux: [auto]
+        include:
+          - os: ubuntu
+            platform: linux
+          - os: windows
+            ls: dir
+            target: x86_64
+            python-architecture: x64
+            interpreter: 3.9 3.10 3.11 3.12 3.13
+          - os: macos
+            target: aarch64
+            interpreter: 3.9 3.10 3.11 3.12 3.13
+          - os: ubuntu
+            platform: linux
+            target: aarch64
+          # musllinux
+          - os: ubuntu
+            platform: linux
+            target: x86_64
+            manylinux: musllinux_1_1
+          - os: ubuntu
+            platform: linux
+            target: aarch64
+            manylinux: musllinux_1_1
+        exclude:
+          - os: windows
+            target: aarch64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: sglang-repo
+
+      - name: Move sgl-model-gateway folder to root and delete sglang-repo
+        run: |
+          mv sglang-repo/sgl-model-gateway/* .
+          rm -rf sglang-repo
+          ls -alt
+        shell: bash
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          architecture: ${{ matrix.python-architecture || 'x64' }}
+
+      - name: Install twine
+        run: pip install -U twine
+
+      - name: Install protoc (macOS)
+        if: matrix.os == 'macos'
+        run: brew install protobuf
+
+      - name: Install protoc (Windows)
+        if: matrix.os == 'windows'
+        run: choco install protoc -y
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: bindings/python
+          target: ${{ matrix.target }}
+          manylinux: ${{ matrix.manylinux || 'auto' }}
+          args: --release --out dist --features vendored-openssl --interpreter ${{ matrix.interpreter || '3.9 3.10 3.11 3.12 3.13 3.14' }}
+          rust-toolchain: stable
+          docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
+          before-script-linux: |
+            # Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC)
+            if command -v yum &> /dev/null; then
+              yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make
+              # Install cross-compilation toolchain for aarch64 if needed
+              if [ "${{ matrix.target }}" = "aarch64" ]; then
+                yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true
+              fi
+            elif command -v apt-get &> /dev/null; then
+              apt-get update && apt-get install -y wget unzip gcc g++ perl make
+              # Install cross-compilation toolchain for aarch64 if needed
+              if [ "${{ matrix.target }}" = "aarch64" ]; then
+                apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true
+              fi
+            fi
+            (cd /tmp && \
+             wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
+             unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
+             rm protoc-32.0-linux-x86_64.zip)
+            protoc --version
+
+      - name: List built packages
+        run: ${{ matrix.ls || 'ls -lh' }} bindings/python/dist/
+
+      - name: Check packages
+        run: twine check --strict bindings/python/dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }}
+          path: bindings/python/dist/
+
+  build-sdist:
+    name: Build SDist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: sglang-repo
+
+      - name: Move sgl-model-gateway folder to root and delete sglang-repo
+        run: |
+          mv sglang-repo/sgl-model-gateway/* .
+          rm -rf sglang-repo
+          ls -alt
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Build SDist
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: bindings/python
+          command: sdist
+          args: --out dist
+          rust-toolchain: stable
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: bindings/python/dist/*.tar.gz
+
+  upload:
+    name: Upload to PyPI
+    if: github.repository == 'sgl-project/sglang'  # Ensure this job only runs for the sgl-project/sglang repository
+    needs: [build, build-sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: Upload to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN_ROUTER }}
+        run: |
+          pip install twine
+          twine upload dist/* --verbose

.github/workflows/release-pypi-nightly.yml

@@ -0,0 +1,175 @@
+name: Release PyPI Nightly Wheels
+
+on:
+  # Run daily at 2 AM UTC
+  schedule:
+    - cron: '0 2 * * *'
+  # Triggered by nightly Docker workflow to use same commit
+  repository_dispatch:
+    types: [nightly-release]
+  # Manual trigger for testing
+  workflow_dispatch:
+    inputs:
+      commit_sha:
+        description: 'Specific commit SHA to build (leave empty for latest)'
+        required: false
+        type: string
+
+concurrency:
+  group: release-pypi-nightly-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-nightly-wheel:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      nightly_version: ${{ steps.build.outputs.nightly_version }}
+      commit_hash: ${{ steps.build.outputs.commit_hash }}
+      build_date: ${{ steps.build.outputs.build_date }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Use commit from: 1) Docker workflow, 2) manual input, 3) latest main
+          ref: ${{ github.event.client_payload.commit_sha || inputs.commit_sha || github.sha }}
+          fetch-depth: 0  # Need full history for setuptools-scm
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install build dependencies
+        run: |
+          pip install build wheel setuptools setuptools-scm
+
+      # Needed by setuptools-rust to build the bundled native gRPC extension
+      # (rust/sglang-grpc) when `python -m build` builds the sglang wheel.
+      - name: Install protoc
+        run: sudo bash scripts/ci/utils/install_protoc.sh
+
+      - name: Build wheel
+        id: build
+        run: |
+          cd python
+          cp ../README.md ../LICENSE .
+
+          TAG=$(python3 ../python/tools/get_version_tag.py)
+          HASH="g$(git rev-parse --short HEAD)"
+          BUILD_DATE=$(date -u +%Y%m%d)
+
+          # Increment patch version for nightlies (e.g., v0.5.9 -> 0.5.10)
+          # Must always increment so nightly > latest tag per PEP 440 ordering:
+          #   X.Y.Z.devN < X.Y.Z.rcN < X.Y.Z < X.Y.(Z+1).devN
+          VERSION=${TAG#v}  # Remove 'v' prefix
+          MAJOR=$(echo "$VERSION" | cut -d. -f1)
+          MINOR=$(echo "$VERSION" | cut -d. -f2)
+          PATCH_RAW=$(echo "$VERSION" | cut -d. -f3)
+          # Strip pre-release suffixes (rc0, post1, etc.) to get numeric patch
+          PATCH=$(echo "$PATCH_RAW" | sed 's/[^0-9].*//')
+          NEXT_PATCH=$((PATCH + 1))
+          NEXT_VERSION="${MAJOR}.${MINOR}.${NEXT_PATCH}"
+
+          # Use date-based dev number for correct chronological sorting
+          # e.g., 0.5.9.dev20260215+g4cf4f0859 > 0.5.9.dev20260214+g45a4697d4
+          FORCE_VERSION="${NEXT_VERSION}.dev${BUILD_DATE}+${HASH}"
+          echo "Forcing nightly version to: $FORCE_VERSION"
+          export SETUPTOOLS_SCM_PRETEND_VERSION="$FORCE_VERSION"
+
+          # Build wheel
+          python3 -m build --wheel
+
+          # Extract version from built wheel filename
+          WHEEL_FILE=$(ls dist/*.whl)
+          NIGHTLY_VERSION=$(echo "$WHEEL_FILE" | sed 's/.*sglang-\(.*\)-py3.*/\1/')
+
+          # Get commit info
+          COMMIT_HASH=$(git rev-parse --short HEAD)
+          BUILD_DATE=$(date -u +%Y-%m-%d)
+
+          echo "Built wheel: $WHEEL_FILE"
+          echo "Nightly version: ${NIGHTLY_VERSION}"
+          echo "Commit: ${COMMIT_HASH}"
+          echo "Build date: ${BUILD_DATE}"
+
+          echo "nightly_version=${NIGHTLY_VERSION}" >> $GITHUB_OUTPUT
+          echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT
+          echo "build_date=${BUILD_DATE}" >> $GITHUB_OUTPUT
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-wheel
+          path: python/dist/*.whl
+          retention-days: 7
+
+  release-nightly:
+    needs: build-nightly-wheel
+    runs-on: ubuntu-latest
+    environment: 'prod'
+    strategy:
+      fail-fast: false
+      # The wheel is CUDA-agnostic and built once — we just register the same
+      # artifact under cu129/sglang/ and cu130/sglang/ wheel indexes so users
+      # can install via either --extra-index-url. Serialize because both matrix
+      # runs clone and push to the same sgl-whl branch.
+      max-parallel: 1
+      matrix:
+        cuda_version: ['129', '130']
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download wheel artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: nightly-wheel
+          path: dist/
+
+      - name: List downloaded wheels
+        run: |
+          echo "Downloaded wheel:"
+          ls -lh dist/
+
+      - name: Create GitHub Release for nightly wheel
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: nightly-${{ needs.build-nightly-wheel.outputs.build_date }}-${{ needs.build-nightly-wheel.outputs.commit_hash }}
+          name: Nightly Build ${{ needs.build-nightly-wheel.outputs.build_date }} (${{ needs.build-nightly-wheel.outputs.commit_hash }})
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          prerelease: true
+          body: |
+            Nightly build from commit ${{ github.sha }}
+            Build date: ${{ needs.build-nightly-wheel.outputs.build_date }}
+            Version: ${{ needs.build-nightly-wheel.outputs.nightly_version }}
+          files: |
+            dist/*.whl
+
+      - name: Clone wheel index repository
+        run: |
+          git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Update wheel index
+        run: |
+          python3 scripts/update_nightly_whl_index.py \
+            --commit-hash ${{ needs.build-nightly-wheel.outputs.commit_hash }} \
+            --nightly-version ${{ needs.build-nightly-wheel.outputs.nightly_version }} \
+            --cuda-version ${{ matrix.cuda_version }} \
+            --build-date ${{ needs.build-nightly-wheel.outputs.build_date }}
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git add -A
+          git diff --staged --quiet || git commit -m "Update cu${{ matrix.cuda_version }} nightly wheel index for commit ${{ needs.build-nightly-wheel.outputs.commit_hash }}"
+          git push

.github/workflows/release-pypi-pr.yml

@@ -0,0 +1,178 @@
+name: Release PyPI PR Wheels
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'PR number to build wheel for (works with both internal and fork PRs)'
+        required: true
+        type: string
+
+concurrency:
+  group: build-pr-wheel-${{ github.event.inputs.pr_number }}
+  cancel-in-progress: true
+
+jobs:
+  build-pr-wheel:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      wheel_version: ${{ steps.gen_version.outputs.wheel_version }}
+      commit_hash: ${{ steps.gen_version.outputs.commit_hash }}
+      build_date: ${{ steps.gen_version.outputs.build_date }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: refs/pull/${{ inputs.pr_number }}/head
+          fetch-depth: 0  # Need full history for version generation
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Generate PR wheel version
+        id: gen_version
+        run: |
+          LATEST_TAG=$(python3 python/tools/get_version_tag.py)
+          BASE_VERSION=${LATEST_TAG#v}
+          echo "Latest release tag: ${LATEST_TAG}"
+
+          # Get commit info
+          COMMIT_HASH=$(git rev-parse --short HEAD)
+          COMMIT_COUNT=$(git rev-list --count HEAD)
+
+          # Get current date in YYYY-MM-DD format
+          BUILD_DATE=$(date -u +%Y-%m-%d)
+
+          # Always use pr-{number} format for suffix
+          SUFFIX="pr-${{ inputs.pr_number }}"
+
+          # Generate PR wheel version following PEP 440
+          # Format: {base_version}.dev{commit_count}+pr-{number}.g{commit_hash}
+          WHEEL_VERSION="${BASE_VERSION}.dev${COMMIT_COUNT}+${SUFFIX}.g${COMMIT_HASH}"
+
+          echo "Base version: ${BASE_VERSION}"
+          echo "PR wheel version: ${WHEEL_VERSION}"
+          echo "Commit: ${COMMIT_HASH}"
+          echo "Build date: ${BUILD_DATE}"
+
+          echo "wheel_version=${WHEEL_VERSION}" >> $GITHUB_OUTPUT
+          echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT
+          echo "base_version=${BASE_VERSION}" >> $GITHUB_OUTPUT
+          echo "build_date=${BUILD_DATE}" >> $GITHUB_OUTPUT
+
+      - name: Update pyproject.toml with PR wheel version
+        run: |
+          cd python
+          WHEEL_VERSION="${{ steps.gen_version.outputs.wheel_version }}"
+
+          # Update pyproject.toml to use static version instead of dynamic
+          # Remove 'version' from dynamic list and add static version
+          sed -i 's/dynamic = \["version"\]/dynamic = []/' pyproject.toml
+          sed -i "/^name = \"sglang\"/a version = \"${WHEEL_VERSION}\"" pyproject.toml
+
+          # Verify update
+          echo "Updated version in pyproject.toml:"
+          grep "^version" pyproject.toml
+          grep "^dynamic" pyproject.toml
+
+      - name: Install build dependencies
+        run: |
+          cd python
+          pip install build wheel setuptools
+
+      - name: Build wheel
+        run: |
+          cd python
+          cp ../README.md ../LICENSE .
+          python3 -m build --wheel
+
+          # List built wheels
+          echo "Built wheel:"
+          ls -lh dist/
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-wheel-${{ inputs.pr_number }}
+          path: python/dist/*.whl
+          retention-days: 30
+
+  release-pr-wheel:
+    needs: build-pr-wheel
+    runs-on: ubuntu-latest
+    environment: 'prod'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download wheel artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: pr-wheel-${{ inputs.pr_number }}
+          path: dist/
+
+      - name: List downloaded wheels
+        run: |
+          echo "Downloaded wheel:"
+          ls -lh dist/
+
+      - name: Create GitHub Release for PR wheel
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: pr-${{ inputs.pr_number }}-${{ needs.build-pr-wheel.outputs.build_date }}-${{ needs.build-pr-wheel.outputs.commit_hash }}
+          name: "PR #${{ inputs.pr_number }} Build (${{ needs.build-pr-wheel.outputs.commit_hash }})"
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          prerelease: true
+          body: |
+            PR wheel build from PR #${{ inputs.pr_number }}
+            Commit: ${{ github.sha }}
+            Build date: ${{ needs.build-pr-wheel.outputs.build_date }}
+            Version: ${{ needs.build-pr-wheel.outputs.wheel_version }}
+
+            **Installation via index (pip):**
+            ```bash
+            pip install sglang==${{ needs.build-pr-wheel.outputs.wheel_version }} --index-url https://sgl-project.github.io/whl/pr/
+            ```
+
+            **Installation via index (uv):**
+            ```bash
+            uv pip install sglang==${{ needs.build-pr-wheel.outputs.wheel_version }} --index-url https://sgl-project.github.io/whl/pr/ --extra-index-url https://pypi.org/simple --index-strategy unsafe-best-match
+            ```
+
+            **Direct installation:**
+            ```bash
+            pip install https://github.com/sgl-project/whl/releases/download/pr-${{ inputs.pr_number }}-${{ needs.build-pr-wheel.outputs.build_date }}-${{ needs.build-pr-wheel.outputs.commit_hash }}/sglang-${{ needs.build-pr-wheel.outputs.wheel_version }}-py3-none-any.whl
+            ```
+          files: |
+            dist/*.whl
+
+      - name: Clone wheel index repository
+        run: |
+          git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Update wheel index
+        run: |
+          python3 scripts/update_pr_whl_index.py \
+            --pr-number ${{ inputs.pr_number }} \
+            --commit-hash ${{ needs.build-pr-wheel.outputs.commit_hash }} \
+            --wheel-version ${{ needs.build-pr-wheel.outputs.wheel_version }} \
+            --build-date ${{ needs.build-pr-wheel.outputs.build_date }}
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git add -A
+          git diff --staged --quiet || git commit -m "Update PR wheel index for PR #${{ inputs.pr_number }} (commit ${{ needs.build-pr-wheel.outputs.commit_hash }})"
+          git push

.github/workflows/release-pypi.yml

@@ -0,0 +1,116 @@
+name: Release PyPI
+on:
+  push:
+    tags:
+      - 'v[0-9]+.*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version to publish (e.g. v0.5.11). Overrides setuptools-scm.'
+        required: true
+        type: string
+
+jobs:
+  build:
+    if: github.repository == 'sgl-project/sglang'
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        arch: [x86_64, aarch64]
+        include:
+          - arch: x86_64
+            runner: x64-docker-build-node
+          - arch: aarch64
+            runner: arm-docker-build-node
+    runs-on: ${{ matrix.runner }}
+    steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts behind that actions/checkout cannot remove,
+      # causing EACCES on rmdir. Wipe them via a throwaway root container.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Required for setuptools-scm to determine version from tags
+          submodules: "recursive"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # Needed by setuptools-rust to build the bundled native gRPC extension
+      # (rust/sglang-grpc) when `python -m build` builds the sglang wheel.
+      - name: Install protoc
+        run: sudo bash scripts/ci/utils/install_protoc.sh
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build wheel
+        env:
+          # When triggered manually, pin the wheel version to the supplied
+          # input (strip leading `v`) so setuptools-scm doesn't derive a
+          # `.devN+gHASH` version from the branch HEAD.
+          RELEASE_VERSION: ${{ inputs.version }}
+        run: |
+          cd python
+          cp ../README.md ../LICENSE .
+          pip install build wheel setuptools setuptools-scm setuptools-rust
+          if [ -n "$RELEASE_VERSION" ]; then
+            export SETUPTOOLS_SCM_PRETEND_VERSION="${RELEASE_VERSION#v}"
+            echo "Pinning wheel version to $SETUPTOOLS_SCM_PRETEND_VERSION"
+          fi
+          python3 -m build --wheel
+
+      # PyPI rejects plain `linux_x86_64` / `linux_aarch64` platform tags;
+      # auditwheel rewrites the wheel's platform tag to a `manylinux_*` tag
+      # and bundles any external native deps. The runner's glibc determines
+      # the lowest acceptable manylinux policy.
+      - name: Repair wheel for manylinux
+        run: |
+          cd python
+          pip install auditwheel patchelf
+          mkdir -p dist-repaired
+          python3 -m auditwheel repair dist/*.whl -w dist-repaired/
+          rm dist/*.whl
+          mv dist-repaired/*.whl dist/
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-py${{ matrix.python-version }}-${{ matrix.arch }}
+          path: python/dist/*.whl
+
+  publish:
+    needs: build
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    environment: "prod"
+    steps:
+      - name: Download all wheels
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          merge-multiple: true
+          pattern: wheel-*
+
+      - name: List wheels
+        run: ls -lh dist/
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Upload to pypi
+        run: |
+          pip install twine
+          # --skip-existing makes partial reruns safe (e.g. one arch failed,
+          # rerun only the failed shard without re-uploading already-published wheels).
+          python3 -m twine upload --skip-existing dist/*.whl -u __token__ -p ${{ secrets.PYPI_TOKEN }}

.github/workflows/release-tag.yml

@@ -0,0 +1,68 @@
+name: Release Tag
+# Creates a git tag to trigger release workflows (PyPI, Docker)
+# Use this after testing on a release branch is complete
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to tag (without v prefix, e.g., 0.5.7)'
+        required: true
+        type: string
+      ref:
+        description: 'Branch or commit to tag (e.g., release/v0.5.7, main, or commit SHA)'
+        required: false
+        default: 'main'
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  create-tag:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    environment: 'prod'
+    steps:
+      - name: Validate version format
+        run: |
+          VERSION="${{ github.event.inputs.version }}"
+          if [ -z "$VERSION" ]; then
+            echo "::error::Version is required"
+            exit 1
+          fi
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::Invalid version format: $VERSION (expected: X.Y.Z or X.Y.Z.postN)"
+            exit 1
+          fi
+          echo "Version validated: v$VERSION"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.ref }}
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check if tag already exists
+        run: |
+          TAG="v${{ github.event.inputs.version }}"
+          if git rev-parse "$TAG" >/dev/null 2>&1; then
+            echo "::error::Tag $TAG already exists"
+            exit 1
+          fi
+          echo "Tag $TAG does not exist, proceeding..."
+
+      - name: Create and push tag
+        run: |
+          TAG="v${{ github.event.inputs.version }}"
+          REF="${{ github.event.inputs.ref }}"
+
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+
+          echo "Creating tag $TAG on ref $REF (commit: $(git rev-parse HEAD))"
+          git tag -a "$TAG" -m "Release $TAG"
+          git push origin "$TAG"
+
+          echo "::notice::Successfully created and pushed tag $TAG"
+          echo "This will trigger the release workflows (PyPI, Docker)"

.github/workflows/release-whl-deepgemm.yml

@@ -0,0 +1,330 @@
+name: Release sgl-deep-gemm
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Wheel version (e.g. 0.1.0, 0.1.1rc0)"
+        type: string
+        required: true
+      target:
+        type: choice
+        description: "Build target (default: all)"
+        required: false
+        default: 'all'
+        options:
+          - 'all'
+          - 'cu129'
+          - 'cu130'
+      branch:
+        description: "DeepGEMM branch to build from (default: dev)"
+        type: string
+        required: false
+        default: 'dev'
+
+concurrency:
+  group: release-sgl-deepgemm-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Must match TORCH_VER in docker/sgl-deep-gemm.Dockerfile — the wheel's
+  # pre-compiled _C.so links against this torch ABI.
+  TORCH_VER: "2.11.0"
+
+jobs:
+  build-cu129-matrix:
+    if: |
+      github.repository == 'sgl-project/sglang' &&
+      (github.event.inputs.target == 'all' || github.event.inputs.target == 'cu129')
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        cuda-version: ["12.9"]
+        arch: [x86_64, aarch64]
+        include:
+          - arch: x86_64
+            runner: x64-kernel-build-node
+          - arch: aarch64
+            runner: arm-kernel-build-node
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - uses: actions/checkout@v4
+
+      - name: Checkout DeepGEMM
+        uses: actions/checkout@v4
+        with:
+          repository: sgl-project/DeepGEMM
+          ref: ${{ inputs.branch || 'dev' }}
+          path: DeepGEMM
+          submodules: recursive
+
+      - name: Set wheel version
+        run: |
+          echo -n "${{ inputs.version }}" > DeepGEMM/sgl_deep_gemm/VERSION
+          cat DeepGEMM/sgl_deep_gemm/VERSION
+
+      - name: Build wheel
+        run: |
+          chmod +x ./scripts/build_sgl_deep_gemm.sh ./scripts/rename_sgl_deep_gemm_whl.sh
+          ./scripts/build_sgl_deep_gemm.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" "${{ github.workspace }}/DeepGEMM" "${{ matrix.arch }}"
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: deepgemm-wheel-cuda${{ matrix.cuda-version }}-${{ matrix.arch }}
+          path: DeepGEMM/dist/*.whl
+
+  test-cu129:
+    needs: build-cu129-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch_label: sm90
+            runner: 8-gpu-h200
+            wheel_arch: x86_64
+          - arch_label: sm100
+            runner: 8-gpu-b200
+            wheel_arch: x86_64
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Checkout DeepGEMM
+        uses: actions/checkout@v4
+        with:
+          repository: sgl-project/DeepGEMM
+          ref: ${{ inputs.branch || 'dev' }}
+          path: DeepGEMM
+          submodules: recursive
+
+      - name: Download wheel
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          merge-multiple: true
+          pattern: deepgemm-wheel-cuda12.9-${{ matrix.wheel_arch }}
+
+      - name: Install wheel and deps
+        env:
+          CU_TAG: cu129
+        run: |
+          python3 -m pip install --upgrade pip
+          # deep_gemm imports torch before loading _C.so, so torch must preload the
+          # libcudart the cu129 wheel links. Pin the CUDA-matched torch (see sgl-deep-gemm.Dockerfile).
+          python3 -m pip install "torch==${TORCH_VER}" --index-url "https://download.pytorch.org/whl/${CU_TAG}" --force-reinstall
+          python3 -m pip install numpy
+          python3 -m pip install dist/*.whl
+          python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
+
+      - name: Run DeepGEMM test suite
+        run: |
+          chmod +x "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"
+          "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"
+
+  release-cu129:
+    needs: [build-cu129-matrix, test-cu129]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          merge-multiple: true
+          pattern: deepgemm-wheel-cuda12.9-*
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ inputs.version }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_deepgemm_whl_index.py --cuda 129
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}"
+          git push
+
+  build-cu130-matrix:
+    if: |
+      github.repository == 'sgl-project/sglang' &&
+      (github.event.inputs.target == 'all' || github.event.inputs.target == 'cu130')
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        cuda-version: ["13.0"]
+        arch: [x86_64, aarch64]
+        include:
+          - arch: x86_64
+            runner: x64-kernel-build-node
+          - arch: aarch64
+            runner: arm-kernel-build-node
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - uses: actions/checkout@v4
+
+      - name: Checkout DeepGEMM
+        uses: actions/checkout@v4
+        with:
+          repository: sgl-project/DeepGEMM
+          ref: ${{ inputs.branch || 'dev' }}
+          path: DeepGEMM
+          submodules: recursive
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set wheel version
+        run: |
+          echo -n "${{ inputs.version }}" > DeepGEMM/sgl_deep_gemm/VERSION
+          cat DeepGEMM/sgl_deep_gemm/VERSION
+
+      - name: Build wheel
+        run: |
+          chmod +x ./scripts/build_sgl_deep_gemm.sh ./scripts/rename_sgl_deep_gemm_whl.sh
+          ./scripts/build_sgl_deep_gemm.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" "${{ github.workspace }}/DeepGEMM" "${{ matrix.arch }}"
+
+      # PyPI upload moved to release-cu130 so it is gated on test-cu130.
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: deepgemm-wheel-cuda${{ matrix.cuda-version }}-${{ matrix.arch }}
+          path: DeepGEMM/dist/*.whl
+
+      - name: Upload PyPI artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: deepgemm-pypi-cuda${{ matrix.cuda-version }}-${{ matrix.arch }}
+          path: DeepGEMM/dist-pypi/*.whl
+
+  test-cu130:
+    needs: build-cu130-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch_label: sm90
+            runner: 8-gpu-h200
+            wheel_arch: x86_64
+          - arch_label: sm100
+            runner: 8-gpu-b200
+            wheel_arch: x86_64
+          - arch_label: sm100-aarch64
+            runner: 4-gpu-gb300
+            wheel_arch: aarch64
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Checkout DeepGEMM
+        uses: actions/checkout@v4
+        with:
+          repository: sgl-project/DeepGEMM
+          ref: ${{ inputs.branch || 'dev' }}
+          path: DeepGEMM
+          submodules: recursive
+
+      - name: Download wheel
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          merge-multiple: true
+          pattern: deepgemm-wheel-cuda13.0-${{ matrix.wheel_arch }}
+
+      - name: Install wheel and deps
+        env:
+          CU_TAG: cu130
+        run: |
+          python3 -m pip install --upgrade pip
+          # deep_gemm imports torch before loading _C.so, so torch must preload the
+          # libcudart the cu130 wheel links. Pin the CUDA-matched torch (see sgl-deep-gemm.Dockerfile).
+          python3 -m pip install "torch==${TORCH_VER}" --index-url "https://download.pytorch.org/whl/${CU_TAG}" --force-reinstall
+          python3 -m pip install numpy
+          python3 -m pip install dist/*.whl
+          python3 -c "import deep_gemm; print('deep_gemm:', deep_gemm.__file__)"
+
+      - name: Run DeepGEMM test suite
+        run: |
+          chmod +x "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"
+          "${{ github.workspace }}/DeepGEMM/sgl_deep_gemm/run_tests.sh"
+
+  release-cu130:
+    needs: [build-cu130-matrix, test-cu130]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          merge-multiple: true
+          pattern: deepgemm-wheel-cuda13.0-*
+
+      - name: Download PyPI artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist-pypi/
+          merge-multiple: true
+          pattern: deepgemm-pypi-cuda13.0-*
+
+      - name: Upload to PyPI
+        run: |
+          pip install --upgrade twine "packaging>=24.2"
+          python3 -m twine upload --skip-existing dist-pypi/* -u __token__ -p ${{ secrets.SGL_DEEP_GEMM_PYPI_TOKEN }}
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ inputs.version }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_deepgemm_whl_index.py --cuda 130
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update sgl-deep-gemm whl index for v${{ inputs.version }}"
+          git push

.github/workflows/release-whl-kernel.yml

@@ -0,0 +1,515 @@
+name: Release SGLang Kernels
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - sgl-kernel/python/sgl_kernel/version.py
+  workflow_dispatch:
+    inputs:
+      target:
+        type: choice
+        description: 'Build target'
+        required: false
+        default: 'all'
+        options:
+          - 'all'
+          - 'cu129'
+          - 'cu130'
+          - 'rocm700'
+          - 'rocm720'
+          - 'musa43'
+      tag_name:
+        description: "Version number, must be in the form of vX.Y.Z (e.g. v0.4.0)"
+        type: string
+        required: false
+      pr_number:
+        description: "PR number to build from (e.g. 12345)"
+        type: string
+        required: false
+
+concurrency:
+  group: release-sglang-kernels-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # cu130 is the PyPI-released variant; cu129 wheels are published only to the
+  # sgl-project/whl index (consumed via `pip install ...+cu129` for the legacy
+  # cuda 12.9 path), not to PyPI.
+  build-cu129-matrix:
+    if: |
+      github.repository == 'sgl-project/sglang' &&
+      (github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'cu129')
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        cuda-version: ["12.9"]
+        arch: [x86_64, aarch64]
+        include:
+          - arch: x86_64
+            runner: x64-kernel-build-node
+          - arch: aarch64
+            runner: arm-kernel-build-node
+    runs-on: ${{ matrix.runner }}
+    steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build wheels
+        run: |
+          cd sgl-kernel
+          chmod +x ./build.sh
+          ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" ${{ matrix.arch == 'aarch64' && 'aarch64' || '' }}
+        env:
+          BUILD_JOBS: 64
+          NVCC_THREADS: 8
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}${{ matrix.arch == 'aarch64' && '-aarch64' || '' }}
+          path: sgl-kernel/dist/*
+
+  release-cu129:
+    needs: build-cu129-matrix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-*
+
+      - name: Set tag name
+        id: set_tag_name
+        run: |
+          if [ -z "${{ inputs.tag_name }}" ]; then
+            TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)"
+            echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
+          else
+            echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.set_tag_name.outputs.tag_name }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            sgl-kernel/dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_kernel_whl_index.py --cuda 129
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update whl index"
+          git push
+
+  build-cu130-matrix:
+    if: |
+      github.repository == 'sgl-project/sglang' &&
+      (github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'cu130')
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        cuda-version: ["13.0"]
+        arch: [x86_64, aarch64]
+        include:
+          - arch: x86_64
+            runner: x64-kernel-build-node
+          - arch: aarch64
+            runner: arm-kernel-build-node
+    runs-on: ${{ matrix.runner }}
+    steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build wheels
+        run: |
+          cd sgl-kernel
+          chmod +x ./build.sh
+          ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" ${{ matrix.arch == 'aarch64' && 'aarch64' || '' }}
+        env:
+          BUILD_JOBS: 64
+          NVCC_THREADS: 8
+
+      - name: Strip +cu130 local version for PyPI upload
+        working-directory: sgl-kernel
+        run: |
+          set -eux
+          pip install wheel
+          mkdir -p dist-pypi
+          for w in dist/*.whl; do
+            tmp=$(mktemp -d)
+            python3 -m wheel unpack "$w" --dest "$tmp"
+            unpacked=$(find "$tmp" -mindepth 1 -maxdepth 1 -type d | head -1)
+            info=$(find "$unpacked" -maxdepth 1 -type d -name "*.dist-info" | head -1)
+            meta="$info/METADATA"
+            orig=$(grep '^Version:' "$meta" | head -1 | sed 's/^Version:[[:space:]]*//')
+            new=$(echo "$orig" | sed 's/+cu[0-9]\+$//')
+            if [ "$orig" != "$new" ]; then
+              sed -i "s/^Version:.*/Version: ${new}/" "$meta"
+              old_base=$(basename "$info")
+              new_base="${old_base/${orig}/${new}}"
+              mv "$info" "$(dirname "$info")/${new_base}"
+            fi
+            python3 -m wheel pack "$unpacked" --dest-dir dist-pypi
+            rm -rf "$tmp"
+          done
+          ls -lh dist-pypi/
+
+      - name: Upload to PyPI
+        working-directory: sgl-kernel
+        run: |
+          pip install twine
+          python3 -m twine upload --skip-existing dist-pypi/* -u __token__ -p ${{ secrets.PYPI_TOKEN_SGLANG_KERNEL }}
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}${{ matrix.arch == 'aarch64' && '-aarch64' || '' }}
+          path: sgl-kernel/dist/*
+
+  release-cu130:
+    needs: build-cu130-matrix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-*
+
+      - name: Set tag name
+        id: set_tag_name
+        run: |
+          if [ -z "${{ inputs.tag_name }}" ]; then
+            TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)"
+            echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
+          else
+            echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.set_tag_name.outputs.tag_name }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            sgl-kernel/dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_kernel_whl_index.py --cuda 130
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update whl index"
+          git push
+
+  build-rocm-matrix:
+    if: |
+      github.repository == 'sgl-project/sglang' &&
+      (github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'rocm700' || github.event.inputs.target == 'rocm720')
+    runs-on: amd-docker-scale
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        rocm-version: ["700", "720"]
+    steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build wheels
+        run: |
+          cp 3rdparty/amd/wheel/sgl-kernel/* sgl-kernel/
+          cd sgl-kernel
+          chmod +x ./build_rocm.sh
+          ./build_rocm.sh "${{ matrix.rocm-version }}"
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-python${{ matrix.python-version }}-rocm${{ matrix.rocm-version }}
+          path: sgl-kernel/dist/*
+
+  release-rocm700:
+    needs: build-rocm-matrix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-*-rocm700
+
+      - name: Set tag name
+        id: set_tag_name
+        run: |
+          if [ -z "${{ inputs.tag_name }}" ]; then
+            TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)"
+            echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
+          else
+            echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.set_tag_name.outputs.tag_name }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            sgl-kernel/dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_kernel_whl_index.py --rocm 700
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update whl index"
+          git push
+
+  release-rocm720:
+    needs: build-rocm-matrix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || '' }}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-*-rocm720
+
+      - name: Set tag name
+        id: set_tag_name
+        run: |
+          if [ -z "${{ inputs.tag_name }}" ]; then
+            TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)"
+            echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
+          else
+            echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.set_tag_name.outputs.tag_name }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            sgl-kernel/dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_kernel_whl_index.py --rocm 720
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update whl index"
+          git push
+
+  build-musa43:
+    if: |
+      github.repository == 'sgl-project/sglang' &&
+      (github.event_name == 'push' || github.event.inputs.target == 'all' || github.event.inputs.target == 'musa43')
+    runs-on: kernel-build-node-musa
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+        musa-version: ["43"]
+    steps:
+      # Self-hosted build nodes retain the workspace across jobs. Prior builds
+      # leave root-owned artifacts under sgl-kernel/build/ that actions/checkout
+      # cannot remove, causing EACCES on rmdir. Wipe them via a throwaway root
+      # container before checkout recreates the workspace.
+      - name: Clean workspace (remove root-owned files from prior runs)
+        run: |
+          docker run --rm -v "${{ github.workspace }}:/workspace" alpine:3 \
+            sh -c 'rm -rf /workspace/..?* /workspace/.[!.]* /workspace/*' || true
+
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+
+      - name: Clean stale torchada extension locks
+        run: |
+          if ps -eo comm=,args= | grep -E '(^|[[:space:]])(mcc|ninja)([[:space:]]|$)|torchada_cpp_ops' | grep -v grep; then
+            echo "::error::Active torchada extension build detected; refusing to remove lock files"
+            exit 1
+          fi
+          find "${HOME}/.cache/torch_extensions" \
+            -path '*/torchada_cpp_ops/lock' \
+            -type f \
+            -print \
+            -delete 2>/dev/null || true
+
+      - name: Build wheels
+        run: |
+          cd sgl-kernel
+          mv pyproject_musa.toml pyproject.toml
+          python setup_musa.py sdist bdist_wheel
+
+      - name: Rename MUSA wheels
+        run: |
+          bash scripts/ci/musa/rename_wheels_musa.sh ${{ matrix.musa-version }} sgl-kernel/dist
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-python${{ matrix.python-version }}-musa${{ matrix.musa-version }}
+          path: sgl-kernel/dist/*
+
+  release-musa43:
+    needs: build-musa43
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: sgl-kernel/dist/
+          merge-multiple: true
+          pattern: wheel-*
+
+      - name: Set tag name
+        id: set_tag_name
+        run: |
+          if [ -z "${{ inputs.tag_name }}" ]; then
+            TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)"
+            echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
+          else
+            echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.set_tag_name.outputs.tag_name }}
+          repository: sgl-project/whl
+          token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+          files: |
+            sgl-kernel/dist/*
+
+      - name: Clone wheel index
+        run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
+        env:
+          WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
+
+      - name: Update wheel index
+        run: python3 scripts/update_kernel_whl_index.py --musa 43
+
+      - name: Push wheel index
+        run: |
+          cd sgl-whl
+          git config --local user.name "sglang-bot"
+          git config --local user.email "sglangbot@gmail.com"
+          git add -A
+          git commit -m "update whl index"
+          git push

.github/workflows/rerun-test.yml

@@ -0,0 +1,349 @@
+name: Rerun Test
+run-name: ${{ inputs.pr_head_sha && format('[rerun-test] {0} {1}', inputs.test_command, inputs.pr_head_sha) || format('[rerun-test] {0}', inputs.test_command) }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "Dispatch mode: cuda | multimodal_gen | cpu"
+        required: true
+        type: choice
+        options:
+          - cuda
+          - multimodal_gen
+          - cpu
+      test_command:
+        description: "Test command(s) to run, one per line (e.g. 'registered/core/test_srt_endpoint.py TestSRTEndpoint.test_simple_decode')"
+        required: true
+        type: string
+      runs_on:
+        description: "GHA runner label (cuda/multimodal_gen only; ignored for cpu)"
+        required: false
+        type: string
+        default: ""
+      install_script:
+        description: "Install script path (cuda only). E.g. scripts/ci/cuda/ci_install_dependency.sh"
+        required: false
+        type: string
+        default: ""
+      install_timeout:
+        description: "Install-step timeout minutes (cuda only)"
+        required: false
+        type: string
+        default: "20"
+      grace_blackwell:
+        description: "Set GRACE_BLACKWELL for the install step (cuda only)"
+        required: false
+        type: string
+        default: "0"
+      rdma_devices:
+        description: "SGLANG_CI_RDMA_ALL_DEVICES csv (cuda only; empty = unset)"
+        required: false
+        type: string
+        default: ""
+      pr_head_sha:
+        description: "PR head SHA to checkout (for /rerun-test on fork PRs)"
+        required: false
+        type: string
+        default: ""
+      reply_comment_id:
+        description: "Reply comment ID to write back result to"
+        required: false
+        type: string
+        default: ""
+      reply_marker:
+        description: "Per-batch marker for locating the line in reply comment"
+        required: false
+        type: string
+        default: ""
+
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
+  SGLANG_CUDA_COREDUMP: "1"
+  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
+  # TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.
+  FORCE_REBUILD_DEEPEP: '1'
+
+permissions:
+  actions: write
+  contents: read
+  issues: read
+
+jobs:
+  rerun-test-cuda:
+    if: inputs.mode == 'cuda'
+    runs-on: ${{ inputs.runs_on }}
+    timeout-minutes: 120
+    permissions:
+      contents: read
+      issues: write
+    env:
+      RUNNER_LABELS: ${{ inputs.runs_on }}
+      SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.rdma_devices }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_head_sha || github.sha }}
+
+      - name: Mark runner picked up
+        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
+            source /etc/profile.d/sglang-ci.sh
+          fi
+          python3 scripts/ci/utils/update_rerun_test_status.py \
+            --comment-id "${{ inputs.reply_comment_id }}" \
+            --marker "${{ inputs.reply_marker }}" \
+            --status running \
+            --repo "${{ github.repository }}"
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies
+        timeout-minutes: ${{ fromJson(inputs.install_timeout) }}
+        env:
+          GRACE_BLACKWELL: ${{ inputs.grace_blackwell }}
+        run: |
+          if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
+            source /etc/profile.d/sglang-ci.sh
+          fi
+          # Fail fast on an empty install_script. `bash <empty>` is a silent
+          # no-op that "succeeds" without installing sglang, so the test step
+          # then dies with `ModuleNotFoundError: No module named 'sglang'`.
+          # /rerun-test resolves install_script from runner_configs.yml; a manual
+          # workflow_dispatch must pass it explicitly (it can't be derived from
+          # runs_on, which is shared across configs, e.g. default vs deepep).
+          if [ -z "${{ inputs.install_script }}" ]; then
+            echo "::error::install_script is required for cuda mode (empty would silently skip installing sglang). Pass e.g. scripts/ci/cuda/ci_install_dependency.sh"
+            exit 1
+          fi
+          bash ${{ inputs.install_script }}
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
+            source /etc/profile.d/sglang-ci.sh
+          fi
+          cmds=()
+          while IFS= read -r cmd; do
+            [ -z "$cmd" ] && continue
+            cmds+=("$cmd")
+          done <<< "${{ inputs.test_command }}"
+          total=${#cmds[@]}
+          suite_start=$SECONDS
+          for idx in "${!cmds[@]}"; do
+            i=$((idx + 1))
+            cmd="${cmds[$idx]}"
+            echo ""
+            echo "."
+            echo "Begin ($i/$total): python3 $cmd"
+            echo "."
+            file_start=$SECONDS
+            # base-a perf tests (fwd_occupancy) need async assert off; match
+            # _pr-test-stage.yml's base-a carve-out.
+            test_file="${cmd%% *}"
+            test_file="${test_file%%::*}"
+            async_assert=true
+            if grep -qE 'register_cuda_ci\(.*stage[[:space:]]*=[[:space:]]*"base-a"' "test/$test_file" 2>/dev/null; then
+              async_assert=false
+            fi
+            (cd test/ && SGLANG_ENABLE_ASYNC_ASSERT=$async_assert python3 $cmd -f) || exit 1
+            elapsed=$(( SECONDS - file_start ))
+            echo "."
+            echo "End ($i/$total): elapsed=${elapsed}s"
+            echo "."
+            echo ""
+          done
+          total_elapsed=$(( SECONDS - suite_start ))
+          echo "All $total test(s) passed in ${total_elapsed}s"
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  rerun-test-multimodal-gen:
+    if: inputs.mode == 'multimodal_gen'
+    runs-on: ${{ inputs.runs_on }}
+    timeout-minutes: 120
+    permissions:
+      contents: read
+      issues: write
+    env:
+      RUNNER_LABELS: ${{ inputs.runs_on }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_head_sha || github.sha }}
+
+      - name: Mark runner picked up
+        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 scripts/ci/utils/update_rerun_test_status.py \
+            --comment-id "${{ inputs.reply_comment_id }}" \
+            --marker "${{ inputs.reply_marker }}" \
+            --status running \
+            --repo "${{ github.repository }}"
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Install dependencies (diffusion)
+        timeout-minutes: 20
+        run: bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          cmds=()
+          while IFS= read -r cmd; do
+            [ -z "$cmd" ] && continue
+            cmds+=("$cmd")
+          done <<< "${{ inputs.test_command }}"
+          total=${#cmds[@]}
+          suite_start=$SECONDS
+          for idx in "${!cmds[@]}"; do
+            i=$((idx + 1))
+            cmd="${cmds[$idx]}"
+            echo ""
+            echo "."
+            echo "Begin ($i/$total): python3 -m pytest $cmd -x"
+            echo "."
+            file_start=$SECONDS
+            python3 -m pytest $cmd -x || exit 1
+            elapsed=$(( SECONDS - file_start ))
+            echo "."
+            echo "End ($i/$total): elapsed=${elapsed}s"
+            echo "."
+            echo ""
+          done
+          total_elapsed=$(( SECONDS - suite_start ))
+          echo "All $total test(s) passed in ${total_elapsed}s"
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: failure()
+
+  rerun-test-cpu:
+    if: inputs.mode == 'cpu'
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+          df -h
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_head_sha || github.sha }}
+
+      - name: Mark runner picked up
+        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 scripts/ci/utils/update_rerun_test_status.py \
+            --comment-id "${{ inputs.reply_comment_id }}" \
+            --marker "${{ inputs.reply_marker }}" \
+            --status running \
+            --repo "${{ github.repository }}"
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      # Needed by setuptools-rust to build the bundled native gRPC extension
+      # (rust/sglang-grpc) when installing the main `sglang` wheel from source.
+      - name: Install protoc + Rust toolchain
+        timeout-minutes: 10
+        run: bash scripts/ci/utils/install_rust_protoc.sh
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        env:
+          UV_SYSTEM_PYTHON: "1"
+        run: |
+          uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow
+
+      - name: Run test
+        timeout-minutes: 60
+        run: |
+          cd test/
+          cmds=()
+          while IFS= read -r cmd; do
+            [ -z "$cmd" ] && continue
+            cmds+=("$cmd")
+          done <<< "${{ inputs.test_command }}"
+          total=${#cmds[@]}
+          suite_start=$SECONDS
+          for idx in "${!cmds[@]}"; do
+            i=$((idx + 1))
+            cmd="${cmds[$idx]}"
+            echo ""
+            echo "."
+            echo "Begin ($i/$total): python3 $cmd"
+            echo "."
+            file_start=$SECONDS
+            python3 $cmd -f || exit 1
+            elapsed=$(( SECONDS - file_start ))
+            echo "."
+            echo "End ($i/$total): elapsed=${elapsed}s"
+            echo "."
+            echo ""
+          done
+          total_elapsed=$(( SECONDS - suite_start ))
+          echo "All $total test(s) passed in ${total_elapsed}s"
+
+  write-back-result:
+    needs: [rerun-test-cuda, rerun-test-multimodal-gen, rerun-test-cpu]
+    if: always() && inputs.reply_comment_id != '' && inputs.reply_marker != ''
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    concurrency:
+      group: rerun-test-writeback-${{ inputs.reply_comment_id }}
+      cancel-in-progress: false
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Write back result to reply comment
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [[ "${{ needs.rerun-test-cuda.result }}" == "success" \
+             || "${{ needs.rerun-test-multimodal-gen.result }}" == "success" \
+             || "${{ needs.rerun-test-cpu.result }}" == "success" ]]; then
+            STATUS=success
+          else
+            STATUS=failure
+          fi
+          python3 scripts/ci/utils/update_rerun_test_status.py \
+            --comment-id "${{ inputs.reply_comment_id }}" \
+            --marker "${{ inputs.reply_marker }}" \
+            --status "$STATUS" \
+            --repo "${{ github.repository }}"

.github/workflows/retag-docker.yml

@@ -0,0 +1,30 @@
+name: Retag Docker Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      source_tag:
+        description: "Existing image tag (e.g., v0.4.7-cu129-amd64)"
+        required: true
+      target_tag:
+        description: "New tag to apply (e.g., latest)"
+        required: true
+
+jobs:
+  retag:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-22.04
+    environment: "prod"
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Retag image
+        run: |
+          echo "Retagging lmsysorg/sglang:${{ inputs.source_tag }} -> lmsysorg/sglang:${{ inputs.target_tag }}"
+          docker buildx imagetools create \
+            -t lmsysorg/sglang:${{ inputs.target_tag }} \
+            lmsysorg/sglang:${{ inputs.source_tag }}

.github/workflows/runner-utilization.yml

@@ -0,0 +1,68 @@
+name: Runner Utilization Report
+
+on:
+  schedule:
+    - cron: '0 8 * * *'  # Daily at 8 AM UTC
+  pull_request:
+    paths:
+      - '.github/workflows/runner-utilization.yml'
+      - 'scripts/ci/utils/runner_utilization_report.py'
+  workflow_dispatch:
+    inputs:
+      hours:
+        description: 'Time window in hours'
+        required: false
+        default: '24'
+        type: string
+      filter:
+        description: 'Filter runner labels (e.g., 5090, h200)'
+        required: false
+        type: string
+
+# Sandbox PRs (e.g. tom/kv_canary) push dozens of times in a row. Without this
+# group, every push spawns another 24h API scan that races the previous ones
+# and drains the shared 15k/hr installation token quota. Cancel-in-progress
+# keeps at most one Runner Utilization run per ref alive at a time.
+concurrency:
+  group: runner-utilization-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  report:
+    name: Generate Report
+    # Skip fork PRs entirely, and require the `run-ci` label on same-repo PRs
+    # so unlabeled iteration pushes (e.g. long-lived sandbox branches) don't
+    # fire the 24h API scan. schedule and workflow_dispatch always run.
+    if: >-
+      github.event_name != 'pull_request' ||
+      (github.event.pull_request.head.repo.full_name == github.repository &&
+       contains(github.event.pull_request.labels.*.name, 'run-ci'))
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Unit-test report logic
+        # Pure-logic tests (stdlib only, no API calls) — guards the
+        # queue-time accounting and status/link rendering before the
+        # (API-heavy) report runs.
+        run: |
+          python -m unittest discover -s scripts/ci/utils \
+            -p 'test_runner_utilization_report.py' -v
+
+      - name: Generate Utilization Report
+        timeout-minutes: 30
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # PR-trigger is just a script smoke check; scan a 20-minute window
+          # (0.34h) to bound API cost. schedule / dispatch run the full report.
+          python scripts/ci/utils/runner_utilization_report.py \
+            --repo ${{ github.repository }} \
+            --hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \
+            ${{ inputs.filter && format('--filter {0}', inputs.filter) || '' }}

.github/workflows/slash-command-handler.yml

@@ -0,0 +1,100 @@
+name: Slash Command Handler
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+permissions:
+  contents: read
+  pull-requests: write # Required to add labels and reactions
+  actions: write       # Required to rerun workflows
+  issues: write        # Required for comment reactions in some contexts
+
+jobs:
+  slash_command:
+    # Only run if it is a PR and the comment contains a recognized command
+    # Use contains() since startsWith() can't handle leading whitespace/newlines
+    if: >
+      github.event.issue.pull_request &&
+      (contains(github.event.comment.body, '/tag-run-ci-label') ||
+       contains(github.event.comment.body, '/rerun-failed-ci') ||
+       contains(github.event.comment.body, '/tag-and-rerun-ci') ||
+       contains(github.event.comment.body, '/rerun-stage') ||
+       contains(github.event.comment.body, '/rerun-group') ||
+       contains(github.event.comment.body, '/rerun-test'))
+    runs-on: ubuntu-latest
+
+    steps:
+      # SECURITY: This workflow runs on issue_comment trigger with elevated permissions
+      # (pull-requests: write, actions: write). For non-fork PRs, we can safely checkout
+      # the PR branch to allow testing changes to this handler. For fork PRs, we MUST
+      # stay on main to prevent untrusted code execution with these elevated permissions.
+      - name: Get PR details
+        id: pr
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_DATA=$(gh pr view ${{ github.event.issue.number }} --repo ${{ github.repository }} --json headRefName,headRepositoryOwner) || {
+            echo "::error::Failed to fetch PR data"
+            exit 1
+          }
+          # Use 'empty' filter to handle null/missing values (e.g., deleted forks)
+          HEAD_OWNER=$(echo "$PR_DATA" | jq -r '.headRepositoryOwner.login // empty')
+          REPO_OWNER="${{ github.repository_owner }}"
+          # Treat missing/null owner as fork for security (fail-safe)
+          if [[ -z "$HEAD_OWNER" || "$HEAD_OWNER" != "$REPO_OWNER" ]]; then
+            IS_FORK="true"
+          else
+            IS_FORK="false"
+          fi
+          echo "is_fork=$IS_FORK" >> $GITHUB_OUTPUT
+          echo "ref=$(echo "$PR_DATA" | jq -r '.headRefName')" >> $GITHUB_OUTPUT
+          echo "pr_ref=refs/pull/${{ github.event.issue.number }}/head" >> $GITHUB_OUTPUT
+          echo "PR owner: $HEAD_OWNER, Repo owner: $REPO_OWNER, Is fork: $IS_FORK"
+
+      - name: Check commenter permission for fork PRs
+        id: perm
+        if: steps.pr.outputs.is_fork == 'true'
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PERM=$(gh api repos/${{ github.repository }}/collaborators/${{ github.event.comment.user.login }}/permission --jq '.permission') || {
+            PERM="none"
+            echo "::warning::Failed to check commenter permission, defaulting to none"
+          }
+          if [[ "$PERM" == "admin" || "$PERM" == "maintain" || "$PERM" == "write" ]]; then
+            echo "safe_to_checkout_pr=true" >> $GITHUB_OUTPUT
+          else
+            echo "safe_to_checkout_pr=false" >> $GITHUB_OUTPUT
+          fi
+          echo "Commenter ${{ github.event.comment.user.login }} permission: $PERM"
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          # For non-fork PRs: checkout PR branch by name
+          # For fork PRs with trusted commenter: checkout via refs/pull/N/head
+          # For fork PRs with untrusted commenter: stay on main for security
+          ref: ${{ steps.pr.outputs.is_fork == 'false' && steps.pr.outputs.ref || (steps.perm.outputs.safe_to_checkout_pr == 'true' && steps.pr.outputs.pr_ref || '') }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          pip install PyGithub PyYAML
+
+      - name: Handle Slash Command
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_FULL_NAME: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.issue.number }}
+          COMMENT_ID: ${{ github.event.comment.id }}
+          COMMENT_BODY: ${{ github.event.comment.body }}
+          USER_LOGIN: ${{ github.event.comment.user.login }}
+        run: |
+          python scripts/ci/utils/slash_command_handler.py

.github/workflows/stress-test.yml

@@ -0,0 +1,44 @@
+name: Stress Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      num_prompts:
+        description: 'Number of prompts per model'
+        required: true
+        default: '50000'
+        type: string
+      duration_minutes:
+        description: 'Timeout per model in minutes'
+        required: true
+        default: '45'
+        type: string
+
+jobs:
+  stress-test:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: 8-gpu-h200
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run stress tests
+        timeout-minutes: 210
+        env:
+          NUM_PROMPTS: ${{ inputs.num_prompts }}
+          DURATION_MINUTES: ${{ inputs.duration_minutes }}
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite stress
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: stress-test-results
+          path: |
+            stress_test_*.jsonl

.github/workflows/sync-lmsys-sglang-blogs.yml

@@ -0,0 +1,60 @@
+name: Sync LMSYS SGLang blogs
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 */12 * * *"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync:
+    if: github.repository == 'sgl-project/sglang' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Sync blog cards
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: python docs_new/scripts/update_lmsys_sglang_blogs.py
+
+      - name: Open or update pull request
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+        run: |
+          set -euo pipefail
+
+          if git diff --quiet -- docs_new/index.mdx; then
+            echo "No blog card changes; nothing to do."
+            exit 0
+          fi
+
+          BRANCH="bot/lmsys-blog-cards"
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+
+          # Recreate the bot branch as "current main + card diff" and overwrite
+          # any previous state, so the PR always shows a clean single-commit diff.
+          git checkout -B "$BRANCH"
+          git add docs_new/index.mdx
+          git commit -m "docs: sync LMSYS SGLang blog cards"
+          git push --force origin "$BRANCH"
+
+          # Open a PR only if one is not already open for this branch; otherwise
+          # the force-push above has already updated the existing PR in place.
+          if [ -z "$(gh pr list --head "$BRANCH" --base main --state open --json number --jq '.[].number')" ]; then
+            gh pr create \
+              --base main \
+              --head "$BRANCH" \
+              --title "docs: sync LMSYS SGLang blog cards" \
+              --body $'Automated sync of the LMSYS SGLang blog cards in `docs_new/index.mdx`.\n\n🤖 Generated with GitHub Actions'
+          fi

.github/workflows/trivy-scan-dev.yml

@@ -0,0 +1,88 @@
+name: Trivy Scan Dev Docker Images
+
+on:
+  # Run daily after nightly dev builds (which run at midnight UTC)
+  schedule:
+    - cron: "0 6 * * *"
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Image tag to scan (e.g., dev, dev-cu13, latest)"
+        required: false
+        default: ""
+
+jobs:
+  scan:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: x64-docker-build-node
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        tag: ${{ inputs.tag && fromJSON(format('["{0}"]', inputs.tag)) || fromJSON('["dev", "dev-cu12"]') }}
+    steps:
+      - name: Cleanup workspace (remove root-owned files from prior runs)
+        run: sudo rm -rf "$GITHUB_WORKSPACE"/* || true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@v0.35.0
+        with:
+          image-ref: 'docker.io/lmsysorg/sglang:${{ matrix.tag }}'
+          scanners: 'vuln'
+          format: 'sarif'
+          output: 'trivy-results-${{ matrix.tag }}.sarif'
+          severity: 'CRITICAL,HIGH'
+          ignore-unfixed: true
+          skip-dirs: 'usr/local/go,opt/nvidia'
+
+      - name: Upload Trivy scan results to GitHub Security
+        uses: github/codeql-action/upload-sarif@v4
+        if: always() && hashFiles(format('trivy-results-{0}.sarif', matrix.tag)) != ''
+        with:
+          sarif_file: 'trivy-results-${{ matrix.tag }}.sarif'
+          category: 'trivy-${{ matrix.tag }}'
+
+      - name: Run Trivy (table output for logs)
+        if: success()
+        uses: aquasecurity/trivy-action@v0.35.0
+        with:
+          image-ref: 'docker.io/lmsysorg/sglang:${{ matrix.tag }}'
+          scanners: 'vuln'
+          format: 'table'
+          severity: 'CRITICAL,HIGH'
+          ignore-unfixed: true
+          skip-dirs: 'usr/local/go,opt/nvidia'
+
+      - name: Scan summary
+        if: always()
+        run: |
+          IMAGE="docker.io/lmsysorg/sglang:${{ matrix.tag }}"
+          SARIF="trivy-results-${{ matrix.tag }}.sarif"
+
+          echo "## Trivy Scan: \`${{ matrix.tag }}\`" >> "$GITHUB_STEP_SUMMARY"
+
+          if [ ! -f "${SARIF}" ]; then
+            echo "**Status:** Scan failed — no SARIF output produced" >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          VULN_COUNT=$(python3 -c "
+          import json
+          data = json.load(open('${SARIF}'))
+          print(sum(len(run.get('results', [])) for run in data.get('runs', [])))
+          ")
+
+          echo "- **Image**: \`${IMAGE}\`" >> "$GITHUB_STEP_SUMMARY"
+          echo "- **Findings**: ${VULN_COUNT}" >> "$GITHUB_STEP_SUMMARY"
+
+          if [ "${VULN_COUNT}" = "0" ]; then
+            echo "- **Result**: No CRITICAL/HIGH unfixed vulnerabilities found" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "- **Result**: Found ${VULN_COUNT} finding(s) — check the Security tab for details" >> "$GITHUB_STEP_SUMMARY"
+          fi

.github/workflows/weekly-test-nvidia.yml

@@ -0,0 +1,50 @@
+name: Weekly Test (Nvidia)
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'  # Run every Sunday at midnight UTC
+  workflow_dispatch:
+    inputs:
+      job_filter:
+        description: 'Select which job to run (leave empty or "all" to run all jobs)'
+        required: false
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - 'weekly-test-8-gpu-h200'
+
+concurrency:
+  group: weekly-test-nvidia-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  SGLANG_IS_IN_CI: true
+  SGLANG_ENABLE_ASYNC_ASSERT: true
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 300
+
+jobs:
+  # Weekly tests - 8 GPU H200
+  weekly-test-8-gpu-h200:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'weekly-test-8-gpu-h200')
+    runs-on: 8-gpu-h200
+    timeout-minutes: 120
+    env:
+      RUNNER_LABELS: 8-gpu-h200
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh
+
+      - name: Run weekly 8-GPU H200 tests
+        timeout-minutes: 120
+        env:
+          GPU_CONFIG: "8-gpu-h200"
+          IS_H200: "1"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite weekly-8-gpu-h200 --nightly --continue-on-error --timeout-per-file 7200

.github/workflows/weekly-update-est-time.yml

@@ -0,0 +1,80 @@
+name: Weekly Update Est Time
+
+on:
+  schedule:
+    - cron: '0 0 * * 1'  # Monday 00:00 UTC
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-est-time:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Update est_time values
+        env:
+          MODEL_URL: "https://raw.githubusercontent.com/sgl-project/sglang-ci-stats/main/model.json"
+        run: |
+          python scripts/ci/update_est_time.py \
+            --model-url "$MODEL_URL" \
+            --summary-file /tmp/est_time_summary.md
+
+      - name: Check for changes
+        id: changes
+        run: |
+          if git diff --quiet; then
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+            echo "No est_time changes detected"
+          else
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            echo "Est_time changes detected:"
+            git diff --stat
+          fi
+
+      - name: Create PR
+        if: steps.changes.outputs.has_changes == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT_FOR_PULL_REQUEST }}
+        run: |
+          git config user.name "sglang-bot"
+          git config user.email "sglang-bot@users.noreply.github.com"
+
+          BRANCH_NAME="bot/update-est-time-$(date +%Y%m%d)"
+          git checkout -b "$BRANCH_NAME"
+
+          git add -A
+          git commit -m "chore: update CI test est_time from recent run data"
+
+          git push origin "$BRANCH_NAME"
+
+          {
+            echo "## Summary"
+            echo
+            echo "Refreshes \`est_time\` literals from [\`sgl-project/sglang-ci-stats\`](https://github.com/sgl-project/sglang-ci-stats)'s \`model.json\` (per-(suite, file) p90 over recent successful CI runs on \`main\`)."
+            echo
+            echo "This keeps the LPT load-balancing algorithm accurate for partitioning tests across parallel CI jobs, and serves as the static fallback when \`compute_partitions\` cannot fetch the live model at PR time."
+            echo
+            if [ -f /tmp/est_time_summary.md ]; then
+              cat /tmp/est_time_summary.md
+              echo
+            fi
+            echo "🤖 Generated with GitHub Actions"
+          } > /tmp/pr_body.md
+
+          gh pr create \
+            --title "chore: update CI test est_time values" \
+            --body-file /tmp/pr_body.md \
+            --base main \
+            --head "$BRANCH_NAME"

.github/workflows/xpu-ci-job-monitor.yml

@@ -0,0 +1,256 @@
+name: XPU CI Job Monitor
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily at midnight UTC
+  pull_request:
+    paths:
+      - '.github/workflows/xpu-ci-job-monitor.yml'
+      - 'scripts/ci/utils/xpu_job_monitor.py'
+  workflow_dispatch:
+    inputs:
+      hours:
+        description: 'Time window in hours'
+        required: false
+        default: '24'
+        type: string
+      job_filter:
+        description: 'Job name filter (leave empty for all XPU jobs)'
+        required: false
+        type: string
+
+# Bound the API cost when the same ref pushes repeatedly. See note in
+# runner-utilization.yml for the original incident this guards against.
+concurrency:
+  group: xpu-ci-job-monitor-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  fetch-actions-data:
+    name: Fetch Actions Snapshot
+    # Skip fork PRs entirely, and require the `run-ci` label on same-repo PRs.
+    # schedule and workflow_dispatch always run.
+    if: >-
+      github.event_name != 'pull_request' ||
+      (github.event.pull_request.head.repo.full_name == github.repository &&
+       contains(github.event.pull_request.labels.*.name, 'run-ci'))
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Select workflows for snapshot
+        id: select-workflows
+        run: |
+          if [[ -n "${{ inputs.job_filter }}" ]]; then
+            echo "workflows=pr-test-xpu.yml" >> "$GITHUB_OUTPUT"
+          else
+            echo "workflows=pr-test-xpu.yml,nightly-test-intel.yml" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Fetch Actions data snapshot
+        timeout-minutes: 30
+        run: |
+          # PR-trigger is just a script smoke check; scan a 20-minute window
+          # (0.34h) to bound API cost. schedule / dispatch run the full report.
+          python scripts/ci/utils/xpu_job_monitor.py \
+            --repo ${{ github.repository }} \
+            --workflow "${{ steps.select-workflows.outputs.workflows }}" \
+            --hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \
+            --dump-data-file actions-job-snapshot.json
+
+      - name: Upload Actions data snapshot
+        uses: actions/upload-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: actions-job-snapshot.json
+          if-no-files-found: error
+
+  # Single job filter mode
+  custom-report:
+    name: Custom Job Report
+    if: ${{ inputs.job_filter }}
+    needs: fetch-actions-data
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Custom Job Report
+        timeout-minutes: 30
+        run: |
+          python scripts/ci/utils/xpu_job_monitor.py \
+            --repo ${{ github.repository }} \
+            --job "${{ inputs.job_filter }}" \
+            --workflow "pr-test-xpu.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # Parse workflow files to get job names dynamically
+  parse-workflows:
+    name: Parse Workflow Jobs
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    outputs:
+      pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
+      nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Parse workflow files
+        id: parse
+        run: |
+          # Parse pr-test-xpu.yml and extract job names (exclude utility jobs)
+          # Excluded: check-changes, pr-gate, wait-for-stage-a, finish
+          pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-xpu.yml | \
+            grep -v -E '^(check-changes|pr-gate|wait-for-stage-a|finish)$' | \
+            jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
+          echo "PR jobs: $pr_jobs"
+
+          # Parse nightly-test-intel.yml and extract job names (exclude utility jobs)
+          # Excluded: check-all-jobs
+          nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-intel.yml | \
+            grep -v -E '^(check-all-jobs)$' | \
+            jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT
+          echo "Nightly jobs: $nightly_jobs"
+
+  # PR CI reports using dynamic matrix
+  pr-ci-reports:
+    name: PR - ${{ matrix.job_name }}
+    needs: [parse-workflows, fetch-actions-data]
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_jobs) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Report
+        timeout-minutes: 15
+        run: |
+          python scripts/ci/utils/xpu_job_monitor.py \
+            --repo ${{ github.repository }} \
+            --job "${{ matrix.job_name }}" \
+            --workflow "pr-test-xpu.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # Nightly XPU test reports using dynamic matrix
+  nightly-reports:
+    name: Nightly - ${{ matrix.job_name }}
+    needs: [parse-workflows, fetch-actions-data]
+    if: ${{ !inputs.job_filter }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_jobs) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Nightly Report
+        timeout-minutes: 15
+        run: |
+          python scripts/ci/utils/xpu_job_monitor.py \
+            --repo ${{ github.repository }} \
+            --job "${{ matrix.job_name }}" \
+            --workflow "nightly-test-intel.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary
+
+  # Runner fleet report - cross-workflow runner analytics in a single pass
+  runner-fleet-report:
+    name: Runner Fleet Report
+    if: ${{ !inputs.job_filter }}
+    needs: fetch-actions-data
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install tabulate
+
+      - name: Download Actions data snapshot
+        uses: actions/download-artifact@v4
+        with:
+          name: actions-job-snapshot
+          path: ci-data
+
+      - name: Generate Runner Fleet Report
+        timeout-minutes: 30
+        run: |
+          python scripts/ci/utils/xpu_job_monitor.py \
+            --repo ${{ github.repository }} \
+            --runner-report \
+            --workflow "pr-test-xpu.yml,nightly-test-intel.yml" \
+            --hours ${{ inputs.hours || '24' }} \
+            --input-data-file ci-data/actions-job-snapshot.json \
+            --summary