chore: import upstream snapshot with attribution

scripts/ci/amd/amd_ci_exec.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -euo pipefail
+
+# Detect GPU family from hostname (e.g., linux-mi35x-gpu-1-xxxxx-runner-zzzzz)
+HOSTNAME_VALUE=$(hostname)
+GPU_FAMILY=""
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_FAMILY="${BASH_REMATCH[1]}"
+  echo "Detected GPU family from hostname: ${GPU_FAMILY}"
+else
+  echo "Warning: could not parse GPU family from '${HOSTNAME_VALUE}'"
+fi
+
+WORKDIR="/sglang-checkout/test"
+declare -A ENV_MAP=(
+  [SGLANG_IS_IN_CI_AMD]=1
+  [SGLANG_IS_IN_CI]=1
+  # Disabled on AMD: the async-assert probes (#27461) fire torch._assert_async in
+  # the MXFP4 EAGLE-MTP decode path and abort the queue with an HSA hardware
+  # exception.
+  [SGLANG_ENABLE_ASYNC_ASSERT]=0
+  [SGLANG_USE_AITER]=1
+)
+
+# Conditionally add GPU_ARCHS only for mi35x
+if [[ "${GPU_FAMILY}" == "mi35x" ]]; then
+  ENV_MAP[GPU_ARCHS]="gfx950"
+fi
+
+# Parse -w/--workdir and -e ENV=VAL
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -w|--workdir)
+      WORKDIR="$2"
+      shift 2
+      ;;
+    -e)
+      IFS="=" read -r key val <<< "$2"
+      ENV_MAP["$key"]="$val"
+      shift 2
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+# Build final ENV_ARGS
+ENV_ARGS=()
+for key in "${!ENV_MAP[@]}"; do
+  ENV_ARGS+=("-e" "$key=${ENV_MAP[$key]}")
+done
+
+# Run docker exec with retry logic for HuggingFace network/download issues
+# When HF model downloads fail due to network timeouts or rate limits,
+# retrying with HF_HUB_OFFLINE=1 uses cached models from previous downloads.
+#
+# First attempt: normal mode (allows HF downloads)
+if docker exec \
+  -w "$WORKDIR" \
+  "${ENV_ARGS[@]}" \
+  ci_sglang "$@"; then
+  exit 0
+else
+  FIRST_EXIT_CODE=$?
+fi
+
+echo "First attempt failed with exit code $FIRST_EXIT_CODE"
+
+# Skip retry for test failures that won't be fixed by offline mode:
+#   - Exit 1: Test assertion failures (accuracy below threshold)
+#   - Exit 137 (128+9): Process killed by OOM
+#   - Exit 255: Test suite completed with test errors
+# Only retry for other exit codes (e.g., network timeouts, HF download failures)
+if [[ "$FIRST_EXIT_CODE" -eq 1 || "$FIRST_EXIT_CODE" -eq 137 || "$FIRST_EXIT_CODE" -eq 255 ]]; then
+  echo "Exit code $FIRST_EXIT_CODE indicates test failure (not network issue), not retrying"
+  exit $FIRST_EXIT_CODE
+fi
+
+echo "Retrying with HF_HUB_OFFLINE=1 (offline mode to use cached models)..."
+
+# Second attempt: force HF offline mode to avoid network timeouts
+docker exec \
+  -w "$WORKDIR" \
+  "${ENV_ARGS[@]}" \
+  -e HF_HUB_OFFLINE=1 \
+  ci_sglang "$@"

scripts/ci/amd/amd_ci_install_dependency.sh

@@ -0,0 +1,356 @@
+#!/bin/bash
+set -euo pipefail
+HOSTNAME_VALUE=$(hostname)
+GPU_ARCH="mi30x"   # default
+SKIP_TT_DEPS=""
+SKIP_SGLANG_BUILD=""
+SKIP_AITER_BUILD=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-aiter-build) SKIP_AITER_BUILD="1"; shift;;
+    --skip-sglang-build) SKIP_SGLANG_BUILD="1"; shift;;
+    --skip-test-time-deps) SKIP_TT_DEPS="1"; shift;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS] [OPTIONAL_DEPS]"
+      echo "Options:"
+      echo "  --skip-sglang-build         Don't build checkout sglang, use what was shipped with the image"
+      echo "  --skip-aiter-build          Don't build aiter, use what was shipped with the image"
+      echo "  --skip-test-time-deps       Don't build miscellaneous dependencies"
+      exit 0
+      ;;
+    *) break ;;
+  esac
+done
+
+OPTIONAL_DEPS="${1:-}"
+
+# Build python extras
+EXTRAS="dev_hip,tracing"
+if [ -n "$OPTIONAL_DEPS" ]; then
+    EXTRAS="dev_hip,tracing,${OPTIONAL_DEPS}"
+fi
+echo "Installing python extras: [${EXTRAS}]"
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_ARCH="${BASH_REMATCH[1]}"
+  echo "Detected GPU architecture from hostname: ${GPU_ARCH}"
+else
+  echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}"
+fi
+
+# Install the required dependencies in CI.
+# Fix permissions on pip cache, ignore errors from concurrent access or missing temp files
+docker exec ci_sglang chown -R root:root /sgl-data/pip-cache 2>/dev/null || true
+docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade pip
+
+# Helper function to install with retries and fallback PyPI mirror
+install_with_retry() {
+  local max_attempts=3
+  local cmd="$@"
+
+  for attempt in $(seq 1 $max_attempts); do
+    echo "Attempt $attempt/$max_attempts: $cmd"
+    if eval "$cmd"; then
+      echo "Success!"
+      return 0
+    fi
+
+    if [ $attempt -lt $max_attempts ]; then
+      echo "Failed, retrying in 5 seconds..."
+      sleep 5
+      # Try with alternative PyPI index on retry
+      if [[ "$cmd" =~ "pip install" ]] && [ $attempt -eq 2 ]; then
+        cmd="$cmd --index-url https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com"
+        echo "Using fallback PyPI mirror: $cmd"
+      fi
+    fi
+  done
+
+  echo "Failed after $max_attempts attempts"
+  return 1
+}
+
+# The anthropic SDK passes `socket_options` to httpx.HTTPTransport, which only
+# exists in httpx>=0.25.0. The CI image ships an older httpx, and several deps
+# installed below (lmms-eval, aiter's requirements.txt, etc.) can pull a stale
+# httpx back in, so test_anthropic_server fails with:
+#   TypeError: HTTPTransport.__init__() got an unexpected keyword argument 'socket_options'
+# Call this as the LAST pip operation so nothing can downgrade httpx afterwards.
+ensure_httpx() {
+  install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade 'httpx>=0.25.0'
+}
+
+# Helper function to git clone with retries
+git_clone_with_retry() {
+  local repo_url="$1"
+  local dest_dir="${2:-}"
+  local branch_args="${3:-}"
+  local max_attempts=3
+
+  for attempt in $(seq 1 $max_attempts); do
+    echo "Git clone attempt $attempt/$max_attempts: $repo_url"
+
+    # prevent from partial clone
+    if [ -n "$dest_dir" ] && [ -d "$dest_dir" ]; then
+      rm -rf "$dest_dir"
+    fi
+
+    if git \
+      -c http.lowSpeedLimit=1000 \
+      -c http.lowSpeedTime=30 \
+      clone --depth 1 ${branch_args:+$branch_args} "$repo_url" "$dest_dir"; then
+      echo "Git clone succeeded."
+      return 0
+    fi
+
+    if [ $attempt -lt $max_attempts ]; then
+      echo "Git clone failed, retrying in 5 seconds..."
+      sleep 5
+    fi
+  done
+
+  echo "Git clone failed after $max_attempts attempts: $repo_url"
+  return 1
+}
+
+# Install checkout sglang
+if [ -n "$SKIP_SGLANG_BUILD" ]; then
+  echo "Didn't build checkout SGLang"
+else
+  docker exec ci_sglang pip uninstall sgl-kernel -y || true
+  docker exec ci_sglang pip uninstall sglang-kernel -y || true
+  docker exec ci_sglang pip uninstall sglang -y || true
+  # Clear Python cache to ensure latest code is used
+  docker exec ci_sglang find /opt/venv -name "*.pyc" -delete || true
+  docker exec ci_sglang find /opt/venv -name "__pycache__" -type d -exec rm -rf {} + || true
+  # Also clear cache in sglang-checkout
+  docker exec ci_sglang find /sglang-checkout -name "*.pyc" -delete || true
+  docker exec ci_sglang find /sglang-checkout -name "__pycache__" -type d -exec rm -rf {} + || true
+  docker exec -w /sglang-checkout/sgl-kernel ci_sglang bash -c "rm -f pyproject.toml && mv pyproject_rocm.toml pyproject.toml && python3 setup_rocm.py install"
+
+  docker exec ci_sglang bash -c 'rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml'
+  install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]"
+fi
+
+if [[ -n "${SKIP_TT_DEPS}" ]]; then
+  echo "Didn't build lmms_eval, human-eval, and others"
+else
+  # For lmms_evals evaluating MMMU
+  # Clone on host (with retry), then copy into the container. The checkout is
+  # owned by the runner (non-root); mark it safe so setuptools_scm /
+  # vcs_versioning can run `git` introspection during pip install.
+  git_clone_with_retry https://github.com/EvolvingLMMs-Lab/lmms-eval.git lmms-eval "--branch v0.4.1"
+  docker cp lmms-eval ci_sglang:/
+  docker exec ci_sglang git config --global --add safe.directory /lmms-eval
+  install_with_retry docker exec -w /lmms-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e .
+
+  git_clone_with_retry https://github.com/akao-amd/human-eval.git human-eval
+  docker cp human-eval ci_sglang:/
+  install_with_retry docker exec -w /human-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e .
+
+  mkdir -p dummy-grok
+  cat > dummy-grok/config.json << 'EOF'
+  {
+    "architectures": [
+      "Grok1ModelForCausalLM"
+    ],
+    "embedding_multiplier_scale": 78.38367176906169,
+    "output_multiplier_scale": 0.5773502691896257,
+    "vocab_size": 131072,
+    "hidden_size": 6144,
+    "intermediate_size": 32768,
+    "max_position_embeddings": 8192,
+    "num_experts_per_tok": 2,
+    "num_local_experts": 8,
+    "num_attention_heads": 48,
+    "num_hidden_layers": 64,
+    "num_key_value_heads": 8,
+    "head_dim": 128,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 10000.0,
+    "model_type": "mixtral",
+    "torch_dtype": "bfloat16"
+  }
+EOF
+  docker exec -w / ci_sglang mkdir -p /dummy-grok
+  docker cp ./dummy-grok/config.json ci_sglang:/dummy-grok/config.json
+
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache huggingface_hub[hf_xet]
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache pytest
+
+  # Install cache-dit for qwen_image_t2i_cache_dit_enabled test (added in PR 16204)
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade 'cache-dit==1.3.0' || echo "cache-dit installation failed"
+
+  # Install accelerate for distributed training and inference support
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache accelerate || echo "accelerate installation failed"
+fi
+
+# -----------------------
+# MORI
+# The CI image bakes MORI at the docker/rocm.Dockerfile-pinned commit; when a PR
+# bumps MORI_COMMIT the image is not rebuilt, so reinstall MORI here the same way
+# the Dockerfile does. Only ENABLE_MORI=1 images ship /sgl-workspace/mori.
+if docker exec ci_sglang test -d /sgl-workspace/mori; then
+  MORI_REPO=$(grep -E '^[[:space:]]*ARG[[:space:]]+MORI_REPO=' docker/rocm.Dockerfile | head -n1 | sed 's/.*MORI_REPO="\([^"]*\)".*/\1/')
+  MORI_COMMIT=$(grep -E '^[[:space:]]*ARG[[:space:]]+MORI_COMMIT=' docker/rocm.Dockerfile | head -n1 | sed 's/.*MORI_COMMIT="\([^"]*\)".*/\1/')
+
+  if [[ "${GPU_ARCH}" == "mi35x" ]]; then
+    MORI_GPU_ARCHS="gfx950"
+  else
+    MORI_GPU_ARCHS="gfx942"
+  fi
+
+  echo "[MORI] Reinstalling MORI ${MORI_COMMIT} (MORI_GPU_ARCHS=${MORI_GPU_ARCHS})"
+  docker exec ci_sglang bash -c "
+    set -euo pipefail
+    export MORI_GPU_ARCHS='${MORI_GPU_ARCHS}'
+    rm -rf /sgl-workspace/mori
+    git clone '${MORI_REPO}' /sgl-workspace/mori
+    cd /sgl-workspace/mori
+    git checkout '${MORI_COMMIT}'
+    git submodule update --init --recursive
+    python3 setup.py develop
+    python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf
+    ldconfig
+  "
+  echo "[MORI] Done."
+fi
+
+if [[ -n "${SKIP_AITER_BUILD}" ]]; then
+  ensure_httpx
+  exit 0
+fi
+
+# Detect AITER version
+#############################################
+# Detect correct AITER_COMMIT for this runner
+# + Check mismatch
+# + Rebuild AITER if needed
+#############################################
+
+echo "[CI-AITER-CHECK] === AITER VERSION CHECK START ==="
+
+DOCKERFILE="docker/rocm.Dockerfile"
+
+# GPU_ARCH
+GPU_ARCH="${GPU_ARCH:-mi30x}"
+echo "[CI-AITER-CHECK] Runner GPU_ARCH=${GPU_ARCH}"
+
+#############################################
+# 1. Extract AITER_COMMIT from correct Dockerfile block
+#############################################
+if [[ "${GPU_ARCH}" == "mi35x" ]]; then
+    echo "[CI-AITER-CHECK] Using gfx950 block from Dockerfile..."
+    REPO_AITER_COMMIT=$(grep -F -A20 'FROM $BASE_IMAGE_950 AS gfx950' docker/rocm.Dockerfile \
+                        | grep 'AITER_COMMIT_DEFAULT=' \
+                        | head -n1 \
+                        | sed 's/.*AITER_COMMIT_DEFAULT="\([^"]*\)".*/\1/')
+else
+    echo "[CI-AITER-CHECK] Using gfx942 block from Dockerfile..."
+    REPO_AITER_COMMIT=$(grep -F -A20 'FROM $BASE_IMAGE_942 AS gfx942' docker/rocm.Dockerfile \
+                        | grep 'AITER_COMMIT_DEFAULT=' \
+                        | head -n1 \
+                        | sed 's/.*AITER_COMMIT_DEFAULT="\([^"]*\)".*/\1/')
+fi
+
+
+if [[ -z "${REPO_AITER_COMMIT}" ]]; then
+    echo "[CI-AITER-CHECK] ERROR: Failed to extract AITER_COMMIT from Dockerfile."
+    exit 1
+fi
+
+echo "[CI-AITER-CHECK] Dockerfile expects AITER_COMMIT=${REPO_AITER_COMMIT}"
+
+#############################################
+# 2. Check container pre-installed AITER version
+#############################################
+IMAGE_AITER_VERSION=$(docker exec ci_sglang bash -c "pip show amd-aiter 2>/dev/null | grep '^Version:' | awk '{print \$2}'" || echo "none")
+IMAGE_AITER_VERSION="v${IMAGE_AITER_VERSION}"
+echo "[CI-AITER-CHECK] AITER version inside CI image: ${IMAGE_AITER_VERSION}"
+
+#############################################
+# 3. Decide rebuild
+#############################################
+NEED_REBUILD="false"
+
+if [[ -n "${AITER_COMMIT_OVERRIDE:-}" ]]; then
+    echo "[CI-AITER-CHECK] AITER_COMMIT_OVERRIDE=${AITER_COMMIT_OVERRIDE} → forcing rebuild"
+    REPO_AITER_COMMIT="${AITER_COMMIT_OVERRIDE}"
+    NEED_REBUILD="true"
+elif [[ "${IMAGE_AITER_VERSION}" == "vnone" || "${IMAGE_AITER_VERSION}" == "v" ]]; then
+    echo "[CI-AITER-CHECK] No AITER found in image → rebuild needed"
+    NEED_REBUILD="true"
+elif [[ "${IMAGE_AITER_VERSION}" == "${REPO_AITER_COMMIT}" ]]; then
+    echo "[CI-AITER-CHECK] AITER version matches"
+elif [[ "${IMAGE_AITER_VERSION}" =~ (dev|\+g[0-9a-f]+) ]]; then
+    # Dev/patched version (contains 'dev' or git hash) → preserve it
+    echo "[CI-AITER-CHECK] Dev/patched version detected: ${IMAGE_AITER_VERSION} → skipping rebuild"
+else
+    echo "[CI-AITER-CHECK] Version mismatch: image=${IMAGE_AITER_VERSION}, repo=${REPO_AITER_COMMIT}"
+    NEED_REBUILD="true"
+fi
+
+
+#############################################
+# 4. Rebuild AITER if needed
+#############################################
+if [[ "${NEED_REBUILD}" == "true" ]]; then
+    echo "[CI-AITER-CHECK] === AITER REBUILD START ==="
+
+    # uninstall existing aiter
+    docker exec ci_sglang pip uninstall -y amd-aiter || true
+
+    # delete old aiter directory
+    docker exec ci_sglang rm -rf /sgl-workspace/aiter
+
+    # clone a fresh copy to /sgl-workspace/aiter
+    docker exec ci_sglang git clone https://github.com/ROCm/aiter.git /sgl-workspace/aiter
+
+    # checkout correct version and install requirements
+    # Use `checkout -f` so the smudge-filter-induced "dirty" working tree from
+    # AITER's .gitattributes (*.csv text eol=lf, added in ROCm/aiter#3370) does
+    # not block switching to commits that predate that rule. The working tree
+    # was just produced by `rm -rf` + fresh `git clone` above, so there are no
+    # real user changes to preserve.
+    docker exec ci_sglang bash -c "
+        cd /sgl-workspace/aiter && \
+        git fetch --all && \
+        git checkout -f ${REPO_AITER_COMMIT} && \
+        git submodule update --init --recursive && \
+        pip install -r requirements.txt
+    "
+
+    if [[ "${GPU_ARCH}" == "mi35x" ]]; then
+        GPU_ARCH_LIST="gfx950"
+    else
+        GPU_ARCH_LIST="gfx942"
+    fi
+    echo "[CI-AITER-CHECK] GPU_ARCH_LIST=${GPU_ARCH_LIST}"
+
+    # build AITER
+    docker exec ci_sglang bash -c "
+        cd /sgl-workspace/aiter && \
+        AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${GPU_ARCH_LIST} python3 setup.py develop
+    "
+
+    echo "[CI-AITER-CHECK] === AITER REBUILD COMPLETE ==="
+fi
+
+echo "[CI-AITER-CHECK] === AITER VERSION CHECK END ==="
+
+# Must be the final pip operation: force httpx>=0.25.0 so the anthropic SDK can
+# construct its httpx transport (see ensure_httpx definition above).
+ensure_httpx
+
+
+# # Clear pre-built AITER kernels from Docker image to avoid segfaults
+# # The Docker image may contain pre-compiled kernels incompatible with the current environment
+# echo "Clearing pre-built AITER kernels from Docker image..."
+# docker exec ci_sglang find /sgl-workspace/aiter/aiter/jit -name "*.so" -delete 2>/dev/null || true
+# docker exec ci_sglang ls -la /sgl-workspace/aiter/aiter/jit/ 2>/dev/null || echo "jit dir empty or not found"
+
+# # Pre-build AITER kernels to avoid timeout during tests
+# echo "Warming up AITER JIT kernels..."
+# docker exec -e SGLANG_USE_AITER=1 ci_sglang python3 /sglang-checkout/scripts/ci/amd/amd_ci_warmup_aiter.py || echo "AITER warmup completed (some kernels may not be available)"

scripts/ci/amd/amd_ci_start_container.sh

@@ -0,0 +1,341 @@
+#!/bin/bash
+set -euo pipefail
+
+# Get version from git tags
+SGLANG_VERSION="v0.5.5"   # Default version, will be overridden if git tags are found
+
+# Fetch tags from origin to ensure we have the latest
+if git fetch --tags origin; then
+  # Use the shared helper so stable/post releases sort above rc tags.
+  VERSION_FROM_TAG=$(python3 python/tools/get_version_tag.py --tag-only || true)
+  if [ -n "$VERSION_FROM_TAG" ]; then
+    SGLANG_VERSION="$VERSION_FROM_TAG"
+    echo "Using SGLang version from git tags: $SGLANG_VERSION"
+  else
+    echo "Warning: No version tags found; using default $SGLANG_VERSION" >&2
+  fi
+else
+  echo "Warning: Failed to fetch tags from origin; using default $SGLANG_VERSION" >&2
+fi
+
+
+# Default base tags (can be overridden by command line arguments)
+ROCM_VERSION="rocm700"
+DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
+
+# Parse command line arguments
+MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
+MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}"
+CUSTOM_IMAGE=""
+BUILD_FROM_DOCKERFILE=""
+GPU_ARCH_BUILD=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --mi30x-base-tag) MI30X_BASE_TAG="$2"; shift 2;;
+    --mi35x-base-tag) MI35X_BASE_TAG="$2"; shift 2;;
+    --custom-image) CUSTOM_IMAGE="$2"; shift 2;;
+    --build-from-dockerfile) BUILD_FROM_DOCKERFILE="1"; shift;;
+    --gpu-arch) GPU_ARCH_BUILD="$2"; shift 2;;
+    --rocm-version)
+      ROCM_VERSION="$2"
+      MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+      MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+      echo "Using ROCm version override: ${ROCM_VERSION}"
+      shift 2;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS]"
+      echo "Options:"
+      echo "  --mi30x-base-tag TAG       Override MI30x base image tag"
+      echo "  --mi35x-base-tag TAG       Override MI35x base image tag"
+      echo "  --custom-image IMAGE       Use a specific Docker image directly"
+      echo "  --build-from-dockerfile    Build image from docker/rocm.Dockerfile"
+      echo "  --gpu-arch ARCH            GPU architecture for Dockerfile build (e.g., gfx950-rocm720)"
+      echo "  --rocm-version VERSION     Override ROCm version for image lookup (e.g., rocm720)"
+      echo ""
+      echo "Environment:"
+      echo "  ENABLE_CACHE_HOST=1|0"
+      echo "      Mount /home/runner/sglang-data to /sgl-data. Defaults to 0."
+      exit 0
+      ;;
+    *) echo "Unknown option $1"; exit 1;;
+  esac
+done
+
+
+
+# Detect GPU architecture from the Kubernetes runner hostname
+HOSTNAME_VALUE=$(hostname)
+GPU_ARCH="mi30x"   # default
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_ARCH="${BASH_REMATCH[1]}"
+  echo "Detected GPU architecture from hostname: ${GPU_ARCH}"
+else
+  echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}"
+fi
+
+# Normalise / collapse architectures we don't yet build specifically for
+case "${GPU_ARCH}" in
+  mi35x)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi35x image."
+    ;;
+  mi30x|mi300|mi325)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi30x image."
+    GPU_ARCH="mi30x"
+    ;;
+  *)
+    echo "Runner architecture '${GPU_ARCH}' unrecognised; defaulting to mi30x image." >&2
+    GPU_ARCH="mi30x"
+    ;;
+esac
+
+
+# Set up DEVICE_FLAG based on Kubernetes pod info
+if [[ -f /etc/podinfo/gha-render-devices ]]; then
+  DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
+else
+  DEVICE_FLAG="--device /dev/dri"
+fi
+
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub to avoid anonymous pull rate limits.
+# Credentials are optional; when absent we fall back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub…"
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
+
+# Find the latest image
+find_latest_image() {
+  local gpu_arch=$1
+  local base_tag days_back image_tag image_id remote_tags
+
+  case "${gpu_arch}" in
+      mi30x) base_tag="${MI30X_BASE_TAG}" ;;
+      mi35x) base_tag="${MI35X_BASE_TAG}" ;;
+      *)     echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
+  esac
+
+  # First, check local cache on the runner.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
+    if [[ -n "$image_id" ]]; then
+      echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # If not found locally, fall back to pulling from public registry.
+  # We intentionally do not probe ${LOCAL_DOCKER_REGISTRY} here with
+  # `docker manifest inspect --insecure` because that command runs in the
+  # runner pod's network namespace, which on every observed AMD scale set
+  # cannot reach 10.44.14.109:5000 (every probe either fast-fails with TLS
+  # reject or hits a 30s TCP timeout, multiplied across 7 daily candidates).
+  # The actual local-registry pull still happens in the call site below via
+  # `docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}"`, which goes through the
+  # docker daemon on the host and inherits its insecure-registries config.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
+    if docker manifest inspect "rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
+      echo "Found available image: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # Docker Hub's `name=` filter is fuzzy; only accept official version tags.
+  echo "Exact version not found. Searching remote registry for versioned ${ROCM_VERSION}-${gpu_arch} images…" >&2
+  for days_back in {0..6}; do
+    local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
+    local sgl_tag_regex="^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-${target_date}$"
+    remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | while read -r tag; do
+      if [[ "${tag}" =~ ${sgl_tag_regex} ]]; then
+        echo "${tag}"
+        break
+      fi
+    done || true)
+    if [[ -n "$remote_tags" ]]; then
+      echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
+      echo "rocm/sgl-dev:${remote_tags}"
+      return 0
+    fi
+  done
+
+  echo "No recent images found. Searching cached local versioned images matching ROCm+arch…" >&2
+  local any_local
+  any_local=$(docker images --format '{{.Repository}}:{{.Tag}}' --filter "reference=rocm/sgl-dev:v*-${ROCM_VERSION}-${gpu_arch}-*" | while read -r image; do
+    local tag="${image#rocm/sgl-dev:}"
+    if [[ "${tag}" =~ ^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-[0-9]{8}$ ]]; then
+      echo "${image}"
+    fi
+  done | sort -r | head -n 1)
+  if [[ -n "$any_local" ]]; then
+      echo "Using cached fallback image: ${any_local}" >&2
+      echo "${any_local}"
+      return 0
+  fi
+
+  echo "Error: no ${gpu_arch} image found in the last 7 days for base ${base_tag}" >&2
+  echo "Using hard-coded fallback for ${ROCM_VERSION}…" >&2
+  case "${ROCM_VERSION}" in
+    rocm720)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260211-preview"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
+      fi
+      ;;
+    rocm700)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
+      fi
+      ;;
+    *)
+      echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
+      return 1
+      ;;
+  esac
+}
+
+# Determine which image to use
+if [[ -n "${CUSTOM_IMAGE}" ]]; then
+  # Use explicitly provided custom image
+  IMAGE="${CUSTOM_IMAGE}"
+  echo "Using custom image: ${IMAGE}"
+  if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
+    docker pull "${IMAGE}"
+  else
+    retry_with_backoff 6 docker pull "${IMAGE}"
+  fi
+elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
+  # Build image from Dockerfile
+  if [[ -z "${GPU_ARCH_BUILD}" ]]; then
+    echo "Error: --gpu-arch is required when using --build-from-dockerfile" >&2
+    exit 1
+  fi
+
+  DOCKERFILE_DIR="${GITHUB_WORKSPACE:-$PWD}/docker"
+  DOCKERFILE="${DOCKERFILE_DIR}/rocm.Dockerfile"
+
+  if [[ ! -f "${DOCKERFILE}" ]]; then
+    echo "Error: Dockerfile not found at ${DOCKERFILE}" >&2
+    exit 1
+  fi
+
+  IMAGE="sglang-ci:${GPU_ARCH_BUILD}-$(date +%Y%m%d)"
+  echo "Building Docker image from ${DOCKERFILE} with GPU_ARCH=${GPU_ARCH_BUILD}..."
+
+  # Pass full GPU_ARCH (e.g., gfx950-rocm720) - Dockerfile handles stripping suffix
+  docker build \
+    --build-arg GPU_ARCH="${GPU_ARCH_BUILD}" \
+    --build-arg SGL_BRANCH="main" \
+    -t "${IMAGE}" \
+    -f "${DOCKERFILE}" \
+    "${DOCKERFILE_DIR}"
+  echo "Successfully built image: ${IMAGE}"
+else
+  # Find the latest pre-built image
+  IMAGE=$(find_latest_image "${GPU_ARCH}")
+  # Try the local docker registry first (avoids Docker Hub rate limits and is
+  # faster on the LAN); if that fails for any reason, fall back to the
+  # public registry with exponential-backoff retries. Capture stderr so the
+  # real failure reason (TLS handshake, 404, connection refused, etc.) is
+  # visible in the job log instead of being silently swallowed.
+  if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
+    echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
+    docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
+  else
+    echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
+    printf '%s\n' "${local_pull_output}" | sed 's/^/  [local-pull] /' >&2
+    retry_with_backoff 6 docker pull "${IMAGE}"
+  fi
+fi
+
+CACHE_HOST=/home/runner/sglang-data
+ENABLE_CACHE_HOST="${ENABLE_CACHE_HOST:-0}"
+case "${ENABLE_CACHE_HOST,,}" in
+  1|true|yes|on|pvc|persistent)
+    if [[ ! -d "$CACHE_HOST" ]]; then
+      echo "Error: ENABLE_CACHE_HOST=1 but ${CACHE_HOST} does not exist." >&2
+      exit 1
+    fi
+    CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
+    echo "Mounting persistent CI data: ${CACHE_HOST} -> /sgl-data"
+    ;;
+  0|false|no|off|"")
+    CACHE_VOLUME=""
+    echo "Not mounting ${CACHE_HOST}; /sgl-data will be container-local."
+    ;;
+  *)
+    echo "Error: unsupported ENABLE_CACHE_HOST='${ENABLE_CACHE_HOST}'" >&2
+    echo "Use 1/true/pvc/persistent or 0/false/off." >&2
+    exit 1
+    ;;
+esac
+
+echo "Launching container: ci_sglang"
+docker run -dt --user root --device=/dev/kfd ${DEVICE_FLAG} \
+  --ulimit nofile=65536:65536 \
+  -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
+  $CACHE_VOLUME \
+  --group-add video \
+  --shm-size 32g \
+  --cap-add=SYS_PTRACE \
+  -e HF_TOKEN="${HF_TOKEN:-}" \
+  -e HF_HOME=/sgl-data/hf-cache \
+  -e HF_HUB_ETAG_TIMEOUT=300 \
+  -e HF_HUB_DOWNLOAD_TIMEOUT=300 \
+  -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
+  -e MIOPEN_CUSTOM_CACHE_DIR=/sgl-data/miopen-cache \
+  -e PYTHONPATH="/opt/tilelang:${PYTHONPATH:-}" \
+  --security-opt seccomp=unconfined \
+  -w /sglang-checkout \
+  --name ci_sglang \
+  "${IMAGE}"
+
+docker exec ci_sglang mkdir -p \
+  /sgl-data/hf-cache/hub \
+  /sgl-data/pip-cache \
+  /sgl-data/miopen-cache \
+  /sgl-data/aiter-kernels
+
+# The checkout is owned by the runner (non-root) but the container runs as
+# root.  Git >= 2.35.2 rejects cross-user repos; mark the mount as safe so
+# setuptools-scm / vcs_versioning can resolve the package version.
+docker exec ci_sglang git config --global --add safe.directory /sglang-checkout

scripts/ci/amd/amd_ci_start_container_disagg.sh

@@ -0,0 +1,331 @@
+#!/bin/bash
+set -euo pipefail
+
+# Get version from git tags
+SGLANG_VERSION="v0.5.5"   # Default version, will be overridden if git tags are found
+
+# Fetch tags from origin to ensure we have the latest
+if git fetch --tags origin; then
+  # Use the shared helper so stable/post releases sort above rc tags.
+  VERSION_FROM_TAG=$(python3 python/tools/get_version_tag.py --tag-only || true)
+  if [ -n "$VERSION_FROM_TAG" ]; then
+    SGLANG_VERSION="$VERSION_FROM_TAG"
+    echo "Using SGLang version from git tags: $SGLANG_VERSION"
+  else
+    echo "Warning: No version tags found; using default $SGLANG_VERSION" >&2
+  fi
+else
+  echo "Warning: Failed to fetch tags from origin; using default $SGLANG_VERSION" >&2
+fi
+
+
+# Default base tags (can be overridden by command line arguments)
+ROCM_VERSION="rocm700"
+DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
+
+# Parse command line arguments
+MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
+MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --mi30x-base-tag) MI30X_BASE_TAG="$2"; shift 2;;
+    --mi35x-base-tag) MI35X_BASE_TAG="$2"; shift 2;;
+    --rocm-version)
+      ROCM_VERSION="$2"
+      MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+      MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+      echo "Using ROCm version override: ${ROCM_VERSION}"
+      shift 2;;
+    -h|--help)
+      echo "Usage: $0 [--mi30x-base-tag TAG] [--mi35x-base-tag TAG] [--rocm-version VERSION]"
+      exit 0
+      ;;
+    *) echo "Unknown option $1"; exit 1;;
+  esac
+done
+
+
+
+# Detect GPU architecture from the Kubernetes runner hostname
+HOSTNAME_VALUE=$(hostname)
+GPU_ARCH="mi30x"   # default
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_ARCH="${BASH_REMATCH[1]}"
+  echo "Detected GPU architecture from hostname: ${GPU_ARCH}"
+else
+  echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}"
+fi
+
+# Normalise / collapse architectures we don’t yet build specifically for
+case "${GPU_ARCH}" in
+  mi35x)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi35x image."
+    ;;
+  mi30x|mi300|mi325)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi30x image."
+    GPU_ARCH="mi30x"
+    ;;
+  *)
+    echo "Runner architecture '${GPU_ARCH}' unrecognised; defaulting to mi30x image." >&2
+    GPU_ARCH="mi30x"
+    ;;
+esac
+
+
+# Set up DEVICE_FLAG based on Kubernetes pod info
+if [[ -f /etc/podinfo/gha-render-devices ]]; then
+  DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
+else
+  DEVICE_FLAG="--device /dev/dri"
+fi
+
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub to avoid anonymous pull rate limits.
+# Credentials are optional; when absent we fall back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub…"
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
+
+# Find the latest image
+find_latest_image() {
+  local gpu_arch=$1
+  local base_tag days_back image_tag image_id remote_tags
+
+  case "${gpu_arch}" in
+      mi30x) base_tag="${MI30X_BASE_TAG}" ;;
+      mi35x) base_tag="${MI35X_BASE_TAG}" ;;
+      *)     echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
+  esac
+
+  # First, check local cache on the runner.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
+    if [[ -n "$image_id" ]]; then
+      echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # If not found locally, fall back to pulling from public registry.
+  # See amd_ci_start_container.sh for why we don't probe
+  # ${LOCAL_DOCKER_REGISTRY} with `docker manifest inspect --insecure` from
+  # the runner pod's network namespace; the actual local-registry pull
+  # happens at the call site below via the docker daemon on the host.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
+    if docker manifest inspect "rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
+      echo "Found available image: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # Docker Hub's `name=` filter is fuzzy; only accept official version tags.
+  echo "Exact version not found. Searching remote registry for versioned ${ROCM_VERSION}-${gpu_arch} images…" >&2
+  for days_back in {0..6}; do
+    local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
+    local sgl_tag_regex="^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-${target_date}$"
+    remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | while read -r tag; do
+      if [[ "${tag}" =~ ${sgl_tag_regex} ]]; then
+        echo "${tag}"
+        break
+      fi
+    done || true)
+    if [[ -n "$remote_tags" ]]; then
+      echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
+      echo "rocm/sgl-dev:${remote_tags}"
+      return 0
+    fi
+  done
+
+  echo "No recent images found. Searching cached local versioned images matching ROCm+arch…" >&2
+  local any_local
+  any_local=$(docker images --format '{{.Repository}}:{{.Tag}}' --filter "reference=rocm/sgl-dev:v*-${ROCM_VERSION}-${gpu_arch}-*" | while read -r image; do
+    local tag="${image#rocm/sgl-dev:}"
+    if [[ "${tag}" =~ ^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-[0-9]{8}$ ]]; then
+      echo "${image}"
+    fi
+  done | sort -r | head -n 1)
+  if [[ -n "$any_local" ]]; then
+      echo "Using cached fallback image: ${any_local}" >&2
+      echo "${any_local}"
+      return 0
+  fi
+
+  echo "Error: no ${gpu_arch} image found in the last 7 days for base ${base_tag}" >&2
+  echo "Using hard-coded fallback for ${ROCM_VERSION}…" >&2
+  case "${ROCM_VERSION}" in
+    rocm720)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260211-preview"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
+      fi
+      ;;
+    rocm700)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
+      fi
+      ;;
+    *)
+      echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
+      return 1
+      ;;
+  esac
+}
+
+# Pull and run the latest image
+IMAGE=$(find_latest_image "${GPU_ARCH}")
+# Try the local docker registry first (avoids Docker Hub rate limits and is
+# faster on the LAN); if that fails for any reason, fall back to the
+# public registry with exponential-backoff retries. Capture stderr so the
+# real failure reason (TLS handshake, 404, connection refused, etc.) is
+# visible in the job log instead of being silently swallowed.
+if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
+  echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
+  docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
+else
+  echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
+  printf '%s\n' "${local_pull_output}" | sed 's/^/  [local-pull] /' >&2
+  retry_with_backoff 6 docker pull "${IMAGE}"
+fi
+
+# CACHE_HOST=/home/runner/sgl-data
+CACHE_HOST=/home/runner/temp-sglang-data
+if [[ -d "$CACHE_HOST" ]]; then
+    CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
+else
+    CACHE_VOLUME=""
+fi
+
+# Detect libionic library for RDMA support
+LIBIONIC_MOUNT=""
+IONIC_SYMLINK="/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so"
+if [[ -L "$IONIC_SYMLINK" ]]; then
+    LIBIONIC_LIB=$(readlink -f "$IONIC_SYMLINK" 2>/dev/null)
+    if [[ -f "$LIBIONIC_LIB" ]]; then
+        echo "Found libionic library: $LIBIONIC_LIB (resolved from symlink)"
+        LIBIONIC_MOUNT="-v ${LIBIONIC_LIB}:${LIBIONIC_LIB}:ro"
+    else
+        echo "Warning: libionic symlink exists but target does not: $LIBIONIC_LIB"
+    fi
+else
+    # Fallback: try to find directly
+    LIBIONIC_FOUND=$(find /usr/lib/x86_64-linux-gnu -maxdepth 1 -name "libionic.so.*" 2>/dev/null | head -1)
+    if [[ -n "$LIBIONIC_FOUND" ]]; then
+        LIBIONIC_LIB=$(readlink -f "$LIBIONIC_FOUND" 2>/dev/null)
+        if [[ -f "$LIBIONIC_LIB" ]]; then
+            echo "Found libionic library: $LIBIONIC_LIB"
+            LIBIONIC_MOUNT="-v ${LIBIONIC_LIB}:${LIBIONIC_LIB}:ro"
+        else
+            echo "Warning: libionic found but cannot resolve real path: $LIBIONIC_FOUND"
+        fi
+    else
+        echo "Warning: libionic library not found on host, RDMA may not work"
+    fi
+fi
+
+MOUNT_ARGS=""
+
+add_mount_if_exists() {
+    local name=$1
+    local search_pattern=$2
+    local path=$(find /lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu /lib64 /usr/lib64 -name "$search_pattern" -print -quit 2>/dev/null)
+
+    if [ -n "$path" ]; then
+        echo "Found $name at: $path"
+        MOUNT_ARGS="$MOUNT_ARGS -v $path:$path:ro"
+    else
+        echo "WARNING: Could not find $name on host! (Pattern: $search_pattern)"
+    fi
+}
+
+IONIC_LINK="/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so"
+if [ -L "$IONIC_LINK" ]; then
+    IONIC_REAL=$(readlink -f "$IONIC_LINK")
+    if [ -f "$IONIC_REAL" ]; then
+        echo "Ionic Driver: $IONIC_REAL"
+        MOUNT_ARGS="$MOUNT_ARGS -v $IONIC_REAL:$IONIC_REAL:ro"
+    fi
+fi
+
+add_mount_if_exists "libnl-3" "libnl-3.so*"
+add_mount_if_exists "libmnl" "libmnl.so*"
+
+echo "Mount args: $MOUNT_ARGS"
+
+echo "Launching container: ci_sglang"
+docker run -dt --user root \
+  --device=/dev/kfd \
+  --device=/dev/dri \
+  ${DEVICE_FLAG} \
+  -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
+  -v /sys/class/infiniband:/sys/class/infiniband:ro \
+  -v /sys/class/infiniband_verbs:/sys/class/infiniband_verbs:ro \
+  -v /sys/class/net:/sys/class/net:ro \
+  -v /etc/libibverbs.d:/etc/libibverbs.d:ro \
+  -v /usr/lib/x86_64-linux-gnu/libibverbs:/usr/lib/x86_64-linux-gnu/libibverbs:ro \
+  $MOUNT_ARGS \
+  $CACHE_VOLUME \
+  --privileged \
+  --network=host \
+  --ipc=host \
+  --ulimit memlock=-1 \
+  --cap-add=IPC_LOCK \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --group-add video \
+  --group-add rdma \
+  --shm-size 32g \
+  -e HF_TOKEN="${HF_TOKEN:-}" \
+  -e HF_HOME=/sgl-data/hf-cache \
+  -e HF_HUB_ETAG_TIMEOUT=300 \
+  -e HF_HUB_DOWNLOAD_TIMEOUT=300 \
+  -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
+  -e MIOPEN_CUSTOM_CACHE_DIR=/sgl-data/miopen-cache \
+  -w /sglang-checkout \
+  --name ci_sglang \
+  "${IMAGE}"
+
+# The checkout is owned by the runner (non-root) but the container runs as
+# root.  Git >= 2.35.2 rejects cross-user repos; mark the mount as safe so
+# setuptools-scm / vcs_versioning can resolve the package version.
+docker exec ci_sglang git config --global --add safe.directory /sglang-checkout

scripts/ci/amd/amd_ci_warmup_aiter.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""
+Warmup script to pre-build AITER JIT kernels.
+
+This script triggers compilation of commonly used AITER kernels by importing
+the relevant modules and calling functions with sample data. This avoids
+timeouts during actual tests when kernels need to be compiled on first use.
+
+Run this after clearing pre-built AITER kernels from the Docker image.
+"""
+
+import os
+import sys
+import time
+
+# Ensure AITER is enabled
+os.environ["SGLANG_USE_AITER"] = "1"
+
+
+def warmup_aiter_kernels():
+    """Trigger AITER JIT kernel compilation."""
+    import torch
+
+    if not torch.cuda.is_available():
+        print("CUDA/ROCm not available, skipping AITER warmup")
+        return
+
+    print("=" * 60)
+    print("AITER JIT Kernel Warmup")
+    print("=" * 60)
+
+    device = torch.device("cuda:0")
+    start_time = time.time()
+
+    # Warmup module_rmsnorm_quant (small module, ~2MB)
+    # Triggered by rmsnorm2d_fwd when hidden_size <= 8192
+    try:
+        print(
+            "\n[1/5] Warming up module_rmsnorm_quant (rmsnorm2d_fwd, hidden<=8192)..."
+        )
+        from aiter import rmsnorm2d_fwd
+
+        hidden_size = 4096
+        batch_size = 512  # Use larger batch to match CUDA graph capture
+        x = torch.randn(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+        weight = torch.ones(hidden_size, dtype=torch.bfloat16, device=device)
+        eps = 1e-6
+
+        # hidden_size=4096 <= 8192 -> takes rmsnorm() path -> compiles module_rmsnorm_quant
+        _ = rmsnorm2d_fwd(x, weight, eps)
+        torch.cuda.synchronize()
+        print("   module_rmsnorm_quant compiled successfully")
+    except Exception as e:
+        print(f"   module_rmsnorm_quant warmup failed: {e}")
+
+    # Warmup module_rmsnorm (large CK module, ~159MB)
+    # Triggered by rmsnorm2d_fwd_with_add (always uses CK path)
+    # NOTE: rmsnorm2d_fwd_with_add signature is:
+    #   rmsnorm2d_fwd_with_add(out, input, residual_in, residual_out, weight, epsilon)
+    try:
+        print("\n[2/5] Warming up module_rmsnorm (rmsnorm2d_fwd_with_add, CK path)...")
+        from aiter import rmsnorm2d_fwd_with_add
+
+        hidden_size = 4096
+        batch_size = 512
+        x = torch.randn(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+        residual_in = torch.randn(
+            batch_size, hidden_size, dtype=torch.bfloat16, device=device
+        )
+        output = torch.empty_like(x)
+        residual_out = torch.empty_like(x)
+        weight = torch.ones(hidden_size, dtype=torch.bfloat16, device=device)
+        eps = 1e-6
+
+        # This triggers JIT compilation of module_rmsnorm (CK kernels)
+        rmsnorm2d_fwd_with_add(output, x, residual_in, residual_out, weight, eps)
+        torch.cuda.synchronize()
+        print("   module_rmsnorm compiled successfully")
+    except Exception as e:
+        print(f"   module_rmsnorm warmup failed: {e}")
+
+    # Warmup module_rmsnorm via rmsnorm2d_fwd with large hidden_size (CK path)
+    # When hidden_size > 8192, rmsnorm2d_fwd takes the rmsnorm2d_fwd_ck path
+    # which also uses module_rmsnorm (already compiled in step 2, but this
+    # ensures the CK rmsnorm2d_fwd path is exercised as well)
+    try:
+        print("\n[3/5] Warming up rmsnorm2d_fwd CK path (hidden>8192)...")
+        from aiter import rmsnorm2d_fwd
+
+        hidden_size = 16384  # > 8192 to trigger rmsnorm2d_fwd_ck (module_rmsnorm)
+        batch_size = 32
+        x = torch.randn(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+        weight = torch.ones(hidden_size, dtype=torch.bfloat16, device=device)
+        eps = 1e-6
+
+        _ = rmsnorm2d_fwd(x, weight, eps)
+        torch.cuda.synchronize()
+        print("   rmsnorm2d_fwd CK path compiled successfully")
+    except Exception as e:
+        print(f"   rmsnorm2d_fwd CK path warmup skipped: {e}")
+
+    # Warmup rotary embedding kernel if available
+    try:
+        print("\n[4/5] Warming up rotary embedding kernel...")
+        from aiter import rotary_embedding
+
+        head_size = 128
+        seq_len = 32
+        num_heads = 32
+        positions = torch.arange(seq_len, device=device)
+        query = torch.randn(
+            seq_len, num_heads, head_size, dtype=torch.bfloat16, device=device
+        )
+        key = torch.randn(
+            seq_len, num_heads, head_size, dtype=torch.bfloat16, device=device
+        )
+        cos = torch.ones(seq_len, head_size // 2, dtype=torch.bfloat16, device=device)
+        sin = torch.zeros(seq_len, head_size // 2, dtype=torch.bfloat16, device=device)
+
+        _ = rotary_embedding(positions, query, key, head_size, cos, sin, True)
+        torch.cuda.synchronize()
+        print("   Rotary embedding kernel compiled successfully")
+    except Exception as e:
+        print(f"   Rotary embedding warmup skipped (may not be available): {e}")
+
+    # Warmup activation kernels if available
+    try:
+        print("\n[5/5] Warming up activation kernels...")
+        from aiter import silu_and_mul
+
+        hidden_size = 4096
+        batch_size = 512
+        x = torch.randn(
+            batch_size, hidden_size * 2, dtype=torch.bfloat16, device=device
+        )
+        out = torch.empty(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+
+        silu_and_mul(out, x)
+        torch.cuda.synchronize()
+        print("   Activation kernel compiled successfully")
+    except Exception as e:
+        print(f"   Activation warmup skipped (may not be available): {e}")
+
+    elapsed = time.time() - start_time
+    print("\n" + "=" * 60)
+    print(f"AITER warmup completed in {elapsed:.1f}s")
+    print("=" * 60 + "\n")
+
+
+if __name__ == "__main__":
+    warmup_aiter_kernels()

scripts/ci/amd/check_vram_clear.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+check_vram_clear() {
+    local vram_threshold_percent=5  # Allow up to 5% VRAM usage
+    local memory_threshold_mb=500   # Allow up to 500MB memory usage
+
+    if command -v rocm-smi >/dev/null 2>&1; then
+        echo "Checking ROCm GPU VRAM usage..."
+        # Check if any GPU has more than threshold VRAM allocated
+        local high_usage=$(rocm-smi --showmemuse | grep -E "GPU Memory Allocated \(VRAM%\): ([6-9]|[1-9][0-9]|100)")
+        if [ -n "$high_usage" ]; then
+            echo "ERROR: VRAM usage exceeds threshold (${vram_threshold_percent}%) on some GPUs:"
+            echo "$high_usage"
+            rocm-smi --showmemuse
+            return 1
+        else
+            echo "✓ VRAM usage is within acceptable limits on all GPUs"
+            return 0
+        fi
+   fi
+}
+
+# If this script is run directly (not sourced), run the check
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    set -e
+    check_vram_clear
+fi

scripts/ci/amd/ensure_vram_clear.sh

@@ -0,0 +1,269 @@
+#!/bin/bash
+
+# Source the VRAM checking function
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/check_vram_clear.sh"
+
+# Stop and remove every container that holds any /dev/kfd or /dev/dri device.
+# Some failing CI runs leave behind containers other than `ci_sglang` (e.g.
+# from previous AMD jobs that were force-killed mid-run); those still hold
+# VRAM via KFD even though the host pgrep finds nothing.
+stop_all_gpu_containers() {
+    if ! command -v docker >/dev/null 2>&1; then
+        return 0
+    fi
+
+    local all_ids
+    all_ids=$(docker ps -aq 2>/dev/null || true)
+    if [ -z "$all_ids" ]; then
+        echo "No docker containers found on host"
+        return 0
+    fi
+
+    local gpu_ids=""
+    local cid
+    for cid in $all_ids; do
+        # A container is "GPU-attached" if its inspect output mentions any
+        # GPU device or NVIDIA/ROCm GPU capability. Inspecting the raw JSON
+        # (instead of a specific field) survives docker version differences.
+        if docker inspect "$cid" 2>/dev/null \
+            | grep -qE '"PathOnHost":"/dev/(kfd|dri)|"Capabilities":\[\["gpu"\]\]'; then
+            gpu_ids+=" $cid"
+        fi
+    done
+
+    gpu_ids=$(echo "$gpu_ids" | tr ' ' '\n' | grep -E '^[a-f0-9]+$' || true)
+    if [ -z "$gpu_ids" ]; then
+        echo "No GPU-attached docker containers found on host"
+        return 0
+    fi
+
+    echo "Found GPU-attached containers, stopping them:"
+    for cid in $gpu_ids; do
+        docker ps -a --filter "id=$cid" --format '  {{.ID}} {{.Image}} {{.Status}} {{.Names}}' 2>/dev/null || true
+    done
+    echo "$gpu_ids" | xargs -r docker stop --time 5 2>/dev/null || true
+    echo "$gpu_ids" | xargs -r docker rm -f 2>/dev/null || true
+}
+
+# Find and kill any host process that holds an open handle to /dev/kfd or
+# /dev/dri/renderD*. This is far more reliable than `rocm-smi --showpids`,
+# which only sees processes that registered a HSA queue (zombies and
+# processes that crashed mid-init are invisible to it).
+kill_processes_holding_gpu_devices() {
+    local signal=${1:-TERM}
+    local pids=""
+
+    # If neither tool is present we silently degrade to a no-op, which used
+    # to look identical in the log to "no holders found" and made the
+    # script's failures very confusing. Emit a loud warning so the runner
+    # owner knows why GPU device cleanup isn't happening.
+    if ! command -v fuser >/dev/null 2>&1 && ! command -v lsof >/dev/null 2>&1; then
+        echo "WARNING: neither fuser nor lsof installed on the host;" \
+             "cannot detect processes holding /dev/kfd or /dev/dri/renderD*."
+        echo "         Install psmisc (for fuser) or lsof on the runner host" \
+             "to enable device-fd-based cleanup."
+        return 0
+    fi
+
+    if command -v fuser >/dev/null 2>&1; then
+        # `fuser` prints PIDs to stdout, names to stderr; collect everything
+        # that has any handle on KFD or render nodes.
+        pids+=" $(fuser /dev/kfd 2>/dev/null || true)"
+        for dev in /dev/dri/renderD*; do
+            [ -e "$dev" ] || continue
+            pids+=" $(fuser "$dev" 2>/dev/null || true)"
+        done
+    fi
+
+    if command -v lsof >/dev/null 2>&1; then
+        pids+=" $(lsof -t /dev/kfd 2>/dev/null || true)"
+        for dev in /dev/dri/renderD*; do
+            [ -e "$dev" ] || continue
+            pids+=" $(lsof -t "$dev" 2>/dev/null || true)"
+        done
+    fi
+
+    pids=$(echo "$pids" | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -u || true)
+    if [ -z "$pids" ]; then
+        return 0
+    fi
+
+    local self_pid=$$
+    echo "Processes holding /dev/kfd or /dev/dri/renderD*:"
+    for pid in $pids; do
+        # Skip our own PID and any of our ancestors so we don't suicide.
+        if [ "$pid" = "$self_pid" ]; then
+            continue
+        fi
+        local cmd
+        cmd=$(ps -p "$pid" -o pid,ppid,stat,cmd --no-headers 2>/dev/null || true)
+        if [ -z "$cmd" ]; then
+            continue
+        fi
+        echo "  $cmd"
+        # If it's a zombie, kill the parent instead — kill -9 on a zombie is
+        # a no-op, the only way to reap it is to make its parent reap it.
+        local stat
+        stat=$(ps -p "$pid" -o stat= 2>/dev/null | tr -d ' ')
+        if [[ "$stat" == Z* ]]; then
+            local ppid
+            ppid=$(ps -p "$pid" -o ppid= 2>/dev/null | tr -d ' ')
+            if [ -n "$ppid" ] && [ "$ppid" != "1" ] && [ "$ppid" != "$self_pid" ]; then
+                echo "    -> $pid is a zombie, sending SIG$signal to parent $ppid"
+                kill "-$signal" "$ppid" 2>/dev/null || true
+            fi
+            continue
+        fi
+        kill "-$signal" "$pid" 2>/dev/null || true
+    done
+}
+
+# Print rich diagnostics that explain *why* VRAM is still allocated when no
+# obvious owner exists. Helpful when zombies / other namespaces hold memory.
+dump_gpu_diagnostics() {
+    echo "=== GPU device file holders (fuser) ==="
+    if command -v fuser >/dev/null 2>&1; then
+        fuser -v /dev/kfd 2>&1 || true
+        for dev in /dev/dri/renderD* /dev/dri/card*; do
+            [ -e "$dev" ] || continue
+            fuser -v "$dev" 2>&1 || true
+        done
+    else
+        echo "fuser not installed"
+    fi
+
+    echo "=== GPU device file holders (lsof) ==="
+    if command -v lsof >/dev/null 2>&1; then
+        lsof /dev/kfd 2>/dev/null || true
+        lsof /dev/dri/renderD* 2>/dev/null || true
+    else
+        echo "lsof not installed"
+    fi
+
+    echo "=== Zombie processes on host ==="
+    ps -eo pid,ppid,stat,etime,cmd 2>/dev/null | awk 'NR==1 || $3 ~ /^Z/ {print}'
+
+    echo "=== Docker containers on host ==="
+    if command -v docker >/dev/null 2>&1; then
+        docker ps -a --format 'table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Names}}' 2>/dev/null || true
+    fi
+
+    echo "=== rocm-smi --showpids ==="
+    timeout 30 rocm-smi --showpids 2>&1 || echo "rocm-smi --showpids timed out"
+
+    echo "=== rocm-smi --showmemuse ==="
+    timeout 30 rocm-smi --showmemuse 2>&1 || echo "rocm-smi --showmemuse timed out"
+}
+
+ensure_vram_clear() {
+    local max_retries=3
+    local retry_count=0
+
+    # Log host information for debugging
+    echo "=== Host Information ==="
+    echo "Hostname: $(hostname)"
+    echo "Host IP: $(hostname -I 2>/dev/null || echo 'N/A')"
+    echo "Date: $(date)"
+    echo "Mode: rocm"
+    echo "========================"
+    echo "Running in ROCm mode"
+
+    # Always stop the well-known CI container first (best-effort).
+    echo "Stopping any existing ci_sglang container..."
+    docker stop ci_sglang 2>/dev/null || true
+    docker rm -f ci_sglang 2>/dev/null || true
+
+    # Show initial GPU status
+    echo "=== Initial GPU Memory Status ==="
+    rocm-smi --showmemuse
+    echo "=================================="
+
+    # Fast path: if the runner is already clean, skip the cleanup loop
+    # entirely so healthy jobs don't pay the ~35s/attempt cleanup cost.
+    if check_vram_clear; then
+        echo "✓ VRAM is already clear; skipping cleanup."
+        return 0
+    fi
+
+    while [ $retry_count -lt $max_retries ]; do
+        echo "=== Cleanup Attempt $((retry_count + 1))/$max_retries ==="
+
+        # Step 1: kill SGLang-named processes on the host (cheap, fast).
+        # NOTE: host pgrep cannot see PIDs inside a container's PID
+        # namespace, so in CI this almost never matches anything; the
+        # heavy lifting is done by step 2 below. Kept as a fast early
+        # cleanup for the rare case where something runs on the host.
+        echo "Killing SGLang processes..."
+        pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' \
+            | xargs -r kill -9 2>/dev/null || true
+
+        # Step 2: aggressive cleanup. Run on EVERY attempt — the previous
+        # version skipped this on attempt 1, which made attempt 1 a near
+        # no-op for the most common failure mode (a leftover container
+        # holding VRAM, invisible to host pgrep).
+        echo "Performing aggressive cleanup..."
+
+        # 2a. Stop ALL GPU-attached containers, not just ci_sglang. A
+        # leftover container from a previous job will keep VRAM held even
+        # though `pgrep` on the host shows nothing.
+        stop_all_gpu_containers
+
+        # 2b. SIGTERM anything that has /dev/kfd or /dev/dri/renderD* open.
+        # `lsof`/`fuser` see processes that `rocm-smi --showpids` misses
+        # (notably zombies and processes outside our PID namespace).
+        echo "Sending SIGTERM to processes holding GPU device files..."
+        kill_processes_holding_gpu_devices TERM
+        sleep 5
+
+        # 2c. SIGKILL anything still holding GPU device files.
+        echo "Sending SIGKILL to remaining holders..."
+        kill_processes_holding_gpu_devices KILL
+
+        # 2d. Best-effort: also kill anything `rocm-smi --showpids` reports.
+        # Handles both the legacy "PID: <n>" line format and the modern
+        # tabular format (`<pid>\t<name>\t<gpus>\t...`); the previous
+        # `grep 'PID:'` matched nothing on ROCm 5+ tabular output.
+        rocm-smi --showpids 2>/dev/null \
+            | awk '/^PID:[[:space:]]*[0-9]+/ {print $2} /^[0-9]+/ {print $1}' \
+            | xargs -r kill -9 2>/dev/null || true
+
+        echo "Waiting 30 seconds for VRAM to clear..."
+        sleep 30
+
+        # Step 3: re-check.
+        echo "Checking VRAM status..."
+        if check_vram_clear; then
+            echo "✓ VRAM cleanup successful after $((retry_count + 1)) attempts"
+            return 0
+        else
+            echo "✗ VRAM still not clear after attempt $((retry_count + 1))"
+            # Step 4: dump diagnostics on every failed attempt so the next
+            # attempt's logs already explain WHY cleanup didn't work.
+            # Without this we'd only see what's holding the GPU at the very
+            # end, which makes triage much harder.
+            echo "--- Diagnostics for failed attempt $((retry_count + 1)) ---"
+            dump_gpu_diagnostics
+            echo "--- End of diagnostics for attempt $((retry_count + 1)) ---"
+            retry_count=$((retry_count + 1))
+        fi
+    done
+
+    # Failed after all retries — diagnostics for the last cleanup attempt
+    # were already dumped above; just print the actionable hint.
+    echo "=== FAILED: VRAM cleanup unsuccessful after $max_retries attempts ==="
+    echo "(See diagnostics above for the final attempt.)"
+    echo "=================================================================="
+    echo "Hint: if no host process / container holds the GPU but VRAM is"
+    echo "still allocated, this is almost certainly a zombie KFD context"
+    echo "(see ROCm/aiter#2061). The node will need to be rebooted before"
+    echo "subsequent jobs can succeed."
+    echo "=================================================================="
+    return 1
+}
+
+# If this script is run directly (not sourced), run the ensure function
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    set -e
+    ensure_vram_clear "$@"
+fi

scripts/ci/amd/test_rccl_multi_gpu.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Simple RCCL test for multi-GPU communication.
+This test verifies that RCCL can initialize and communicate across multiple GPUs.
+"""
+
+import os
+import sys
+
+import torch
+import torch.distributed as dist
+
+
+def test_rccl_allreduce():
+    """Test basic RCCL allreduce operation across all GPUs."""
+    if not torch.cuda.is_available():
+        print("CUDA not available, skipping test")
+        sys.exit(1)
+
+    # Initialize process group with NCCL (RCCL on AMD)
+    dist.init_process_group(backend="nccl")
+
+    rank = dist.get_rank()
+    world_size = dist.get_world_size()
+
+    print(f"[Rank {rank}/{world_size}] Initialized successfully")
+
+    # Set device
+    device = torch.device(f"cuda:{rank}")
+    torch.cuda.set_device(device)
+
+    print(f"[Rank {rank}] Device: {torch.cuda.get_device_name(device)}")
+    print(
+        f"[Rank {rank}] Device memory: {torch.cuda.get_device_properties(device).total_memory / 1e9:.2f} GB"
+    )
+
+    # Create a tensor and perform allreduce
+    tensor = torch.ones(1000, device=device) * rank
+    print(f"[Rank {rank}] Before allreduce: tensor sum = {tensor.sum().item()}")
+
+    dist.all_reduce(tensor, op=dist.ReduceOp.SUM)
+
+    expected_sum = sum(range(world_size)) * 1000
+    actual_sum = tensor.sum().item()
+
+    print(
+        f"[Rank {rank}] After allreduce: tensor sum = {actual_sum}, expected = {expected_sum}"
+    )
+
+    if abs(actual_sum - expected_sum) < 0.1:
+        print(f"[Rank {rank}] ✓ RCCL allreduce test PASSED")
+        dist.destroy_process_group()
+        sys.exit(0)
+    else:
+        print(f"[Rank {rank}] ✗ RCCL allreduce test FAILED")
+        dist.destroy_process_group()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    test_rccl_allreduce()

scripts/ci/check_no_docs_changes.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Reject staged changes under the legacy docs/ tree."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+ERROR_MESSAGE = """\
+Changes under the legacy docs/ directory are not allowed.
+
+The documentation has been migrated. Please make documentation updates in the
+corresponding location under docs_new/ instead.
+"""
+
+LEGACY_DOCS_ALLOWLIST = {
+    "docs/_static/css/custom_log.css",
+    "docs/_static/js/deprecation_banner.js",
+    "docs/conf.py",
+    # Has relative links into the source tree that the offline lychee check
+    # validates, so it must be updated when the linked source files move.
+    "docs/developer_guide/development_jit_kernel_guide.md",
+}
+
+
+def staged_paths() -> list[str]:
+    result = subprocess.run(
+        [
+            "git",
+            "diff",
+            "--cached",
+            "--name-only",
+            "--diff-filter=ACMRDTUXB",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+
+
+def main() -> int:
+    paths = sys.argv[1:] or staged_paths()
+    docs_paths = sorted(
+        path
+        for path in paths
+        if (path == "docs" or path.startswith("docs/"))
+        and path not in LEGACY_DOCS_ALLOWLIST
+    )
+
+    if not docs_paths:
+        return 0
+
+    print(ERROR_MESSAGE, file=sys.stderr)
+    print("Detected legacy docs/ changes:", file=sys.stderr)
+    for path in docs_paths:
+        print(f"  - {path}", file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ci/check_no_registered_tests_in_package.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""
+Pre-commit hook: reject CI-registered tests that live inside the importable
+`sglang` package (python/sglang/).
+
+Registered tests and benchmarks must live under test/registered/ (e.g.
+test/registered/jit/ for JIT kernel tests and test/registered/jit/benchmark/
+for JIT kernel benchmarks) so they are not shipped in the wheel and are
+collected by run_suite.py's registered glob. A registered file placed inside
+the package would be shipped to users AND silently dropped by run_suite.py
+(which no longer globs the package) -- it would never run in CI. This guard
+turns that silent skip into a hard failure.
+
+Reuses ut_parse_one_file() from ci_register.py (AST-based) so the registry
+detection matches run_suite.py's collect_tests() exactly.
+"""
+
+import glob
+import importlib.util
+import os
+import sys
+
+# Markers whose mere presence in the source is worth an AST parse. Anything
+# without one of these strings cannot register a test, so we skip parsing it.
+_MARKERS = (
+    "register_cuda_ci",
+    "register_amd_ci",
+    "register_cpu_ci",
+    "register_npu_ci",
+    "register_xpu_ci",
+    "register_musa_ci",
+)
+
+
+def main() -> int:
+    # Import ci_register directly to avoid pulling in all of sglang.
+    spec = importlib.util.spec_from_file_location(
+        "ci_register",
+        os.path.join("python", "sglang", "test", "ci", "ci_register.py"),
+    )
+    ci_register = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(ci_register)
+
+    offenders = []
+    for f in sorted(glob.glob("python/sglang/**/*.py", recursive=True)):
+        try:
+            with open(f, "r", encoding="utf-8") as fh:
+                source = fh.read()
+        except (OSError, UnicodeDecodeError):
+            continue
+        if not any(marker in source for marker in _MARKERS):
+            continue
+        try:
+            registries, _has_main_entry = ci_register.ut_parse_one_file(f)
+        except Exception:
+            # A malformed register call still indicates a misplaced test.
+            offenders.append(f)
+            continue
+        if registries:
+            offenders.append(f)
+
+    if offenders:
+        print(
+            "ERROR: CI-registered test(s)/benchmark(s) found inside the sglang package:"
+        )
+        print(
+            "  Registered tests and benchmarks must live under test/registered/\n"
+            "  (e.g. test/registered/jit/ for JIT kernel tests and\n"
+            "  test/registered/jit/benchmark/ for JIT kernel benchmarks) so they\n"
+            "  are not shipped in the wheel and are collected by run_suite.py.\n"
+        )
+        for f in offenders:
+            print(f"  {f}")
+        print()
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/check_registered_tests.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+Pre-commit hook: validate CI registry calls under test/registered/.
+
+1. Every test file must contain a CI registry call (register_cuda_ci,
+   register_amd_ci, etc.).
+2. A CUDA test must register its PR-test suite via the modern
+   `stage=`/`runner_config=` form. The legacy single-string `suite=` is reserved
+   for the nightly/stress/weekly families (and for AMD/CPU/NPU suites); any other
+   CUDA `suite=` resolves to a name no PR-test workflow invokes, so the test
+   silently never runs. Two shapes are rejected:
+     a. `{stage}-test-{runner_config}` -- the modern name stuffed back into the
+        legacy form. Reported with the exact stage/runner split to use.
+     b. an older `{stage}-{runner_config}` PR-test name (e.g. the pre-migration
+        `base-b-kernel-unit-1-gpu-large`) -- no longer matches any workflow
+        suite at all.
+   The modern form resolves to the identical suite (CIRegistry.effective_suite
+   is f"{stage}-test-{runner_config}") and is /rerun-test-able.
+
+Reuses ut_parse_one_file() from ci_register.py (AST-based parsing)
+to match the same logic used by run_suite.py's collect_tests().
+"""
+
+import glob
+import importlib.util
+import os
+import re
+import sys
+
+# Suite names of the form `{stage}-test-{runner_config}` are exactly what the
+# modern stage=/runner_config= form produces, so a legacy suite= carrying this
+# shape is always expressible (and should be expressed) the modern way.
+_MODERN_SHAPE = re.compile(r"^(.+)-test-(.+)$")
+
+# The only suite families a CUDA registry may keep on the legacy single-string
+# `suite=` form. Everything else is a PR-test/base stage that must use the
+# modern stage=/runner_config= form (otherwise its effective_suite matches no
+# suite the PR-test workflows invoke, and the test silently never runs).
+_LEGACY_CUDA_PREFIXES = ("nightly", "stress", "weekly")
+
+
+def main() -> int:
+    # Import ci_register directly to avoid pulling in all of sglang
+    spec = importlib.util.spec_from_file_location(
+        "ci_register",
+        os.path.join("python", "sglang", "test", "ci", "ci_register.py"),
+    )
+    ci_register = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(ci_register)
+    cuda = ci_register.HWBackend.CUDA
+
+    # Same filter as run_suite.py: skip conftest.py, __init__.py, and utils.py
+    files = sorted(
+        f
+        for f in glob.glob("test/registered/**/*.py", recursive=True)
+        if os.path.basename(f) not in ("conftest.py", "__init__.py", "utils.py")
+    )
+    if not files:
+        return 0
+
+    missing = []
+    legacy_shape = []  # (file, suite, stage, runner_config) -- has a -test- split
+    non_dispatchable = []  # (file, suite) -- legacy CUDA suite no workflow invokes
+    for f in files:
+        try:
+            registries, _has_main_entry = ci_register.ut_parse_one_file(f)
+        except Exception:
+            # Skip files that can't be parsed (syntax errors, etc.)
+            continue
+        if len(registries) == 0:
+            missing.append(f)
+            continue
+        for r in registries:
+            # Pure legacy form on a CUDA registry: suite set, stage/runner unset.
+            if not (
+                r.backend == cuda
+                and r.suite is not None
+                and r.stage is None
+                and r.runner_config is None
+            ):
+                continue
+            # nightly/stress/weekly are the only CUDA suites allowed to stay on
+            # the legacy single-string form.
+            if r.suite.split("-", 1)[0] in _LEGACY_CUDA_PREFIXES:
+                continue
+            m = _MODERN_SHAPE.match(r.suite)
+            if m:
+                legacy_shape.append((f, r.suite, m.group(1), m.group(2)))
+            else:
+                non_dispatchable.append((f, r.suite))
+
+    exit_code = 0
+    if missing:
+        print("ERROR: Files in test/registered/ missing CI registry call:")
+        print("  Move manual-only tests to test/manual/.\n")
+        for f in missing:
+            print(f"  {f}")
+        print()
+        exit_code = 1
+    if legacy_shape:
+        print(
+            "ERROR: CUDA test(s) register a `{stage}-test-{runner_config}`-shaped "
+            'suite via the legacy `suite="..."` form, which is not dispatchable '
+            "via /rerun-test. Switch to the modern `stage=`/`runner_config=` form "
+            "(same stage, same runner):\n"
+        )
+        for f, suite, stage, runner_config in legacy_shape:
+            print(
+                f"  {f}\n"
+                f'    suite="{suite}"'
+                f'  ->  stage="{stage}", runner_config="{runner_config}"'
+            )
+        print()
+        exit_code = 1
+    if non_dispatchable:
+        print(
+            'ERROR: CUDA test(s) register a legacy `suite="..."` that is neither a '
+            "nightly/stress/weekly suite nor the modern `stage=`/`runner_config=` "
+            "form. This name matches no suite the PR-test workflows invoke, so the "
+            "test silently never runs. Switch to the modern form:\n"
+        )
+        for f, suite in non_dispatchable:
+            print(
+                f"  {f}\n"
+                f'    suite="{suite}"'
+                f'  ->  stage="...", runner_config="..."'
+            )
+        print()
+        exit_code = 1
+
+    return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/check_workflow_job_names.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Check that required status check job names are unique across workflows.
+
+Duplicate job names on the same commit allow a passing job in one workflow
+to satisfy a required status check meant for a different workflow, bypassing
+branch protection.
+
+See: https://github.com/sgl-project/sglang/pull/20208 for an example where
+pr-test-npu.yml's "pr-test-finish" job (which passed) caused GitHub to treat
+the required "pr-test-finish" check (from pr-test.yml, which failed) as met.
+"""
+
+import glob
+import sys
+from collections import defaultdict
+
+import yaml
+
+# Job names used as required status checks in branch protection.
+# These MUST be unique across all workflow files.
+PROTECTED_JOB_NAMES = {
+    "pr-test-finish",
+    "lint",
+}
+
+
+def main() -> int:
+    workflows = sorted(glob.glob(".github/workflows/*.yml"))
+    job_to_files: dict[str, list[str]] = defaultdict(list)
+
+    for wf in workflows:
+        with open(wf, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        if not data or "jobs" not in data:
+            continue
+        for job in data["jobs"]:
+            if job in PROTECTED_JOB_NAMES:
+                job_to_files[job].append(wf)
+
+    duplicates = {job: files for job, files in job_to_files.items() if len(files) > 1}
+
+    if not duplicates:
+        return 0
+
+    print("ERROR: Required status check job names must be unique across workflows.")
+    print("Duplicates allow branch protection bypass via auto-merge.\n")
+    for job, files in sorted(duplicates.items()):
+        print(f"  Job '{job}' appears in:")
+        for f in files:
+            print(f"    - {f}")
+        print()
+
+    print("Fix: rename the job in non-primary workflows to avoid collision.")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/cuda/ci_cleanup_venv.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Remove the per-job uv venv created by ci_install_dependency.sh.
+#
+# Meant to run in a post-job workflow step with `if: always()` so the venv is
+# destroyed even on job failure/cancel. Runner-level safety net: a cron or
+# startup task should also purge stale /tmp/sglang-ci-* directories to catch
+# cancelled or crashed jobs that never reached this cleanup.
+
+# Best-effort cleanup: never fail the job.
+set +e
+set -u
+
+# Bound the persistent uv cache (~/.cache/uv, bind-mounted and shared across all
+# runner containers). Nothing else evicts it, so it grows unbounded — on the 5090
+# hosts it reached ~500 GB and filled the disk, failing jobs with ENOSPC at
+# dependency install. Prune only under real disk pressure so healthy jobs pay
+# nothing (just a df check) and no rebuildable wheel is dropped until it matters:
+# `uv cache prune --ci` keeps downloaded wheels + sdist archives (no re-download)
+# but drops built wheels, so a later install may recompile source-built packages.
+# Best effort; runs regardless of venv mode; never fails the job.
+if command -v uv >/dev/null 2>&1; then
+    cache_dir="$(uv cache dir 2>/dev/null || echo "${HOME:-/root}/.cache/uv")"
+    [ -d "$cache_dir" ] || cache_dir=/
+    used="$(df --output=pcent "$cache_dir" 2>/dev/null | tr -dc '0-9')"
+    if [ "${used:-0}" -ge 85 ]; then
+        echo "uv cache filesystem at ${used}% — pruning"
+        uv cache prune --ci >/dev/null 2>&1 || true
+    fi
+fi
+
+# Skip entirely when venv mode is disabled — no /tmp/sglang-ci-* dir exists
+# and there's nothing to sweep. Matches the USE_VENV parsing in
+# ci_install_dependency.sh (accepts 1/true/yes, case-insensitive).
+USE_VENV_RAW="${USE_VENV:-true}"
+case "$(printf '%s' "$USE_VENV_RAW" | tr '[:upper:]' '[:lower:]')" in
+    1 | true | yes) ;;
+    *)
+        echo "USE_VENV=${USE_VENV_RAW}: skipping venv cleanup"
+        exit 0
+        ;;
+esac
+
+# Prefer the path propagated via GITHUB_ENV. Fallback: glob for any venv from
+# this run+job (covers the case where install crashed before exporting the path).
+if [ -n "${SGLANG_CI_VENV_PATH:-}" ] && [ -d "$SGLANG_CI_VENV_PATH" ]; then
+    if rm -rf "$SGLANG_CI_VENV_PATH"; then
+        echo "Cleaned up venv: $SGLANG_CI_VENV_PATH"
+    else
+        echo "::warning::Failed to remove $SGLANG_CI_VENV_PATH — runner cron should sweep /tmp/sglang-ci-*"
+    fi
+else
+    matched=0
+    for venv in /tmp/sglang-ci-${GITHUB_RUN_ID:-unknownrun}-${GITHUB_JOB:-unknownjob}-*; do
+        [ -d "$venv" ] || continue
+        matched=1
+        if rm -rf "$venv"; then
+            echo "Cleaned up venv (via glob): $venv"
+        else
+            echo "::warning::Failed to remove $venv — runner cron should sweep /tmp/sglang-ci-*"
+        fi
+    done
+    [ "$matched" -eq 0 ] && echo "No venv to clean for run=${GITHUB_RUN_ID:-?} job=${GITHUB_JOB:-?}"
+fi
+
+# Sweep stale venvs from cancelled/crashed jobs that never reached cleanup.
+# Any /tmp/sglang-ci-* dir older than 4 hours is considered orphaned.
+stale_count=0
+for venv in /tmp/sglang-ci-*; do
+    [ -d "$venv" ] || continue
+    if find "$venv" -maxdepth 0 -mmin +240 -print -quit | grep -q .; then
+        rm -rf "$venv" && stale_count=$((stale_count + 1))
+    fi
+done
+[ "$stale_count" -gt 0 ] && echo "Swept $stale_count stale venv(s) older than 4h"
+
+exit 0

scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# Install flashinfer-jit-cache with caching and retry logic (flashinfer.ai can have transient DNS issues).
+# The jit-cache wheel is 1.2+ GB, so we skip the download entirely if already installed.
+#
+# Required environment (caller must export or set):
+#   UNINSTALL_JIT_CACHE          — literal true/false (skip download when false)
+#   FLASHINFER_PYTHON_REQUIRED   — e.g. from python/pyproject.toml (flashinfer_python)
+#   CU_VERSION                   — e.g. cu130
+#   PIP_CMD                      — e.g. "pip" or "uv pip"
+#   PIP_INSTALL_SUFFIX           — extra pip args for this runner
+set -euxo pipefail
+
+: "${UNINSTALL_JIT_CACHE:?must be set}"
+: "${FLASHINFER_PYTHON_REQUIRED:?must be set}"
+: "${CU_VERSION:?must be set}"
+: "${PIP_CMD:?must be set}"
+
+FLASHINFER_JIT_CACHE_INSTALLED=false
+if [ "$UNINSTALL_JIT_CACHE" = false ]; then
+    FLASHINFER_JIT_CACHE_INSTALLED=true
+    echo "flashinfer-jit-cache already at correct version, skipping download"
+fi
+
+if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then
+    FLASHINFER_CACHE_DIR="${HOME}/.cache/flashinfer-wheels"
+    mkdir -p "${FLASHINFER_CACHE_DIR}"
+
+    FLASHINFER_WHEEL_PATTERN="flashinfer_jit_cache-${FLASHINFER_PYTHON_REQUIRED}+${CU_VERSION}*.whl"
+    CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
+
+    if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
+        echo "Found cached flashinfer wheel: $CACHED_WHEEL"
+        if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
+            FLASHINFER_JIT_CACHE_INSTALLED=true
+            echo "Successfully installed flashinfer-jit-cache from cache"
+        else
+            echo "Failed to install from cache, will try downloading..."
+            rm -f "$CACHED_WHEEL"
+        fi
+    fi
+
+    if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then
+        for i in {1..5}; do
+            # Download wheel to cache directory (use pip directly as uv pip doesn't support download)
+            if timeout 600 pip download "flashinfer-jit-cache==${FLASHINFER_PYTHON_REQUIRED}" \
+                --index-url "https://flashinfer.ai/whl/${CU_VERSION}" \
+                -d "${FLASHINFER_CACHE_DIR}"; then
+
+                CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
+                if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
+                    if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
+                        FLASHINFER_JIT_CACHE_INSTALLED=true
+                        echo "Successfully downloaded and installed flashinfer-jit-cache"
+                        break
+                    fi
+                else
+                    echo "Warning: Download succeeded but wheel file not found"
+                fi
+            fi
+            echo "Attempt $i to download flashinfer-jit-cache failed, retrying in 10 seconds..."
+            sleep 10
+        done
+    fi
+fi
+
+if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then
+    echo "ERROR: Failed to install flashinfer-jit-cache after 5 attempts"
+    exit 1
+fi

scripts/ci/cuda/ci_install_deepep.sh

@@ -0,0 +1,177 @@
+#!/bin/bash
+# Install the dependency in CI.
+set -euxo pipefail
+
+# Source (not bash) so that venv activation, $PIP_CMD, $CU_VERSION, $NVCC_VER, and
+# $PIP_INSTALL_SUFFIX all propagate into this shell. Without sourcing, the subshell
+# exits and this script would fall back to system Python.
+#
+# Note: any `exit N` or `set -e` trip inside the sourced script terminates *this*
+# script too (bash runs sourced commands in the current shell, so `exit` is not
+# caught by `if`/`||`). The real error message appears upstream in the log.
+# shellcheck disable=SC1091
+source scripts/ci/cuda/ci_install_dependency.sh
+
+# In venv mode, PIP_CMD must be set by the sourced script. If it isn't, the
+# source chain is broken and we'd silently fall back to system `pip` below —
+# exactly the split-install bug the migration is meant to prevent.
+if [ -z "${PIP_CMD:-}" ]; then
+    echo "FATAL:PIP_CMD is unset after sourcing ci_install_dependency.sh"
+    exit 1
+fi
+
+export GDRCOPY_HOME=/usr/src/gdrdrv-2.5.1/
+export CUDA_HOME=/usr/local/cuda
+
+GRACE_BLACKWELL=${GRACE_BLACKWELL:-0}
+# Detect architecture
+ARCH=$(uname -m)
+if [ "$ARCH" != "x86_64" ] && [ "$ARCH" != "aarch64" ]; then
+    echo "Unsupported architecture: $ARCH"
+    exit 1
+fi
+
+if [ "${FORCE_REBUILD_DEEPEP:-0}" = "1" ]; then
+    echo "FORCE_REBUILD_DEEPEP=1; uninstalling any cached deep_ep before rebuild."
+    ${PIP_UNINSTALL_CMD:-pip uninstall -y} deep_ep ${PIP_UNINSTALL_SUFFIX:-} || true
+elif python3 -c "import deep_ep" >/dev/null 2>&1; then
+    echo "deep_ep is already installed or importable. Skipping installation."
+    exit 0
+fi
+
+# Install system dependencies
+# Use fallback logic in case apt fails due to unrelated broken packages on the runner
+DEEPEP_SYSTEM_DEPS="curl wget git sudo rdma-core infiniband-diags openssh-server perftest libibumad3 libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils libnl-3-200 libnl-route-3-200 librdmacm1 build-essential cmake"
+apt-get install -y --no-install-recommends $DEEPEP_SYSTEM_DEPS || {
+    echo "Warning: apt-get install failed, checking if required packages are available..."
+    for pkg in $DEEPEP_SYSTEM_DEPS; do
+        if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+            echo "ERROR: Required package $pkg is not installed and apt-get failed"
+            exit 1
+        fi
+    done
+    echo "All required packages are already installed, continuing..."
+}
+
+# Install GDRCopy
+rm -rf /opt/gdrcopy && mkdir -p /opt/gdrcopy
+cd /opt/gdrcopy
+git clone https://github.com/NVIDIA/gdrcopy.git .
+git checkout v2.5.1
+apt-get update || true  # May fail due to unrelated broken packages
+GDRCOPY_DEPS_1="nvidia-dkms-580"
+GDRCOPY_DEPS_2="build-essential devscripts debhelper fakeroot pkg-config dkms"
+GDRCOPY_DEPS_3="check libsubunit0 libsubunit-dev python3-venv"
+for deps_group in "$GDRCOPY_DEPS_1" "$GDRCOPY_DEPS_2" "$GDRCOPY_DEPS_3"; do
+    apt-get install -y --no-install-recommends $deps_group || {
+        echo "Warning: apt-get install failed for '$deps_group', checking if packages are available..."
+        for pkg in $deps_group; do
+            if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+                echo "ERROR: Required package $pkg is not installed and apt-get failed"
+                exit 1
+            fi
+        done
+        echo "All required packages from '$deps_group' are already installed, continuing..."
+    }
+done
+cd packages
+CUDA=/usr/local/cuda ./build-deb-packages.sh
+dpkg -i gdrdrv-dkms_*.deb
+dpkg -i libgdrapi_*.deb
+dpkg -i gdrcopy-tests_*.deb
+dpkg -i gdrcopy_*.deb
+
+# Set up library paths based on architecture
+LIB_PATH="/usr/lib/$ARCH-linux-gnu"
+if [ ! -e "$LIB_PATH/libmlx5.so" ]; then
+    ln -s $LIB_PATH/libmlx5.so.1 $LIB_PATH/libmlx5.so
+fi
+apt-get update || true
+apt-get install -y --no-install-recommends libfabric-dev || {
+    if ! dpkg -l libfabric-dev 2>/dev/null | grep -q "^ii"; then
+        echo "ERROR: Required package libfabric-dev is not installed and apt-get failed"
+        exit 1
+    fi
+    echo "libfabric-dev is already installed, continuing..."
+}
+
+# Install DeepEP
+DEEPEP_DIR=/root/.cache/deepep
+rm -rf ${DEEPEP_DIR}
+if [ "$GRACE_BLACKWELL" = "1" ]; then
+    GRACE_BLACKWELL_DEEPEP_BRANCH=hybrid-ep
+    git clone https://github.com/deepseek-ai/DeepEP.git -b ${GRACE_BLACKWELL_DEEPEP_BRANCH} ${DEEPEP_DIR} && \
+    pushd ${DEEPEP_DIR} && \
+    git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6 && \
+    sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh && \
+    popd
+else
+    git clone https://github.com/deepseek-ai/DeepEP.git ${DEEPEP_DIR} && \
+    pushd ${DEEPEP_DIR} && \
+    git checkout 9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee && \
+    popd
+fi
+
+cd ${DEEPEP_DIR}
+if [ "$GRACE_BLACKWELL" = "1" ]; then
+    # Resolve the toolkit CUDA version. Preference order:
+    #   1. $NVCC_VER inherited from the sourced ci_install_dependency.sh
+    #      (both scripts agree on the detected value, no re-detection cost).
+    #   2. Local `nvcc --version` (authoritative — container toolkit).
+    #   3. `nvidia-smi` (host driver; last resort).
+    if [ -n "${NVCC_VER:-}" ]; then
+        CUDA_VERSION="$NVCC_VER"
+    elif command -v nvcc >/dev/null 2>&1; then
+        CUDA_VERSION=$(nvcc --version | grep -oP 'release \K[0-9]+\.[0-9]+')
+    else
+        CUDA_VERSION=$(nvidia-smi | grep "CUDA Version" | head -n1 | awk '{print $9}' || true)
+    fi
+    if [ -z "${CUDA_VERSION:-}" ]; then
+        echo "FATAL: could not determine CUDA toolkit version (NVCC_VER unset, nvcc missing, nvidia-smi empty)"
+        exit 1
+    fi
+    if [ "$CUDA_VERSION" = "12.8" ]; then
+        CHOSEN_TORCH_CUDA_ARCH_LIST='10.0'
+    elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then
+        # CUDA > 12.8 supports sm_103 (Blackwell)
+        CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3'
+    else
+        echo "Unsupported CUDA version for Grace Blackwell: $CUDA_VERSION" && exit 1
+    fi && \
+    if [ "${CUDA_VERSION%%.*}" = "13" ]; then \
+        sed -i "/^    include_dirs = \['csrc\/'\]/a\    include_dirs.append('${CUDA_HOME}/include/cccl')" setup.py; \
+    fi
+    TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" ${PIP_CMD:-pip} install --no-build-isolation . ${PIP_INSTALL_SUFFIX:-}
+else
+    # CUDA 13.0 puts CCCL headers in /usr/local/cuda/include/cccl/ but nvshmem
+    # includes them as <cuda/__cccl_config> expecting /usr/local/cuda/include/cuda/.
+    # Add the cccl path to setup.py include_dirs so the compiler finds them.
+    NVCC_MAJOR=$(nvcc --version 2>/dev/null | grep -oP 'release \K[0-9]+' || echo "0")
+    if [ "$NVCC_MAJOR" = "13" ]; then
+        sed -i "/^    include_dirs = \['csrc\/'\]/a\    include_dirs.append('${CUDA_HOME:-/usr/local/cuda}/include/cccl')" setup.py
+    fi
+
+    # Build for both Hopper (sm_90) and Blackwell (sm_100) so the same wheel
+    # runs on H200 and B200 runners. Mirrors the CUDA-version-keyed list in
+    # docker/Dockerfile's DeepEP build stage.
+    if [ -n "${NVCC_VER:-}" ]; then
+        CUDA_VERSION="$NVCC_VER"
+    elif command -v nvcc >/dev/null 2>&1; then
+        CUDA_VERSION=$(nvcc --version | grep -oP 'release \K[0-9]+\.[0-9]+')
+    else
+        CUDA_VERSION=$(nvidia-smi | grep "CUDA Version" | head -n1 | awk '{print $9}' || true)
+    fi
+    if [ -z "${CUDA_VERSION:-}" ]; then
+        echo "FATAL: could not determine CUDA toolkit version (NVCC_VER unset, nvcc missing, nvidia-smi empty)"
+        exit 1
+    fi
+    if [ "$CUDA_VERSION" = "12.8" ]; then
+        CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0'
+    elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then
+        # CUDA > 12.8 supports sm_103 (Blackwell)
+        CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0;10.3'
+    else
+        CHOSEN_TORCH_CUDA_ARCH_LIST='9.0'
+    fi
+    TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" python3 setup.py install
+fi

scripts/ci/cuda/ci_install_dependency.sh

@@ -0,0 +1,613 @@
+#!/bin/bash
+# Install dependencies for CUDA CI jobs.
+#
+# CU_VERSION (default: cu130) controls PyTorch index URL, FlashInfer JIT cache
+# index, and nvrtc variant selection.
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+
+# ---------------------------------------------------------------------------
+# Timing helper
+# ---------------------------------------------------------------------------
+SECONDS=0
+_CI_MARK_PREV=${SECONDS}
+
+mark_step_done() {
+    local label=$1
+    local now=${SECONDS}
+    local step=$((now - _CI_MARK_PREV))
+    printf '\n[STEP DONE] %s,  step: %ss,  total: %ss,  date: %s\n' \
+        "${label}" "${step}" "${now}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+    _CI_MARK_PREV=${now}
+}
+
+# ---------------------------------------------------------------------------
+# Functions
+# ---------------------------------------------------------------------------
+
+configure_environment() {
+    # CU_VERSION controls PyTorch index URL, FlashInfer JIT cache index, and
+    # nvrtc variant selection (cu12 vs cu13).
+    CU_VERSION="${CU_VERSION:-cu130}"
+    CU_STRIP="${CU_VERSION#cu}"
+    CU_MAJOR="${CU_STRIP:0:2}"
+
+    OPTIONAL_DEPS="${1:-}"
+
+    # Whether to create a uv venv (set USE_VENV=1). Default: 0.
+    USE_VENV="${USE_VENV:-0}"
+    echo "USE_VENV=${USE_VENV}"
+
+    python3 -m pip install --upgrade pip
+    if ! command -v uv >/dev/null 2>&1; then
+        pip install uv
+    fi
+
+    SYS_PYTHON_VER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+
+    if [ "$USE_VENV" = "1" ]; then
+        UV_VENV="/tmp/sglang-ci-${GITHUB_RUN_ID:-norun}-${GITHUB_JOB:-nojob}-$$"
+        uv venv "$UV_VENV" --python "python${SYS_PYTHON_VER}" --seed
+        # shellcheck disable=SC1091
+        source "$UV_VENV/bin/activate"
+        [ "${VIRTUAL_ENV:-}" = "$UV_VENV" ] || { echo "FATAL: venv activation did not set VIRTUAL_ENV correctly"; exit 1; }
+        [ "$(command -v python3)" = "$UV_VENV/bin/python3" ] || { echo "FATAL: python3 still resolves outside venv (got $(command -v python3))"; exit 1; }
+
+        if [ -n "${GITHUB_ENV:-}" ]; then
+            # Self-heal: see install_rustup.sh for context on missing _runner_file_commands/.
+            mkdir -p "$(dirname "$GITHUB_ENV")" 2>/dev/null || true
+            echo "VIRTUAL_ENV=$UV_VENV" >> "$GITHUB_ENV" || true
+            echo "SGLANG_CI_VENV_PATH=$UV_VENV" >> "$GITHUB_ENV" || true
+            echo "BASH_ENV=$UV_VENV/env.sh" >> "$GITHUB_ENV" || true
+            touch "$UV_VENV/env.sh"
+        fi
+        if [ -n "${GITHUB_PATH:-}" ]; then
+            mkdir -p "$(dirname "$GITHUB_PATH")" 2>/dev/null || true
+            echo "$UV_VENV/bin" >> "$GITHUB_PATH" || true
+        fi
+    else
+        echo "USE_VENV=0: skipping uv venv creation, installing into system Python"
+        UV_VENV=""
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+detect_host() {
+    ARCH=$(uname -m)
+    echo "Detected architecture: ${ARCH}"
+
+    if [ "${IS_BLACKWELL+set}" = set ]; then
+        case "$IS_BLACKWELL" in 1 | true | yes) IS_BLACKWELL=1 ;; *) IS_BLACKWELL=0 ;; esac
+        echo "IS_BLACKWELL=${IS_BLACKWELL} (manually set via environment)"
+    else
+        IS_BLACKWELL=0
+        if command -v nvidia-smi >/dev/null 2>&1; then
+            while IFS= read -r cap; do
+                major="${cap%%.*}"
+                if [ "${major:-0}" -ge 10 ] 2>/dev/null; then
+                    IS_BLACKWELL=1
+                    break
+                fi
+            done <<< "$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null || true)"
+        fi
+        echo "IS_BLACKWELL=${IS_BLACKWELL} (auto-detected via nvidia-smi)"
+    fi
+
+    if [ "${USE_UV+set}" != set ]; then
+        if [ "$IS_BLACKWELL" = "1" ]; then
+            USE_UV=false
+        else
+            USE_UV=true
+        fi
+    fi
+    case "$(printf '%s' "$USE_UV" | tr '[:upper:]' '[:lower:]')" in 1 | true | yes) USE_UV=1 ;; *) USE_UV=0 ;; esac
+    echo "USE_UV=${USE_UV}"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+kill_existing_processes() {
+    python3 "${REPO_ROOT}/python/sglang/cli/killall.py"
+    KILLALL_EXIT=$?
+    echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}"
+
+    if [ $KILLALL_EXIT -ne 0 ]; then
+        echo "ERROR: killall.py detected uncleanable GPU memory. Aborting CI."
+        exit 1
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+cleanup_stale_shm() {
+    # Reclaim /dev/shm segments leaked by SIGKILLed processes from earlier
+    # jobs; leaked segments accumulate until the tmpfs fills and scheduler
+    # init dies with SIGBUS. Runs right after killall so every dead creator's
+    # segments are reclaimable. The module is dependency-free and runnable by
+    # path, so this works before sglang is installed.
+    SGLANG_IS_IN_CI=true python3 "${REPO_ROOT}/python/sglang/srt/utils/stale_shm_cleanup.py" || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_apt_packages() {
+    apt-get update || true
+    CI_APT_PACKAGES=(
+        python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config
+        libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils
+        ffmpeg libavcodec-dev libavformat-dev libavutil-dev libswscale-dev
+    )
+    apt-get install -y --no-install-recommends "${CI_APT_PACKAGES[@]}" || {
+        echo "Warning: apt-get install failed, checking if required packages are available..."
+        for pkg in "${CI_APT_PACKAGES[@]}"; do
+            if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+                echo "ERROR: Required package $pkg is not installed and apt-get failed"
+                exit 1
+            fi
+        done
+        echo "All required packages are already installed, continuing..."
+    }
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+clean_site_packages() {
+    # Clear torch compilation cache
+    python3 -c 'import os, shutil, tempfile, getpass; cache_dir = os.environ.get("TORCHINDUCTOR_CACHE_DIR") or os.path.join(tempfile.gettempdir(), "torchinductor_" + getpass.getuser()); shutil.rmtree(cache_dir, ignore_errors=True)'
+
+    # Remove broken dist-info directories (missing METADATA per PEP 376)
+    SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])")
+    if [ -d "$SITE_PACKAGES" ]; then
+        { set +x; } 2>/dev/null
+        find "$SITE_PACKAGES" -maxdepth 1 -name "*.dist-info" -type d | while read -r d; do
+            if [ ! -f "$d/METADATA" ]; then
+                echo "Removing broken dist-info: $d"
+                rm -rf "$d"
+            fi
+        done
+        set -x
+    fi
+
+    # Install protoc + Rust toolchain (needed by setuptools-rust, e.g. the native gRPC extension)
+    bash "${SCRIPT_DIR}/../utils/install_rust_protoc.sh"
+    export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+setup_pip_toolchain() {
+    python3 -m pip install --upgrade pip
+
+    if [ "$USE_VENV" != "1" ]; then
+        export UV_SYSTEM_PYTHON=1
+    fi
+
+    export UV_LINK_MODE=copy
+    PIP_CMD="uv pip"
+    PIP_INSTALL_SUFFIX="--index-strategy unsafe-best-match"
+    PIP_UNINSTALL_CMD="uv pip uninstall"
+    PIP_UNINSTALL_SUFFIX=""
+
+    $PIP_UNINSTALL_CMD sgl-kernel sglang-kernel sglang sgl-fa4 flash-attn-4 $PIP_UNINSTALL_SUFFIX || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+remove_stale_cuda12_nvidia_wheels() {
+    if [ "$CU_MAJOR" != "13" ]; then
+        mark_step_done "${FUNCNAME[0]}"
+        return
+    fi
+
+    mapfile -t STALE_CUDA12_NVIDIA_WHEELS < <(
+        python3 -m pip list --format=freeze | sed -n 's/^\(nvidia-.*-cu12\)==.*/\1/p'
+    )
+    if [ ${#STALE_CUDA12_NVIDIA_WHEELS[@]} -eq 0 ]; then
+        echo "No stale CUDA 12 NVIDIA wheels found for ${CU_VERSION} job"
+        mark_step_done "${FUNCNAME[0]}"
+        return
+    fi
+
+    echo "Removing stale CUDA 12 NVIDIA wheels from ${CU_VERSION} job: ${STALE_CUDA12_NVIDIA_WHEELS[*]}"
+    $PIP_UNINSTALL_CMD "${STALE_CUDA12_NVIDIA_WHEELS[@]}" $PIP_UNINSTALL_SUFFIX
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+uninstall_stale_flashinfer() {
+    # Keep flashinfer packages if version matches to avoid re-downloading:
+    # - flashinfer-cubin: 150+ MB
+    # - flashinfer-jit-cache: 1.2+ GB
+    FLASHINFER_PYTHON_REQUIRED=$(grep -Po -m1 'flashinfer_python(\[[^]]+\])?==\K[0-9A-Za-z\.\-]+' python/pyproject.toml || echo "")
+    # flashinfer-cubin is no longer a pyproject dependency (installed explicitly below), tracks the same version as flashinfer_python
+    FLASHINFER_CUBIN_REQUIRED="$FLASHINFER_PYTHON_REQUIRED"
+    FLASHINFER_CUBIN_INSTALLED=$(pip show flashinfer-cubin 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    FLASHINFER_JIT_INSTALLED=$(pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//' || echo "")
+    FLASHINFER_JIT_CU_VERSION=$(pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+//p' || echo "")
+
+    UNINSTALL_CUBIN=true
+    UNINSTALL_JIT_CACHE=true
+
+    if [ "$FLASHINFER_CUBIN_INSTALLED" = "$FLASHINFER_CUBIN_REQUIRED" ] && [ -n "$FLASHINFER_CUBIN_REQUIRED" ]; then
+        echo "flashinfer-cubin==${FLASHINFER_CUBIN_REQUIRED} already installed, keeping it"
+        UNINSTALL_CUBIN=false
+    else
+        echo "flashinfer-cubin version mismatch (installed: ${FLASHINFER_CUBIN_INSTALLED:-none}, required: ${FLASHINFER_CUBIN_REQUIRED}), reinstalling"
+    fi
+
+    if [ "$FLASHINFER_JIT_INSTALLED" = "$FLASHINFER_PYTHON_REQUIRED" ] && [ -n "$FLASHINFER_PYTHON_REQUIRED" ]; then
+        echo "flashinfer-jit-cache==${FLASHINFER_PYTHON_REQUIRED} already installed, keeping it"
+        UNINSTALL_JIT_CACHE=false
+    else
+        echo "flashinfer-jit-cache version mismatch (installed: ${FLASHINFER_JIT_INSTALLED:-none}, required: ${FLASHINFER_PYTHON_REQUIRED}), will reinstall"
+    fi
+
+    if [ "$UNINSTALL_JIT_CACHE" = false ] && [ "$FLASHINFER_JIT_CU_VERSION" != "$CU_VERSION" ]; then
+        echo "flashinfer-jit-cache CUDA version mismatch (installed: ${FLASHINFER_JIT_CU_VERSION:-none}, required: ${CU_VERSION}), will reinstall"
+        UNINSTALL_JIT_CACHE=true
+    fi
+
+    FLASHINFER_UNINSTALL="flashinfer-python"
+    [ "$UNINSTALL_CUBIN" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-cubin"
+    [ "$UNINSTALL_JIT_CACHE" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-jit-cache"
+    $PIP_UNINSTALL_CMD $FLASHINFER_UNINSTALL $PIP_UNINSTALL_SUFFIX || true
+    $PIP_UNINSTALL_CMD opencv-python opencv-python-headless $PIP_UNINSTALL_SUFFIX || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_sglang() {
+    EXTRAS="dev,runai,tracing"
+    if [ -n "$OPTIONAL_DEPS" ]; then
+        EXTRAS="dev,runai,tracing,${OPTIONAL_DEPS}"
+    fi
+    echo "Installing python extras: [${EXTRAS}]"
+    $PIP_CMD install -e "python[${EXTRAS}]" $PIP_INSTALL_SUFFIX
+
+    # Defensive: some runners ended up with nvidia-cusparselt-cu13 metadata
+    # present but libcusparseLt.so.0 missing on disk, breaking any torch import.
+    # If the file is missing, force-reinstall the wheel before downstream steps.
+    SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])")
+    if [ ! -f "$SITE_PACKAGES/nvidia/cusparselt/lib/libcusparseLt.so.0" ] \
+       && pip show nvidia-cusparselt-cu13 >/dev/null 2>&1; then
+        echo "WARNING: nvidia-cusparselt-cu13 metadata present but libcusparseLt.so.0 missing — reinstalling"
+        $PIP_CMD install --reinstall nvidia-cusparselt-cu13 $PIP_INSTALL_SUFFIX
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_sglang_kernel() {
+    SGL_KERNEL_VERSION_FROM_KERNEL=$(grep -Po '(?<=^version = ")[^"]*' sgl-kernel/pyproject.toml)
+    SGL_KERNEL_VERSION_FROM_SRT=$(grep -Po -m1 '(?<=sglang-kernel==)[0-9A-Za-z\.\-]+' python/pyproject.toml)
+    echo "SGL_KERNEL_VERSION_FROM_KERNEL=${SGL_KERNEL_VERSION_FROM_KERNEL} SGL_KERNEL_VERSION_FROM_SRT=${SGL_KERNEL_VERSION_FROM_SRT}"
+
+    if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ -d "sgl-kernel/dist" ]; then
+        ls -alh sgl-kernel/dist
+        if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+            WHEEL_ARCH="aarch64"
+        else
+            WHEEL_ARCH="x86_64"
+        fi
+        KERNEL_WHL=$(ls sgl-kernel/dist/sglang_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}+${CU_VERSION}-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl 2>/dev/null | head -1 || true)
+        if [ -z "$KERNEL_WHL" ]; then
+            echo "ERROR: No matching sgl-kernel wheel found in sgl-kernel/dist/ for version ${SGL_KERNEL_VERSION_FROM_KERNEL} arch ${WHEEL_ARCH} cuda ${CU_VERSION}"
+            ls -alh sgl-kernel/dist/
+            exit 1
+        fi
+        echo "Installing sgl-kernel wheel: $KERNEL_WHL"
+        $PIP_CMD install "$KERNEL_WHL" --force-reinstall $PIP_INSTALL_SUFFIX
+    else
+        if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ ! -d "sgl-kernel/dist" ]; then
+            echo "ERROR: CUSTOM_BUILD_SGL_KERNEL=true but sgl-kernel/dist not found."
+            echo "This usually happens when rerunning a stage without the sgl-kernel-build-wheels job."
+            echo "Please re-run the full workflow using /tag-and-rerun-ci to rebuild the kernel."
+            exit 1
+        fi
+    fi
+
+    # Reinstall torch with matching CUDA version if needed
+    # TODO: Remove after torch 2.11 where cu13 is enabled by default
+    REINSTALL_TORCH=false
+    if TORCH_CUDA_VER=$(python3 -c "import torch; v=torch.version.cuda; parts=v.split('.'); print(f'cu{parts[0]}{parts[1]}')" 2>&1); then
+        echo "Detected torch CUDA version: ${TORCH_CUDA_VER}"
+    else
+        TORCH_IMPORT_ERROR="${TORCH_CUDA_VER}"
+        TORCH_CUDA_VER=""
+        echo "WARNING: importing torch failed while probing CUDA version; force-reinstalling torch packages."
+        printf '%s\n' "${TORCH_IMPORT_ERROR}"
+        REINSTALL_TORCH=true
+    fi
+    TORCHAUDIO_CUDA_VER=$(pip show torchaudio 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+\(cu[0-9][0-9]*\)$/\1/p' || true)
+    TORCHVISION_CUDA_VER=$(pip show torchvision 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+\(cu[0-9][0-9]*\)$/\1/p' || true)
+    if [ "${TORCH_CUDA_VER}" != "${CU_VERSION}" ]; then
+        REINSTALL_TORCH=true
+    else
+        for cuda_ver in "${TORCHAUDIO_CUDA_VER}" "${TORCHVISION_CUDA_VER}"; do
+            if [ -n "${cuda_ver}" ] && [ "${cuda_ver}" != "${CU_VERSION}" ]; then
+                REINSTALL_TORCH=true
+                break
+            fi
+        done
+    fi
+    if [ "${REINSTALL_TORCH}" = true ]; then
+        TORCH_VER=$(pip show torch 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//')
+        TORCHAUDIO_VER=$(pip show torchaudio 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//')
+        TORCHVISION_VER=$(pip show torchvision 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//')
+        if [ -z "${TORCH_VER}" ] || [ -z "${TORCHAUDIO_VER}" ] || [ -z "${TORCHVISION_VER}" ]; then
+            echo "ERROR: could not determine installed torch package versions before reinstall."
+            pip show torch torchaudio torchvision || true
+            exit 1
+        fi
+        echo "Reinstalling torch==${TORCH_VER} torchaudio==${TORCHAUDIO_VER} torchvision==${TORCHVISION_VER} from ${CU_VERSION} index to match torch..."
+        $PIP_CMD install "torch==${TORCH_VER}" "torchaudio==${TORCHAUDIO_VER}" "torchvision==${TORCHVISION_VER}" --index-url "https://download.pytorch.org/whl/${CU_VERSION}" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    fi
+
+    if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" != "true" ]; then
+        # install_sglang above pulls sglang-kernel from PyPI, whose default wheel
+        # tracks one CUDA version (currently cu130). Force-reinstall from the
+        # CU_VERSION-matched sglang wheel index so runners on a different CUDA
+        # (e.g. h20 / cu129) get a wheel linked against the right libnvrtc.
+        $PIP_CMD install "sglang-kernel==${SGL_KERNEL_VERSION_FROM_SRT}" --index-url "https://docs.sglang.ai/whl/${CU_VERSION}/" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    else
+        echo "CUSTOM_BUILD_SGL_KERNEL=true: keeping freshly built sgl-kernel wheel."
+    fi
+    SGL_DEEP_GEMM_VERSION=$(grep -Po -m1 '(?<=sgl-deep-gemm==)[0-9A-Za-z\.\-]+' python/pyproject.toml)
+    if [ "$CU_MAJOR" = "13" ]; then
+        $PIP_CMD install "sgl-deep-gemm==${SGL_DEEP_GEMM_VERSION}" --force-reinstall $PIP_INSTALL_SUFFIX
+    else
+        $PIP_CMD install "https://github.com/sgl-project/whl/releases/download/v${SGL_DEEP_GEMM_VERSION}/sgl_deep_gemm-${SGL_DEEP_GEMM_VERSION}+cu129-py3-none-manylinux2014_$(uname -m).whl" --force-reinstall $PIP_INSTALL_SUFFIX
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_sglang_router() {
+    $PIP_CMD install sglang-router $PIP_INSTALL_SUFFIX
+    $PIP_CMD list
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_flashinfer_cubin() {
+    if [ "$UNINSTALL_CUBIN" = false ]; then
+        echo "flashinfer-cubin==${FLASHINFER_CUBIN_REQUIRED} already installed, skipping install"
+    else
+        # flashinfer-cubin is CUDA-version-agnostic, unlike jit-cache, so its index-url has no cu${CU_VERSION} suffix
+        $PIP_CMD install "flashinfer-cubin==${FLASHINFER_CUBIN_REQUIRED}" --index-url https://flashinfer.ai/whl $PIP_INSTALL_SUFFIX
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+download_flashinfer_cache() {
+    UNINSTALL_JIT_CACHE="$UNINSTALL_JIT_CACHE" \
+        FLASHINFER_PYTHON_REQUIRED="$FLASHINFER_PYTHON_REQUIRED" \
+        CU_VERSION="$CU_VERSION" \
+        PIP_CMD="$PIP_CMD" \
+        PIP_INSTALL_SUFFIX="$PIP_INSTALL_SUFFIX" \
+        bash "${SCRIPT_DIR}/ci_download_flashinfer_jit_cache.sh"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+force_reinstall_cutlass_dsl_libs_cu13() {
+    # nvidia-cutlass-dsl[cu13] has additive PyPI extras: installing it pulls in
+    # both -libs-base and -libs-cu13. The two wheels ship intentionally-different
+    # content for the same paths (cutlass/_mlir/dialects/_gpu_ops_gen.py and
+    # cutlass/_mlir/_mlir_libs/_cutlass_ir.cpython-*.so) -- each Python wrapper
+    # is paired with a matching pybind11 .so. If install order leaves the .py
+    # from one wheel and the .so from the other, GPUModuleOp.__init__ raises
+    # TypeError: incompatible function arguments at kernel-compile time.
+    #
+    # Force-reinstall -libs-cu13 LAST so both files come from the same wheel
+    # (BOTH-cu13 state), eliminating the mismatch. The version is parsed from
+    # pyproject.toml so this stays in sync with whatever nvidia-cutlass-dsl
+    # version the project pins.
+    if [ "$CU_MAJOR" != "13" ]; then
+        return
+    fi
+
+    CUTLASS_DSL_VERSION=$(grep -Po -m1 'nvidia-cutlass-dsl(\[[^]]+\])?==\K[0-9A-Za-z\.\-]+' "${REPO_ROOT}/python/pyproject.toml" || echo "")
+    if [ -z "$CUTLASS_DSL_VERSION" ]; then
+        echo "WARNING: could not detect nvidia-cutlass-dsl version from pyproject.toml; skipping libs-cu13 force-reinstall"
+        return
+    fi
+
+    $PIP_CMD install --force-reinstall --no-deps "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}" $PIP_INSTALL_SUFFIX
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+stabilize_flashinfer_jit_paths() {
+    # In venv mode, FlashInfer JIT writes build.ninja with hardcoded -isystem
+    # paths. Per-job venvs get unique paths, but the JIT cache is shared on the
+    # host mount. Fix by symlinking venv copies to a stable host-mounted path.
+    if [ "$USE_VENV" != "1" ]; then
+        return
+    fi
+
+    STABLE_FI_DIR="${HOME}/.cache/flashinfer/_stable_src"
+
+    # Clear stale cached_ops (keep valid compiled kernels)
+    if [ -d "${HOME}/.cache/flashinfer" ]; then
+        STALE_COUNT=0
+        while IFS= read -r ninja_file; do
+            STALE_PATH=$(grep -o '/tmp/sglang-ci-[^ ]*\|flashinfer-src' "$ninja_file" 2>/dev/null | head -1 || true)
+            if [ -n "$STALE_PATH" ]; then
+                if echo "$STALE_PATH" | grep -q "flashinfer-src" || [ ! -d "$STALE_PATH" ]; then
+                    rm -rf "$(dirname "$ninja_file")"
+                    STALE_COUNT=$((STALE_COUNT + 1))
+                fi
+            fi
+        done < <(find "${HOME}/.cache/flashinfer" -name "build.ninja" -type f 2>/dev/null)
+        echo "Cleaned $STALE_COUNT stale FlashInfer cached_ops (kept valid ones)"
+    fi
+
+    # Copy source files to stable path and symlink venv copies there
+    FI_DATA=$(python3 -c "import flashinfer, os; print(os.path.join(os.path.dirname(flashinfer.__file__), 'data'))")
+    TVM_INC=$(python3 -c "import tvm_ffi, os; print(os.path.join(os.path.dirname(tvm_ffi.__file__), 'include'))")
+
+    FI_VERSION="${FLASHINFER_PYTHON_REQUIRED}"
+    if [ ! -d "$STABLE_FI_DIR/flashinfer-data" ] || [ "$(cat "$STABLE_FI_DIR/.version" 2>/dev/null)" != "$FI_VERSION" ]; then
+        rm -rf "$STABLE_FI_DIR"
+        mkdir -p "$STABLE_FI_DIR"
+        cp -a "$FI_DATA" "$STABLE_FI_DIR/flashinfer-data"
+        cp -a "$TVM_INC" "$STABLE_FI_DIR/tvm-ffi-include"
+        echo "$FI_VERSION" > "$STABLE_FI_DIR/.version"
+        echo "Copied flashinfer source files to stable path: $STABLE_FI_DIR (version=$FI_VERSION)"
+    else
+        echo "Stable flashinfer source path up to date (version=$FI_VERSION)"
+    fi
+
+    rm -rf "$FI_DATA"
+    ln -s "$STABLE_FI_DIR/flashinfer-data" "$FI_DATA"
+    TVM_INC_PARENT=$(dirname "$TVM_INC")
+    rm -rf "$TVM_INC_PARENT/include"
+    ln -s "$STABLE_FI_DIR/tvm-ffi-include" "$TVM_INC_PARENT/include"
+    echo "Symlinked venv flashinfer/tvm_ffi -> $STABLE_FI_DIR"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_extra_deps() {
+    MOONCAKE_VERSION="0.3.11.post1"
+    NIXL_VERSION="1.3.0"
+    if [ "$CU_MAJOR" = "13" ]; then
+        MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
+        MOONCAKE_STALE_PKG="mooncake-transfer-engine"
+        NIXL_BIN_NAME="nixl-cu13"
+        EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc"
+    else
+        MOONCAKE_PKG="mooncake-transfer-engine==${MOONCAKE_VERSION}"
+        MOONCAKE_STALE_PKG="mooncake-transfer-engine-cuda13"
+        NIXL_BIN_NAME="nixl-cu12"
+        EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc-cu12"
+    fi
+    # Both variants own the same mooncake/ package files and bin/ scripts
+    # (mooncake_master, etc.). Uninstalling the stale variant deletes shared
+    # files that the live variant's RECORD still references, so we force a
+    # reinstall to restore them — pip would otherwise see "already satisfied"
+    # and skip.
+    if pip show ${MOONCAKE_STALE_PKG} >/dev/null 2>&1; then
+        $PIP_UNINSTALL_CMD ${MOONCAKE_STALE_PKG} $PIP_UNINSTALL_SUFFIX || true
+        $PIP_CMD install ${MOONCAKE_PKG} --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    fi
+    $PIP_CMD install ${MOONCAKE_PKG} ${EXTRA_NVIDIA_SPECS} py-spy scipy huggingface_hub[hf_xet] pytest $PIP_INSTALL_SUFFIX
+
+    NIXL_INSTALLED=$(pip show nixl 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    NIXL_BIN_INSTALLED=$(pip show "${NIXL_BIN_NAME}" 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    if [ "$NIXL_INSTALLED" = "$NIXL_VERSION" ] && [ "$NIXL_BIN_INSTALLED" = "$NIXL_VERSION" ]; then
+        echo "nixl==${NIXL_VERSION} and ${NIXL_BIN_NAME}==${NIXL_VERSION} already installed, keeping them"
+    else
+        echo "nixl mismatch (meta: ${NIXL_INSTALLED:-none}, ${NIXL_BIN_NAME}: ${NIXL_BIN_INSTALLED:-none}, required: ${NIXL_VERSION}); installing"
+        # Meta stub owns the nixl import path; install only the CUDA binary for
+        # this runner's torch CUDA major. --no-deps avoids pulling the other CUDA
+        # variant; leave any other variant already on the runner image untouched.
+        $PIP_CMD install "nixl==${NIXL_VERSION}" "${NIXL_BIN_NAME}==${NIXL_VERSION}" \
+            --no-deps --force-reinstall $PIP_INSTALL_SUFFIX
+    fi
+
+    if [ "$IS_BLACKWELL" != "1" ]; then
+        git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+        $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX
+        # lmms-eval v0.5 pulls antlr4-python3-runtime==4.7.2, clobbering the
+        # 4.9.3 that sgl-eval's latex2sympy2_extended needs (4.7.2 ImportError
+        # at sgl-eval import). Pin it back so the nightly sgl-eval path works.
+        $PIP_CMD install "antlr4-python3-runtime==4.9.3" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    fi
+    $PIP_CMD uninstall xformers || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_test_tools() {
+    # Download kernels from kernels community
+    kernels download python || true
+    kernels lock python || true
+    [ -e "${HOME}/.cache/sglang" ] && [ ! -d "${HOME}/.cache/sglang" ] && rm -f "${HOME}/.cache/sglang"
+    mkdir -p "${HOME}/.cache/sglang/"
+    mv python/kernels.lock "${HOME}/.cache/sglang/" || true
+
+    # Install human-eval (subshell keeps cd local)
+    $PIP_CMD install "setuptools==70.0.0" $PIP_INSTALL_SUFFIX
+    [ -d human-eval ] || git clone https://github.com/merrymercy/human-eval.git
+    (
+        cd human-eval
+        $PIP_CMD install -e . --no-build-isolation $PIP_INSTALL_SUFFIX
+    )
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+prepare_runner() {
+    bash "${SCRIPT_DIR}/prepare_runner.sh"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+setup_ld_library_path() {
+    # NVIDIA pip packages and torch ship .so files under site-packages that are
+    # not on the default LD_LIBRARY_PATH.
+    SITE_PACKAGES=$(python3 -c "import site, sys; print(site.getsitepackages()[0])")
+    NVIDIA_LIBS=$(find "$SITE_PACKAGES" -path "*/nvidia/*/lib" -type d 2>/dev/null | tr '\n' ':')
+    TORCH_LIB="$SITE_PACKAGES/torch/lib"
+    VENV_LD="${NVIDIA_LIBS}${TORCH_LIB}"
+    export LD_LIBRARY_PATH="${VENV_LD}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+    if [ "$USE_VENV" = "1" ] && [ -n "$UV_VENV" ]; then
+        echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH\"" >> "$UV_VENV/env.sh"
+    fi
+    if [ -n "${GITHUB_ENV:-}" ]; then
+        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$GITHUB_ENV" || echo "WARNING: GITHUB_ENV write failed; LD_LIBRARY_PATH will be set via BASH_ENV instead"
+    fi
+    echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+verify_imports() {
+    $PIP_CMD list
+    python3 -c "import torch; print(torch.version.cuda)"
+    python3 -c "import cutlass; import cutlass.cute;"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+main() {
+    configure_environment "$@"
+    detect_host
+    kill_existing_processes
+    cleanup_stale_shm
+    install_apt_packages
+    clean_site_packages
+    setup_pip_toolchain
+    remove_stale_cuda12_nvidia_wheels
+    uninstall_stale_flashinfer
+    install_sglang
+    # Diffusion B200 CI imports torch inside install_sglang_kernel after removing
+    # stale CUDA 12 NVIDIA wheels, so opt into one early LD_LIBRARY_PATH refresh.
+    if [ "${SGLANG_CI_EARLY_LD_LIBRARY_PATH:-0}" = "1" ]; then
+        setup_ld_library_path
+    fi
+    install_sglang_kernel
+    install_sglang_router
+    install_flashinfer_cubin
+    download_flashinfer_cache
+    force_reinstall_cutlass_dsl_libs_cu13
+    stabilize_flashinfer_jit_paths
+    install_extra_deps
+    install_test_tools
+    prepare_runner
+    setup_ld_library_path
+    verify_imports
+}
+
+main "$@"

scripts/ci/cuda/ci_install_gateway_dependencies.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Install dependencies for the sgl-model-gateway CI jobs.
+#
+# Gateway-specific apt deps are installed here; protoc and the Rust toolchain
+# are delegated to the shared installer (the toolchain version is pinned by
+# sgl-model-gateway/rust-toolchain.toml, picked up automatically on first
+# `cargo` invocation).
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+GATEWAY_APT_PACKAGES=(libssl-dev pkg-config redis-server)
+APT_OPTS=(
+    -y
+    -o "Acquire::Retries=5"
+    -o "Acquire::http::Timeout=30"
+    -o "Acquire::https::Timeout=30"
+)
+SUDO=""
+command -v sudo >/dev/null 2>&1 && SUDO="sudo"
+
+# GH-hosted runners' Azure Ubuntu mirrors flake periodically. Retry the
+# whole install with backoff so we don't fail the whole CI on a 1-min
+# DNS hiccup at apt-mirrors.txt → azure.archive.ubuntu.com.
+for attempt in 1 2 3 4 5; do
+    if $SUDO apt-get update "${APT_OPTS[@]}" \
+       && $SUDO apt-get install "${APT_OPTS[@]}" "${GATEWAY_APT_PACKAGES[@]}"; then
+        break
+    fi
+    if [ "$attempt" = 5 ]; then
+        echo "apt-get install failed after 5 attempts; giving up." >&2
+        exit 1
+    fi
+    sleep $((attempt * 15))
+done
+
+bash "${SCRIPT_DIR}/../utils/install_rust_protoc.sh"
+
+# Make cargo/rustc/protoc visible in this shell.
+. "$HOME/.cargo/env"
+
+rustc --version
+cargo --version
+protoc --version

scripts/ci/cuda/ci_start_disaggregation_servers.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+set -euo pipefail
+
+# Optional: set DISAGG_READY_FILE to a filepath; when all servers are healthy, the script will
+# create this file as a readiness signal (useful for CI to proceed to next steps).
+DISAGG_READY_FILE="${DISAGG_READY_FILE:-}"
+
+MODEL_PATH="/raid/models/meta-llama/Llama-3.1-8B-Instruct"
+
+# Function to find the first available active IB device
+find_active_ib_device() {
+    for device in mlx5_{0..11}; do
+        if ibv_devinfo $device >/dev/null 2>&1; then
+            state=$(ibv_devinfo $device | grep "state:" | head -1 | awk '{print $2}')
+            if [[ "$state" == "PORT_ACTIVE" ]]; then
+                echo "$device"
+                return 0
+            fi
+        fi
+    done
+    echo "No active IB device found" >&2
+    return 1
+}
+
+# Get the first available active IB device
+DEVICE=$(find_active_ib_device)
+echo "Using IB device: $DEVICE"
+
+# Launch prefill servers on GPU 0–3
+for i in {0..3}; do
+  PORT=$((30001 + i))
+  BOOTSTRAP_PORT=$((9001 + i))
+  HOST="127.0.0.$((i + 1))"
+  echo "Launching PREFILL server on GPU $i at $HOST:$PORT (bootstrap: $BOOTSTRAP_PORT)"
+  CUDA_VISIBLE_DEVICES=$i \
+  python3 -m sglang.launch_server \
+    --model-path "$MODEL_PATH" \
+    --disaggregation-mode prefill \
+    --host "$HOST" \
+    --port "$PORT" \
+    --disaggregation-ib-device "$DEVICE" \
+    --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" &
+done
+
+# Launch decode servers on GPU 4–7
+for i in {4..7}; do
+  PORT=$((30001 + i))
+  HOST="127.0.0.$((i + 1))"
+  echo "Launching DECODE server on GPU $i at $HOST:$PORT"
+  CUDA_VISIBLE_DEVICES=$i \
+  python3 -m sglang.launch_server \
+    --model-path "$MODEL_PATH" \
+    --disaggregation-mode decode \
+    --host "$HOST" \
+    --port "$PORT" \
+    --disaggregation-ib-device "$DEVICE" \
+    --base-gpu-id 0 &
+done
+
+# Wait for disaggregation servers to initialize
+echo "Waiting for disaggregation servers to initialize..."
+
+# Health check with 5-minute timeout
+TIMEOUT=300
+START_TIME=$(date +%s)
+
+echo "Checking health of all 8 servers..."
+while true; do
+    CURRENT_TIME=$(date +%s)
+    ELAPSED=$((CURRENT_TIME - START_TIME))
+
+    if [ $ELAPSED -ge $TIMEOUT ]; then
+        echo "❌ Timeout: Servers did not become healthy within 5 minutes"
+        exit 1
+    fi
+
+    HEALTHY_COUNT=0
+    # Check all 8 servers (127.0.0.1-8:30001-30008)
+    for i in {1..8}; do
+        if curl -s -f "http://127.0.0.$i:$((30000 + i))/health" >/dev/null 2>&1; then
+            HEALTHY_COUNT=$((HEALTHY_COUNT + 1))
+        fi
+    done
+
+    echo "Healthy servers: $HEALTHY_COUNT/8 (elapsed: ${ELAPSED}s)"
+
+    if [ $HEALTHY_COUNT -eq 8 ]; then
+        echo "✅ All 8 servers are healthy!"
+        # Emit readiness signal file if requested
+        if [ -n "$DISAGG_READY_FILE" ]; then
+            echo "Creating readiness flag: $DISAGG_READY_FILE"
+            # Ensure parent dir exists; ignore errors
+            mkdir -p "$(dirname "$DISAGG_READY_FILE")" 2>/dev/null || true
+            touch "$DISAGG_READY_FILE"
+        fi
+        break
+    else
+        sleep 10  # Wait 10 seconds before next check
+    fi
+done
+
+# Don't launch router here - just keep servers running
+echo "✅ All disaggregation servers are ready and waiting for router connections"
+
+# Keep the script running
+wait

scripts/ci/cuda/prepare_runner.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Prepare the CI runner by cleaning up stale HuggingFace cache artifacts and validating models
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "Preparing CI runner..."
+echo ""
+
+# Clean up stale HuggingFace cache artifacts from previous failed downloads
+python3 "${SCRIPT_DIR}/../utils/cleanup_hf_cache.py"
+echo ""
+
+# Pre-validate cached models and write markers for offline mode
+# This allows tests to run with HF_HUB_OFFLINE=1 for models that are fully cached
+python3 "${SCRIPT_DIR}/../utils/prevalidate_cached_models.py"
+echo ""
+
+echo "CI runner preparation complete!"

scripts/ci/cuda/warmup_deep_gemm.py

@@ -0,0 +1,603 @@
+"""
+Lightweight DeepGEMM JIT compilation warmup without loading model weights.
+
+Reads model config.json from HF cache to derive kernel shapes, then compiles
+DeepGEMM kernels directly. This avoids the expensive model weight loading step
+that the full `sglang.compile_deep_gemm` requires.
+
+Supports DeepSeek V2/V3 family models. Falls back to `sglang.compile_deep_gemm`
+for unsupported architectures.
+
+Usage:
+    python3 scripts/ci/cuda/warmup_deep_gemm.py \
+        deepseek-ai/DeepSeek-V3-0324:8 \
+        deepseek-ai/DeepSeek-V3.2:8
+"""
+
+import hashlib
+import json
+import os
+import signal
+import subprocess
+import sys
+import threading
+import time
+from math import ceil
+from pathlib import Path
+from typing import Dict, List
+
+# Shared with warmup_server.py. Wipe alongside /root/.cache/deep_gemm if you
+# clear the DeepGEMM JIT cache — a stale marker → in-test JIT compile.
+MARKER_DIR = os.path.join(os.path.expanduser("~"), ".cache", "sglang", "warmup_markers")
+
+# Outer cap for stuck fallback subprocesses; CRASH_MARKERS abort sooner.
+FALLBACK_TIMEOUT_SEC = 600
+
+# Per-model launch flags forwarded to `sglang.compile_deep_gemm`. DeepGEMM
+# cache key includes per-rank N/K (depends on tp/dp/ep) — must match each
+# model's `other_args` in test/registered/ or warmed shapes won't be hit.
+FALLBACK_ARGS: Dict[str, List[str]] = {
+    "deepseek-ai/DeepSeek-V3.2": ["--dp", "8", "--enable-dp-attention"],
+    "zai-org/GLM-5-FP8": ["--dp", "8", "--enable-dp-attention"],
+    "XiaomiMiMo/MiMo-V2-Flash": [
+        "--dp",
+        "2",
+        "--enable-dp-attention",
+        "--attention-backend",
+        "fa3",
+    ],
+    # --mm-enable-dp-encoder is required: without it DP0 runs the vision
+    # encoder alone and DP1 deadlocks at the next collective.
+    "XiaomiMiMo/MiMo-V2.5": [
+        "--dp",
+        "2",
+        "--enable-dp-attention",
+        "--mm-enable-dp-encoder",
+        "--attention-backend",
+        "fa3",
+        "--mm-attention-backend",
+        "fa3",
+    ],
+}
+
+# compile_deep_gemm polls /v1/models for the full timeout even after a TP rank
+# dies; the watcher uses these to kill the group within seconds instead.
+CRASH_MARKERS = (
+    "Scheduler hit an exception",
+    "Received sigquit from a child",
+)
+
+# Configure DeepGEMM cache before importing deep_gemm
+os.environ["DG_JIT_CACHE_DIR"] = os.getenv(
+    "SGLANG_DG_CACHE_DIR",
+    os.path.join(os.path.expanduser("~"), ".cache", "deep_gemm"),
+)
+os.environ["DG_JIT_USE_NVRTC"] = os.getenv("SGL_DG_USE_NVRTC", "0")
+
+BLOCK_SIZE = 128
+
+
+def get_config_json(model_name):
+    """Load config.json for a cached model from HF cache."""
+    cache_dir = os.environ.get(
+        "HF_HOME", os.path.join(os.path.expanduser("~"), ".cache", "huggingface")
+    )
+    hub_dir = os.path.join(cache_dir, "hub")
+    safe_name = "models--" + model_name.replace("/", "--")
+    snapshots_dir = os.path.join(hub_dir, safe_name, "snapshots")
+
+    if not os.path.isdir(snapshots_dir):
+        return None
+
+    snapshots = sorted(
+        Path(snapshots_dir).iterdir(), key=lambda p: p.stat().st_mtime, reverse=True
+    )
+    for snapshot in snapshots:
+        config_path = snapshot / "config.json"
+        if config_path.exists():
+            with open(config_path) as f:
+                return json.load(f)
+    return None
+
+
+def is_deepseek_v2v3(config):
+    """Check if a model is from the DeepSeek V2/V3 family."""
+    architectures = config.get("architectures", [])
+    model_type = config.get("model_type", "")
+    return any(
+        "DeepseekV2" in a or "DeepseekV3" in a for a in architectures
+    ) or model_type in ("deepseek_v2", "deepseek_v3")
+
+
+def compute_deepseek_v2v3_shapes(config, tp):
+    """Compute all DeepGEMM (kernel_type, N, K, num_groups) for DeepSeek V2/V3.
+
+    Shape derivation based on:
+    - MoE: python/sglang/srt/layers/moe/fused_moe_triton/layer.py
+    - MLA: python/sglang/srt/models/deepseek_v2.py
+    - FP8: python/sglang/srt/layers/quantization/fp8_kernel.py
+    """
+    shapes = []
+
+    hidden_size = config["hidden_size"]
+    num_attention_heads = config.get("num_attention_heads", 128)
+    kv_lora_rank = config.get("kv_lora_rank", 512)
+    qk_nope_head_dim = config.get("qk_nope_head_dim", 128)
+    v_head_dim = config.get("v_head_dim", 128)
+    n_routed_experts = config.get("n_routed_experts", 0)
+    n_shared_experts = config.get("n_shared_experts", 0)
+    moe_intermediate_size = config.get("moe_intermediate_size", 0)
+
+    num_local_heads = num_attention_heads // tp
+    # Shared expert fusion is enabled by default (disable_shared_experts_fusion=False)
+    # so the FusedMoE weight tensor includes shared experts
+    num_local_experts = n_routed_experts + n_shared_experts
+
+    # --- MoE expert GEMM shapes ---
+    # FusedMoE shards intermediate_size across TP ranks (column parallel for gate/up,
+    # row parallel for down). All experts are replicated on each TP rank.
+    if n_routed_experts > 0 and moe_intermediate_size > 0:
+        moe_inter_per_tp = moe_intermediate_size // tp
+
+        # Gate-Up projection: (tokens, hidden_size) @ (experts, 2*inter_per_tp, hidden_size)^T
+        # Both masked and contiguous paths are used at runtime
+        shapes.append(("MASKED", moe_inter_per_tp * 2, hidden_size, num_local_experts))
+        shapes.append(("CONTIG", moe_inter_per_tp * 2, hidden_size, num_local_experts))
+
+        # Down projection: (tokens, inter_per_tp) @ (experts, hidden_size, inter_per_tp)^T
+        shapes.append(("MASKED", hidden_size, moe_inter_per_tp, num_local_experts))
+        shapes.append(("CONTIG", hidden_size, moe_inter_per_tp, num_local_experts))
+
+    # --- MLA attention GEMM shapes (masked grouped GEMM) ---
+    if kv_lora_rank > 0 and num_local_heads > 0:
+        # Q_nope -> compressed K: (heads, m, qk_nope_head_dim) @ (heads, kv_lora_rank, qk_nope_head_dim)^T
+        shapes.append(("MASKED", kv_lora_rank, qk_nope_head_dim, num_local_heads))
+
+        # Attention output -> V: (heads, m, kv_lora_rank) @ (heads, v_head_dim, kv_lora_rank)^T
+        shapes.append(("MASKED", v_head_dim, kv_lora_rank, num_local_heads))
+
+    # --- kv_b_proj (non-grouped GEMM via FP8 kernel) ---
+    # ColumnParallelLinear(kv_lora_rank, num_heads * (qk_nope + v_head_dim))
+    # Per TP rank: N = num_local_heads * (qk_nope_head_dim + v_head_dim)
+    if kv_lora_rank > 0 and num_local_heads > 0:
+        kv_b_proj_n = num_local_heads * (qk_nope_head_dim + v_head_dim)
+        shapes.append(("NORMAL", kv_b_proj_n, kv_lora_rank, 1))
+
+    return shapes
+
+
+def get_architecture_key(config, tp):
+    """Key for dedup: models with same key share DeepGEMM kernels."""
+    if config is None:
+        return None
+    fields = [
+        config.get("hidden_size", 0),
+        config.get("moe_intermediate_size", 0),
+        config.get("n_routed_experts", 0),
+        config.get("n_shared_experts", 0),
+        config.get("num_attention_heads", 0),
+        config.get("kv_lora_rank", 0),
+        config.get("qk_nope_head_dim", 0),
+        config.get("v_head_dim", 0),
+        tp,
+    ]
+    return tuple(fields)
+
+
+def compute_m_list(fast_warmup=False, chunked_prefill_size=8192):
+    """Compute the list of M values to compile (matches compile_utils.py logic)."""
+    m_list = []
+    if fast_warmup:
+        m_list += list(range(1, 1025))
+        next_m, sample_step = 1024, 2
+        max_prefill_bs = min(chunked_prefill_size, 32 * 1024)
+        while next_m < max_prefill_bs:
+            m_list += list(range(next_m, 2 * next_m, sample_step))
+            next_m *= 2
+            sample_step *= 2
+        m_list.append(max_prefill_bs)
+        m_list = sorted(set(m_list))
+    else:
+        m_max = 16 * 1024
+        if chunked_prefill_size > 8192:
+            m_max = chunked_prefill_size * 2
+        m_max = min(128 * 1024, m_max)
+        m_list = list(range(1, m_max + 1))
+    return m_list
+
+
+def _empty_token_fp8(size):
+    """Create FP8 token tensor + per-block scale tensor."""
+    import torch
+
+    *dims, k = size
+    return (
+        torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
+        torch.empty((*dims, ceil(k / BLOCK_SIZE)), device="cuda", dtype=torch.float32),
+    )
+
+
+def _empty_block_fp8(size):
+    """Create FP8 block tensor + per-block scale tensor."""
+    import torch
+
+    *dims, n, k = size
+    return (
+        torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
+        torch.empty(
+            (*dims, ceil(n / BLOCK_SIZE), ceil(k / BLOCK_SIZE)),
+            device="cuda",
+            dtype=torch.float32,
+        ),
+    )
+
+
+def get_memory_requirement(kernel_type, max_m, n, k, num_groups):
+    """Estimate GPU memory needed in GB for compilation buffers."""
+    _GB = 1 << 30
+    if kernel_type == "NORMAL":
+        return (max_m * k + n * k + max_m * n * 2) / _GB
+    elif kernel_type == "CONTIG":
+        return (max_m * k + num_groups * n * k + max_m * 4 + max_m * n * 2) / _GB
+    elif kernel_type == "MASKED":
+        return (
+            num_groups * max_m * k
+            + num_groups * n * k
+            + num_groups * 4
+            + num_groups * max_m * n * 2
+        ) / _GB
+    return 0
+
+
+def compile_one_shape(kernel_type, n, k, num_groups, m_list):
+    """Compile DeepGEMM kernels for one (kernel_type, N, K, num_groups) shape."""
+    import deep_gemm
+    import torch
+    from tqdm import tqdm
+
+    # Filter M list for contiguous layout alignment
+    if kernel_type == "CONTIG":
+        m_alignment = deep_gemm.get_mk_alignment_for_contiguous_layout()
+        m_list = sorted(set(m for m in m_list if m % m_alignment == 0))
+
+    if not m_list:
+        return
+
+    max_m = max(m_list)
+
+    # Reduce max_m if not enough GPU memory
+    mem_free = torch.cuda.mem_get_info()[0] / (1 << 30)
+    mem_required = get_memory_requirement(kernel_type, max_m, n, k, num_groups)
+    if mem_required > mem_free:
+        while (
+            get_memory_requirement(kernel_type, max_m, n, k, num_groups) > mem_free
+            and max_m > 4096
+        ):
+            max_m //= 2
+        print(
+            f"  Memory {mem_free:.1f}GB < required {mem_required:.1f}GB, "
+            f"reducing max_m to {max_m}"
+        )
+        m_list = [m for m in m_list if m <= max_m]
+
+    get_compile_mode = getattr(deep_gemm, "get_compile_mode", None)
+    set_compile_mode = getattr(deep_gemm, "set_compile_mode", None)
+    old_mode = get_compile_mode() if get_compile_mode is not None else None
+    if set_compile_mode is not None:
+        set_compile_mode(1)
+    try:
+        if kernel_type == "NORMAL":
+            lhs_q, lhs_s = _empty_token_fp8((max_m, k))
+            rhs_q, rhs_s = _empty_block_fp8((n, k))
+            out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16)
+            for m in tqdm(m_list, desc=f"  NORMAL N={n} K={k}"):
+                deep_gemm.fp8_gemm_nt((lhs_q[:m], lhs_s[:m]), (rhs_q, rhs_s), out[:m])
+
+        elif kernel_type == "CONTIG":
+            lhs_q, lhs_s = _empty_token_fp8((max_m, k))
+            rhs_q, rhs_s = _empty_block_fp8((num_groups, n, k))
+            m_indices = torch.zeros((max_m,), device="cuda", dtype=torch.int32)
+            out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16)
+            for m in tqdm(m_list, desc=f"  CONTIG N={n} K={k} G={num_groups}"):
+                deep_gemm.m_grouped_fp8_gemm_nt_contiguous(
+                    (lhs_q[:m], lhs_s[:m]),
+                    (rhs_q, rhs_s),
+                    out[:m],
+                    m_indices[:m],
+                )
+
+        elif kernel_type == "MASKED":
+            lhs_q, lhs_s = _empty_token_fp8((num_groups, max_m, k))
+            rhs_q, rhs_s = _empty_block_fp8((num_groups, n, k))
+            masked_m = torch.zeros((num_groups,), device="cuda", dtype=torch.int32)
+            out = torch.empty(
+                (num_groups, max_m, n), device="cuda", dtype=torch.bfloat16
+            )
+            for m in tqdm(m_list, desc=f"  MASKED N={n} K={k} G={num_groups}"):
+                deep_gemm.fp8_m_grouped_gemm_nt_masked(
+                    (lhs_q, lhs_s),
+                    (rhs_q, rhs_s),
+                    out,
+                    masked_m=masked_m,
+                    expected_m=m,
+                )
+    finally:
+        if set_compile_mode is not None and old_mode is not None:
+            set_compile_mode(old_mode)
+
+    torch.cuda.current_stream().synchronize()
+    torch.cuda.empty_cache()
+
+
+def compile_shapes_lightweight(shapes, m_list):
+    """Compile all DeepGEMM shapes directly (no model loading)."""
+    for i, (kernel_type, n, k, num_groups) in enumerate(shapes, 1):
+        print(f"\n[{i}/{len(shapes)}] {kernel_type} N={n} K={k} G={num_groups}")
+        t0 = time.time()
+        compile_one_shape(kernel_type, n, k, num_groups, m_list)
+        elapsed = time.time() - t0
+        print(f"  Done in {elapsed:.1f}s")
+
+
+def _kill_pg_and_wait(proc):
+    """SIGTERM the subprocess's process group, escalate to SIGKILL if needed."""
+    try:
+        os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+    except (ProcessLookupError, OSError):
+        pass
+    try:
+        return proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+        except (ProcessLookupError, OSError):
+            pass
+        try:
+            return proc.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            return -1
+
+
+def get_version_key():
+    """Hash of Python + Triton + PyTorch versions; invalidates markers on upgrade."""
+    parts = [sys.version]
+    try:
+        import triton  # noqa: WPS433
+
+        parts.append(f"triton={triton.__version__}")
+    except ImportError:
+        parts.append("triton=none")
+    try:
+        import torch  # noqa: WPS433
+
+        parts.append(f"torch={torch.__version__}")
+    except ImportError:
+        parts.append("torch=none")
+    return hashlib.sha256("|".join(parts).encode()).hexdigest()[:12]
+
+
+def get_fallback_marker_path(model, tp, extra_args):
+    """Marker path for one (model, tp, extra_args) fallback invocation."""
+    args_blob = json.dumps(list(extra_args))
+    args_hash = hashlib.md5(args_blob.encode()).hexdigest()[:8]
+    safe_model = model.replace("/", "--")
+    return os.path.join(
+        MARKER_DIR,
+        f"deepgemm_fallback_{safe_model}_tp{tp}_{args_hash}_{get_version_key()}.done",
+    )
+
+
+def check_fallback_marker(model, tp, extra_args):
+    return os.path.exists(get_fallback_marker_path(model, tp, extra_args))
+
+
+def write_fallback_marker(model, tp, extra_args):
+    marker = get_fallback_marker_path(model, tp, extra_args)
+    os.makedirs(os.path.dirname(marker), exist_ok=True)
+    Path(marker).write_text(
+        json.dumps(
+            {
+                "model": model,
+                "tp": tp,
+                "extra_args": list(extra_args),
+                "version_key": get_version_key(),
+                "timestamp": time.time(),
+            }
+        )
+    )
+    print(f"  Wrote marker: {marker}")
+
+
+def fallback_compile_deep_gemm(model, tp):
+    """Fall back to full sglang.compile_deep_gemm (loads model weights).
+
+    Runs in its own process group so a hung subprocess (e.g. one TP rank
+    crashes and the rest deadlock on NCCL collectives) can be killed
+    cleanly without leaking children. Watches subprocess output for crash
+    markers so a deterministic failure aborts in seconds rather than burning
+    the full FALLBACK_TIMEOUT_SEC.
+    """
+    extra_args = FALLBACK_ARGS.get(model, [])
+    print(
+        f"Falling back to full compile_deep_gemm for {model} "
+        f"(tp={tp}, extra_args={extra_args})..."
+    )
+    cmd = [
+        sys.executable,
+        "-m",
+        "sglang.compile_deep_gemm",
+        "--model",
+        model,
+        "--tp",
+        str(tp),
+        "--trust-remote-code",
+        "--model-loader-extra-config",
+        '{"enable_multithread_load": true, "num_threads": 64}',
+        # Cap compile_deep_gemm's own /v1/models polling loop so it gives up
+        # before our outer timeout has to SIGTERM it.
+        "--timeout",
+        str(FALLBACK_TIMEOUT_SEC),
+        *extra_args,
+    ]
+
+    crashed = threading.Event()
+    proc = subprocess.Popen(
+        cmd,
+        preexec_fn=os.setsid,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        bufsize=1,
+        text=True,
+    )
+
+    def _watch():
+        # Stream child output to our stdout while scanning for crash markers.
+        for line in proc.stdout:
+            sys.stdout.write(line)
+            sys.stdout.flush()
+            if not crashed.is_set() and any(m in line for m in CRASH_MARKERS):
+                crashed.set()
+
+    watcher = threading.Thread(target=_watch, daemon=True)
+    watcher.start()
+
+    deadline = time.monotonic() + FALLBACK_TIMEOUT_SEC
+    while True:
+        rc = proc.poll()
+        if rc is not None:
+            watcher.join(timeout=2)
+            if rc != 0:
+                print(f"Warning: fallback failed for {model} (exit code {rc})")
+            return rc == 0
+        if crashed.is_set():
+            print(
+                f"Warning: detected crash marker in {model} (tp={tp}) subprocess; "
+                "killing process group and continuing."
+            )
+            _kill_pg_and_wait(proc)
+            return False
+        if time.monotonic() >= deadline:
+            print(
+                f"Warning: fallback timed out after {FALLBACK_TIMEOUT_SEC}s for "
+                f"{model} (tp={tp}); killing process group and continuing."
+            )
+            _kill_pg_and_wait(proc)
+            return False
+        time.sleep(2)
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
+        print("Usage: warmup_deep_gemm.py model1:tp1 [model2:tp2 ...]")
+        print("\nDerives DeepGEMM kernel shapes from config.json without loading model")
+        print(
+            "weights. Falls back to full compile_deep_gemm for unknown architectures."
+        )
+        sys.exit(0)
+
+    # Parse model:tp pairs
+    model_tp_pairs = []
+    for arg in sys.argv[1:]:
+        if ":" not in arg:
+            print(f"Error: expected model:tp format, got '{arg}'")
+            sys.exit(1)
+        model, tp_str = arg.rsplit(":", 1)
+        model_tp_pairs.append((model, int(tp_str)))
+
+    fast_warmup = os.environ.get("SGLANG_JIT_DEEPGEMM_FAST_WARMUP", "0").lower() in (
+        "1",
+        "true",
+    )
+    print(f"=== DeepGEMM Lightweight Warmup ({len(model_tp_pairs)} model(s)) ===")
+    print(f"    Fast warmup: {fast_warmup}")
+    print(
+        f"    Cache dir: {os.environ.get('DG_JIT_CACHE_DIR', '~/.cache/deep_gemm')}\n"
+    )
+
+    # Load configs and deduplicate by architecture
+    seen_keys = {}
+    to_process = []  # (model, tp, config_or_None, shapes_or_None)
+
+    for model, tp in model_tp_pairs:
+        config = get_config_json(model)
+        if config is None:
+            print(f"  SKIP   {model} (tp={tp}): config.json not in HF cache")
+            continue
+
+        # Models with FALLBACK_ARGS launch with extra dp/ep/dp-attention flags
+        # that change per-rank N/K. The lightweight path doesn't model those —
+        # it computes attention shapes assuming TP-only sharding — so dedup'ing
+        # such a model to a no-override DeepSeek V2/V3 lookalike (e.g. V3.2 →
+        # V3-0324) silently picks the wrong attention shapes for the test's
+        # actual launch config. Force these through fallback so the populated
+        # cache matches the real test.
+        has_fallback_override = model in FALLBACK_ARGS
+
+        key = get_architecture_key(config, tp)
+        if key in seen_keys and not has_fallback_override:
+            print(f"  DEDUP  {model} (tp={tp}): same shapes as {seen_keys[key]}")
+            continue
+
+        if is_deepseek_v2v3(config) and not has_fallback_override:
+            shapes = compute_deepseek_v2v3_shapes(config, tp)
+            seen_keys[key] = model
+            to_process.append((model, tp, config, shapes))
+            print(f"  FOUND  {model} (tp={tp}): {len(shapes)} DeepGEMM shape(s)")
+        else:
+            seen_keys[key] = model
+            to_process.append((model, tp, config, None))
+            if has_fallback_override:
+                print(
+                    f"  FOUND  {model} (tp={tp}): forced fallback (extra args "
+                    f"{FALLBACK_ARGS[model]})"
+                )
+            else:
+                arch = config.get("architectures", ["unknown"])
+                print(
+                    f"  FOUND  {model} (tp={tp}): unknown arch {arch}, "
+                    "will use fallback"
+                )
+
+    if not to_process:
+        print("\nNo models to process. Done.")
+        return
+
+    m_list = compute_m_list(fast_warmup=fast_warmup)
+    print(f"\nM list: {len(m_list)} values (range {min(m_list)}-{max(m_list)})")
+
+    for model, tp, config, shapes in to_process:
+        print(f"\n{'=' * 60}")
+        print(f"Model: {model} (tp={tp})")
+        print(f"{'=' * 60}")
+
+        if shapes is None:
+            # Fallback path: full sglang.compile_deep_gemm with the test's launch
+            # flags. Loading model weights inside that subprocess is the dominant
+            # cost (45-170s/model) and dwarfs the actual DeepGEMM compile, so we
+            # skip the whole fallback when the marker says we already populated
+            # this cache. Cache is keyed on (model, tp, extra_args, version_key).
+            extra_args = FALLBACK_ARGS.get(model, [])
+            if check_fallback_marker(model, tp, extra_args):
+                print(
+                    f"  SKIP fallback (warm marker found): {model} (tp={tp}, "
+                    f"extra_args={extra_args})"
+                )
+                continue
+            if fallback_compile_deep_gemm(model, tp):
+                write_fallback_marker(model, tp, extra_args)
+            continue
+
+        # Print shape summary
+        for kernel_type, n, k, num_groups in shapes:
+            print(f"  {kernel_type:8s} N={n:<6d} K={k:<6d} G={num_groups}")
+
+        t0 = time.time()
+        compile_shapes_lightweight(shapes, m_list)
+        elapsed = time.time() - t0
+        print(f"\nCompleted {model} in {elapsed:.1f}s")
+
+    print("\nDeepGEMM lightweight warmup complete.")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/cuda/warmup_server.py

@@ -0,0 +1,335 @@
+"""
+Full server warmup to pre-warm Triton autotuning and CUDA graph capture.
+
+On cold H200 nodes (new nodes or after container recreation), CUDA graph capture
+triggers Triton autotuning which takes ~330s per server launch. This script
+launches actual servers with CUDA graphs enabled to cache the autotuned kernels,
+so subsequent test launches are fast (~30-60s).
+
+Uses marker files to skip warmup on already-warm nodes. Marker files are
+invalidated when Python, Triton, or PyTorch versions change.
+
+Usage:
+    python3 scripts/ci/cuda/warmup_server.py \
+        deepseek-ai/DeepSeek-V3-0324:8
+"""
+
+import hashlib
+import json
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+# Reuse helpers from warmup_deep_gemm (same directory)
+sys.path.insert(0, os.path.dirname(__file__))
+from warmup_deep_gemm import get_architecture_key, get_config_json
+
+MARKER_DIR = os.path.join(os.path.expanduser("~"), ".cache", "sglang", "warmup_markers")
+HEALTH_POLL_INTERVAL = 10  # seconds between health checks
+SERVER_STARTUP_TIMEOUT = 900  # 15 min max to wait for server ready
+DEFAULT_PORT = 39876
+
+
+def get_version_key():
+    """Hash of Python + Triton + PyTorch versions to invalidate markers on upgrades."""
+    parts = [sys.version]
+    try:
+        import triton
+
+        parts.append(f"triton={triton.__version__}")
+    except ImportError:
+        parts.append("triton=none")
+    try:
+        import torch
+
+        parts.append(f"torch={torch.__version__}")
+    except ImportError:
+        parts.append("torch=none")
+    return hashlib.sha256("|".join(parts).encode()).hexdigest()[:12]
+
+
+def get_marker_path(model, tp):
+    """Get the marker file path for a model:tp pair."""
+    version_key = get_version_key()
+    safe_model = model.replace("/", "--")
+    return os.path.join(
+        MARKER_DIR, f"server_warmup_{safe_model}_tp{tp}_{version_key}.done"
+    )
+
+
+def check_marker(model, tp):
+    """Check if warmup marker exists (node already warm)."""
+    marker = get_marker_path(model, tp)
+    return os.path.exists(marker)
+
+
+def write_marker(model, tp):
+    """Write warmup marker after successful warmup."""
+    marker = get_marker_path(model, tp)
+    os.makedirs(os.path.dirname(marker), exist_ok=True)
+    Path(marker).write_text(
+        json.dumps(
+            {
+                "model": model,
+                "tp": tp,
+                "version_key": get_version_key(),
+                "timestamp": time.time(),
+            }
+        )
+    )
+    print(f"  Wrote marker: {marker}")
+
+
+def kill_server(proc):
+    """Kill server process tree."""
+    if proc.poll() is None:
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+        except (ProcessLookupError, OSError):
+            pass
+        try:
+            proc.wait(timeout=15)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+            except (ProcessLookupError, OSError):
+                pass
+            try:
+                proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                pass
+
+    # sglang's scheduler_TP* and detokenizer workers spawn through
+    # multiprocessing with their own session/process group, so they escape
+    # killpg on launch_server and stay alive holding GPU memory after a
+    # readiness-timeout or unclean exit. Kill any survivors by name so the
+    # next model (or the next CI step) starts with empty GPUs.
+    for pattern in ("sglang::scheduler", "sglang::detokenizer"):
+        try:
+            subprocess.run(
+                ["pkill", "-9", "-f", pattern],
+                timeout=5,
+                check=False,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        except (FileNotFoundError, subprocess.TimeoutExpired):
+            pass
+    # Let the driver release device memory before the caller measures it.
+    time.sleep(2)
+
+
+def wait_for_server(base_url, proc, timeout):
+    """Poll /health_generate until server is ready or timeout."""
+    import requests
+
+    start = time.time()
+    while time.time() - start < timeout:
+        ret = proc.poll()
+        if ret is not None:
+            return False, f"Server exited with code {ret}"
+        try:
+            resp = requests.get(f"{base_url}/health_generate", timeout=5)
+            if resp.status_code == 200:
+                return True, None
+        except requests.RequestException:
+            pass
+        time.sleep(HEALTH_POLL_INTERVAL)
+    return False, "Timed out waiting for server"
+
+
+def send_generate_request(base_url):
+    """Send one /generate request to exercise the full inference path."""
+    import requests
+
+    payload = {
+        "input_ids": [0, 1, 2, 3],
+        "sampling_params": {
+            "max_new_tokens": 8,
+            "temperature": 0,
+        },
+    }
+    try:
+        resp = requests.post(f"{base_url}/generate", json=payload, timeout=120)
+        if resp.status_code == 200:
+            print("  Generate request succeeded")
+        else:
+            print(f"  Warning: generate request returned {resp.status_code}")
+    except requests.RequestException as e:
+        print(f"  Warning: generate request failed: {e}")
+
+
+def warmup_one_model(model, tp, port):
+    """Launch server, wait for ready, send one request, then kill."""
+    base_url = f"http://127.0.0.1:{port}"
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "sglang.launch_server",
+        "--model-path",
+        model,
+        "--tp",
+        str(tp),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(port),
+        "--trust-remote-code",
+        "--model-loader-extra-config",
+        '{"enable_multithread_load": true, "num_threads": 64}',
+    ]
+
+    # Use a temp file for server output to avoid pipe buffer deadlock
+    # (server logs can exceed the 64KB pipe buffer during CUDA graph capture)
+    log_file = tempfile.NamedTemporaryFile(
+        mode="w", prefix="warmup_server_", suffix=".log", delete=False
+    )
+    log_path = log_file.name
+
+    print(f"  Launching server: {' '.join(cmd)}")
+    print(f"  Server log: {log_path}")
+    proc = subprocess.Popen(
+        cmd,
+        stdout=log_file,
+        stderr=subprocess.STDOUT,
+        preexec_fn=os.setsid,
+    )
+
+    try:
+        # Wait for server to be ready (includes CUDA graph capture)
+        print(
+            f"  Waiting for server (timeout={SERVER_STARTUP_TIMEOUT}s, "
+            f"polling every {HEALTH_POLL_INTERVAL}s)..."
+        )
+        ok, err = wait_for_server(base_url, proc, SERVER_STARTUP_TIMEOUT)
+        if not ok:
+            print(f"  Warning: server not ready: {err}")
+            return False
+
+        print("  Server ready, sending generate request...")
+        send_generate_request(base_url)
+        return True
+
+    finally:
+        # Surface the tail of the server log so CI captures validation
+        # messages, exceptions, and warmup progress (the launch_server
+        # subprocess writes stdout/stderr to the tempfile, not our stdout).
+        try:
+            log_file.flush()
+            with open(log_path) as f:
+                lines = f.readlines()
+            print(f"  --- server log tail ({len(lines)} lines, last 30) ---")
+            for line in lines[-30:]:
+                print(f"    | {line.rstrip()}")
+            print("  --- end server log ---")
+        except Exception:
+            pass
+
+        print("  Killing server...")
+        kill_server(proc)
+        log_file.close()
+        try:
+            os.unlink(log_path)
+        except OSError:
+            pass
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
+        print("Usage: warmup_server.py model1:tp1 [model2:tp2 ...]")
+        print(
+            "\nLaunches full servers with CUDA graphs enabled to pre-warm"
+            " Triton autotuning."
+        )
+        print("Skips instantly on warm nodes (marker file exists).")
+        sys.exit(0)
+
+    # Parse model:tp pairs
+    model_tp_pairs = []
+    for arg in sys.argv[1:]:
+        if ":" not in arg:
+            print(f"Error: expected model:tp format, got '{arg}'")
+            sys.exit(1)
+        model, tp_str = arg.rsplit(":", 1)
+        model_tp_pairs.append((model, int(tp_str)))
+
+    print(f"=== Server CUDA Graph Warmup ({len(model_tp_pairs)} model(s)) ===")
+    print(f"    Marker dir: {MARKER_DIR}")
+    print(f"    Version key: {get_version_key()}\n")
+
+    # Deduplicate by architecture and check markers
+    seen_keys = {}
+    to_warmup = []
+
+    for model, tp in model_tp_pairs:
+        # Check marker first (fast path)
+        if check_marker(model, tp):
+            print(f"  SKIP   {model} (tp={tp}): already warm (marker exists)")
+            continue
+
+        # Architecture dedup
+        config = get_config_json(model)
+        if config is not None:
+            key = get_architecture_key(config, tp)
+            if key in seen_keys:
+                print(
+                    f"  DEDUP  {model} (tp={tp}): same architecture as {seen_keys[key]}"
+                )
+                continue
+            seen_keys[key] = model
+
+        to_warmup.append((model, tp))
+        print(f"  QUEUE  {model} (tp={tp}): needs warmup")
+
+    if not to_warmup:
+        print("\nAll models already warm. Done.")
+        return
+
+    print(f"\n{len(to_warmup)} model(s) to warm up.\n")
+
+    port = DEFAULT_PORT
+    for i, (model, tp) in enumerate(to_warmup, 1):
+        print(f"\n{'=' * 60}")
+        print(f"[{i}/{len(to_warmup)}] {model} (tp={tp})")
+        print(f"{'=' * 60}")
+
+        t0 = time.time()
+        success = warmup_one_model(model, tp, port)
+        elapsed = time.time() - t0
+
+        if success:
+            print(f"  Completed in {elapsed:.0f}s")
+            write_marker(model, tp)
+            # Also write markers for dedup'd models that share this architecture
+            config = get_config_json(model)
+            if config is not None:
+                key = get_architecture_key(config, tp)
+                for other_model, other_tp in model_tp_pairs:
+                    if (other_model, other_tp) == (model, tp):
+                        continue
+                    other_config = get_config_json(other_model)
+                    if other_config is not None:
+                        other_key = get_architecture_key(other_config, other_tp)
+                        if other_key == key and not check_marker(other_model, other_tp):
+                            write_marker(other_model, other_tp)
+                            print(
+                                f"  Also marked {other_model} (tp={other_tp}) as warm (same arch)"
+                            )
+        else:
+            print(
+                f"  Warning: warmup failed after {elapsed:.0f}s (non-fatal, tests will still work)"
+            )
+
+        # Use a different port for the next model to avoid bind conflicts
+        port += 100
+
+    print("\nServer CUDA graph warmup complete.")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/list_stage_models.py

@@ -0,0 +1,688 @@
+#!/usr/bin/env python3
+"""Build a per-stage model inventory for NVIDIA (CUDA) CI.
+
+Emits a mapping `CI suite -> [HuggingFace model ids]` so the models a stage
+exercises can be pre-warmed into a runner cache. The mapping is produced by
+*static analysis* of the registered test files (no GPU, no sglang import), so
+it can run on a plain runner and stays fresh per commit.
+
+How `suite -> files` is resolved
+    Reuses the AST registry parser (`ut_parse_one_file` in ci_register.py, the
+    same one `run_suite.py` uses): registered test files call
+    `register_<backend>_ci(...)`; we group each file under its
+    `effective_suite` for the requested backend (the property falls back to a
+    legacy single-string `suite=` when `stage=`/`runner_config=` are unset).
+
+How `file -> models` is resolved (best effort, recall-favoring)
+    - A constant table built from `python/sglang/test/**/*.py` module-level
+      assignments (`DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/..."`, including
+      tuple/list values) plus each test file's own module-level constants.
+    - Inline HuggingFace-id string literals in the file (f-string fragments are
+      skipped: they are partial/dynamic and would yield truncated ids).
+    - `ast.Name` references that resolve to a known model constant.
+    Anything we cannot resolve is reported per suite as `unresolved_files`, and
+    any file we cannot parse is reported in `parse_failures`, so recall gaps are
+    visible rather than silent. A `--overrides` JSON file supplies models for
+    dynamic cases and trims false positives.
+
+How `runner label -> models` is aggregated
+    Registration/prewarm decisions are made per GH runner *label* (a runner's
+    `runs-on` tag), not per suite. Each suite's runner_config maps to a label
+    via scripts/ci/runner_configs.yml (several configs can share one label,
+    e.g. `4-gpu-h100` and `deepep-4-gpu-h100`), so `runner_labels` carries the
+    per-label UNION -- the set a runner registered under that label must have
+    cached before it takes jobs. Suites without a mappable runner_config are
+    listed in `unmapped_suites`.
+
+Usage:
+    python3 scripts/ci/list_stage_models.py --backend cuda \
+        --commit "$GITHUB_SHA" --output models-per-stage.json --markdown out.md
+"""
+
+import argparse
+import ast
+import glob
+import importlib.util
+import json
+import os
+import re
+import subprocess
+import sys
+from typing import Dict, List, Optional, Set, Tuple
+
+# A HuggingFace repo id is `namespace/name`: exactly one slash, each side
+# starting alphanumeric and made of alnum plus `.`, `_`, `-`. Note `.` is kept
+# because real ids carry it (e.g. `RedHatAI/Llama-3.2-3B-quantized.w8a8`).
+MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$")
+
+# `namespace/...` values that look like model ids but are MIME types or similar.
+_MIME_NAMESPACES = frozenset(
+    {
+        "application",
+        "audio",
+        "example",
+        "font",
+        "image",
+        "message",
+        "model",
+        "multipart",
+        "text",
+        "video",
+    }
+)
+
+# Trailing extensions that mark a file path or weight file, not a model name.
+# (Real ids use suffixes like `-GGUF`/`.w8a8`, not these dotted extensions.)
+_FILE_EXTENSIONS = (
+    ".py",
+    ".json",
+    ".txt",
+    ".md",
+    ".rst",
+    ".yaml",
+    ".yml",
+    ".sh",
+    ".cu",
+    ".cuh",
+    ".cpp",
+    ".cc",
+    ".h",
+    ".hpp",
+    ".so",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".csv",
+    ".log",
+    ".safetensors",
+    ".bin",
+    ".h5",
+    ".gguf",
+    ".pt",
+    ".pth",
+    ".onnx",
+)
+
+# Non-test helper files under test/registered/ (skipped by basename, matching
+# scripts/ci/check_registered_tests.py). run_suite.py skips `cpu/utils.py` by
+# path; excluding every `utils.py` by basename is a superset that drops no
+# CUDA-registered test (the other `utils.py` registers CPU only).
+_NON_TEST_BASENAMES = frozenset({"conftest.py", "__init__.py", "utils.py"})
+
+
+def looks_like_model_id(value: str, deny: Optional[Set[str]] = None) -> bool:
+    """Heuristic: does ``value`` look like a HuggingFace repo id?
+
+    Recall-favoring (a few false positives are cheap for cache-warming) but
+    drops the obvious non-models: MIME types, file paths, numeric ratios.
+    """
+    if deny and value in deny:
+        return False
+    if not MODEL_ID_RE.match(value):
+        return False
+    if not any(c.isalpha() for c in value):  # e.g. "2/3"
+        return False
+    namespace, name = value.split("/", 1)
+    if len(namespace) < 2 or len(name) < 2:  # e.g. "N/A"; real ids have longer parts
+        return False
+    if namespace.lower() in _MIME_NAMESPACES:
+        return False
+    if name.lower().endswith(_FILE_EXTENSIONS):
+        return False
+    return True
+
+
+def _string_values(node: ast.AST) -> List[str]:
+    """String constants directly held by ``node`` (a Constant, Tuple, or List)."""
+    if isinstance(node, ast.Constant):
+        return [node.value] if isinstance(node.value, str) else []
+    if isinstance(node, (ast.Tuple, ast.List)):
+        out: List[str] = []
+        for elt in node.elts:
+            if isinstance(elt, ast.Constant) and isinstance(elt.value, str):
+                out.append(elt.value)
+        return out
+    return []
+
+
+def extract_constants_from_source(
+    source: str, deny: Optional[Set[str]] = None
+) -> Dict[str, Set[str]]:
+    """Module-level ``NAME = "<model id>"`` (and tuple/list) assignments.
+
+    Handles both bare ``Assign`` and annotated ``AnnAssign`` (``NAME: str =
+    ...``). Returns ``{constant_name: {model_id, ...}}``; only model-shaped
+    values are kept, so referencing a non-model constant later contributes
+    nothing.
+    """
+    table: Dict[str, Set[str]] = {}
+    tree = ast.parse(source)
+    for stmt in tree.body:
+        if isinstance(stmt, ast.Assign):
+            targets, value = stmt.targets, stmt.value
+        elif isinstance(stmt, ast.AnnAssign) and stmt.value is not None:
+            targets, value = [stmt.target], stmt.value
+        else:
+            continue
+        models = {v for v in _string_values(value) if looks_like_model_id(v, deny)}
+        if not models:
+            continue
+        for target in targets:
+            if isinstance(target, ast.Name):
+                table.setdefault(target.id, set()).update(models)
+    return table
+
+
+def extract_models_from_source(
+    source: str,
+    const_table: Dict[str, Set[str]],
+    deny: Optional[Set[str]] = None,
+) -> Set[str]:
+    """All model ids reachable from ``source``: inline literals + name refs.
+
+    ``const_table`` is the merged (global + local) constant lookup. Name
+    references resolve against it, so an imported ``DEFAULT_*`` constant is
+    picked up even though its value is defined elsewhere. String fragments
+    inside f-strings are skipped -- they are partial/dynamic and would yield
+    truncated, non-existent ids (e.g. ``f"org/model-{ver}"``).
+    """
+    tree = ast.parse(source)
+    fstring_fragments = {
+        id(part)
+        for node in ast.walk(tree)
+        if isinstance(node, ast.JoinedStr)
+        for part in node.values
+        if isinstance(part, ast.Constant)
+    }
+    found: Set[str] = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Constant) and isinstance(node.value, str):
+            if id(node) in fstring_fragments:
+                continue
+            if looks_like_model_id(node.value, deny):
+                found.add(node.value)
+        elif isinstance(node, ast.Name) and node.id in const_table:
+            found.update(m for m in const_table[node.id] if not (deny and m in deny))
+    return found
+
+
+def build_global_constant_table(
+    repo_root: str, deny: Optional[Set[str]] = None
+) -> Tuple[Dict[str, Set[str]], Dict[str, str]]:
+    """Constant table from every module under ``python/sglang/test/``.
+
+    These shared helpers (e.g. test_utils, lora_utils) define the ``DEFAULT_*``
+    model constants test files reference by name. Returns ``(table, errors)``
+    where ``errors`` maps any unparsable helper to its exception string -- a
+    broken shared helper drops constants across many suites, so the gap must be
+    surfaced rather than swallowed.
+    """
+    table: Dict[str, Set[str]] = {}
+    errors: Dict[str, str] = {}
+    pattern = os.path.join(repo_root, "python", "sglang", "test", "**", "*.py")
+    for path in glob.glob(pattern, recursive=True):
+        try:
+            with open(path, encoding="utf-8") as f:
+                source = f.read()
+            local = extract_constants_from_source(source, deny)
+        except (OSError, SyntaxError) as exc:
+            rel = os.path.relpath(path, repo_root)
+            errors[rel] = f"{type(exc).__name__}: {exc}"
+            print(
+                f"WARNING: could not parse constant source {rel}; its model "
+                f"constants are EXCLUDED: {errors[rel]}",
+                file=sys.stderr,
+            )
+            continue
+        for name, models in local.items():
+            table.setdefault(name, set()).update(models)
+    return table, errors
+
+
+def _load_ci_register(repo_root: str):
+    """Import ci_register.py by path, sidestepping the heavy `sglang` package."""
+    spec = importlib.util.spec_from_file_location(
+        "ci_register",
+        os.path.join(repo_root, "python", "sglang", "test", "ci", "ci_register.py"),
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def collect_suite_files(
+    repo_root: str, backend_name: str, include_disabled: bool = False
+) -> Tuple[
+    Dict[str, List[str]],
+    Dict[str, bool],
+    Dict[str, str],
+    Dict[str, Optional[str]],
+]:
+    """Map ``effective_suite -> [relative test file]`` for one backend.
+
+    By default only enabled (`disabled is None`) registries are grouped, since a
+    disabled suite does not run and thus needs no cache warming. Pass
+    ``include_disabled=True`` to also group disabled registries (useful to see
+    what a suite *would* download once re-enabled). Returns the mapping,
+    ``{suite: is_nightly}``, ``{file: parse error}`` for files whose registry
+    could not be parsed (their models are excluded -- surfaced, not silently
+    dropped), and ``{suite: runner_config}`` (None for legacy single-string
+    ``suite=`` registrations, which carry no runner_config).
+    """
+    ci_register = _load_ci_register(repo_root)
+    backend = getattr(ci_register.HWBackend, backend_name.upper())
+
+    pattern = os.path.join(repo_root, "test", "registered", "**", "*.py")
+    files = sorted(
+        f
+        for f in glob.glob(pattern, recursive=True)
+        if os.path.basename(f) not in _NON_TEST_BASENAMES
+    )
+
+    suite_files: Dict[str, List[str]] = {}
+    suite_nightly: Dict[str, bool] = {}
+    suite_runner_config: Dict[str, Optional[str]] = {}
+    errors: Dict[str, str] = {}
+    for path in files:
+        rel = os.path.relpath(path, repo_root)
+        # Narrow catch: SyntaxError (bad source), ValueError (malformed
+        # registration, raised by RegistryVisitor), OSError (vanished file). A
+        # broader failure (e.g. AttributeError from a parser API drift) should
+        # crash loudly rather than silently empty the inventory.
+        try:
+            registries, _ = ci_register.ut_parse_one_file(path)
+        except (SyntaxError, ValueError, OSError) as exc:
+            errors[rel] = f"{type(exc).__name__}: {exc}"
+            print(
+                f"WARNING: could not parse {rel}; its models are EXCLUDED from "
+                f"the inventory: {errors[rel]}",
+                file=sys.stderr,
+            )
+            continue
+        for r in registries:
+            if r.backend != backend:
+                continue
+            if r.disabled is not None and not include_disabled:
+                continue
+            suite = r.effective_suite
+            if suite is None:
+                continue
+            if rel not in suite_files.setdefault(suite, []):
+                suite_files[suite].append(rel)
+            suite_nightly[suite] = suite_nightly.get(suite, False) or bool(r.nightly)
+            # Modern registrations name the suite `{stage}-test-{runner_config}`,
+            # so every registry in a suite shares one runner_config; legacy
+            # `suite=` registrations have none (stays None).
+            if r.runner_config is not None:
+                suite_runner_config[suite] = r.runner_config
+            else:
+                suite_runner_config.setdefault(suite, None)
+    return suite_files, suite_nightly, errors, suite_runner_config
+
+
+def load_overrides(path: Optional[str]) -> Dict[str, object]:
+    """Read the overrides JSON: ``by_file``, ``by_suite``, ``deny``,
+    ``suite_labels`` (all optional).
+
+    ``suite_labels`` maps a legacy ``suite=`` registration (which carries no
+    runner_config) to the GH runner label(s) its dispatching workflow
+    hardcodes in ``runs-on`` -- a LIST, since one suite can run on several
+    labels. A present-but-null key is treated as its default, so a hand-edit
+    like ``"deny": null`` does not blow up downstream iteration.
+    """
+    overrides: Dict[str, object] = {
+        "by_file": {},
+        "by_suite": {},
+        "deny": [],
+        "suite_labels": {},
+    }
+    if not path:
+        return overrides
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"overrides file not found: {path}")
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    for key in ("by_file", "by_suite", "deny", "suite_labels"):
+        if data.get(key) is not None:
+            overrides[key] = data[key]
+    return overrides
+
+
+# One runner_config entry in runner_configs.yml: a two-space-indented key with
+# a flow-style (inline `{...}`) mapping. This is the file's documented shape;
+# entries are matched line-by-line so the tool stays stdlib-only (the workflow
+# installs nothing, so PyYAML is not available).
+_RUNNER_CONFIG_LINE_RE = re.compile(r"^  ([A-Za-z0-9_-]+):\s*\{(.*)\}\s*$")
+_RUNS_ON_RE = re.compile(r"\bruns_on:\s*([^,}\s]+)")
+
+# runner_configs.yml uses this placeholder for the dynamically-selected b200
+# runner label (resolved at workflow-load time by runner_configs.py --map).
+# The inventory keeps it literal unless --b200-runner substitutes it, so the
+# consumer can see the group is dynamic rather than silently guessing a label.
+B200_SENTINEL = "$b200_runner"
+
+
+def load_runner_labels(path: str) -> Dict[str, str]:
+    """Parse ``{runner_config: runs_on label}`` out of runner_configs.yml.
+
+    The mapping is what turns per-suite model sets into per-runner-LABEL sets:
+    a runner is registered under a `runs_on` label (several runner_configs can
+    share one, e.g. `4-gpu-h100` and `deepep-4-gpu-h100` both run on
+    `4-gpu-h100`), so a runner's cache must cover the union of every suite
+    that can land on its label. Raises ValueError on an entry without
+    `runs_on` or a file with no entries at all -- a format drift must fail
+    the workflow loudly, not silently empty the label aggregation.
+    """
+    labels: Dict[str, str] = {}
+    in_section = False
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("runner_configs:"):
+                in_section = True
+                continue
+            if in_section and line.strip() and not line.startswith(" "):
+                break  # next top-level key
+            if not in_section:
+                continue
+            m = _RUNNER_CONFIG_LINE_RE.match(line)
+            if not m:
+                continue
+            name, body = m.group(1), m.group(2)
+            runs_on = _RUNS_ON_RE.search(body)
+            if not runs_on:
+                raise ValueError(f"{path}: runner_config {name!r} has no runs_on field")
+            labels[name] = runs_on.group(1)
+    if not labels:
+        raise ValueError(
+            f"{path}: no runner_configs entries parsed -- format drift? "
+            f"(expected two-space-indented `name: {{...}}` lines under "
+            f"a `runner_configs:` key)"
+        )
+    return labels
+
+
+def build_inventory(
+    repo_root: str,
+    backend_name: str,
+    overrides: Dict[str, object],
+    commit: str,
+    include_disabled: bool = False,
+    b200_runner: Optional[str] = None,
+) -> Dict[str, object]:
+    deny: Set[str] = set(overrides.get("deny", []))  # type: ignore[arg-type]
+    by_file: Dict[str, List[str]] = overrides.get("by_file", {})  # type: ignore[assignment]
+    by_suite: Dict[str, List[str]] = overrides.get("by_suite", {})  # type: ignore[assignment]
+    suite_labels_override: Dict[str, List[str]] = overrides.get("suite_labels", {})  # type: ignore[assignment]
+
+    global_table, table_errors = build_global_constant_table(repo_root, deny)
+    suite_files, suite_nightly, registry_errors, suite_runner_config = (
+        collect_suite_files(repo_root, backend_name, include_disabled)
+    )
+
+    runner_labels_map: Dict[str, str] = {}
+    runner_configs_path = os.path.join(repo_root, "scripts", "ci", "runner_configs.yml")
+    if os.path.exists(runner_configs_path):
+        runner_labels_map = load_runner_labels(runner_configs_path)
+    else:
+        print(
+            f"WARNING: {runner_configs_path} not found; every suite will be "
+            f"reported as unmapped_suites (no per-runner-label aggregation).",
+            file=sys.stderr,
+        )
+
+    # Resolve models once per file (a file can belong to several suites).
+    file_models: Dict[str, Set[str]] = {}
+    extract_errors: Dict[str, str] = {}
+    for files in suite_files.values():
+        for rel in files:
+            if rel in file_models:
+                continue
+            abs_path = os.path.join(repo_root, rel)
+            try:
+                with open(abs_path, encoding="utf-8") as f:
+                    source = f.read()
+                local_table = extract_constants_from_source(source, deny)
+                merged = dict(global_table)
+                for name, models in local_table.items():
+                    merged.setdefault(name, set()).update(models)
+                resolved = extract_models_from_source(source, merged, deny)
+            except (OSError, SyntaxError) as exc:
+                extract_errors[rel] = f"{type(exc).__name__}: {exc}"
+                print(
+                    f"WARNING: could not extract models from {rel}; treating it "
+                    f"as unresolved: {extract_errors[rel]}",
+                    file=sys.stderr,
+                )
+                resolved = set()
+            resolved.update(by_file.get(rel, []))
+            file_models[rel] = resolved
+
+    suites: Dict[str, object] = {}
+    all_models: Set[str] = set()
+    for suite in sorted(suite_files):
+        models: Set[str] = set(by_suite.get(suite, []))
+        unresolved: List[str] = []
+        for rel in suite_files[suite]:
+            resolved = file_models.get(rel, set())
+            if resolved:
+                models.update(resolved)
+            else:
+                unresolved.append(rel)
+        all_models.update(models)
+        suites[suite] = {
+            "nightly": suite_nightly.get(suite, False),
+            "runner_config": suite_runner_config.get(suite),
+            "models": sorted(models),
+            "test_file_count": len(suite_files[suite]),
+            "unresolved_files": sorted(unresolved),
+        }
+
+    # Per-runner-LABEL aggregation: registration/prewarm decisions are made per
+    # GH runner label (what a runner is registered with), and several suites --
+    # via several runner_configs -- can route to one label. A runner's cache
+    # must cover the UNION of every suite that can land on it. Label
+    # resolution order: an explicit `suite_labels` override (legacy suites
+    # whose runs-on lives hardcoded in their dispatching workflow; may name
+    # several labels), else runner_config -> runner_configs.yml. Suites we
+    # cannot map land in `unmapped_suites` -- visible, never silently
+    # dropped, same contract as unresolved_files.
+    label_models: Dict[str, Set[str]] = {}
+    label_suites: Dict[str, List[str]] = {}
+    unmapped_suites: List[str] = []
+    for suite in sorted(suites):
+        labels = suite_labels_override.get(suite)
+        if labels is None:
+            rc = suite_runner_config.get(suite)
+            label = runner_labels_map.get(rc) if rc is not None else None
+            labels = [label] if label is not None else []
+        if not labels:
+            unmapped_suites.append(suite)
+            continue
+        for label in labels:
+            if label == B200_SENTINEL and b200_runner:
+                label = b200_runner
+            label_models.setdefault(label, set()).update(suites[suite]["models"])
+            label_suites.setdefault(label, []).append(suite)
+    runner_labels: Dict[str, object] = {
+        label: {
+            "models": sorted(label_models[label]),
+            "suites": label_suites[label],
+        }
+        for label in sorted(label_models)
+    }
+
+    parse_failures = {}
+    parse_failures.update(table_errors)
+    parse_failures.update(registry_errors)
+    parse_failures.update(extract_errors)
+
+    return {
+        "generated_at_commit": commit,
+        "backend": backend_name.lower(),
+        "suite_count": len(suites),
+        "model_count": len(all_models),
+        "runner_label_count": len(runner_labels),
+        "all_models": sorted(all_models),
+        "parse_failures": dict(sorted(parse_failures.items())),
+        "runner_labels": runner_labels,
+        "unmapped_suites": unmapped_suites,
+        "suites": suites,
+    }
+
+
+def render_markdown(inventory: Dict[str, object]) -> str:
+    suites: Dict[str, dict] = inventory["suites"]  # type: ignore[assignment]
+    failures = inventory.get("parse_failures") or {}
+    runner_labels: Dict[str, dict] = inventory.get("runner_labels") or {}  # type: ignore[assignment]
+    unmapped = inventory.get("unmapped_suites") or []
+    lines = [
+        f"## NVIDIA CI model inventory (`{inventory['backend']}`)",
+        "",
+        f"- Commit: `{inventory['generated_at_commit']}`",
+        f"- Suites: **{inventory['suite_count']}**, "
+        f"distinct models: **{inventory['model_count']}**, "
+        f"runner labels: **{inventory.get('runner_label_count', 0)}**",
+    ]
+    if failures:
+        lines.append(
+            f"- ⚠️ Unparsable files: **{len(failures)}** (see `parse_failures`)"
+        )
+    if unmapped:
+        lines.append(
+            f"- ⚠️ Suites with no runner label: **{len(unmapped)}** "
+            f"({', '.join(f'`{s}`' for s in unmapped)})"
+        )
+    if runner_labels:
+        lines += [
+            "",
+            "### Per runner label (prewarm a runner's cache with this union)",
+            "",
+            "| Runner label | Suites | Models |",
+            "| --- | ---: | --- |",
+        ]
+        for label in sorted(runner_labels):
+            info = runner_labels[label]
+            models = ", ".join(info["models"]) if info["models"] else "_(none)_"
+            lines.append(f"| `{label}` | {len(info['suites'])} | {models} |")
+    lines += [
+        "",
+        "### Per suite",
+        "",
+        "| Suite | Nightly | Models | Unresolved files |",
+        "| --- | :---: | --- | ---: |",
+    ]
+    for suite in sorted(suites):
+        info = suites[suite]
+        models = ", ".join(info["models"]) if info["models"] else "_(none)_"
+        nightly = "✓" if info["nightly"] else ""
+        lines.append(
+            f"| `{suite}` | {nightly} | {models} | {len(info['unresolved_files'])} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def resolve_commit(arg: Optional[str], repo_root: str) -> str:
+    if arg:
+        return arg
+    env = os.environ.get("GITHUB_SHA")
+    if env:
+        return env
+    try:
+        return subprocess.check_output(
+            ["git", "-C", repo_root, "rev-parse", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return "unknown"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--repo-root",
+        default=".",
+        help="Repository root (default: current directory).",
+    )
+    parser.add_argument(
+        "--backend",
+        default="cuda",
+        help="Hardware backend name (default: cuda).",
+    )
+    parser.add_argument(
+        "--overrides",
+        default=os.path.join("scripts", "ci", "stage_models_overrides.json"),
+        help="Path to the overrides JSON (relative paths resolve under "
+        "--repo-root). A warning is printed if it does not exist.",
+    )
+    parser.add_argument("--commit", default=None, help="Commit sha to record.")
+    parser.add_argument(
+        "--include-disabled",
+        action="store_true",
+        help="Also include suites whose tests are currently disabled.",
+    )
+    parser.add_argument(
+        "--b200-runner",
+        default=None,
+        help="Concrete runner label to substitute for the $b200_runner "
+        "placeholder in runner_configs.yml (default: keep the placeholder "
+        "as the runner_labels key, marking the group as dynamically routed).",
+    )
+    parser.add_argument(
+        "--output", default=None, help="Write JSON here (default: stdout)."
+    )
+    parser.add_argument(
+        "--markdown", default=None, help="Also write a Markdown summary here."
+    )
+    args = parser.parse_args()
+
+    repo_root = os.path.abspath(args.repo_root)
+    # Resolve a relative overrides path against repo_root (not cwd) so the flag
+    # works when invoked from elsewhere with --repo-root.
+    overrides_path = args.overrides
+    if not os.path.isabs(overrides_path):
+        overrides_path = os.path.join(repo_root, overrides_path)
+    if not os.path.exists(overrides_path):
+        print(
+            f"WARNING: overrides file not found at {overrides_path}; "
+            f"proceeding without overrides.",
+            file=sys.stderr,
+        )
+        overrides_path = None
+    overrides = load_overrides(overrides_path)
+    commit = resolve_commit(args.commit, repo_root)
+
+    inventory = build_inventory(
+        repo_root,
+        args.backend,
+        overrides,
+        commit,
+        args.include_disabled,
+        b200_runner=args.b200_runner,
+    )
+    payload = json.dumps(inventory, indent=2, sort_keys=False) + "\n"
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(payload)
+        print(
+            f"Wrote {args.output}: {inventory['suite_count']} suites, "
+            f"{inventory['runner_label_count']} runner labels, "
+            f"{inventory['model_count']} distinct models, "
+            f"{len(inventory['parse_failures'])} parse failures, "
+            f"{len(inventory['unmapped_suites'])} unmapped suites.",
+            file=sys.stderr,
+        )
+    else:
+        sys.stdout.write(payload)
+
+    if args.markdown:
+        with open(args.markdown, "w", encoding="utf-8") as f:
+            f.write(render_markdown(inventory))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/musa/musa_install_dependency.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -euo pipefail
+
+# Parse command line arguments
+OPTIONAL_DEPS=""
+SKIP_SGLANG_BUILD=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-sglang-build) SKIP_SGLANG_BUILD="1"; shift;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS] [OPTIONAL_DEPS]"
+      echo "Options:"
+      echo "  --skip-sglang-build         Don't build checkout sglang, use what was shipped with the image"
+      exit 0
+      ;;
+    *)
+      OPTIONAL_DEPS="$1"
+      shift
+      ;;
+  esac
+done
+
+PIP_INSTALL="python3 -m pip install --no-cache-dir"
+${PIP_INSTALL} --upgrade pip setuptools torchada --user
+
+echo "Checking stale torchada extension locks..."
+active_torchada_builds="$(
+    pgrep -af '(^|[[:space:]/])(mcc|ninja)([[:space:]]|$)|torchada_cpp_ops' 2>/dev/null \
+        | awk -v self="$$" '$1 != self'
+)" || true
+if [ -n "$active_torchada_builds" ]; then
+    echo "$active_torchada_builds"
+    echo "::error::Active torchada extension build detected; refusing to remove lock files"
+    exit 1
+fi
+torch_extensions_dir="${HOME}/.cache/torch_extensions"
+if [ -d "$torch_extensions_dir" ]; then
+    find "$torch_extensions_dir" \
+        -path '*/torchada_cpp_ops/lock' \
+        -type f \
+        -print \
+        -delete
+fi
+
+WHL_DIR="/sglang-checkout/whl"
+if [ -d "$WHL_DIR" ] && compgen -G "${WHL_DIR}"/*.whl > /dev/null; then
+    echo "Uninstall old packages based on wheel METADATA..."
+    PKGS=$(
+      for whl in "${WHL_DIR}"/*.whl; do
+        meta_file=$(zipinfo -1 "$whl" | awk '/\.dist-info\/METADATA$/ {print; exit}')
+        [ -n "$meta_file" ] || continue
+        unzip -p "$whl" "$meta_file" 2>/dev/null | sed -n 's/^Name: //p' | head -n1
+      done | sort -u
+    )
+    for pkg in $PKGS; do
+      echo "Uninstalling $pkg"
+      pip uninstall -y "$pkg" || true
+    done
+    echo "Installing wheel files without dependency resolution..."
+    ${PIP_INSTALL} "${WHL_DIR}"/*.whl --user
+fi
+
+if [ -n "$SKIP_SGLANG_BUILD" ]; then
+    echo "Didn't build checkout SGLang"
+    exit 0
+else
+    pip uninstall sgl-kernel -y || true
+    pip uninstall sglang -y || true
+    # Clear Python cache to ensure latest code is used (works for any env: venv, system, conda)
+    REPO_ROOT="${GITHUB_WORKSPACE:-$(pwd)}"
+    find "$REPO_ROOT" -name "*.pyc" -delete 2>/dev/null || true
+    find "$REPO_ROOT" -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true
+
+    rm -f "${REPO_ROOT}/python/pyproject.toml" && mv "${REPO_ROOT}/python/pyproject_other.toml" "${REPO_ROOT}/python/pyproject.toml"
+    cd "${REPO_ROOT}" && ${PIP_INSTALL} -v -e "python[dev_musa]" --user
+
+    cd "${REPO_ROOT}/sgl-kernel"
+    rm -f pyproject.toml && mv pyproject_musa.toml pyproject.toml && MTGPU_TARGET=mp_31 python3 setup_musa.py install --user
+    echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+fi

scripts/ci/musa/rename_wheels_musa.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# Align MUSA wheel filenames (+musa43/...) with internal METADATA Version and
+# WHEEL tags after build. Two drifts need fixing in lockstep:
+#   - METADATA `Version:` must carry the `+musa<suffix>` local version, or
+#     recent pip versions reject the wheel with "inconsistent version".
+#   - WHEEL `Tag:` must be `manylinux2014_*` when the filename says so;
+#     leaving it as `linux_*` can trip installers that re-derive the platform.
+# Unpack → patch WHEEL/METADATA → wheel pack (RECORD regenerated; no hand-editing).
+#
+# Usage:
+#   rename_wheels_musa.sh <musa_suffix> [wheel_dir]
+# Example:
+#   rename_wheels_musa.sh 43 sgl-kernel/dist
+set -euxo pipefail
+
+if [[ $# -lt 1 || $# -gt 2 ]]; then
+  echo "Usage: $0 <musa_suffix> [wheel_dir]" >&2
+  exit 1
+fi
+
+MUSA_SUFFIX="+musa$1"
+WHEEL_DIR="${2:-dist}"
+
+patch_wheel_platform_tags() {
+    local wheel_file="$1"
+    # Line-end anchors: "linux_x86_64" is a substring of "manylinux2014_x86_64", so
+    # unanchored global replace corrupts tags on a second run.
+    sed -i \
+        -e 's/-linux_x86_64$/-manylinux2014_x86_64/' \
+        -e 's/-linux_aarch64$/-manylinux2014_aarch64/' \
+        "$wheel_file"
+}
+
+wheel_files=("$WHEEL_DIR"/*.whl)
+if [[ ! -e "${wheel_files[0]}" ]]; then
+  echo "No wheel files found in ${WHEEL_DIR}/, nothing to rename."
+  exit 0
+fi
+
+for wheel in "${wheel_files[@]}"; do
+    [[ -f "$wheel" ]] || continue
+
+    intermediate_wheel="$wheel"
+    case "$wheel" in
+        *-linux_x86_64.whl)
+            intermediate_wheel="${wheel%-linux_x86_64.whl}-manylinux2014_x86_64.whl"
+            ;;
+        *-linux_aarch64.whl)
+            intermediate_wheel="${wheel%-linux_aarch64.whl}-manylinux2014_aarch64.whl"
+            ;;
+    esac
+    if [[ "$wheel" != "$intermediate_wheel" ]]; then
+        mv -- "$wheel" "$intermediate_wheel"
+        wheel="$intermediate_wheel"
+    fi
+
+    TMPDIR=$(mktemp -d)
+    trap 'rm -rf -- "$TMPDIR"' ERR
+
+    "${PYTHON:-python3}" -m wheel unpack "$wheel" --dest "$TMPDIR"
+    # `find | head -1` succeeds with empty stdout when there are no matches —
+    # `set -e` won't catch that. Assert each path is real so a malformed wheel
+    # surfaces with a useful message instead of a downstream `sed: /WHEEL` error.
+    UNPACKED=$(find "$TMPDIR" -mindepth 1 -maxdepth 1 -type d | head -1)
+    [[ -d "$UNPACKED" ]] || { echo "ERROR: wheel unpack produced no top-level dir for $wheel" >&2; exit 1; }
+    DIST_INFO=$(find "$UNPACKED" -maxdepth 1 -type d -name "*.dist-info" | head -1)
+    [[ -d "$DIST_INFO" ]] || { echo "ERROR: no *.dist-info under $UNPACKED (malformed wheel?): $wheel" >&2; exit 1; }
+    WHEEL_META="${DIST_INFO}/WHEEL"
+    METADATA_FILE="${DIST_INFO}/METADATA"
+    [[ -f "$WHEEL_META" && -f "$METADATA_FILE" ]] || { echo "ERROR: missing WHEEL or METADATA in $DIST_INFO" >&2; exit 1; }
+
+    patch_wheel_platform_tags "$WHEEL_META"
+
+    ORIG_VERSION=$(grep '^Version:' "$METADATA_FILE" | head -1 | sed 's/^Version:[[:space:]]*//')
+    # Empty ORIG_VERSION would fall through the `+musa` check below and silently
+    # produce `Version: +musa43` — a broken release. Fail loud instead.
+    [[ -n "$ORIG_VERSION" ]] || { echo "ERROR: no 'Version:' line in $METADATA_FILE" >&2; exit 1; }
+    if [[ "$ORIG_VERSION" == *"$MUSA_SUFFIX"* ]]; then
+        echo "Skipping $wheel: version in METADATA is already suffixed."
+        rm -rf "$TMPDIR"
+        trap - ERR
+        continue
+    fi
+    NEW_VERSION="${ORIG_VERSION}${MUSA_SUFFIX}"
+    sed -i "s/^Version:.*/Version: ${NEW_VERSION}/" "$METADATA_FILE"
+    # `sed -i` exits 0 even when the pattern matched zero lines. Verify the
+    # rewrite actually landed before we publish.
+    grep -qx "Version: ${NEW_VERSION}" "$METADATA_FILE" || { echo "ERROR: METADATA Version rewrite did not land in $METADATA_FILE" >&2; exit 1; }
+
+    OLD_BASE=$(basename "$DIST_INFO")
+    NEW_BASE="${OLD_BASE/${ORIG_VERSION}/${NEW_VERSION}}"
+    # `${var/pat/repl}` silently leaves var unchanged if pat is empty or absent.
+    [[ "$NEW_BASE" != "$OLD_BASE" ]] || { echo "ERROR: dist-info dir '$OLD_BASE' did not contain ORIG_VERSION='$ORIG_VERSION'" >&2; exit 1; }
+    mv "$DIST_INFO" "${UNPACKED}/${NEW_BASE}"
+
+    rm -f "$wheel"
+    "${PYTHON:-python3}" -m wheel pack "$UNPACKED" --dest-dir "$WHEEL_DIR"
+    rm -rf "$TMPDIR"
+    trap - ERR
+done
+
+echo "MUSA wheel renaming completed."

scripts/ci/npu/npu_ci_install_dependency.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+set -euo pipefail
+
+PIP_INSTALL="python3 -m pip install --no-cache-dir"
+UV_PIP_INSTALL="uv pip install "
+DEVICE_TYPE=$1
+
+
+# Install the required dependencies in CI.
+apt update -y && apt install -y \
+    unzip \
+    build-essential \
+    cmake \
+    wget \
+    curl \
+    net-tools \
+    zlib1g-dev \
+    lld \
+    clang \
+    locales \
+    ccache \
+    libgl1-mesa-glx \
+    libgl1-mesa-dri \
+    ca-certificates \
+    libgl1 \
+    libglib2.0-0
+update-ca-certificates
+${PIP_INSTALL} --upgrade pip
+${PIP_INSTALL} uv
+export UV_NO_CACHE=true
+export UV_SYSTEM_PYTHON=true
+export UV_INDEX_STRATEGY=unsafe-best-match
+
+# Install Rust toolchain (needed by crates built via setuptools-rust, e.g. the
+# native gRPC extension bundled into the sglang wheel).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+bash "${SCRIPT_DIR}/../utils/install_rustup.sh"
+export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}"
+
+${UV_PIP_INSTALL} pybind11 pyyaml decorator scipy attrs psutil
+
+
+### Install MemFabric
+${UV_PIP_INSTALL} memfabric-hybrid==1.0.8
+
+
+### Install PyTorch and PTA
+PYTORCH_VERSION="2.10.0"
+TORCHVISION_VERSION="0.25.0"
+TORCHAUDIO_VERSION="2.10.0"
+${UV_PIP_INSTALL} torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION} torchaudio==${TORCHAUDIO_VERSION} --index-url ${TORCH_CACHE_URL:="https://download.pytorch.org/whl/cpu"} --extra-index-url ${PYPI_CACHE_URL:="https://pypi.org/simple/"}
+PTA_URL="https://gitcode.com/Ascend/pytorch/releases/download/v26.0.0-pytorch2.10.0/torch_npu-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl"
+# GitCode does not allow UV downloads.
+${PIP_INSTALL} ${PTA_URL}
+
+### Install zbal
+${UV_PIP_INSTALL} memfabric-zbal==1.1.1
+
+### Install Triton-Ascend
+${PIP_INSTALL} triton-ascend==3.2.1.dev20260530 --extra-index-url=https://mirrors.huaweicloud.com/ascend/repos/pypi/nightly --trusted-host triton-ascend.osinfra.cn
+
+
+### Install sgl-kernel-npu
+SGLANG_KERNEL_NPU_TAG="2026.6.2"
+mkdir sgl-kernel-npu
+(cd sgl-kernel-npu && wget "${GITHUB_PROXY_URL:=""}https://github.com/sgl-project/sgl-kernel-npu/releases/download/${SGLANG_KERNEL_NPU_TAG}/sgl-kernel-npu-${SGLANG_KERNEL_NPU_TAG}-torch${PYTORCH_VERSION}-py311-cann9.0.0-${DEVICE_TYPE}-$(arch).zip" \
+&& unzip ./sgl-kernel-npu-${SGLANG_KERNEL_NPU_TAG}-torch${PYTORCH_VERSION}-py311-cann9.0.0-${DEVICE_TYPE}-$(arch).zip \
+&& ${UV_PIP_INSTALL} ./deep_ep*.whl ./sgl_kernel_npu*.whl \
+&& (cd "$(python3 -m pip show deep-ep | grep -E '^Location:' | awk '{print $2}')" && ln -s deep_ep/deep_ep_cpp*.so))
+
+
+### Install SGLang
+rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml
+${UV_PIP_INSTALL} -v -e "python[dev_npu]"

scripts/ci/npu/npu_log_print.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+
+# Print log information(sglang version, commit sha, sgl-kernel-npu version, sgl-kernel-npu commit sha, npu-smi info and pip list.
+npu-smi info
+pip list
+get_version() {
+    [ -f "$1" ] && python3 -c 'import re, sys; print(sys.argv[2] + " version: v" + re.search(r"__version__\s*=\s*[\"'"'"'](.*?)[\"'"'"']", open(sys.argv[1]).read()).group(1))' "$1" "$2" 2>/dev/null || echo "$2 version: unknown"
+}
+get_version "./python/sglang/version.py" "sglang"
+get_version "./sgl-kernel/python/sgl_kernel/version.py" "sgl_kernel"
+SGLANG_URL="https://github.com/sgl-project/sglang.git"
+SGL_KERNEL_URL="https://github.com/sgl-project/sgl-kernel-npu.git"
+SGLANG_BRANCH="main"
+SGL_KERNEL_BRANCH="main"
+get_sha() {
+    local name="$1"
+    local url="$2"
+    local branch="$3"
+    local sha
+    sha=$(git ls-remote "$url" "refs/heads/$branch" | cut -f1)
+    echo "$name SHA for branch $branch: ${sha:-"Not Found"}"
+}
+get_sha "sglang" "$SGLANG_URL" "$SGLANG_BRANCH"
+get_sha "sgl-kernel" "$SGL_KERNEL_URL" "$SGL_KERNEL_BRANCH"
+chmod +x scripts/ci/npu/npu_log_print.sh

scripts/ci/runner_configs.py

@@ -0,0 +1,60 @@
+"""Emit runner_config setup for GitHub Actions $GITHUB_OUTPUT.
+
+runner_configs.py <runner_config>
+    Per-field `key=value` lines (install / artifact_version /
+    install_timeout / rdma_devices). `runs_on` is intentionally omitted —
+    it carries the `$b200_runner` sentinel and is resolved via --map.
+    Called per stage by _pr-test-stage.yml.
+
+runner_configs.py --map <b200_runner_label>
+    `runs_on_map={json}` — flat dict {runner_config: runs_on}, with
+    `$b200_runner` substituted. Called once by _pr-test-check-changes.yml.
+"""
+
+import json
+import os
+import sys
+
+import yaml
+
+_YAML_PATH = os.path.join(os.path.dirname(__file__), "runner_configs.yml")
+_B200_SENTINEL = "$b200_runner"
+
+
+def load() -> dict:
+    with open(_YAML_PATH) as f:
+        return yaml.safe_load(f)["runner_configs"]
+
+
+def _emit_single(rc: str) -> None:
+    # runs_on goes through --map (resolves $b200_runner). Suppress it here so a
+    # consumer can't accidentally read the raw sentinel value.
+    cfg = load().get(rc)
+    if cfg is None:
+        sys.exit(f"unknown runner_config: {rc!r}")
+    for key, value in cfg.items():
+        if key == "runs_on":
+            continue
+        print(f"{key}={value}")
+
+
+def _emit_map(b200_runner: str) -> None:
+    runs_on = {
+        name: (b200_runner if cfg.get("runs_on") == _B200_SENTINEL else cfg["runs_on"])
+        for name, cfg in load().items()
+    }
+    print(f"runs_on_map={json.dumps(runs_on, separators=(',', ':'))}")
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    if len(args) == 1:
+        _emit_single(args[0])
+    elif len(args) == 2 and args[0] == "--map":
+        _emit_map(args[1])
+    else:
+        sys.exit(
+            "usage:\n"
+            "  runner_configs.py <runner_config>\n"
+            "  runner_configs.py --map <b200_runner_label>"
+        )

scripts/ci/runner_configs.yml

@@ -0,0 +1,35 @@
+# Per-runner-config CUDA setup details. Single source of truth for the
+# `runner_config` field on `register_cuda_ci(...)` calls. Consumed by
+# scripts/ci/runner_configs.py (CLI wrapper), which is in turn called by
+# .github/workflows/_pr-test-stage.yml.
+#
+# Each runner_config carries:
+#   - install: install script path
+#   - artifact_version: actions/download-artifact major version
+#   - install_timeout: install-step wall-clock cap (minutes), enforced via
+#     `timeout-minutes:` on the install step in _pr-test-stage.yml
+#   - grace_blackwell (optional): exported as GRACE_BLACKWELL for the install
+#     step. Used by GB300 DeePEP setup.
+#   - runs_on: GHA runner label for the stage's `runs-on:`. The literal
+#     `$b200_runner` is substituted at workflow-load time with the dynamic
+#     b200 runner tag from check-changes (see runner_configs.py --map).
+#   - rdma_devices (optional): exported as SGLANG_CI_RDMA_ALL_DEVICES env
+#     to the stage job; absent means unset.
+
+_anchors:
+  default_install: &default scripts/ci/cuda/ci_install_dependency.sh
+  deepep_install:  &deepep  scripts/ci/cuda/ci_install_deepep.sh
+
+runner_configs:
+  1-gpu-small:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-5090 }
+  1-gpu-large:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-h100 }
+  2-gpu-large:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 2-gpu-h100 }
+  4-gpu-b200:        { install: *default, artifact_version: v6, install_timeout: "20", runs_on: $b200_runner }
+  4-gpu-gb300:       { install: *deepep,  artifact_version: v6, install_timeout: "20", grace_blackwell: "1", runs_on: 4-gpu-gb300 }
+  4-gpu-h100:        { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 }
+  8-gpu-h200:        { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 }
+  8-gpu-b200:        { install: *default, artifact_version: v6, install_timeout: "20", runs_on: 8-gpu-b200 }
+  8-gpu-h20:         { install: *deepep,  artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h20, rdma_devices: "mlx5_1,mlx5_2,mlx5_3,mlx5_4" }
+  deepep-4-gpu-h100: { install: *deepep,  artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 }
+  deepep-4-gpu-b200: { install: *deepep,  artifact_version: v6, install_timeout: "20", runs_on: $b200_runner }
+  deepep-8-gpu-h200: { install: *deepep,  artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 }

scripts/ci/slurm/analyze_logs_with_modal.py

@@ -0,0 +1,376 @@
+#!/usr/bin/env python3
+"""Analyze srtslurm logs with opencode inside a Modal sandbox.
+
+This script accepts either:
+- a local log directory
+- a `.tar.gz` bundle such as `multinode_server_logs.tar.gz`
+
+It uploads the logs into an ephemeral Modal sandbox, installs and runs
+opencode with an analysis prompt, and prints the resulting markdown.
+
+Example:
+  uv run --with modal python scripts/ci/slurm/analyze_logs_with_modal.py \
+    --tarball /tmp/multinode_server_logs.tar.gz \
+    --job-id 4645
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import re
+import shlex
+import shutil
+import tarfile
+import tempfile
+from pathlib import Path
+
+try:
+    import modal
+except ImportError:  # pragma: no cover - runtime guard for local usage
+    modal = None
+
+
+logger = logging.getLogger("slurm_log_analysis")
+
+SANDBOX_TIMEOUT = 600
+DEFAULT_MODAL_SECRET_NAME = "or"
+DEFAULT_MODEL = "openrouter/minimax/minimax-m2.7"
+DEFAULT_REPOS = [
+    "https://github.com/sgl-project/sglang.git",
+]
+PROMPT_PATH = Path(__file__).with_name("log_analysis_prompt.md")
+
+# Matches common API key / token patterns (sk-..., ak-..., as-..., key-..., etc.)
+_SECRET_PATTERN = re.compile(
+    r"""(?:"""
+    r"""(?:sk|ak|as|key|token|secret|bearer)[-_][A-Za-z0-9_\-]{16,}"""
+    r"""|"""
+    r"""(?:OPENROUTER_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|ANTHROPIC_API_KEY)"""
+    r"""[=:]\s*\S+"""
+    r""")""",
+    re.IGNORECASE,
+)
+
+
+def sanitize(text: str) -> str:
+    """Redact strings that look like API keys or secrets."""
+    if not text:
+        return text
+    sanitized = _SECRET_PATTERN.sub("[REDACTED]", text)
+    # Also redact any env var values we know are secrets
+    for var in ("OPENROUTER_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"):
+        val = os.environ.get(var)
+        if val and len(val) > 8:
+            sanitized = sanitized.replace(val, "[REDACTED]")
+    return sanitized
+
+
+def configure_logging(verbose: bool) -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s: %(message)s",
+    )
+    logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+
+
+def extract_tarball(tarball: Path, destination: Path) -> None:
+    with tarfile.open(tarball, "r:gz") as archive:
+        # Python 3.14 changes the default extraction behavior. Use the
+        # data filter when available so extraction remains explicit.
+        if "data" in tarfile._NAMED_FILTERS:  # type: ignore[attr-defined]
+            archive.extractall(destination, filter="data")
+        else:
+            archive.extractall(destination)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Analyze a srtslurm log bundle with opencode in Modal."
+    )
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument(
+        "--tarball",
+        type=Path,
+        help="Path to a local multinode_server_logs.tar.gz bundle.",
+    )
+    source.add_argument(
+        "--log-dir",
+        type=Path,
+        help="Path to an unpacked log directory.",
+    )
+    parser.add_argument(
+        "--job-id",
+        default="unknown",
+        help="Job identifier used in the report header and logs.",
+    )
+    parser.add_argument(
+        "--model",
+        default=DEFAULT_MODEL,
+        help="Model selector to pass to opencode run.",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="Optional path to write the markdown analysis.",
+    )
+    parser.add_argument(
+        "--repo-url",
+        action="append",
+        dest="repo_urls",
+        help=(
+            "Optional extra repo URL to clone into the sandbox for context. "
+            "Can be specified multiple times."
+        ),
+    )
+    parser.add_argument(
+        "--timeout-seconds",
+        type=int,
+        default=SANDBOX_TIMEOUT,
+        help="Sandbox lifetime in seconds.",
+    )
+    parser.add_argument(
+        "--modal-secret-name",
+        default=DEFAULT_MODAL_SECRET_NAME,
+        help="Modal secret name that provides OPENROUTER_API_KEY to the sandbox.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging.",
+    )
+    return parser.parse_args()
+
+
+def build_sandbox_image() -> modal.Image:
+    if modal is None:
+        raise RuntimeError(
+            "The 'modal' package is required. Run this script with "
+            "`uv run --with modal python ...` or install modal locally."
+        )
+
+    return (
+        modal.Image.debian_slim(python_version="3.12")
+        .apt_install("bash", "curl", "git", "gh")
+        .run_commands(
+            "curl -fsSL https://opencode.ai/install | bash",
+        )
+        .env(
+            {
+                "PATH": "/root/.opencode/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            }
+        )
+    )
+
+
+def prepare_log_dir(args: argparse.Namespace) -> tuple[Path, Path | None]:
+    if args.log_dir:
+        if not args.log_dir.is_dir():
+            raise FileNotFoundError(f"log directory not found: {args.log_dir}")
+        return args.log_dir.resolve(), None
+
+    assert args.tarball is not None
+    if not args.tarball.is_file():
+        raise FileNotFoundError(f"tarball not found: {args.tarball}")
+
+    temp_dir = Path(tempfile.mkdtemp(prefix="sglang_logs_"))
+    extract_tarball(args.tarball, temp_dir)
+    return temp_dir.resolve(), temp_dir
+
+
+def build_prompt(job_id: str, repo_urls: list[str]) -> str:
+    skill_content = PROMPT_PATH.read_text()
+    repo_lines = []
+    for repo_url in repo_urls:
+        repo_name = repo_url.rsplit("/", 1)[-1].removesuffix(".git")
+        repo_lines.append(f"- **{repo_name} repo**: `/workspace/repos/{repo_name}/`")
+    repo_section = (
+        "\n".join(repo_lines) if repo_lines else "- No extra repos were requested."
+    )
+
+    return f"""{skill_content}
+
+---
+
+## Your Environment
+
+- **Logs**: `/workspace/logs/`
+- **GitHub CLI**: `gh` is installed and authenticated.
+{repo_section}
+
+## Job
+
+You are analyzing job `{job_id}`. Follow Steps 1–5 in the prompt above.
+
+**You MUST write the final markdown report to `/workspace/logs/ai_analysis.md`.**
+This is a hard requirement. Do not just print the report to stdout. Use your
+file-writing tool to create `/workspace/logs/ai_analysis.md` with the full
+analysis. The downstream pipeline reads this file.
+
+**You MUST file GitHub issues when the root cause is clear (Category A or B).**
+Do not skip issue filing. The whole point of this system is automated triage.
+"""
+
+
+def upload_tree(sandbox: modal.Sandbox, log_dir: Path) -> None:
+    log_files = [path for path in log_dir.rglob("*") if path.is_file()]
+    logger.info("Uploading %d log files into the sandbox", len(log_files))
+    for index, log_file in enumerate(log_files, start=1):
+        rel_path = log_file.relative_to(log_dir)
+        remote_path = str(Path("/workspace/logs") / rel_path)
+        sandbox.mkdir(str(Path(remote_path).parent), parents=True)
+        sandbox.filesystem.write_bytes(log_file.read_bytes(), remote_path)
+        if index % 10 == 0 or index == len(log_files):
+            logger.info("Uploaded %d/%d files", index, len(log_files))
+
+
+def clone_context_repos(sandbox: modal.Sandbox, repo_urls: list[str]) -> None:
+    if not repo_urls:
+        return
+
+    sandbox.mkdir("/workspace/repos", parents=True)
+
+    for repo_url in repo_urls:
+        repo_name = repo_url.rsplit("/", 1)[-1].removesuffix(".git")
+        logger.info("Cloning %s into the sandbox", repo_name)
+        sandbox.exec(
+            "git",
+            "clone",
+            "--depth",
+            "100",
+            repo_url,
+            f"/workspace/repos/{repo_name}",
+        ).wait()
+
+
+def read_optional_file(sandbox: modal.Sandbox, path: str) -> str | None:
+    try:
+        return sandbox.filesystem.read_text(path)
+    except Exception:
+        return None
+
+
+def run_opencode_analysis(
+    *,
+    log_dir: Path,
+    job_id: str,
+    model: str,
+    repo_urls: list[str],
+    timeout_seconds: int,
+    modal_secret_name: str,
+) -> str:
+    prompt = build_prompt(job_id, repo_urls)
+    app = modal.App.lookup("sglang-log-analyzer", create_if_missing=True)
+    sandbox = modal.Sandbox.create(
+        app=app,
+        image=build_sandbox_image(),
+        timeout=timeout_seconds,
+        secrets=[modal.Secret.from_name(modal_secret_name)],
+    )
+    logger.info("Created Modal sandbox %s", sandbox.object_id)
+
+    try:
+        sandbox.mkdir("/workspace/logs", parents=True)
+        sandbox.mkdir("/workspace/repos", parents=True)
+
+        clone_context_repos(sandbox, repo_urls)
+        upload_tree(sandbox, log_dir)
+
+        sandbox.filesystem.write_text(prompt, "/workspace/prompt.txt")
+
+        runner_script = f"""#!/bin/bash
+set -uo pipefail
+cd /workspace
+if opencode run \\
+  --dangerously-skip-permissions \\
+  --dir /workspace/logs \\
+  -m {shlex.quote(model)} \\
+  "$(cat /workspace/prompt.txt)" \\
+  < /dev/null \\
+  > /workspace/logs/opencode.stdout \\
+  2> /workspace/logs/opencode.stderr; then
+  echo 0 > /workspace/logs/opencode.exitcode
+else
+  echo $? > /workspace/logs/opencode.exitcode
+fi
+ls -la /workspace/logs > /workspace/logs/log_dir_listing.txt
+"""
+        sandbox.filesystem.write_text(runner_script, "/workspace/run_opencode.sh")
+        sandbox.exec("chmod", "+x", "/workspace/run_opencode.sh").wait()
+
+        logger.info("Running opencode analysis")
+        process = sandbox.exec(
+            "bash",
+            "/workspace/run_opencode.sh",
+        )
+        process.wait()
+
+        stderr = process.stderr.read()
+        if stderr:
+            logger.warning("runner stderr: %s", sanitize(stderr[:500]))
+
+        exitcode = read_optional_file(sandbox, "/workspace/logs/opencode.exitcode")
+        opencode_stdout = (
+            read_optional_file(sandbox, "/workspace/logs/opencode.stdout") or ""
+        )
+        opencode_stderr = read_optional_file(sandbox, "/workspace/logs/opencode.stderr")
+        log_dir_listing = read_optional_file(
+            sandbox, "/workspace/logs/log_dir_listing.txt"
+        )
+        try:
+            ai_analysis = read_optional_file(sandbox, "/workspace/logs/ai_analysis.md")
+            if ai_analysis and ai_analysis.strip():
+                return sanitize(ai_analysis)
+
+            if opencode_stdout.strip():
+                return sanitize(opencode_stdout)
+
+            raise RuntimeError("opencode completed without producing analysis output")
+        except Exception as exc:
+            stdout = process.stdout.read()
+            if stdout:
+                return sanitize(stdout)
+            details = [
+                f"opencode analysis did not produce a usable report: {exc}",
+                f"exitcode={exitcode!r}",
+                f"stdout_preview={sanitize(opencode_stdout[:500])!r}",
+                f"stderr_preview={sanitize((opencode_stderr or '')[:500])!r}",
+                f"log_dir_listing={sanitize((log_dir_listing or '')[:500])!r}",
+            ]
+            raise RuntimeError(" ".join(details)) from exc
+    finally:
+        sandbox.terminate()
+
+
+def main() -> int:
+    args = parse_args()
+    configure_logging(args.verbose)
+
+    repo_urls = list(DEFAULT_REPOS)
+    if args.repo_urls:
+        repo_urls.extend(args.repo_urls)
+
+    log_dir, cleanup_dir = prepare_log_dir(args)
+    try:
+        analysis = run_opencode_analysis(
+            log_dir=log_dir,
+            job_id=args.job_id,
+            model=args.model,
+            repo_urls=repo_urls,
+            timeout_seconds=args.timeout_seconds,
+            modal_secret_name=args.modal_secret_name,
+        )
+    finally:
+        if cleanup_dir is not None:
+            shutil.rmtree(cleanup_dir, ignore_errors=True)
+
+    print(analysis)
+    if args.output:
+        args.output.write_text(analysis)
+        logger.info("Wrote analysis to %s", args.output)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ci/slurm/generate_matrix.py

@@ -0,0 +1,98 @@
+"""
+Reads nightly-configs.yaml and generates one matrix entry per recipe YAML,
+where each srt-slurm recipe runs its full concurrency sweep as a single Slurm job.
+
+conc-list in the config is documentation only and is not used to split jobs.
+
+Output: JSON array written to stdout, consumed by the workflow setup job as
+a dynamic matrix via fromJson(needs.setup.outputs.matrix).
+
+Usage:
+    python3 generate_matrix.py <path-to-nightly-configs.yaml> --runner <label> [--filter NAMES]
+
+Example:
+    python3 generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200
+    python3 generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200 \\
+        --filter dsr1-fp8-1k1k-max-tpt,dsr1-fp4-1k1k-mid-curve
+"""
+
+import argparse
+import json
+import sys
+
+import yaml
+
+
+def seq_len_str(isl, osl):
+    def fmt(n):
+        return f"{n // 1024}k" if n % 1024 == 0 else str(n)
+
+    return f"{fmt(isl)}{fmt(osl)}"
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("config_file", help="Path to nightly-configs.yaml")
+    parser.add_argument(
+        "--runner",
+        required=True,
+        help="Filter configs by runner label (e.g. gb200, b200)",
+    )
+    parser.add_argument(
+        "--filter",
+        default="",
+        help=(
+            "Optional comma-separated list of matrix entry names to include "
+            "(e.g. 'dsr1-fp8-1k1k-max-tpt'). Names must match exactly."
+        ),
+    )
+    args = parser.parse_args()
+
+    with open(args.config_file) as f:
+        data = yaml.safe_load(f)
+
+    matrix = []
+    for exp_name, exp in data.items():
+        if exp["runner"] != args.runner:
+            continue
+
+        for seq_cfg in exp["seq-len-configs"]:
+            isl, osl = seq_cfg["isl"], seq_cfg["osl"]
+            sl = seq_len_str(isl, osl)
+
+            for entry in seq_cfg["search-space"]:
+                config_file = entry["config_file"]
+                topology = config_file.rsplit("/", 1)[-1].replace(".yaml", "")
+
+                matrix.append(
+                    {
+                        "name": f"{exp['model-prefix']}-{exp['precision']}-{sl}-{topology}",
+                        "exp_name": exp_name,
+                        "model": exp["model"],
+                        "model_prefix": exp["model-prefix"],
+                        "model_path": exp.get("model_path", ""),
+                        "precision": exp["precision"],
+                        "isl": str(isl),
+                        "osl": str(osl),
+                        "config_file": config_file,
+                    }
+                )
+
+    wanted = [n.strip() for n in args.filter.split(",") if n.strip()]
+    if wanted:
+        available = [e["name"] for e in matrix]
+        unknown = [n for n in wanted if n not in available]
+        if unknown:
+            print(
+                f"ERROR: unknown config name(s): {', '.join(unknown)}. "
+                f"Available for runner '{args.runner}': {', '.join(available)}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        matrix = [e for e in matrix if e["name"] in wanted]
+
+    print(json.dumps(matrix))
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/slurm/launch_gb200.sh

@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+# Launch a dynamo-sglang benchmark job on the GB200 cluster via srt-slurm.
+#
+# Required environment variables (set by the GitHub Actions workflow):
+#   FRAMEWORK         - must be "dynamo-sglang"
+#   MODEL             - HuggingFace model ID (used as fallback if no local path)
+#   MODEL_PREFIX      - short prefix: "dsr1"
+#   PRECISION         - "fp8" or "fp4"
+#   ISL               - input sequence length (e.g. "1024")
+#   OSL               - output sequence length (e.g. "1024")
+#   CONFIG_FILE       - path relative to srt-slurm repo root (e.g. recipes/gb200-fp8/1k1k/low-latency.yaml)
+#   RESULT_FILENAME   - prefix for output JSON filenames
+#   RUNNER_NAME       - GitHub Actions runner name (used to tag the Slurm job)
+#   SQUASH_FILE       - path to pre-imported sglang enroot squash file on Lustre
+#   NGINX_SQUASH_FILE - path to pre-imported nginx enroot squash file on Lustre
+#   SLURM_PARTITION   - Slurm partition (default: batch)
+#   SLURM_ACCOUNT     - Slurm account  (default: sglang)
+#   SRT_SLURM_BRANCH  - branch of srt-slurm repo to check out
+#   GITHUB_WORKSPACE  - set automatically by GitHub Actions
+#   MATRIX_CONFIG_NAME- matrix entry name (e.g. dsr1-fp4-1k1k-mid-curve); used in S3 prefix
+#   S3_BUCKET         - MinIO bucket for benchmark log uploads
+#   S3_ENDPOINT_URL   - MinIO endpoint URL (e.g. https://minio.<host>.nip.io)
+#   AWS_ACCESS_KEY_ID - writer access key for S3_BUCKET (via GH secrets)
+#   AWS_SECRET_ACCESS_KEY - writer secret key for S3_BUCKET (via GH secrets)
+
+set -euo pipefail
+set -x
+
+# ---------------------------------------------------------------------------
+# Validate required vars
+# ---------------------------------------------------------------------------
+: "${FRAMEWORK:?}"
+: "${MODEL_PREFIX:?}"
+: "${PRECISION:?}"
+: "${ISL:?}"
+: "${OSL:?}"
+: "${CONFIG_FILE:?}"
+: "${RESULT_FILENAME:?}"
+: "${RUNNER_NAME:?}"
+: "${SQUASH_FILE:?}"
+: "${NGINX_SQUASH_FILE:?}"
+: "${GITHUB_WORKSPACE:?}"
+
+SLURM_PARTITION="${SLURM_PARTITION:-batch}"
+SLURM_ACCOUNT="${SLURM_ACCOUNT:-sglang}"
+SRT_SLURM_BRANCH="${SRT_SLURM_BRANCH:-sglang-nightly-regression}"
+
+# ---------------------------------------------------------------------------
+# Resolve local model paths on Lustre (avoids re-downloading on each run)
+# ---------------------------------------------------------------------------
+if [[ "$MODEL_PREFIX" == "dsr1" && "$PRECISION" == "fp8" ]]; then
+    MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528"
+    SRT_SLURM_MODEL_PREFIX="dsr1-fp8"
+elif [[ "$MODEL_PREFIX" == "dsr1" && "$PRECISION" == "fp4" ]]; then
+    MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528-fp4-v2/"
+    SRT_SLURM_MODEL_PREFIX="dsr1-fp4"
+else
+    MODEL_PATH="$MODEL"
+    SRT_SLURM_MODEL_PREFIX="$MODEL_PREFIX"
+fi
+
+# ---------------------------------------------------------------------------
+# Set up per-runner Lustre workspace (cleaned before each run, accessible
+# to both the runner and compute nodes)
+# ---------------------------------------------------------------------------
+LUSTRE_WORKSPACE="/mnt/lustre01/users-public/sglang-ci/workspace/${RUNNER_NAME}"
+rm -rf "$LUSTRE_WORKSPACE"
+mkdir -p "$LUSTRE_WORKSPACE"
+
+# ---------------------------------------------------------------------------
+# Clone and set up srt-slurm
+# ---------------------------------------------------------------------------
+SRT_REPO_DIR="$LUSTRE_WORKSPACE/srt-slurm"
+
+git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+cd "$SRT_REPO_DIR"
+git checkout "$SRT_SLURM_BRANCH"
+echo "--- srt-slurm last commit ---"
+git log -1 --format="commit %H%nauthor %an%ndate   %ad%nsubject %s" --date=iso
+
+curl -LsSf https://astral.sh/uv/install.sh | sh
+source "$HOME/.local/bin/env"
+
+uv venv
+source .venv/bin/activate
+uv pip install -e .
+
+if ! command -v srtctl &>/dev/null; then
+    echo "ERROR: srtctl installation failed"
+    exit 1
+fi
+
+# ---------------------------------------------------------------------------
+# Generate srtslurm.yaml
+# ---------------------------------------------------------------------------
+SRTCTL_ROOT="$SRT_REPO_DIR"
+
+: "${S3_BUCKET:?S3_BUCKET must be set}"
+: "${S3_ENDPOINT_URL:?S3_ENDPOINT_URL must be set}"
+: "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID must be set}"
+: "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY must be set}"
+: "${MATRIX_CONFIG_NAME:?MATRIX_CONFIG_NAME must be set}"
+: "${GITHUB_RUN_ID:?GITHUB_RUN_ID must be set}"
+: "${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT must be set}"
+
+# Map the GitHub trigger into a friendlier top-level prefix: cron/manual.
+case "${GITHUB_EVENT_NAME:-}" in
+    schedule)          TRIGGER=cron ;;
+    workflow_dispatch) TRIGGER=manual ;;
+    *)                 TRIGGER="${GITHUB_EVENT_NAME:-unknown}" ;;
+esac
+
+# Format ISL/OSL as "1k1k" / "1k8k" / "8k1k" etc. for the S3 prefix, so logs
+# group naturally by sequence-length bucket under each run.
+fmt_seq_len() {
+    local n=$1
+    if (( n % 1024 == 0 )); then echo "$((n / 1024))k"; else echo "$n"; fi
+}
+SEQ_LEN="$(fmt_seq_len "$ISL")$(fmt_seq_len "$OSL")"
+
+S3_PREFIX="${TRIGGER}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${SEQ_LEN}/${MATRIX_CONFIG_NAME}"
+
+cat > srtslurm.yaml <<EOF
+# SRT SLURM configuration for SGLang GB200 nightly CI
+default_account: "${SLURM_ACCOUNT}"
+default_partition: "${SLURM_PARTITION}"
+default_time_limit: "6:00:00"
+
+gpus_per_node: 4
+network_interface: ""
+
+srtctl_root: "${SRTCTL_ROOT}"
+
+model_paths:
+  "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}"
+
+containers:
+  dynamo-sglang: ${SQUASH_FILE}
+  nginx: ${NGINX_SQUASH_FILE}
+  nginx-sqsh: ${NGINX_SQUASH_FILE}
+
+# srt-slurm postprocess uploads /logs to this bucket after each Slurm job.
+# Credentials are read from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars,
+# not written to disk. srt-slurm appends /<date>/<slurm-job-id>/ after prefix.
+reporting:
+  s3:
+    bucket: "${S3_BUCKET}"
+    prefix: "${S3_PREFIX}"
+    endpoint_url: "${S3_ENDPOINT_URL}"
+EOF
+
+echo "--- srtslurm.yaml ---"
+cat srtslurm.yaml
+echo "--- S3 log upload: s3://${S3_BUCKET}/${S3_PREFIX}/ ---"
+
+make setup ARCH=aarch64
+
+# ---------------------------------------------------------------------------
+# Patch job name and submit via srtctl
+# ---------------------------------------------------------------------------
+sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE"
+
+SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \
+    --tags "gb200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},sglang-nightly-$(date +%Y%m%d)" \
+    --setup-script install-torchao.sh 2>&1)
+echo "$SRTCTL_OUTPUT"
+
+JOB_ID=$(echo "$SRTCTL_OUTPUT" | grep -oP '✅ Job \K[0-9]+' || echo "$SRTCTL_OUTPUT" | grep -oP 'Job \K[0-9]+' || true)
+
+if [ -z "$JOB_ID" ]; then
+    echo "ERROR: Could not extract JOB_ID from srtctl output"
+    exit 1
+fi
+
+echo "Submitted Slurm job: $JOB_ID"
+
+set +x
+
+# ---------------------------------------------------------------------------
+# Wait for job and stream logs
+# ---------------------------------------------------------------------------
+LOGS_DIR="outputs/$JOB_ID/logs"
+LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log"
+
+mkdir -p "$LOGS_DIR"
+
+while ! ls "$LOG_FILE" &>/dev/null; do
+    if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then
+        echo "ERROR: Job $JOB_ID failed before creating log file"
+        scontrol show job "$JOB_ID" || true
+        exit 1
+    fi
+    echo "Waiting for job $JOB_ID to start and $LOG_FILE to appear..."
+    sleep 5
+done
+
+(
+    while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do
+        sleep 10
+    done
+) &
+POLL_PID=$!
+
+tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
+
+wait $POLL_PID
+
+set -x
+
+echo "Job $JOB_ID completed. Collecting results..."
+
+# ---------------------------------------------------------------------------
+# Collect results
+# ---------------------------------------------------------------------------
+if [ ! -d "$LOGS_DIR" ]; then
+    echo "WARNING: Logs directory not found at $LOGS_DIR"
+    exit 1
+fi
+
+cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS"
+tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" .
+
+RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null || true)
+
+if [ -z "$RESULT_SUBDIRS" ]; then
+    echo "ERROR: No result subdirectories found in $LOGS_DIR — benchmark did not produce any output"
+    exit 1
+else
+    RESULT_COUNT=0
+    for result_subdir in $RESULT_SUBDIRS; do
+        CONFIG_NAME=$(basename "$result_subdir")
+        RESULT_FILES=$(find "$result_subdir" -name "results_concurrency_*.json" 2>/dev/null || true)
+        for result_file in $RESULT_FILES; do
+            if [ -f "$result_file" ]; then
+                filename=$(basename "$result_file")
+                concurrency=$(echo "$filename" | sed -n 's/results_concurrency_\([0-9]*\)_gpus_.*/\1/p')
+                gpus=$(echo "$filename" | sed -n 's/results_concurrency_[0-9]*_gpus_\([0-9]*\)_ctx_.*/\1/p')
+                ctx=$(echo "$filename" | sed -n 's/.*_ctx_\([0-9]*\)_gen_.*/\1/p')
+                gen=$(echo "$filename" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
+                DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${CONFIG_NAME}_conc${concurrency}_gpus_${gpus}_ctx_${ctx}_gen_${gen}.json"
+                cp "$result_file" "$DEST"
+                echo "Saved: $DEST"
+                RESULT_COUNT=$((RESULT_COUNT + 1))
+            fi
+        done
+    done
+    if [ "$RESULT_COUNT" -eq 0 ]; then
+        echo "ERROR: Result subdirectories found but no result JSON files produced — benchmark failed"
+        exit 1
+    fi
+fi
+
+echo "Done."

scripts/ci/slurm/launch_mi355x.sh

@@ -0,0 +1,777 @@
+#!/usr/bin/env bash
+# Launch a 2-node 1P1D disaggregation benchmark on the AMD MI355X `amd-sglang`
+# Slurm cluster, then emit per-concurrency result JSONs that
+# scripts/ci/slurm/process_result.py aggregates.
+#
+# salloc's (prefill_workers + decode_workers) nodes -- one server per node --
+# and runs the Docker harness: prefill server(s) on the first nodes, decode
+# server(s) on the rest, a standalone load balancer on the prefill node, then an
+# sglang.bench_serving concurrency sweep over MORI. Default recipe is 1P1D (2
+# nodes); see the drive.sh note on reserving 2P2D / 1P3D / 3P1D.
+#
+# Required environment variables (set by the GitHub Actions workflow):
+#   MODEL              - HuggingFace model id (table label / served model)
+#   MODEL_PREFIX       - short prefix, e.g. dsv4flash
+#   PRECISION          - fp8 / fp4
+#   ISL, OSL           - input / output sequence lengths for the sweep
+#   CONFIG_FILE        - path to the recipe YAML (relative to repo root)
+#   RESULT_FILENAME    - prefix for the emitted result JSONs
+#   MATRIX_CONFIG_NAME - matrix entry name (used in filenames/tags)
+#   GITHUB_WORKSPACE   - set by GitHub Actions; where result JSONs are written
+# Optional:
+#   MODEL_PATH         - local snapshot dir (preferred over downloading MODEL)
+#   SLURM_PARTITION    - default: amd-sglang
+#   SLURM_NODELIST     - optional explicit node pin (else scheduler chooses)
+#   SLURM_EXCLUDE      - optional comma-separated nodes to keep the scheduler
+#                        off (e.g. hosts with a broken RDMA driver)
+#   SGLANG_USE_CHECKOUT_RUNTIME
+#                      - default 1. Reinstall this workflow checkout's Python
+#                        sglang package inside each runtime container, and the
+#                        checkout sglang-router package inside the bench
+#                        container, before launching servers/bench. Set 0 to
+#                        use the image's baked-in packages.
+#   RUNNER_NAME        - GitHub runner name (a built-in default env var)
+#   GITHUB_RUN_ID      - GitHub Actions run id (a built-in default env var)
+#                        The allocation is named
+#                          mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>
+#                        so workflow cleanup can scancel exactly this leg's job
+#                        (full name) or this runner's stale jobs (RUNNER_NAME
+#                        prefix) -- never a blanket `squeue --me`. The run id +
+#                        config make the name unique per matrix leg even if two
+#                        runners happen to share a name.
+#   SLURM_EXCLUSIVE    - request whole nodes (default 1); set 0 to disable
+#   TIME_LIMIT         - salloc time limit, default 02:30:00 (covers server
+#                        load + perf sweep + full GSM8K, under the 180m step cap)
+
+set -euo pipefail
+set -x
+
+: "${MODEL_PREFIX:?}"
+: "${PRECISION:?}"
+: "${ISL:?}"
+: "${OSL:?}"
+: "${CONFIG_FILE:?}"
+: "${RESULT_FILENAME:?}"
+: "${MATRIX_CONFIG_NAME:?}"
+: "${GITHUB_WORKSPACE:?}"
+
+SLURM_PARTITION="${SLURM_PARTITION:-amd-sglang}"
+TIME_LIMIT="${TIME_LIMIT:-02:30:00}"
+MODEL_PATH="${MODEL_PATH:-${MODEL:-}}"
+SGLANG_USE_CHECKOUT_RUNTIME="${SGLANG_USE_CHECKOUT_RUNTIME:-1}"
+case "${SGLANG_USE_CHECKOUT_RUNTIME,,}" in
+    0|false|no|off) SGLANG_USE_CHECKOUT_RUNTIME=0 ;;
+    *) SGLANG_USE_CHECKOUT_RUNTIME=1 ;;
+esac
+
+if [[ -z "$MODEL_PATH" ]]; then
+    echo "ERROR: set MODEL_PATH (local snapshot) or MODEL" >&2
+    exit 1
+fi
+
+# Resolve a HuggingFace cache dir (models--org--name) to its live snapshot dir.
+# Lets nightly-configs / recipes point at the shared cache without hardcoding a
+# snapshot hash; a concrete snapshot dir (or plain dir) is returned unchanged.
+# Used for both MODEL_PATH and an optional speculative draft model path.
+resolve_snapshot() {
+    local p="$1"
+    if [[ -f "$p/refs/main" && -d "$p/snapshots" ]]; then
+        local hash resolved
+        hash="$(cat "$p/refs/main")"
+        resolved="$p/snapshots/$hash"
+        if [[ -n "$hash" && -d "$resolved" ]]; then
+            echo "resolved snapshot: $p -> $resolved" >&2
+            echo "$resolved"
+            return 0
+        fi
+        echo "ERROR: refs/main=$hash but $resolved missing" >&2
+        return 1
+    fi
+    echo "$p"
+}
+MODEL_PATH="$(resolve_snapshot "$MODEL_PATH")" || exit 1
+
+# ---------------------------------------------------------------------------
+# Parse the recipe (runtime + bench + topology) into shell vars.
+# ---------------------------------------------------------------------------
+# Ensure PyYAML is available to the host python used for parsing.
+python3 -c 'import yaml' 2>/dev/null || pip install pyyaml -q 2>/dev/null \
+    || pip install --user pyyaml -q 2>/dev/null || true
+
+# Emit KEY=value lines and eval them (robust single-level command substitution;
+# avoids a nested read<<EOF/$(<<PY) heredoc that misparses on some shells).
+RECIPE_VARS="$(python3 - "$CONFIG_FILE" <<'PY'
+import sys, yaml
+r = yaml.safe_load(open(sys.argv[1]))
+rt = r["runtime"]; b = r["backend"]["sglang_config"]; bn = r["bench"]
+res = r.get("resources", {})
+def emit(k, v): print(f"{k}={v}")
+emit("IMAGE", rt["image"])
+# Attention backend: single (`attention_backend`) or split
+# (`prefill_attention_backend`/`decode_attention_backend`). Empty when absent so
+# the flag is dropped for a model that omits it.
+emit("ATTN", rt.get("attention_backend", ""))
+emit("PATTN", rt.get("prefill_attention_backend", ""))
+emit("DATTN", rt.get("decode_attention_backend", ""))
+emit("IB", rt["ib_devices"])
+emit("PPORT", rt["prefill_port"])
+emit("DPORT", rt["decode_port"])
+emit("PBOOT", rt["prefill_bootstrap_port"])
+emit("DBOOT", rt["decode_bootstrap_port"])
+emit("LBPORT", rt["lb_port"])
+emit("MEMFRAC", rt["mem_fraction_static"])
+emit("PAGE", rt["page_size"])
+emit("MAXREQ", rt["max_running_requests"])
+emit("MAXTOK", rt.get("max_total_tokens", ""))
+emit("CHUNK", rt["chunked_prefill_size"])
+# swa is DSV4-specific; emit empty when a model omits it so the flag is dropped.
+emit("SWA", rt.get("swa_full_tokens_ratio", ""))
+# 1 when the recipe carries a `model:` block (env + server_args written to
+# model_flags.sh); 0 for the DSV4 recipes, which keep the hardcoded DSV4 path.
+emit("HAS_MODEL", 1 if r.get("model") else 0)
+emit("PTP", b["prefill"]["tensor-parallel-size"])
+emit("DTP", b["decode"]["tensor-parallel-size"])
+emit("PEP", b["prefill"].get("expert-parallel-size", 1))
+emit("PDP", b["prefill"].get("data-parallel-size", 1))
+m = r.get("mtp", {}) or {}
+emit("MTP_ENABLED", 1 if m.get("enabled") else 0)
+emit("MTP_ALGO", m.get("algorithm", "EAGLE"))
+emit("MTP_STEPS", m.get("num_steps", 3))
+emit("MTP_TOPK", m.get("eagle_topk", 1))
+emit("MTP_DRAFT", m.get("num_draft_tokens", 4))
+# External draft checkpoint (EAGLE3 etc.); empty for DSV4's built-in EAGLE head.
+emit("MTP_DRAFT_PATH", m.get("draft_model_path", ""))
+# Worker counts double as node counts here: one server per node (TP == GPUs/node).
+# 1P1D today; bumping these reserves 2P2D / 1P3D / 3P1D. Multi-node-per-worker
+# (TP > GPUs/node, needs --dist-init-addr/--nnodes/--node-rank) is out of scope.
+emit("PW", res.get("prefill_workers", 1))
+emit("DW", res.get("decode_workers", 1))
+emit("CONCS", ",".join(str(c) for c in bn["concurrencies"]))
+emit("NPF", bn["num_prompts_factor"])
+emit("RRR", bn["random_range_ratio"])
+acc = bn.get("accuracy", {}) or {}
+emit("ACC_ENABLED", 1 if acc.get("enabled") else 0)
+emit("ACC_SHOTS", acc.get("num_shots", 8))
+emit("ACC_NQ", acc.get("num_questions", 1319))
+emit("ACC_THR", acc.get("threshold", 0.91))
+PY
+)"
+if [[ -z "$RECIPE_VARS" ]]; then
+    echo "ERROR: failed to parse recipe $CONFIG_FILE (empty output from python3/yaml)" >&2
+    exit 1
+fi
+eval "$RECIPE_VARS"
+# Optional image override from workflow_dispatch input.
+if [[ -n "${IMAGE_OVERRIDE:-}" ]]; then
+    IMAGE="$IMAGE_OVERRIDE"
+fi
+echo "recipe: image=$IMAGE attn=${ATTN:-$PATTN/$DATTN} ib=$IB ptp=$PTP dtp=$DTP concs=$CONCS isl=$ISL osl=$OSL"
+
+# ---------------------------------------------------------------------------
+# Shared NFS scratch (visible to login node + compute nodes). Raw bench output
+# lands here; the launcher normalizes it into GITHUB_WORKSPACE afterwards.
+# ---------------------------------------------------------------------------
+WORKDIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}"
+rm -rf "$WORKDIR"; mkdir -p "$WORKDIR"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Stage the workflow checkout on shared NFS so Slurm compute-node containers can
+# reinstall the same code SHA the workflow checked out. The container gets a
+# read-only mount and copies it to /tmp before mutating pyproject.toml.
+CHECKOUT_DOCKER_ARGS="-e SGLANG_USE_CHECKOUT_RUNTIME=$SGLANG_USE_CHECKOUT_RUNTIME"
+if [[ "$SGLANG_USE_CHECKOUT_RUNTIME" == "1" ]]; then
+    CHECKOUT_STAGE="$WORKDIR/checkout"
+    CHECKOUT_SHA="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"
+    echo "Staging checkout runtime: sha=$CHECKOUT_SHA -> $CHECKOUT_STAGE"
+    rm -rf "$CHECKOUT_STAGE"
+    mkdir -p "$CHECKOUT_STAGE"
+    tar --exclude='__pycache__' --exclude='*.pyc' --exclude='.git/config' \
+        -C "$GITHUB_WORKSPACE" -cf - . | tar -C "$CHECKOUT_STAGE" -xf -
+    CHECKOUT_DOCKER_ARGS="$CHECKOUT_DOCKER_ARGS -e SGLANG_CHECKOUT_SHA=$CHECKOUT_SHA -v $CHECKOUT_STAGE:/sglang-checkout:ro"
+else
+    echo "SGLANG_USE_CHECKOUT_RUNTIME=0; using sglang package baked into image."
+fi
+
+# Accuracy-gate helpers (written when enabled). Pre-stage the GSM8K test set on
+# shared NFS from the login node (which has internet) so the in-container eval
+# doesn't depend on compute-node connectivity; fall back to in-container
+# download if the pre-fetch fails.
+if [[ "$ACC_ENABLED" == "1" ]]; then
+    GSM8K_URL="https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
+    curl -fsSL "$GSM8K_URL" -o "$WORKDIR/gsm8k_test.jsonl" 2>/dev/null \
+        && echo "gsm8k dataset staged at $WORKDIR/gsm8k_test.jsonl" \
+        || echo "WARN: gsm8k pre-stage failed; in-container download will be attempted"
+    cat > "$WORKDIR/check_acc.py" <<'PY'
+import sys
+acc, thr = float(sys.argv[1]), float(sys.argv[2])
+print(f"[gsm8k] accuracy={acc:.3f} threshold={thr}")
+sys.exit(0 if acc > thr else 1)
+PY
+fi
+
+# DSV4 load-bearing env (see test/registered/amd/test_deepseek_v4_flash_fp8.py).
+# SGLANG_DSV4_FP4_EXPERTS is precision-driven: true for fp4 weights, false for fp8.
+if [[ "$PRECISION" == "fp4" ]]; then
+    FP4_EXPERTS=true
+else
+    FP4_EXPERTS=false
+fi
+DSV4_ENV=(
+  -e SGLANG_DEFAULT_THINKING=1 -e SGLANG_DSV4_REASONING_EFFORT=max
+  -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -e SGLANG_USE_AITER=1
+  -e SGLANG_USE_ROCM700A=1 -e SGLANG_OPT_USE_FUSED_COMPRESS=true
+  -e SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true
+  -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
+  -e SGLANG_OPT_FP8_WO_A_GEMM=false -e SGLANG_OPT_USE_JIT_INDEXER_METADATA=false
+  -e SGLANG_OPT_USE_TOPK_V2=false -e SGLANG_OPT_USE_AITER_INDEXER=true
+  -e SGLANG_OPT_USE_TILELANG_INDEXER=false -e SGLANG_OPT_USE_TILELANG_MHC_PRE=false
+  -e SGLANG_OPT_USE_TILELANG_MHC_POST=false -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1
+  -e SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false -e SGLANG_ROCM_USE_MULTI_STREAM=false
+  -e AITER_BF16_FP8_MOE_BOUND=0 -e SGLANG_DSV4_FP4_EXPERTS=$FP4_EXPERTS
+)
+DSV4_ENV_STR="${DSV4_ENV[*]}"
+# A recipe carrying a `model:` block supplies its OWN docker env (below), so the
+# DSV4 env must not leak into it; the DSV4 recipes keep the string above.
+[[ "$HAS_MODEL" == "1" ]] && DSV4_ENV_STR=""
+MORI_ENV="-e MORI_DISABLE_AUTO_XGMI=1 -e NCCL_IB_HCA=ionic -e NCCL_IB_GID_INDEX=1 -e NCCL_CROSS_NIC=1"
+
+# Model-specific docker `-e` env + sglang server args from the recipe's optional
+# `model:` block, written as bash arrays to model_flags.sh (sourced by
+# prefill.sh/decode.sh). DSV4 recipes have no `model:` block -> empty arrays, so
+# their generated docker argv is unchanged. Each server arg + its value MUST be a
+# separate YAML list item so shlex.quote keeps "--foo" and "bar" as two tokens.
+python3 - "$CONFIG_FILE" "$WORKDIR/model_flags.sh" <<'PY'
+import shlex, sys, yaml
+r = yaml.safe_load(open(sys.argv[1]))
+model = r.get("model", {}) or {}
+env = model.get("env", {}) or {}
+server_args = model.get("server_args", []) or []
+# YAML true/false parse to Python bool; render lowercase so env values stay
+# byte-identical to shell (`=false`, not `=False`) -- SGLang parsing is
+# case-sensitive for some of these.
+def fmt(v):
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    return str(v)
+env_args = []
+for k, v in env.items():
+    env_args += ["-e", f"{k}={fmt(v)}"]
+def q(items):
+    return " ".join(shlex.quote(fmt(x)) for x in items)
+with open(sys.argv[2], "w") as f:
+    f.write(f"MODEL_ENV_ARGS=({q(env_args)})\n")
+    f.write(f"MODEL_SERVER_ARGS=({q(server_args)})\n")
+PY
+
+# Optional topology / speculative-decode flags driven by the recipe. Base recipes
+# (EP1/DP1, no mtp) leave EXTRA_FLAGS empty, preserving prior behavior exactly.
+EXTRA_FLAGS=""
+(( PDP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --enable-dp-attention --dp-size $PDP"
+(( PEP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --ep-size $PEP"
+[[ -n "$MAXTOK" ]] && EXTRA_FLAGS="$EXTRA_FLAGS --max-total-tokens $MAXTOK"
+if [[ "$MTP_ENABLED" == "1" ]]; then
+    EXTRA_FLAGS="$EXTRA_FLAGS --speculative-algorithm $MTP_ALGO \
+--speculative-num-steps $MTP_STEPS --speculative-eagle-topk $MTP_TOPK \
+--speculative-num-draft-tokens $MTP_DRAFT"
+    # EAGLE3 (and other draft-model algos) need an external draft checkpoint;
+    # built-in EAGLE (DSV4) omits draft_model_path and this stays unset.
+    if [[ -n "$MTP_DRAFT_PATH" ]]; then
+        DRAFT_RESOLVED="$(resolve_snapshot "$MTP_DRAFT_PATH")" || exit 1
+        EXTRA_FLAGS="$EXTRA_FLAGS --speculative-draft-model-path $DRAFT_RESOLVED"
+    fi
+fi
+echo "extra flags: ${EXTRA_FLAGS:-<none>} (pep=$PEP pdp=$PDP mtp=$MTP_ENABLED algo=$MTP_ALGO)"
+
+if [[ "$HAS_MODEL" == "1" ]]; then
+    # Generic path (e.g. Kimi): attention + swa from the recipe, model parsers /
+    # quirks ride MODEL_SERVER_ARGS. Single `--attention-backend` when the recipe
+    # sets `attention_backend`; split `--prefill-/--decode-attention-backend` when
+    # it sets the per-role keys. swa dropped when the recipe omits it.
+    ATTN_FLAGS=""
+    [[ -n "$ATTN" ]]  && ATTN_FLAGS="$ATTN_FLAGS --attention-backend $ATTN"
+    [[ -n "$PATTN" ]] && ATTN_FLAGS="$ATTN_FLAGS --prefill-attention-backend $PATTN"
+    [[ -n "$DATTN" ]] && ATTN_FLAGS="$ATTN_FLAGS --decode-attention-backend $DATTN"
+    SWA_FLAG=""
+    [[ -n "$SWA" ]] && SWA_FLAG=" --swa-full-tokens-ratio $SWA"
+    COMMON_FLAGS="--trust-remote-code --tp $PTP --disable-radix-cache \
+$ATTN_FLAGS --max-running-requests $MAXREQ --page-size $PAGE \
+--mem-fraction-static $MEMFRAC$SWA_FLAG \
+--chunked-prefill-size $CHUNK \
+--disaggregation-transfer-backend mori --disaggregation-ib-device $IB$EXTRA_FLAGS"
+else
+    # DSV4 path: byte-identical to the pre-Kimi launcher.
+    COMMON_FLAGS="--trust-remote-code --tp $PTP --disable-radix-cache \
+--attention-backend $ATTN --max-running-requests $MAXREQ --page-size $PAGE \
+--mem-fraction-static $MEMFRAC --swa-full-tokens-ratio $SWA \
+--chunked-prefill-size $CHUNK --disable-shared-experts-fusion \
+--tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 \
+--disaggregation-transfer-backend mori --disaggregation-ib-device $IB$EXTRA_FLAGS"
+fi
+
+DOCKER_COMMON="--rm --network host --ipc host --shm-size 32g --privileged \
+--security-opt seccomp=unconfined \
+--device /dev/kfd --device /dev/dri --device /dev/infiniband \
+-v /it-share:/it-share:ro -v $HOME:/host_home $CHECKOUT_DOCKER_ARGS"
+
+# ---------------------------------------------------------------------------
+# Write per-role scripts that srun dispatches to each compute node.
+# ---------------------------------------------------------------------------
+# These are UNQUOTED `<<EOF` heredocs, so $MORI_ENV/$DSV4_ENV_STR/$COMMON_FLAGS/
+# $IMAGE/$MODEL_PATH expand now (at generation). model_flags.sh is sourced at
+# runtime for the model's env/server arrays, so the `${MODEL_ENV_ARGS[@]}` /
+# `${MODEL_SERVER_ARGS[@]}` refs are backslash-escaped to survive into the script
+# and expand after `source`. For DSV4 those arrays are empty and $DSV4_ENV_STR is
+# set, so the resulting docker argv is byte-identical to the pre-Kimi launcher.
+cat > "$WORKDIR/install_checkout_sglang.sh" <<'EOF'
+#!/bin/bash
+set -euo pipefail
+
+case "${SGLANG_USE_CHECKOUT_RUNTIME:-1}" in
+  0|false|False|FALSE|no|No|NO|off|Off|OFF)
+    echo "[checkout-sglang] disabled; using image-baked sglang"
+    exit 0
+    ;;
+esac
+
+CHECKOUT_SRC="${CHECKOUT_SRC:-/sglang-checkout}"
+RUNTIME_CHECKOUT="${RUNTIME_CHECKOUT:-/tmp/sglang-checkout-runtime}"
+
+if [[ ! -f "$CHECKOUT_SRC/python/sglang/version.py" ]]; then
+  echo "[checkout-sglang] ERROR: invalid checkout mount: $CHECKOUT_SRC" >&2
+  exit 1
+fi
+
+echo "[checkout-sglang] reinstalling sglang from $CHECKOUT_SRC"
+rm -rf "$RUNTIME_CHECKOUT"
+mkdir -p "$RUNTIME_CHECKOUT"
+tar --exclude='__pycache__' --exclude='*.pyc' \
+  -C "$CHECKOUT_SRC" -cf - . | tar -C "$RUNTIME_CHECKOUT" -xf -
+
+git config --global --add safe.directory "$RUNTIME_CHECKOUT" || true
+
+# The ROCm pyproject variant is the one used by AMD CI. Mutate only the private
+# /tmp copy so prefill/decode/bench never race on the read-only checkout mount.
+rm -f "$RUNTIME_CHECKOUT/python/pyproject.toml"
+cp "$RUNTIME_CHECKOUT/python/pyproject_other.toml" "$RUNTIME_CHECKOUT/python/pyproject.toml"
+for f in README.md LICENSE; do
+  if [[ -f "$RUNTIME_CHECKOUT/$f" && ! -e "$RUNTIME_CHECKOUT/python/$f" ]]; then
+    cp "$RUNTIME_CHECKOUT/$f" "$RUNTIME_CHECKOUT/python/$f"
+  fi
+done
+
+python3 -m pip uninstall -y sglang || true
+python3 -m pip install --no-deps --no-build-isolation -e "$RUNTIME_CHECKOUT/python"
+
+export RUNTIME_CHECKOUT
+export PYTHONPATH="$RUNTIME_CHECKOUT/python:${PYTHONPATH:-}"
+python3 - <<'PY'
+import importlib.metadata
+import os
+import subprocess
+import sglang
+
+checkout = os.environ["RUNTIME_CHECKOUT"]
+expected = os.path.realpath(os.path.join(checkout, "python", "sglang")) + os.sep
+actual = os.path.realpath(os.path.dirname(sglang.__file__)) + os.sep
+try:
+    sha = subprocess.check_output(
+        ["git", "-C", checkout, "rev-parse", "HEAD"], text=True
+    ).strip()
+except Exception:
+    sha = os.environ.get("SGLANG_CHECKOUT_SHA", "unknown")
+
+print(f"[checkout-sglang] sha={sha}")
+print(f"[checkout-sglang] sglang_file={sglang.__file__}")
+print(f"[checkout-sglang] sglang_version={importlib.metadata.version('sglang')}")
+if not actual.startswith(expected):
+    raise SystemExit(f"sglang did not import from checkout: {sglang.__file__}")
+PY
+EOF
+
+cat > "$WORKDIR/install_checkout_router.sh" <<'EOF'
+#!/bin/bash
+set -euo pipefail
+
+case "${SGLANG_USE_CHECKOUT_RUNTIME:-1}" in
+  0|false|False|FALSE|no|No|NO|off|Off|OFF)
+    echo "[checkout-router] disabled; using image-baked sglang-router"
+    python3 - <<'PY' || true
+import importlib.metadata
+import sglang_router
+
+print(f"[checkout-router] sglang_router_file={sglang_router.__file__}")
+print(
+    "[checkout-router] sglang_router_version="
+    f"{importlib.metadata.version('sglang-router')}"
+)
+try:
+    import sglang_router.sglang_router_rs as rs
+
+    print(f"[checkout-router] sglang_router_rs_file={rs.__file__}")
+except Exception as exc:
+    print(f"[checkout-router] sglang_router_rs_import_error={exc}")
+PY
+    exit 0
+    ;;
+esac
+
+RUNTIME_CHECKOUT="${RUNTIME_CHECKOUT:-/tmp/sglang-checkout-runtime}"
+ROUTER_SRC="$RUNTIME_CHECKOUT/sgl-model-gateway/bindings/python"
+WHEEL_DIR="${SGLANG_ROUTER_WHEEL_DIR:-/tmp/sglang-router-wheels}"
+
+if [[ ! -f "$ROUTER_SRC/pyproject.toml" ]]; then
+  echo "[checkout-router] ERROR: invalid router checkout: $ROUTER_SRC" >&2
+  exit 1
+fi
+
+echo "[checkout-router] building sglang-router from $ROUTER_SRC"
+export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-4}"
+python3 -m maturin --version >/dev/null 2>&1 \
+  || python3 -m pip install --no-cache-dir "maturin<1.14"
+
+# Match the ROCm image build recipe when compiling from the checkout copy.
+if [[ -f "$RUNTIME_CHECKOUT/sgl-model-gateway/Cargo.toml" ]]; then
+  sed -i -E 's|^(smg-[a-zA-Z-]+)\s*=\s*"~1\.0\.0"|\1 = "=1.0.0"|' \
+    "$RUNTIME_CHECKOUT/sgl-model-gateway/Cargo.toml"
+fi
+
+rm -rf "$WHEEL_DIR"
+mkdir -p "$WHEEL_DIR"
+(
+  cd "$ROUTER_SRC"
+  ulimit -n 65536 || true
+  python3 -m maturin build --release --features vendored-openssl --out "$WHEEL_DIR"
+)
+
+python3 -m pip uninstall -y sglang-router || true
+python3 -m pip install --force-reinstall --no-deps "$WHEEL_DIR"/*.whl
+
+python3 - <<'PY'
+import importlib.metadata
+import sglang_router
+import sglang_router.sglang_router_rs as rs
+from sglang_router.sglang_router_rs import Router
+
+print(f"[checkout-router] sglang_router_file={sglang_router.__file__}")
+print(
+    "[checkout-router] sglang_router_version="
+    f"{importlib.metadata.version('sglang-router')}"
+)
+print(f"[checkout-router] sglang_router_rs_file={rs.__file__}")
+print(f"[checkout-router] Router={Router}")
+PY
+EOF
+
+cat > "$WORKDIR/prefill_entry.sh" <<EOF
+#!/bin/bash
+set -euo pipefail
+CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
+source "\$CIDIR/model_flags.sh"
+bash "\$CIDIR/install_checkout_sglang.sh"
+if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
+  export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
+fi
+exec python3 -m sglang.launch_server \
+  --model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \
+  $COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
+  --disaggregation-mode prefill --disaggregation-bootstrap-port $PBOOT
+EOF
+
+cat > "$WORKDIR/decode_entry.sh" <<EOF
+#!/bin/bash
+set -euo pipefail
+CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
+source "\$CIDIR/model_flags.sh"
+bash "\$CIDIR/install_checkout_sglang.sh"
+if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
+  export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
+fi
+exec python3 -m sglang.launch_server \
+  --model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \
+  $COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
+  --disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT
+EOF
+
+cat > "$WORKDIR/prefill.sh" <<EOF
+#!/bin/bash
+source "$WORKDIR/model_flags.sh"
+docker rm -f mi355x_prefill 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_prefill \
+  -e HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 $MORI_ENV $DSV4_ENV_STR "\${MODEL_ENV_ARGS[@]}" \
+  $IMAGE bash /host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}/prefill_entry.sh
+EOF
+
+cat > "$WORKDIR/decode.sh" <<EOF
+#!/bin/bash
+source "$WORKDIR/model_flags.sh"
+docker rm -f mi355x_decode 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_decode \
+  -e HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 $MORI_ENV $DSV4_ENV_STR "\${MODEL_ENV_ARGS[@]}" \
+  $IMAGE bash /host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}/decode_entry.sh
+EOF
+
+# Probe payload + validator (separate files to avoid quoting inside the
+# bench.sh `bash -lc '...'` block). One real request exercises the full
+# prefill->decode KV handoff before we commit to the whole sweep.
+cat > "$WORKDIR/probe.json" <<'JSON'
+{"text": "The capital of France is", "sampling_params": {"max_new_tokens": 16, "temperature": 0.0}}
+JSON
+cat > "$WORKDIR/assert_nonempty.py" <<'PY'
+import sys, json
+d = json.load(sys.stdin)
+t = d.get("text", "") if isinstance(d, dict) else ""
+if not (t and t.strip()):
+    print("[probe] empty/invalid output:", str(d)[:200])
+    sys.exit(1)
+print("[probe] ok:", t[:80].replace("\n", " "))
+PY
+
+# Bench script runs on the prefill node; \$PIP/\$DIP injected at srun time.
+cat > "$WORKDIR/bench.sh" <<EOF
+#!/bin/bash
+set -e
+PIP=\$1; DIP=\$2
+docker rm -f mi355x_bench 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_bench \
+  -e PIP=\$PIP -e DIP=\$DIP \
+  $IMAGE bash -lc '
+    CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
+    bash \$CIDIR/install_checkout_sglang.sh
+    if [ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]; then
+      export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
+    else
+      export PYTHONPATH=/sgl-workspace/sglang/python:\${PYTHONPATH:-}
+    fi
+    bash \$CIDIR/install_checkout_router.sh
+    echo "[wait] prefill"; for i in \$(seq 1 600); do curl -sf http://\$PIP:$PPORT/health >/dev/null && break; sleep 5; done
+    echo "[wait] decode";  for i in \$(seq 1 600); do curl -sf http://\$DIP:$DPORT/health >/dev/null && break; sleep 5; done
+    python3 -m sglang_router.launch_router \
+      --pd-disaggregation \
+      --prefill http://\$PIP:$PPORT $PBOOT \
+      --decode http://\$DIP:$DPORT \
+      --host 0.0.0.0 --port $LBPORT \
+      --disable-circuit-breaker &
+    for i in \$(seq 1 30); do curl -sf http://127.0.0.1:$LBPORT/health >/dev/null && break; sleep 2; done
+    echo "[probe] PD end-to-end check via LB"
+    curl -sf -X POST http://127.0.0.1:$LBPORT/generate \
+      -H "content-type: application/json" -d @\$CIDIR/probe.json > \$CIDIR/probe_out.json \
+      || { echo "[probe] request failed -- PD path not serving; aborting before sweep"; exit 1; }
+    python3 \$CIDIR/assert_nonempty.py < \$CIDIR/probe_out.json \
+      || { echo "[probe] empty/invalid generation; aborting before sweep"; exit 1; }
+    # Correctness gate runs BEFORE the perf sweep: if the model is wrong there
+    # is no point spending ~15min measuring how fast it is wrong, so a failure
+    # here exits immediately and the sweep never runs.
+    if [ "$ACC_ENABLED" = "1" ]; then
+      echo "=== GSM8K accuracy gate (num_questions=$ACC_NQ shots=$ACC_SHOTS) ==="
+      DP_ARG=""
+      [ -s \$CIDIR/gsm8k_test.jsonl ] && DP_ARG="--data-path \$CIDIR/gsm8k_test.jsonl"
+      python3 -m sglang.test.few_shot_gsm8k \
+        --num-shots $ACC_SHOTS --num-questions $ACC_NQ --parallel $MAXREQ \
+        --max-new-tokens 512 --host http://127.0.0.1 --port $LBPORT \
+        \$DP_ARG 2>&1 | tee \$CIDIR/gsm8k.log
+      ACC=\$(grep -oE "Accuracy: [0-9.]+" \$CIDIR/gsm8k.log | tail -1 | cut -d" " -f2)
+      [ -n "\$ACC" ] || { echo "[gsm8k] could not parse accuracy from harness output"; exit 1; }
+      python3 \$CIDIR/check_acc.py "\$ACC" "$ACC_THR" || { echo "[gsm8k] accuracy below threshold -- failing before sweep"; exit 1; }
+    fi
+    for C in ${CONCS//,/ }; do
+      echo "=== concurrency=\$C ==="
+      OUT=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}/raw_conc\${C}.json
+      rm -f \$OUT
+      python3 -m sglang.bench_serving --backend sglang \
+        --host 127.0.0.1 --port $LBPORT --model $MODEL_PATH \
+        --dataset-name random --random-input-len $ISL --random-output-len $OSL \
+        --random-range-ratio $RRR --max-concurrency \$C \
+        --num-prompts \$((C*$NPF)) --warmup-requests \$C \
+        --output-file \$OUT || true
+    done
+  '
+EOF
+chmod +x "$WORKDIR"/*.sh
+
+# ---------------------------------------------------------------------------
+# Orchestration drive (runs inside the salloc allocation on the login node).
+# ---------------------------------------------------------------------------
+# drive.sh splits the allocation into the first PW nodes (prefill) and the next
+# DW nodes (decode), launches one server per node, then benches. For 1P1D
+# (PW=DW=1) this is exactly prefill-on-node-A / decode-on-node-B. Larger PW/DW
+# reserve 2P2D / 1P3D / 3P1D: all servers come up, but the load balancer and
+# bench still target the first prefill + first decode (multi-P/D fan-out is the
+# remaining LB piece), so a >1 topology logs an explicit NOTE rather than
+# silently producing partial-coverage numbers.
+cat > "$WORKDIR/drive.sh" <<'DRIVE'
+#!/bin/bash
+set -x
+WORKDIR="$1"; PW="${2:-1}"; DW="${3:-1}"
+mapfile -t NODES < <(scontrol show hostnames "$SLURM_JOB_NODELIST")
+PNODES=("${NODES[@]:0:PW}")
+DNODES=("${NODES[@]:PW:DW}")
+PNODE="${PNODES[0]}"; DNODE="${DNODES[0]}"
+PIP=$(getent ahostsv4 "$PNODE" | head -1 | awk '{print $1}')
+DIP=$(getent ahostsv4 "$DNODE" | head -1 | awk '{print $1}')
+echo "[drive] prefill nodes: ${PNODES[*]} ; decode nodes: ${DNODES[*]}"
+echo "[drive] bench targets prefill=$PNODE($PIP) decode=$DNODE($DIP)"
+if (( PW > 1 || DW > 1 )); then
+  echo "[drive] NOTE: router + bench use the first prefill and first decode only;"
+  echo "[drive]       multi-prefill/multi-decode fan-out is not wired yet (LB work)."
+fi
+# Each server's srun runs here on the login node and returns exactly when its
+# compute-node container exits. Wrap it so the return code lands in a marker
+# file on shared NFS. The monitor then watches for markers instead of polling
+# PIDs -- unambiguous (no zombie/kill -0 guesswork) and it records which role
+# died and with what code. (A hung-but-alive server is NOT caught here; that is
+# bounded by bench.sh's health-wait timeout.)
+rm -f "$WORKDIR"/server_exit_* "$WORKDIR/bench_exit"
+for n in "${PNODES[@]}"; do
+  ( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/prefill.sh" > "$WORKDIR/prefill_$n.log" 2>&1
+    echo "prefill@$n rc=$?" > "$WORKDIR/server_exit_prefill_$n" ) &
+done
+for n in "${DNODES[@]}"; do
+  ( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/decode.sh" > "$WORKDIR/decode_$n.log" 2>&1
+    echo "decode@$n rc=$?" > "$WORKDIR/server_exit_decode_$n" ) &
+done
+sleep 5
+# Bench in the background with its own marker, so the wait loop is purely file
+# based: finish when bench writes its marker, abort if any server marker shows up
+# first (a server died before the sweep completed).
+( srun --overlap -N1 --nodelist="$PNODE" bash "$WORKDIR/bench.sh" "$PIP" "$DIP" > "$WORKDIR/bench.log" 2>&1
+  echo $? > "$WORKDIR/bench_exit" ) &
+BENCH_BG=$!
+# Stream bench output live and poll the markers with xtrace OFF, so the console
+# shows clean benchmark/accuracy output instead of a compgen/sleep trace every
+# 10s. (Mirrors NVIDIA's launch_gb200.sh, which set +x around its log stream.)
+touch "$WORKDIR/bench.log"
+tail -n +1 -F "$WORKDIR/bench.log" 2>/dev/null &
+TAIL_PID=$!
+set +x
+RC=0
+while [[ ! -f "$WORKDIR/bench_exit" ]]; do
+  if compgen -G "$WORKDIR/server_exit_*" > /dev/null; then
+    echo "[drive] ERROR: a server exited early before bench finished:"
+    cat "$WORKDIR"/server_exit_* || true
+    kill "$BENCH_BG" 2>/dev/null || true
+    RC=1
+    break
+  fi
+  sleep 10
+done
+set -x
+kill "$TAIL_PID" 2>/dev/null || true
+[[ "$RC" -eq 0 ]] && RC=$(cat "$WORKDIR/bench_exit" 2>/dev/null || echo 1)
+echo "[drive] bench finished (rc=$RC), tearing down"
+for n in "${PNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_prefill >/dev/null 2>&1 || true; done
+for n in "${DNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_decode  >/dev/null 2>&1 || true; done
+exit "$RC"
+DRIVE
+chmod +x "$WORKDIR/drive.sh"
+
+NODELIST_ARG=()
+[[ -n "${SLURM_NODELIST:-}" ]] && NODELIST_ARG=(--nodelist="$SLURM_NODELIST")
+
+# Request whole nodes so a co-scheduled job can't share a node and skew the
+# benchmark numbers. Toggle off with SLURM_EXCLUSIVE=0 on partitions that
+# disallow --exclusive.
+EXCLUSIVE_ARG=()
+[[ "${SLURM_EXCLUSIVE:-1}" == "1" ]] && EXCLUSIVE_ARG=(--exclusive)
+
+# Keep the scheduler off known-bad nodes (e.g. a host whose ionic RDMA driver
+# ABI mismatches the container, where MORI reports "no active RDMA device" and
+# the disagg server dies on init). Comma-separated node list.
+EXCLUDE_ARG=()
+[[ -n "${SLURM_EXCLUDE:-}" ]] && EXCLUDE_ARG=(--exclude="$SLURM_EXCLUDE")
+
+# One node per prefill/decode worker (TP == GPUs/node). 1P1D -> 2 nodes.
+TOTAL_NODES=$((PW + DW))
+
+# Name the allocation <RUNNER_NAME>-<GITHUB_RUN_ID>-<config> so the workflow's
+# cleanup steps can scancel precisely instead of a blanket `squeue --me` that
+# would kill a concurrent matrix leg. RUNNER_NAME alone is not assumed unique;
+# GITHUB_RUN_ID + config make the name unique per matrix leg regardless.
+JOB_NAME="mi355x-ci-${RUNNER_NAME:-norunner}-${GITHUB_RUN_ID:-0}-${MATRIX_CONFIG_NAME}"
+
+set +e
+salloc -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" \
+    --job-name "$JOB_NAME" -t "$TIME_LIMIT" \
+    bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW"
+SALLOC_RC=$?
+set -e
+
+# bench output already streamed live from drive.sh (tail -F). drive.sh exits
+# non-zero when a server died or bench failed; on failure dump bench.log + the
+# server logs (the actual root cause). We still fall through to normalize
+# whatever raw results the completed concurrencies produced -- partial perf data
+# is worth uploading -- and propagate the failure via the exit code at the end.
+if [[ "$SALLOC_RC" -ne 0 ]]; then
+    echo "ERROR: allocation/bench failed (rc=$SALLOC_RC); bench + server logs:" >&2
+    echo "--- bench.log (tail) ---"; tail -40 "$WORKDIR/bench.log" 2>/dev/null || true
+    for f in "$WORKDIR"/prefill_*.log "$WORKDIR"/decode_*.log; do
+        [[ -f "$f" ]] && { echo "--- $f (tail) ---"; tail -30 "$f"; }
+    done
+fi
+
+# Surface the GSM8K accuracy in the job summary -- it scrolls past in the live
+# log, and the perf table (collect-results/summarize.py) doesn't include it.
+if [[ "$ACC_ENABLED" == "1" && -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    ACC_LINE=$(grep -aoE "Accuracy: [0-9.]+" "$WORKDIR/bench.log" 2>/dev/null | tail -1 || true)
+    {
+        echo "### GSM8K accuracy gate — ${MATRIX_CONFIG_NAME}"
+        echo '```'
+        echo "${ACC_LINE:-Accuracy: <not found in bench.log>}   (threshold > ${ACC_THR})"
+        echo '```'
+    } >> "$GITHUB_STEP_SUMMARY"
+fi
+
+# ---------------------------------------------------------------------------
+# Normalize raw bench_serving output -> process_result.py schema.
+#
+# bench_serving and process_result.py disagree on field names, so we remap the
+# last JSON line of each raw file. If bench_serving ever renames an output
+# field, the KeyError raised here (rather than a silently wrong table) is the
+# signal to update this mapping. Field-by-field:
+#
+#   bench_serving key          ->  process_result.py key       (purpose)
+#   --------------------------     ------------------------     -------------------------
+#   max_concurrency            ->  max_concurrency             (sweep point; falls back to $C)
+#   total_throughput           ->  total_token_throughput      (in+out tok/s, tput_per_gpu)
+#   output_throughput          ->  output_throughput           (out tok/s, output_tput_per_gpu)
+#   median_ttft_ms             ->  median_ttft_ms              (TTFT; /1000 -> s)
+#   median_tpot_ms             ->  median_tpot_ms              (TPOT; -> interactivity)
+#   median_e2e_latency_ms      ->  median_e2el_ms              (E2E latency; /1000 -> s)
+#   (none; injected here)      ->  model_id                    (served model, from $MODEL_PATH)
+# ---------------------------------------------------------------------------
+TOTAL_GPUS=$((PTP + DTP))
+PROCESSED=0
+for C in ${CONCS//,/ }; do
+    RAW="$WORKDIR/raw_conc${C}.json"
+    [[ -f "$RAW" ]] || { echo "WARN: missing $RAW"; continue; }
+    DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${MATRIX_CONFIG_NAME}_conc${C}_gpus_${TOTAL_GPUS}_ctx_${PTP}_gen_${DTP}.json"
+    MODEL_ID="$MODEL_PATH" python3 - "$RAW" "$DEST" "$C" <<'PY'
+import json, os, sys
+raw_path, dest, conc = sys.argv[1], sys.argv[2], int(sys.argv[3])
+line = [l for l in open(raw_path).read().splitlines() if l.strip()][-1]
+r = json.loads(line)
+norm = {
+    "max_concurrency": r.get("max_concurrency") or conc,
+    "model_id": os.environ["MODEL_ID"],
+    "total_token_throughput": r["total_throughput"],
+    "output_throughput": r["output_throughput"],
+    "median_ttft_ms": r["median_ttft_ms"],
+    "median_tpot_ms": r["median_tpot_ms"],
+    "median_e2el_ms": r["median_e2e_latency_ms"],
+}
+json.dump(norm, open(dest, "w"), indent=2)
+print("normalized ->", dest)
+PY
+    PROCESSED=$((PROCESSED + 1))
+done
+
+# Propagate a benchmark/allocation failure even though we emitted partial
+# results above (the workflow uploads them with `always()`).
+if [[ "$SALLOC_RC" -ne 0 ]]; then
+    echo "ERROR: benchmark failed (rc=$SALLOC_RC); emitted $PROCESSED partial result file(s)." >&2
+    exit "$SALLOC_RC"
+fi
+if [[ "$PROCESSED" -eq 0 ]]; then
+    echo "ERROR: no result files produced" >&2
+    exit 1
+fi
+echo "Done. $PROCESSED result file(s) in $GITHUB_WORKSPACE."

scripts/ci/slurm/log_analysis_prompt.md

@@ -0,0 +1,230 @@
+# srtslurm Log Analysis
+
+You are an automated CI failure analyst. Your job is to analyze logs from a
+failed srtslurm job, determine the root cause, and **take action** by filing
+GitHub issues when the cause is clear.
+
+srtslurm is a Python-first orchestration framework for running distributed LLM
+inference benchmarks on SLURM clusters using SGLang and TRTLLM backends.
+
+## Architecture
+
+There are two repos involved:
+
+- **`NVIDIA/srt-slurm`**: The orchestration layer. It owns recipes (YAML configs)
+  that define which flags, environment variables, and topology to use when
+  launching SGLang workers. It controls `srtctl`, worker lifecycle, health
+  checks, and benchmark execution.
+- **`sgl-project/sglang`**: The inference engine. It owns the server, model
+  loading, CUDA kernels, MoE routing, attention backends, and all runtime code.
+
+When a recipe passes flags that SGLang doesn't support together, **that is a
+recipe bug in srt-slurm**, not an sglang bug — even though the error appears in
+SGLang code. The recipe is responsible for only requesting valid combinations.
+
+## Step 1: Read Logs
+
+List the directory contents, then read files in this priority order:
+
+### 1. `sweep_{job_id}.log`
+
+Read this first. It is the orchestration timeline.
+
+Look for:
+- stage transitions
+- worker readiness
+- benchmark start
+- exit codes
+- the last error before teardown
+
+### 2. `config.yaml`
+
+Read this to understand the flags being passed to workers. Pay close attention
+to flags on prefill vs decode workers — they often differ and mismatches are a
+common source of bugs.
+
+### 3. `benchmark.out`
+
+If present, this usually contains the benchmark-side exception or timeout.
+
+### 4. `artifacts/*/logs/aiperf_*.log`
+
+If present, these often contain framework-level initialization failures and
+HTTP/network issues.
+
+### 5. Worker logs
+
+Focus on errors that line up with the failure timestamp:
+- `{node}_prefill_w{N}.out`
+- `{node}_decode_w{N}.out`
+- `{node}_frontend_{N}.out`
+
+### 6. `infra.out`
+
+Use this to confirm infrastructure failures involving NATS, etcd, ports, or
+service health checks.
+
+## Step 2: Correlate Timestamps
+
+This is the most important analysis technique.
+
+Many warnings are harmless. The root cause is usually the error that occurs at
+the same time the orchestration log transitions into failure.
+
+1. Find the failure time in `sweep_{job_id}.log`.
+2. Search other logs for matching timestamps.
+3. Ignore earlier warnings if the job continued past them.
+4. Ignore cleanup/teardown errors — they are consequences, not causes.
+
+## Step 3: Classify the Failure
+
+Determine which category the failure falls into:
+
+### Category A: Recipe/Config Bug → file against `NVIDIA/srt-slurm`
+
+The recipe or config is passing invalid or incompatible flags to SGLang. Examples:
+- Incompatible flag combinations (e.g., `--moe-a2a-backend deepep` with
+  `--fp4-gemm-backend flashinfer_cutedsl` when no fused func exists for that pair)
+- Wrong environment variables for the topology
+- Incorrect worker counts, GPU assignments, or port configs
+- srtctl bugs, health check misconfigurations, orchestration logic errors
+
+**Key signal**: The error is in SGLang code but the `config.yaml` shows the
+recipe chose a flag combination that SGLang doesn't support. The fix belongs in
+the recipe, not in SGLang.
+
+### Category B: SGLang Bug → list suspect PRs (do NOT auto-file)
+
+A genuine bug in SGLang's runtime code. Examples:
+- CUDA OOM, NCCL timeout, or kernel crash with valid flags
+- Model loading failure for a supported model
+- Regression introduced by a recent commit
+
+For these, use `gh` to find recent commits:
+```
+gh api "repos/sgl-project/sglang/commits?since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)&per_page=50" --jq '.[] | "\(.sha[:8]) \(.commit.message | split("\n")[0])"'
+```
+Then check which files each suspect commit touched:
+```
+gh api repos/sgl-project/sglang/commits/<sha> --jq '.files[].filename'
+```
+List suspect PRs in the report. Do NOT auto-file issues against sglang.
+
+### Category C: Infra/Transient → do NOT file any issue
+
+Flaky infrastructure, transient network issues, SLURM scheduling problems.
+Just note it in the report.
+
+## Step 4: Write the Report
+
+Write the report to `/workspace/logs/ai_analysis.md`. This is mandatory.
+
+Use this structure:
+
+```markdown
+## Job Analysis: {job_id}
+
+### Root Cause
+One clear sentence. State the category (A/B/C) and which repo owns the fix.
+
+### Evidence
+- `file:line` — exact error text
+- `config.yaml` — the relevant flags that caused or contributed to the failure
+- Timestamps showing correlation
+
+### Timeline
+| Time | Event |
+|------|-------|
+| ... | ... |
+
+### Noise
+- Warnings that were NOT causal (and why)
+
+### Suspect PRs (sglang)
+(Only for Category B failures)
+- PR #NNNN: "title" — why this commit could be related based on files changed
+
+### Recommended Fix
+Concrete, actionable steps. Not generic advice. Reference specific files,
+flags, or config values that need to change.
+```
+
+## Step 5: File Issues
+
+This step is **mandatory** for Category A and Category B failures. You MUST
+take action — the whole point of this system is to create issues so humans
+can track and fix problems.
+
+### For Category A (recipe/config bugs) → file against `NVIDIA/srt-slurm`
+
+1. First, check for duplicates:
+   ```
+   gh issue list --repo NVIDIA/srt-slurm --search "<key error message>" --limit 5
+   ```
+2. If no duplicate exists, file the issue:
+   ```
+   gh issue create --repo NVIDIA/srt-slurm \
+     --title "<concise title>" \
+     --body "<body>"
+   ```
+
+The issue body MUST include:
+- **Summary**: One sentence describing the failure
+- **Error**: The exact error message and which log file/line it came from
+- **Config**: The relevant flags from `config.yaml` that caused the issue
+- **Job**: The job ID and model/precision/topology
+- **Suggested Fix**: What the recipe should change (e.g., "change
+  `moe-runner-backend` from `flashinfer_cutedsl` to `flashinfer_cutlass`
+  when `moe-a2a-backend` is `deepep`", or "add validation to reject this
+  combination")
+
+### For Category B (sglang bugs) → file against `sgl-project/sglang`
+
+1. First, check for duplicates:
+   ```
+   gh issue list --repo sgl-project/sglang --search "<key error message>" --limit 5
+   ```
+2. If no duplicate exists, file the issue:
+   ```
+   gh issue create --repo sgl-project/sglang \
+     --title "<concise title>" \
+     --body "<body>"
+   ```
+
+The issue body MUST include:
+- **Summary**: One sentence describing the failure
+- **Error**: The exact error message, traceback, and which log file it came from
+- **Repro context**: Model, precision, topology, relevant flags from `config.yaml`
+- **Suspect commits**: List any recent commits that may have caused this, with
+  links (e.g., `https://github.com/sgl-project/sglang/commit/<sha>`)
+- **Suggested Fix**: If you can identify the fix from reading the sglang source
+  in `/workspace/repos/sglang/`, include it. Otherwise, describe what needs to
+  change conceptually.
+
+### For Category C (infra/transient) → do NOT file any issue
+
+Just include the analysis in the report.
+
+## Common Signal Reference
+
+High-signal failures:
+- `NotImplementedError` with runner/backend combinations → Category A
+- `ReadTimeout` / `Connection refused` during benchmark → check if config-caused
+- `CUDA out of memory` → likely Category B (unless config requests too many GPUs)
+- `NCCL timeout` → could be B or C, check if topology is valid
+- `Model not found` → check if recipe has correct model path
+- Benchmark exit code failures → check benchmark.out for details
+
+Low-signal noise (ignore these):
+- dependency resolver warnings
+- cleanup warnings during teardown
+- keep-alive failures AFTER the main crash
+- import warnings unrelated to the active model
+- `pip`/`rustup`/`apt-get` warnings during setup
+
+## Safety
+
+- Do NOT include API keys, tokens, or secrets in issues or the report.
+- Do NOT file issues if you are uncertain about the root cause. Only file when
+  you have concrete evidence.
+- Do NOT file duplicate issues. Always search first.

scripts/ci/slurm/nightly-configs.yaml

@@ -0,0 +1,358 @@
+# Nightly benchmark configurations for srt-slurm powered runners.
+#
+# Structure mirrors InferenceX nvidia-master.yaml but only includes fields
+# actually needed by the runner — prefill/decode topology details are already
+# encoded in each srt-slurm recipe YAML and are not duplicated here.
+#
+# To add/remove concurrencies: edit conc-list for the relevant search-space entry.
+# To add a new runner:         add a new top-level block and create a corresponding
+#                              nightly-test-<runner>.yml workflow.
+# Never edit workflow YAML files directly for these changes.
+
+dsr1-fp8-gb200-dynamo-sglang:
+  model: deepseek-ai/DeepSeek-R1-0528
+  model-prefix: dsr1
+  runner: gb200
+  precision: fp8
+  framework: dynamo-sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1024, 2048, 4096, 6144]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp8/1k1k/max-tpt.yaml
+          config_file: recipes/gb200-fp8/1k1k/max-tpt.yaml
+
+        - conc-list: [4096]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp8/1k1k/ultra-tpt.yaml
+          config_file: recipes/gb200-fp8/1k1k/ultra-tpt.yaml
+
+dsr1-fp4-gb200-dynamo-sglang:
+  model: nvidia/DeepSeek-R1-0528-NVFP4-v2
+  model-prefix: dsr1
+  runner: gb200
+  precision: fp4
+  framework: dynamo-sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [512, 2048, 4096, 8192]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp4/1k1k/mid-curve.yaml
+          config_file: recipes/gb200-fp4/1k1k/mid-curve.yaml
+
+# AMD MI355X 2-node 1P1D disaggregation. Driven by
+# scripts/ci/slurm/launch_mi355x.sh, which reads each recipe's `runtime`,
+# `bench`, and `bench.accuracy` sections. Every nightly runs ALL four
+# DeepSeek-V4 model x precision combos below (full matrix; GitHub
+# strategy.matrix.config, fail-fast: false). Each runs a GSM8K accuracy
+# hard-gate before the perf sweep.
+#
+# model_path points at the shared NFS HuggingFace cache dir (models--org--name);
+# the launcher resolves the live snapshot via refs/main, so no hash is hardcoded.
+dsv4flash-fp8-mi355x-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml
+
+dsv4pro-fp8-mi355x-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d.yaml
+
+dsv4flash-fp4-mi355x-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml
+
+dsv4pro-fp4-mi355x-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d.yaml
+
+# AMD MI355X 2-node 1P1D disaggregation over MORI with extra topology / MTP
+# coverage on the SAME four DeepSeek-V4 model x precision combos as the base
+# MORI blocks above (which stay TP8, no MTP). Three variants per model:
+#   * -mtp        : TP8 + EAGLE MTP (recipe `mtp.enabled`)
+#   * -dp8ep8     : DP-attention 8 + narrow within-node EP8
+#   * -dp8ep8-mtp : DP8 + narrow EP8 + EAGLE MTP
+# launch_mi355x.sh reads expert-/data-parallel-size and the `mtp:` section to
+# append --ep-size / --enable-dp-attention --dp-size / --speculative-* flags.
+
+dsv4flash-fp8-mi355x-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml
+
+dsv4flash-fp8-mi355x-dp8ep8-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml
+
+dsv4flash-fp8-mi355x-dp8ep8-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml
+
+dsv4pro-fp8-mi355x-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        # conc256 excluded: disagg-decode SWA hybrid pool retract->get_cpu_copy
+        # is an upstream NotImplementedError (crashes decode). See recipe.
+        - conc-list: [1, 8, 16, 32, 64, 128]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-mtp.yaml
+
+dsv4pro-fp8-mi355x-dp8ep8-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8.yaml
+
+dsv4pro-fp8-mi355x-dp8ep8-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml
+
+dsv4flash-fp4-mi355x-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml
+
+dsv4flash-fp4-mi355x-dp8ep8-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml
+
+dsv4flash-fp4-mi355x-dp8ep8-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml
+
+dsv4pro-fp4-mi355x-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-mtp.yaml
+
+dsv4pro-fp4-mi355x-dp8ep8-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8.yaml
+
+dsv4pro-fp4-mi355x-dp8ep8-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml
+
+# Kimi-K2.6 (FP8) 2-node 1P1D. Demonstrates the launcher is model-agnostic: all
+# Kimi-specific config lives in the recipe's `model:` block + split attention
+# runtime, with nothing hardcoded in launch_mi355x.sh. Base + EAGLE3 MTP (the
+# MTP leg uses an external draft checkpoint via mtp.draft_model_path).
+kimik26-fp8-mi355x-sglang:
+  model: moonshotai/Kimi-K2.6
+  model-prefix: kimik26
+  model_path: /it-share/model_coverage/models--moonshotai--Kimi-K2.6
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d.yaml
+
+kimik26-fp8-mi355x-mtp-sglang:
+  model: moonshotai/Kimi-K2.6
+  model-prefix: kimik26
+  model_path: /it-share/model_coverage/models--moonshotai--Kimi-K2.6
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml

scripts/ci/slurm/process_result.py

@@ -0,0 +1,117 @@
+"""Process a raw srt-slurm benchmark result JSON into an aggregated format.
+
+Usage (called once per result file):
+    RESULT_FILENAME=<path_without_.json> PREFILL_GPUS=<n> DECODE_GPUS=<n> \\
+        RECIPE_FILE=<path_to_recipe.yaml> python3 process_result.py
+
+Required env vars:
+    RESULT_FILENAME   - path to the result file without the .json extension
+    FRAMEWORK         - e.g. dynamo-sglang
+    PRECISION         - e.g. fp8, fp4
+    MODEL_PREFIX      - short model label, e.g. dsr1
+    ISL               - input sequence length
+    OSL               - output sequence length
+    PREFILL_GPUS      - number of prefill GPUs (extracted from result filename)
+    DECODE_GPUS       - number of decode GPUs (extracted from result filename)
+
+Optional env vars:
+    RECIPE_FILE       - path to the srt-slurm recipe YAML; if set, topology
+                        fields (TP, EP, DP, workers) are parsed from it
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def require(var):
+    val = os.environ.get(var)
+    if val is None:
+        print(f"ERROR: Missing required env var: {var}", file=sys.stderr)
+        sys.exit(1)
+    return val
+
+
+result_filename = require("RESULT_FILENAME")
+framework = require("FRAMEWORK")
+precision = require("PRECISION")
+model_prefix = require("MODEL_PREFIX")
+isl = int(require("ISL"))
+osl = int(require("OSL"))
+prefill_gpus = int(require("PREFILL_GPUS"))
+decode_gpus = int(require("DECODE_GPUS"))
+
+with open(f"{result_filename}.json") as f:
+    raw = json.load(f)
+
+# ---------------------------------------------------------------------------
+# Topology — parse from recipe YAML if available, otherwise default to 0/"-"
+# ---------------------------------------------------------------------------
+prefill_tp = prefill_ep = prefill_dp_attn = 0
+prefill_num_workers = decode_tp = decode_ep = decode_dp_attn = decode_num_workers = 0
+
+recipe_file = os.environ.get("RECIPE_FILE")
+if recipe_file and Path(recipe_file).exists():
+    import yaml
+
+    with open(recipe_file) as f:
+        recipe = yaml.safe_load(f)
+
+    res = recipe.get("resources", {})
+    prefill_num_workers = res.get("prefill_workers", 0)
+    decode_num_workers = res.get("decode_workers", 0)
+
+    sgl = recipe.get("backend", {}).get("sglang_config", {})
+    p = sgl.get("prefill", {})
+    d = sgl.get("decode", {})
+
+    prefill_tp = p.get("tensor-parallel-size", 0)
+    prefill_ep = p.get("expert-parallel-size", 0)
+    prefill_dp_attn = p.get("data-parallel-size", "-")
+    decode_tp = d.get("tensor-parallel-size", 0)
+    decode_ep = d.get("expert-parallel-size", 0)
+    decode_dp_attn = d.get("data-parallel-size", "-")
+
+total_gpus = prefill_gpus + decode_gpus
+
+data = {
+    "hw": os.environ.get("HW", "gb200"),
+    "conc": int(raw["max_concurrency"]),
+    "model": raw["model_id"],
+    "infmax_model_prefix": model_prefix,
+    "framework": framework,
+    "precision": precision,
+    "isl": isl,
+    "osl": osl,
+    "is_multinode": True,
+    "disagg": True,
+    "num_prefill_gpu": prefill_gpus,
+    "num_decode_gpu": decode_gpus,
+    "prefill_num_workers": prefill_num_workers,
+    "prefill_tp": prefill_tp,
+    "prefill_ep": prefill_ep,
+    "prefill_dp_attention": prefill_dp_attn,
+    "decode_num_workers": decode_num_workers,
+    "decode_tp": decode_tp,
+    "decode_ep": decode_ep,
+    "decode_dp_attention": decode_dp_attn,
+    "tput_per_gpu": float(raw["total_token_throughput"]) / total_gpus,
+    "output_tput_per_gpu": float(raw["output_throughput"]) / decode_gpus,
+    "input_tput_per_gpu": (
+        float(raw["total_token_throughput"]) - float(raw["output_throughput"])
+    )
+    / prefill_gpus,
+}
+
+for key, value in raw.items():
+    if key.endswith("_ms"):
+        data[key.replace("_ms", "")] = float(value) / 1000.0
+    if "tpot" in key:
+        data[key.replace("_ms", "").replace("tpot", "intvty")] = 1000.0 / float(value)
+
+out_path = Path(result_filename).parent / f"agg_{Path(result_filename).name}.json"
+with open(out_path, "w") as f:
+    json.dump(data, f, indent=2)
+
+print(f"Written: {out_path}")

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,53 @@
+# MI355X DeepSeek-V4-Flash FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash FP4 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml

@@ -0,0 +1,55 @@
+# MI355X DeepSeek-V4-Flash (FP4) 2-node 1P1D disaggregation recipe.
+#
+# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,61 @@
+# MI355X DeepSeek-V4-Pro FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,52 @@
+# MI355X DeepSeek-V4-Pro FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,61 @@
+# MI355X DeepSeek-V4-Pro FP4 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d.yaml

@@ -0,0 +1,56 @@
+# MI355X DeepSeek-V4-Pro (FP4) 2-node 1P1D disaggregation recipe.
+#
+# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
+# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
+# larger, so mem_fraction_static / max_running_requests may need tuning.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,53 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml

@@ -0,0 +1,54 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path after the perf sweep. Mirrors the
+  # single-node registered test test/registered/amd/test_deepseek_v4_flash_fp8.py
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,61 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,52 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,68 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  # conc256 is excluded for this leg only: at conc256 the disagg-decode SWA
+  # hybrid KV pool fills, the scheduler retracts running requests, and the
+  # retract->offload_kv_cache path calls get_cpu_copy() which is unimplemented
+  # for the SWA hybrid pool (raises NotImplementedError, crashes the decode
+  # scheduler). Raising swa_full_tokens_ratio (tried up to 0.3) does not help
+  # -- usage climbs to fill the larger budget and still retracts. Until the
+  # upstream get_cpu_copy stub is implemented, cap this leg at 128.
+  concurrencies: [1, 8, 16, 32, 64, 128]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d.yaml

@@ -0,0 +1,55 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe.
+#
+# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
+# larger, so mem_fraction_static / max_running_requests may need tuning.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,85 @@
+# MI355X Kimi-K2.6 (FP8) 2-node 1P1D disaggregation recipe (base + EAGLE3 MTP).
+#
+# Self-contained (no inheritance): same as 1p1d.yaml plus the `mtp:` block. Uses
+# EAGLE3 speculative decoding with an EXTERNAL draft checkpoint (unlike DSV4's
+# built-in EAGLE NextN head): the launcher resolves mtp.draft_model_path through
+# the HF-cache snapshot logic and appends --speculative-draft-model-path. The
+# draft dir must live under /it-share (the container's :ro mount). MTP is applied
+# to both prefill and decode. Mirrors the Kimi-K2.6 serving cookbook.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, `model`, `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+# Model-specific docker env + sglang server args (written verbatim via
+# model_flags.sh). Each server arg + value is a SEPARATE list item.
+model:
+  env:
+    SGLANG_USE_AITER: 1
+    SGLANG_ROCM_FUSED_DECODE_MLA: 0
+  server_args:
+    - --model-loader-extra-config
+    - '{"enable_multithread_load": true}'
+    - --watchdog-timeout
+    - 1200
+    - --reasoning-parser
+    - kimi_k2
+    - --tool-call-parser
+    - kimi_k2
+
+# EAGLE3 speculative decoding with an external draft checkpoint.
+mtp:
+  enabled: true
+  algorithm: EAGLE3
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+  draft_model_path: /it-share/model_coverage/models--lightseekorg--kimi-k2.6-eagle3.1-mla
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  # Kimi uses split attention backends (aiter prefill / triton decode), not a
+  # single --attention-backend.
+  prefill_attention_backend: aiter
+  decode_attention_backend: triton
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep. Mirrors the
+  # registered single-node Kimi-K2.6 eval (full GSM8K, 8-shot, accuracy > 0.92).
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.92

scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d.yaml

@@ -0,0 +1,75 @@
+# MI355X Kimi-K2.6 (FP8) 2-node 1P1D disaggregation recipe (base).
+#
+# All Kimi-specific config lives in this recipe's `model:` block (docker env +
+# sglang server args) and `runtime` (split attention backends); nothing about
+# Kimi is hardcoded in launch_mi355x.sh. Mirrors the single-node registered test
+# test/registered/amd/accuracy/mi35x/test_kimi_k26_eval_mi35x.py (TP8, split
+# attention backends, multithread loader, GSM8K > 0.92).
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, `model`, `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+# Model-specific docker env + sglang server args (written verbatim via
+# model_flags.sh). Each server arg + value is a SEPARATE list item.
+model:
+  env:
+    SGLANG_USE_AITER: 1
+    SGLANG_ROCM_FUSED_DECODE_MLA: 0
+  server_args:
+    - --model-loader-extra-config
+    - '{"enable_multithread_load": true}'
+    - --watchdog-timeout
+    - 1200
+    - --reasoning-parser
+    - kimi_k2
+    - --tool-call-parser
+    - kimi_k2
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  # Kimi uses split attention backends (aiter prefill / triton decode), not a
+  # single --attention-backend.
+  prefill_attention_backend: aiter
+  decode_attention_backend: triton
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep. Mirrors the
+  # registered single-node Kimi-K2.6 eval (full GSM8K, 8-shot, accuracy > 0.92).
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.92

scripts/ci/slurm/summarize.py

@@ -0,0 +1,128 @@
+"""Print a markdown summary table from processed benchmark results.
+
+Usage:
+    python3 summarize.py <results_dir>
+
+Reads all agg_*.json files recursively from <results_dir> and prints a
+markdown table to stdout (redirect to $GITHUB_STEP_SUMMARY to publish).
+"""
+
+import json
+import sys
+from pathlib import Path
+
+from tabulate import tabulate
+
+HEADERS = [
+    "Model",
+    "Served Model",
+    "Hardware",
+    "Framework",
+    "Precision",
+    "ISL",
+    "OSL",
+    "Prefill TP",
+    "Prefill EP",
+    "Prefill DP Attn",
+    "Prefill Workers",
+    "Prefill GPUs",
+    "Decode TP",
+    "Decode EP",
+    "Decode DP Attn",
+    "Decode Workers",
+    "Decode GPUs",
+    "Conc",
+    "TTFT (ms)",
+    "TPOT (ms)",
+    "Interactivity (tok/s/user)",
+    "E2EL (s)",
+    "TPUT per GPU",
+    "Output TPUT per GPU",
+    "Input TPUT per GPU",
+]
+
+
+def load_json(path):
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except Exception:
+        return None
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python3 summarize.py <results_dir>")
+        sys.exit(1)
+
+    results_dir = Path(sys.argv[1])
+    results = [
+        r
+        for path in results_dir.rglob("agg_*.json")
+        if (r := load_json(path)) and "is_multinode" in r
+    ]
+
+    if not results:
+        print("No processed result files found.")
+        return
+
+    results.sort(
+        key=lambda r: (
+            r["infmax_model_prefix"],
+            r["hw"],
+            r["framework"],
+            r["precision"],
+            r["isl"],
+            r["osl"],
+            r["prefill_tp"],
+            r["prefill_ep"],
+            r["decode_tp"],
+            r["decode_ep"],
+            r["conc"],
+        )
+    )
+
+    rows = [
+        [
+            r["infmax_model_prefix"],
+            r["model"],
+            r["hw"].upper(),
+            r["framework"].upper(),
+            r["precision"].upper(),
+            r["isl"],
+            r["osl"],
+            r["prefill_tp"],
+            r["prefill_ep"],
+            r["prefill_dp_attention"],
+            r["prefill_num_workers"],
+            r["num_prefill_gpu"],
+            r["decode_tp"],
+            r["decode_ep"],
+            r["decode_dp_attention"],
+            r["decode_num_workers"],
+            r["num_decode_gpu"],
+            r["conc"],
+            f"{r['median_ttft'] * 1000:.4f}",
+            f"{r['median_tpot'] * 1000:.4f}",
+            f"{r['median_intvty']:.4f}",
+            f"{r['median_e2el']:.4f}",
+            f"{r['tput_per_gpu']:.4f}",
+            f"{r['output_tput_per_gpu']:.4f}",
+            f"{r['input_tput_per_gpu']:.4f}",
+        ]
+        for r in results
+    ]
+
+    hw_label = "/".join(sorted({r["hw"].upper() for r in results}))
+    if hw_label == "GB200":
+        # NVIDIA GB200 nightly: keep the original hardcoded title untouched.
+        print("## GB200 Nightly Benchmark Results\n")
+    else:
+        # AMD (e.g. MI355X) nightly: derive the title from the result hardware.
+        print(f"## {hw_label} Nightly Benchmark Results\n")
+    print(tabulate(rows, headers=HEADERS, tablefmt="github"))
+    print()
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/stage_models_overrides.json

@@ -0,0 +1,31 @@
+{
+  "_comment": "Manual overrides for list_stage_models.py. by_file/by_suite ADD models the static scan cannot see (models built dynamically, read from configs, passed via CLI args). deny REMOVES false-positive ids the heuristic mistakes for models. Keys in by_file are repo-relative test paths (e.g. test/registered/foo/test_bar.py). suite_labels maps legacy suite= registrations (no runner_config) to the GH runner label(s) their dispatching workflow hardcodes in runs-on -- a list, because one suite can run on several labels (nightly-8-gpu-common). $b200_runner is the dynamic-b200 placeholder from runner_configs.yml. Deliberately absent: nightly-4-gpu-gb300-* (run as k8s pods, not GHA runners) and nightly-2-gpu (registered but dispatched by no workflow); both stay visible in unmapped_suites.",
+  "by_file": {},
+  "by_suite": {},
+  "suite_labels": {
+    "base-b-kernel-benchmark-1-gpu-large": ["1-gpu-h100"],
+    "base-b-kernel-unit-1-gpu-b200": ["$b200_runner"],
+    "base-b-kernel-unit-1-gpu-large": ["1-gpu-h100"],
+    "base-b-kernel-unit-8-gpu-h200": ["8-gpu-h200"],
+    "nightly-1-gpu": ["1-gpu-h100"],
+    "nightly-4-gpu": ["4-gpu-h100"],
+    "nightly-4-gpu-b200": ["$b200_runner"],
+    "nightly-8-gpu-b200": ["8-gpu-b200"],
+    "nightly-8-gpu-common": ["8-gpu-h200", "8-gpu-b200"],
+    "nightly-8-gpu-h200": ["8-gpu-h200"],
+    "nightly-eval-text-2-gpu": ["2-gpu-h100"],
+    "nightly-eval-vlm-2-gpu": ["2-gpu-h100"],
+    "nightly-kernel-1-gpu": ["1-gpu-h100"],
+    "nightly-kernel-8-gpu-h200": ["8-gpu-h200"],
+    "nightly-perf-text-2-gpu": ["2-gpu-h100"],
+    "nightly-perf-vlm-2-gpu": ["2-gpu-h100"],
+    "nightly-precision-8-gpu-h200": ["8-gpu-h200"],
+    "stress": ["8-gpu-h200"],
+    "weekly-8-gpu-h200": ["8-gpu-h200"]
+  },
+  "deny": [
+    "tok/req",
+    "qwen/qwen3",
+    "qwen/qwen3-vl"
+  ]
+}

scripts/ci/test_list_stage_models.py

@@ -0,0 +1,602 @@
+"""Unit tests for list_stage_models extraction logic.
+
+Pure-logic tests (stdlib only, no GPU, no sglang import) so they run in the
+ci-model-inventory workflow without installing dependencies:
+
+    python -m unittest discover -s scripts/ci -p 'test_list_stage_models.py'
+
+Guards the recall/precision contract of the static model extractor: model-id
+shape filtering, constant-table resolution (incl. tuple values), inline +
+name-reference extraction, override merge semantics, and the suite->files +
+inventory assembly that drives cache-warming.
+"""
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import list_stage_models as lsm  # noqa: E402
+
+# Repo root inferred from this file's location: <root>/scripts/ci/<this>.
+_REPO_ROOT = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+_REAL_CI_REGISTER = os.path.join(
+    _REPO_ROOT, "python", "sglang", "test", "ci", "ci_register.py"
+)
+
+
+def _write(root, rel, content):
+    path = os.path.join(root, rel)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(content)
+
+
+def _make_fake_repo(root, registered, helpers=None):
+    """Build a temp repo: copy the real ci_register.py, write test + helper files.
+
+    ``registered`` maps ``test/registered/...`` relpaths to file content;
+    ``helpers`` maps ``python/sglang/test/...`` relpaths to content.
+    """
+    dst = os.path.join(root, "python", "sglang", "test", "ci", "ci_register.py")
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    shutil.copy(_REAL_CI_REGISTER, dst)
+    for rel, content in registered.items():
+        _write(root, os.path.join("test", "registered", rel), content)
+    for rel, content in (helpers or {}).items():
+        _write(root, os.path.join("python", "sglang", "test", rel), content)
+
+
+class LooksLikeModelId(unittest.TestCase):
+    def test_accepts_real_model_ids(self):
+        for value in (
+            "meta-llama/Llama-3.1-8B-Instruct",
+            "RedHatAI/Llama-3.2-3B-quantized.w8a8",  # dotted suffix is real
+            "cross-encoder/ms-marco-MiniLM-L6-v2",
+            "nvidia/DeepSeek-V3-0324-FP4",
+            "lmsys/sglang-ci-dsv3-test",
+            "Qwen/Qwen2-1.5B-Instruct-GGUF",  # -GGUF suffix, not .gguf extension
+        ):
+            self.assertTrue(lsm.looks_like_model_id(value), value)
+
+    def test_rejects_non_models(self):
+        for value in (
+            "text/plain",  # MIME
+            "application/json",  # MIME
+            "image/png",  # MIME
+            "Text/Plain",  # MIME, case-insensitive
+            "2/3",  # numeric ratio, no letters
+            "N/A",  # single-char sides
+            "configs/model.json",  # path with file extension
+            "org/weights.safetensors",  # weight-file extension
+            "org/model.bin",  # weight-file extension
+            "a/b/c",  # too many slashes
+            "/abs/path",  # leading slash
+            "./relative",  # leading dot
+            "just-a-string",  # no slash
+        ):
+            self.assertFalse(lsm.looks_like_model_id(value), value)
+
+    def test_deny_set_overrides(self):
+        value = "org/looks-like-a-model"
+        self.assertTrue(lsm.looks_like_model_id(value))
+        self.assertFalse(lsm.looks_like_model_id(value, deny={value}))
+
+
+class ConstantTable(unittest.TestCase):
+    def test_single_and_tuple_values(self):
+        source = (
+            'DEFAULT_MODEL = "meta-llama/Llama-3.1-8B-Instruct"\n'
+            'PAIR = ("OPEA/Qwen2.5-0.5B-int4", "Intel/Qwen2-0.5B-int4")\n'
+            'NOT_A_MODEL = "https://example.com/x"\n'
+            "PORT = 30000\n"
+        )
+        table = lsm.extract_constants_from_source(source)
+        self.assertEqual(table["DEFAULT_MODEL"], {"meta-llama/Llama-3.1-8B-Instruct"})
+        self.assertEqual(
+            table["PAIR"], {"OPEA/Qwen2.5-0.5B-int4", "Intel/Qwen2-0.5B-int4"}
+        )
+        self.assertNotIn("NOT_A_MODEL", table)
+        self.assertNotIn("PORT", table)
+
+    def test_annotated_assignment(self):
+        source = 'DEFAULT: str = "meta-llama/Llama-3.1-8B-Instruct"\n'
+        table = lsm.extract_constants_from_source(source)
+        self.assertEqual(table["DEFAULT"], {"meta-llama/Llama-3.1-8B-Instruct"})
+
+    def test_implicit_concatenation(self):
+        # Python folds adjacent string literals at parse time into one Constant.
+        source = 'M = (\n    "meta-llama/"\n    "Llama-3.1-8B-Instruct"\n)\n'
+        table = lsm.extract_constants_from_source(source)
+        self.assertEqual(table["M"], {"meta-llama/Llama-3.1-8B-Instruct"})
+
+    def test_deny_excludes_from_constant_table(self):
+        source = 'BAD = "weird/thing"\nGOOD = "meta-llama/Llama-3.1-8B-Instruct"\n'
+        table = lsm.extract_constants_from_source(source, deny={"weird/thing"})
+        self.assertNotIn("BAD", table)
+        self.assertIn("GOOD", table)
+
+
+class ExtractModels(unittest.TestCase):
+    def test_inline_and_name_reference(self):
+        const_table = {
+            "DEFAULT_MODEL_NAME_FOR_TEST": {"meta-llama/Llama-3.1-8B-Instruct"}
+        }
+        source = (
+            "from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST\n"
+            "class T:\n"
+            "    model = DEFAULT_MODEL_NAME_FOR_TEST\n"
+            '    draft = "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B"\n'
+        )
+        models = lsm.extract_models_from_source(source, const_table)
+        self.assertEqual(
+            models,
+            {
+                "meta-llama/Llama-3.1-8B-Instruct",
+                "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B",
+            },
+        )
+
+    def test_fstring_fragments_are_skipped(self):
+        # An f-string with a placeholder yields a truncated, non-existent id;
+        # it must NOT be picked up.
+        source = 'msg = f"meta-llama/Llama-3.1-8B-Instruct-{suffix}"\n'
+        self.assertEqual(lsm.extract_models_from_source(source, {}), set())
+
+    def test_fstring_name_placeholder_still_resolves(self):
+        # The literal fragment is skipped, but a constant referenced inside the
+        # f-string placeholder is still resolved.
+        const_table = {"DRAFT": {"lmsys/sglang-EAGLE-llama2-chat-7B"}}
+        source = 'path = f"prefix/{DRAFT}/topk"\n'
+        self.assertEqual(
+            lsm.extract_models_from_source(source, const_table),
+            {"lmsys/sglang-EAGLE-llama2-chat-7B"},
+        )
+
+    def test_model_less_file_yields_nothing(self):
+        source = (
+            "import torch\n"
+            "def test_kernel():\n"
+            "    assert torch.cuda.is_available() or True\n"
+        )
+        self.assertEqual(lsm.extract_models_from_source(source, {}), set())
+
+    def test_deny_removes_name_resolved_model(self):
+        # deny must apply to constant-resolved ids too, not just inline literals.
+        const_table = {"M": {"weird/thing"}}
+        source = "x = M\n"
+        self.assertIn(
+            "weird/thing", lsm.extract_models_from_source(source, const_table)
+        )
+        self.assertEqual(
+            lsm.extract_models_from_source(source, const_table, deny={"weird/thing"}),
+            set(),
+        )
+
+
+class Overrides(unittest.TestCase):
+    def test_load_defaults_when_missing(self):
+        ov = lsm.load_overrides(None)
+        self.assertEqual(
+            ov, {"by_file": {}, "by_suite": {}, "deny": [], "suite_labels": {}}
+        )
+
+    def test_null_values_fall_back_to_defaults(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "ov.json")
+            with open(path, "w", encoding="utf-8") as f:
+                f.write('{"deny": null, "by_file": null}')
+            ov = lsm.load_overrides(path)
+            self.assertEqual(
+                ov, {"by_file": {}, "by_suite": {}, "deny": [], "suite_labels": {}}
+            )
+
+
+class CollectSuiteFiles(unittest.TestCase):
+    REG = (
+        "import unittest\n"
+        "from sglang.test.ci.ci_register import register_cuda_ci, register_amd_ci\n"
+        "{calls}\n"
+        'MODEL = "{model}"\n'
+        'if __name__ == "__main__":\n    unittest.main()\n'
+    )
+
+    def _repo(self, tmp):
+        _make_fake_repo(
+            tmp,
+            registered={
+                # enabled CUDA, two suites in one file -> dedupe per suite
+                "a/test_a.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-x", '
+                        'runner_config="1-gpu")\n'
+                        'register_cuda_ci(est_time=1, suite="nightly-y", '
+                        "nightly=True)"
+                    ),
+                    model="meta-llama/Llama-3.1-8B-Instruct",
+                ),
+                # disabled CUDA -> excluded unless include_disabled
+                "b/test_b.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-z", '
+                        'runner_config="1-gpu", disabled="flaky")'
+                    ),
+                    model="Qwen/Qwen3-8B",
+                ),
+                # AMD only -> never in CUDA inventory
+                "c/test_c.py": self.REG.format(
+                    calls='register_amd_ci(est_time=1, suite="nightly-amd")',
+                    model="google/gemma-3-4b-it",
+                ),
+            },
+        )
+
+    def test_enabled_only_by_default(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            suites, nightly, errors, runner_configs = lsm.collect_suite_files(
+                tmp, "cuda"
+            )
+            self.assertEqual(set(suites), {"base-x-test-1-gpu", "nightly-y"})
+            self.assertEqual(errors, {})
+            self.assertTrue(nightly["nightly-y"])
+            self.assertFalse(nightly["base-x-test-1-gpu"])
+            # AMD suite never appears for the CUDA backend.
+            self.assertNotIn("nightly-amd", suites)
+            # Modern registrations carry their runner_config; legacy suite= has none.
+            self.assertEqual(runner_configs["base-x-test-1-gpu"], "1-gpu")
+            self.assertIsNone(runner_configs["nightly-y"])
+
+    def test_include_disabled(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            suites, _, _, _ = lsm.collect_suite_files(
+                tmp, "cuda", include_disabled=True
+            )
+            self.assertIn("base-z-test-1-gpu", suites)
+
+    def test_unparsable_registry_is_surfaced(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            _make_fake_repo(
+                tmp,
+                registered={
+                    # est_time missing -> RegistryVisitor raises ValueError
+                    "d/test_bad.py": (
+                        "from sglang.test.ci.ci_register import register_cuda_ci\n"
+                        'register_cuda_ci(suite="base-x")\n'
+                    ),
+                },
+            )
+            suites, _, errors, _ = lsm.collect_suite_files(tmp, "cuda")
+            self.assertEqual(suites, {})
+            self.assertIn("test/registered/d/test_bad.py", errors)
+
+
+class BuildInventory(unittest.TestCase):
+    def _repo(self, tmp):
+        _make_fake_repo(
+            tmp,
+            registered={
+                # resolves a model via an imported constant
+                "a/test_a.py": (
+                    "import unittest\n"
+                    "from sglang.test.ci.ci_register import register_cuda_ci\n"
+                    "from sglang.test.test_utils import DEFAULT_MODEL\n"
+                    'register_cuda_ci(est_time=1, stage="base-x", '
+                    'runner_config="1-gpu")\n'
+                    "MODEL = DEFAULT_MODEL\n"
+                    'if __name__ == "__main__":\n    unittest.main()\n'
+                ),
+                # model-less -> lands in unresolved_files (same suite as a)
+                "b/test_b.py": (
+                    "import unittest\n"
+                    "from sglang.test.ci.ci_register import register_cuda_ci\n"
+                    'register_cuda_ci(est_time=1, stage="base-x", '
+                    'runner_config="1-gpu")\n'
+                    'if __name__ == "__main__":\n    unittest.main()\n'
+                ),
+            },
+            helpers={"test_utils.py": 'DEFAULT_MODEL = "meta-llama/Llama-3.1-8B"\n'},
+        )
+
+    def test_resolution_and_unresolved(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            inv = lsm.build_inventory(tmp, "cuda", lsm.load_overrides(None), "sha123")
+            self.assertEqual(inv["generated_at_commit"], "sha123")
+            suite = inv["suites"]["base-x-test-1-gpu"]
+            self.assertEqual(suite["models"], ["meta-llama/Llama-3.1-8B"])
+            self.assertEqual(suite["test_file_count"], 2)
+            self.assertEqual(suite["unresolved_files"], ["test/registered/b/test_b.py"])
+            self.assertEqual(inv["model_count"], 1)
+            self.assertEqual(inv["parse_failures"], {})
+
+    def test_by_file_override_adds_and_clears_unresolved(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            overrides = {
+                "by_file": {"test/registered/b/test_b.py": ["org/extra-model"]},
+                "by_suite": {},
+                "deny": [],
+            }
+            inv = lsm.build_inventory(tmp, "cuda", overrides, "sha")
+            suite = inv["suites"]["base-x-test-1-gpu"]
+            self.assertIn("org/extra-model", suite["models"])
+            # by_file supplied a model for test_b -> no longer unresolved.
+            self.assertEqual(suite["unresolved_files"], [])
+
+    def test_by_suite_override_adds_model(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            overrides = {
+                "by_file": {},
+                "by_suite": {"base-x-test-1-gpu": ["org/suite-model"]},
+                "deny": [],
+            }
+            inv = lsm.build_inventory(tmp, "cuda", overrides, "sha")
+            self.assertIn(
+                "org/suite-model", inv["suites"]["base-x-test-1-gpu"]["models"]
+            )
+
+
+class RenderMarkdown(unittest.TestCase):
+    def test_table_and_glyphs(self):
+        inv = {
+            "backend": "cuda",
+            "generated_at_commit": "sha",
+            "suite_count": 2,
+            "model_count": 1,
+            "parse_failures": {},
+            "suites": {
+                "n-suite": {
+                    "nightly": True,
+                    "models": ["org/model"],
+                    "test_file_count": 1,
+                    "unresolved_files": [],
+                },
+                "empty": {
+                    "nightly": False,
+                    "models": [],
+                    "test_file_count": 2,
+                    "unresolved_files": ["x.py", "y.py"],
+                },
+            },
+        }
+        md = lsm.render_markdown(inv)
+        self.assertIn("| `n-suite` | ✓ | org/model | 0 |", md)
+        self.assertIn("| `empty` |  | _(none)_ | 2 |", md)
+
+    def test_parse_failures_line(self):
+        inv = {
+            "backend": "cuda",
+            "generated_at_commit": "sha",
+            "suite_count": 0,
+            "model_count": 0,
+            "parse_failures": {"x.py": "SyntaxError: bad"},
+            "suites": {},
+        }
+        self.assertIn("Unparsable files", lsm.render_markdown(inv))
+
+    def test_runner_label_table_and_unmapped_note(self):
+        inv = {
+            "backend": "cuda",
+            "generated_at_commit": "sha",
+            "suite_count": 1,
+            "model_count": 1,
+            "runner_label_count": 1,
+            "parse_failures": {},
+            "runner_labels": {
+                "1-gpu-h100": {"models": ["org/m"], "suites": ["s1", "s2"]}
+            },
+            "unmapped_suites": ["nightly-legacy"],
+            "suites": {
+                "s1": {"nightly": False, "models": ["org/m"], "unresolved_files": []}
+            },
+        }
+        md = lsm.render_markdown(inv)
+        self.assertIn("Per runner label", md)
+        self.assertIn("| `1-gpu-h100` | 2 | org/m |", md)
+        self.assertIn("no runner label: **1**", md)
+        self.assertIn("`nightly-legacy`", md)
+
+
+_FAKE_RUNNER_CONFIGS_YML = """\
+# comment
+_anchors:
+  default_install: &default scripts/ci/cuda/ci_install_dependency.sh
+
+runner_configs:
+  1-gpu:        { install: *default, artifact_version: v4, runs_on: 1-gpu-h100 }
+  deepep-1-gpu: { install: *default, artifact_version: v4, runs_on: 1-gpu-h100 }
+  4-gpu-b200:   { install: *default, artifact_version: v6, runs_on: $b200_runner }
+"""
+
+
+class LoadRunnerLabels(unittest.TestCase):
+    def _load(self, content):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "runner_configs.yml")
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(content)
+            return lsm.load_runner_labels(path)
+
+    def test_parses_flat_inline_maps(self):
+        labels = self._load(_FAKE_RUNNER_CONFIGS_YML)
+        self.assertEqual(
+            labels,
+            {
+                "1-gpu": "1-gpu-h100",
+                "deepep-1-gpu": "1-gpu-h100",
+                "4-gpu-b200": lsm.B200_SENTINEL,
+            },
+        )
+
+    def test_missing_runs_on_is_loud(self):
+        with self.assertRaises(ValueError):
+            self._load("runner_configs:\n  broken: { install: x }\n")
+
+    def test_empty_or_drifted_format_is_loud(self):
+        with self.assertRaises(ValueError):
+            self._load("something_else:\n  a: 1\n")
+
+    def test_real_repo_file(self):
+        """Anchor the stdlib parser to the actual runner_configs.yml: format
+        drift there must fail these tests (which the workflow runs before
+        generating the inventory), not silently empty the label aggregation."""
+        labels = lsm.load_runner_labels(
+            os.path.join(_REPO_ROOT, "scripts", "ci", "runner_configs.yml")
+        )
+        # Two configs sharing a label is the reason the aggregation exists.
+        self.assertEqual(labels["4-gpu-h100"], "4-gpu-h100")
+        self.assertEqual(labels["deepep-4-gpu-h100"], "4-gpu-h100")
+        self.assertEqual(labels["4-gpu-b200"], lsm.B200_SENTINEL)
+        self.assertGreaterEqual(len(labels), 10)
+
+
+class RunnerLabelAggregation(unittest.TestCase):
+    REG = (
+        "import unittest\n"
+        "from sglang.test.ci.ci_register import register_cuda_ci\n"
+        "{calls}\n"
+        'MODEL = "{model}"\n'
+        'if __name__ == "__main__":\n    unittest.main()\n'
+    )
+
+    def _repo(self, tmp):
+        _make_fake_repo(
+            tmp,
+            registered={
+                # Two suites on runner_configs that share the 1-gpu-h100 label.
+                "a/test_a.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-x", '
+                        'runner_config="1-gpu")'
+                    ),
+                    model="meta-llama/Llama-3.1-8B-Instruct",
+                ),
+                "b/test_b.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-y", '
+                        'runner_config="deepep-1-gpu")'
+                    ),
+                    model="Qwen/Qwen3-8B",
+                ),
+                # Sentinel-labeled config.
+                "c/test_c.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-z", '
+                        'runner_config="4-gpu-b200")'
+                    ),
+                    model="google/gemma-3-4b-it",
+                ),
+                # Legacy suite= with no runner_config -> unmapped.
+                "d/test_d.py": self.REG.format(
+                    calls='register_cuda_ci(est_time=1, suite="nightly-legacy")',
+                    model="openai/gpt-oss-20b",
+                ),
+            },
+        )
+        _write(
+            tmp,
+            os.path.join("scripts", "ci", "runner_configs.yml"),
+            _FAKE_RUNNER_CONFIGS_YML,
+        )
+
+    def test_union_per_label_and_unmapped(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            inv = lsm.build_inventory(tmp, "cuda", lsm.load_overrides(None), "sha")
+            self.assertEqual(inv["runner_label_count"], 2)
+            # Shared label carries the UNION of both suites' models.
+            shared = inv["runner_labels"]["1-gpu-h100"]
+            self.assertEqual(
+                shared["models"],
+                ["Qwen/Qwen3-8B", "meta-llama/Llama-3.1-8B-Instruct"],
+            )
+            self.assertEqual(
+                shared["suites"],
+                ["base-x-test-1-gpu", "base-y-test-deepep-1-gpu"],
+            )
+            # Sentinel stays literal without --b200-runner.
+            self.assertIn(lsm.B200_SENTINEL, inv["runner_labels"])
+            # Legacy suite is visible as unmapped, and still fully present
+            # (with its models) in the per-suite section.
+            self.assertEqual(inv["unmapped_suites"], ["nightly-legacy"])
+            self.assertEqual(
+                inv["suites"]["nightly-legacy"]["models"], ["openai/gpt-oss-20b"]
+            )
+            self.assertEqual(inv["suites"]["nightly-legacy"]["runner_config"], None)
+            self.assertEqual(
+                inv["suites"]["base-x-test-1-gpu"]["runner_config"], "1-gpu"
+            )
+
+    def test_b200_runner_substitution(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            inv = lsm.build_inventory(
+                tmp,
+                "cuda",
+                lsm.load_overrides(None),
+                "sha",
+                b200_runner="4-gpu-b200-dyn",
+            )
+            self.assertNotIn(lsm.B200_SENTINEL, inv["runner_labels"])
+            self.assertEqual(
+                inv["runner_labels"]["4-gpu-b200-dyn"]["models"],
+                ["google/gemma-3-4b-it"],
+            )
+
+    def test_suite_labels_override_maps_legacy_suite(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            overrides = lsm.load_overrides(None)
+            # Legacy suite dispatched by two workflows -> two labels; the
+            # sentinel in an override is substituted like a yml-derived one.
+            overrides["suite_labels"] = {
+                "nightly-legacy": ["1-gpu-h100", "$b200_runner"]
+            }
+            inv = lsm.build_inventory(
+                tmp, "cuda", overrides, "sha", b200_runner="b200-dyn"
+            )
+            self.assertEqual(inv["unmapped_suites"], [])
+            self.assertIn(
+                "openai/gpt-oss-20b", inv["runner_labels"]["1-gpu-h100"]["models"]
+            )
+            self.assertIn(
+                "openai/gpt-oss-20b", inv["runner_labels"]["b200-dyn"]["models"]
+            )
+            self.assertIn(
+                "nightly-legacy", inv["runner_labels"]["1-gpu-h100"]["suites"]
+            )
+
+    def test_missing_yml_leaves_all_suites_unmapped(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            os.remove(os.path.join(tmp, "scripts", "ci", "runner_configs.yml"))
+            inv = lsm.build_inventory(tmp, "cuda", lsm.load_overrides(None), "sha")
+            self.assertEqual(inv["runner_labels"], {})
+            self.assertEqual(len(inv["unmapped_suites"]), 4)
+
+
+class ResolveCommit(unittest.TestCase):
+    def test_explicit_arg_wins(self):
+        self.assertEqual(lsm.resolve_commit("abc", "/nonexistent"), "abc")
+
+    def test_env_fallback(self):
+        with mock.patch.dict(os.environ, {"GITHUB_SHA": "deadbeef"}, clear=True):
+            self.assertEqual(lsm.resolve_commit(None, "/nonexistent"), "deadbeef")
+
+    def test_unknown_when_no_git(self):
+        with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(
+            os.environ, {}, clear=True
+        ):
+            self.assertEqual(lsm.resolve_commit(None, tmp), "unknown")
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/ci/update_est_time.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+"""Refresh est_time literals from sglang-ci-stats/model.json.
+
+Usage:
+    python scripts/ci/update_est_time.py [--dry-run] \\
+        [--model-url URL] [--summary-file PATH]
+"""
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+DEFAULT_MODEL_URL = (
+    "https://raw.githubusercontent.com/sgl-project/sglang-ci-stats/main/model.json"
+)
+
+# AMD / NPU live in separate workflows and are not scraped by sglang-ci-stats.
+BACKENDS = ("cuda", "cpu")
+
+# A change is "significant" if |delta| >= this many seconds AND the relative
+# change is at least SIGNIFICANT_REL_DELTA. Dual threshold filters out both
+# tiny absolute drifts on long tests and small-but-noisy relative swings on
+# short tests.
+SIGNIFICANT_ABS_DELTA = 30
+SIGNIFICANT_REL_DELTA = 0.3
+
+
+def fetch_model(url):
+    """Curl model.json. Fail loudly on network or parse errors -- the
+    weekly workflow will surface the failure rather than silently making
+    a no-op PR."""
+    out = subprocess.run(
+        ["curl", "--fail", "--silent", "--show-error", "--max-time", "30", url],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return json.loads(out.stdout)
+
+
+def make_patterns(suite):
+    """Yield regex objects that match `register_{backend}_ci(est_time=N, ...)`
+    for the given suite, covering both registration styles:
+
+      legacy: register_X_ci(est_time=N, suite="<full-suite>")
+      new:    register_X_ci(est_time=N, stage="<stage>", runner_config="<rc>")
+    """
+    stage_rc = None
+    if "-test-" in suite:
+        stage, _, rc = suite.partition("-test-")
+        stage_rc = (stage, rc)
+    for backend in BACKENDS:
+        yield re.compile(
+            rf"(register_{backend}_ci\(est_time=)(\d+)"
+            rf'(,\s*suite="{re.escape(suite)}")'
+        )
+        if stage_rc is not None:
+            stage, rc = stage_rc
+            yield re.compile(
+                rf"(register_{backend}_ci\(est_time=)(\d+)"
+                rf'(,\s*stage="{re.escape(stage)}",\s*runner_config="{re.escape(rc)}")'
+            )
+
+
+def update_files(model, dry_run=False):
+    """Walk `model.est`, apply each p90 to the matching register call.
+
+    Returns list of (relpath, suite, old, new) for every changed entry.
+    """
+    by_file = defaultdict(list)
+    for suite, files in model.get("est", {}).items():
+        for relpath, p90 in files.items():
+            by_file[relpath].append((suite, p90))
+
+    changes = []
+    for relpath, entries in sorted(by_file.items()):
+        filepath = REPO_ROOT / relpath
+        if not filepath.exists():
+            continue
+        content = filepath.read_text()
+        new_content = content
+
+        for suite, p90 in entries:
+            for pattern in make_patterns(suite):
+                match = pattern.search(new_content)
+                if match is None:
+                    continue
+                old_val = int(match.group(2))
+                if old_val != p90:
+                    new_content = pattern.sub(rf"\g<1>{p90}\3", new_content)
+                    changes.append((relpath, suite, old_val, p90))
+                    print(
+                        f"  {relpath}: suite={suite!r} " f"est_time {old_val} -> {p90}",
+                        file=sys.stderr,
+                    )
+                break  # one (file, suite) -> at most one register call
+
+        if new_content != content and not dry_run:
+            filepath.write_text(new_content)
+
+    return changes
+
+
+def is_significant(old, new):
+    delta = abs(new - old)
+    return (
+        delta >= SIGNIFICANT_ABS_DELTA and delta / max(old, 1) >= SIGNIFICANT_REL_DELTA
+    )
+
+
+def write_summary(changes, summary_file):
+    """Write a markdown summary of significant est_time changes."""
+    sig = [c for c in changes if is_significant(c[2], c[3])]
+    sig.sort(key=lambda c: abs(c[3] - c[2]), reverse=True)
+
+    lines = []
+    if sig:
+        lines.append(
+            f"### Significant est_time changes "
+            f"({len(sig)} of {len(changes)} updates)"
+        )
+        lines.append("")
+        lines.append("| File | Suite | Old (s) | New (s) | Δ |")
+        lines.append("| --- | --- | ---: | ---: | ---: |")
+        for relpath, suite, old, new in sig:
+            delta = new - old
+            sign = "+" if delta > 0 else ""
+            pct = round(delta / max(old, 1) * 100)
+            lines.append(
+                f"| `{Path(relpath).name}` | `{suite}` | "
+                f"{old} | {new} | {sign}{delta} ({sign}{pct}%) |"
+            )
+    else:
+        lines.append(
+            f"_{len(changes)} est_time update(s); none exceeded both "
+            f"±{SIGNIFICANT_ABS_DELTA}s and "
+            f"±{int(SIGNIFICANT_REL_DELTA * 100)}% thresholds._"
+        )
+
+    Path(summary_file).write_text("\n".join(lines) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model-url",
+        default=DEFAULT_MODEL_URL,
+        help="URL of model.json from sglang-ci-stats (file:// is OK for testing)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print changes without modifying files",
+    )
+    parser.add_argument(
+        "--summary-file",
+        default=None,
+        help="Write a markdown summary of significant changes to this path",
+    )
+    args = parser.parse_args()
+
+    print(f"Fetching {args.model_url}", file=sys.stderr)
+    model = fetch_model(args.model_url)
+    print(
+        f"  model fit_window_start={model.get('fit_window_start')} "
+        f"fit_window_days={model.get('fit_window_days')} "
+        f"n_runs={model.get('n_runs')} "
+        f"n_suites={len(model.get('est', {}))}",
+        file=sys.stderr,
+    )
+
+    changes = update_files(model, dry_run=args.dry_run)
+
+    n_files = len({c[0] for c in changes})
+    action = "Would update" if args.dry_run else "Updated"
+    print(
+        f"\n{action} {len(changes)} est_time entries across {n_files} files",
+        file=sys.stderr,
+    )
+
+    if args.summary_file:
+        write_summary(changes, args.summary_file)
+        print(f"Wrote summary to {args.summary_file}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/ci_coverage_report.py

@@ -0,0 +1,649 @@
+#!/usr/bin/env python3
+"""
+CI Coverage Report Generator
+
+Collects all CI test registrations from test/registered/ and generates
+a coverage report organized by folder, backend, and suite.
+
+Usage:
+    python scripts/ci/utils/ci_coverage_report.py [--output-format markdown|json]
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+# Add the ci_register module path directly to avoid heavy sglang imports
+sys.path.insert(
+    0,
+    str(
+        Path(__file__).parent.parent.parent.parent / "python" / "sglang" / "test" / "ci"
+    ),
+)
+
+from ci_register import CIRegistry, HWBackend, ut_parse_one_file
+
+# Display order for backend tables / sections. The list is sourced from
+# HWBackend so a newly-added enum member can never be silently dropped from
+# the report -- if the assert below fires, add the new backend name here in
+# the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead
+# (highest test volume historically), then accelerators that have been
+# wired into the registry more recently (XPU, MUSA).
+BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA")
+assert set(BACKEND_DISPLAY_ORDER) == {
+    b.name for b in HWBackend
+}, "BACKEND_DISPLAY_ORDER is out of sync with HWBackend"
+
+# --------------------------------------------------------------------------- #
+# multimodal_gen test coverage
+#
+# multimodal_gen tests live under python/sglang/multimodal_gen/test/ and use
+# their own run_suite.py / partitioning framework, NOT the register_*_ci()
+# registry used under test/registered/. To surface them in the daily
+# overview we synthesize CIRegistry records from file paths + filename
+# tokens, using the rules below. These are heuristics derived from how the
+# multimodal_gen workflows (pr-test-{musa,npu,amd}.yml, pr-test-multimodal-
+# gen.yml) currently invoke each file -- they MAY drift if a workflow
+# stops/starts running a directory.
+# --------------------------------------------------------------------------- #
+MULTIMODAL_GEN_TEST_DIR = "python/sglang/multimodal_gen/test"
+
+# Subdirectory (relative to MULTIMODAL_GEN_TEST_DIR) -> backends those files
+# run on by default. Empty string is the top-level. Files whose tokenized
+# filename contains an explicit backend marker (see below) override this.
+_MM_GEN_SUBDIR_BACKENDS = {
+    # Top-level helper-style tests (e.g. test_consistency_metrics.py).
+    "": ("CUDA",),
+    # server/ top-level: pr-test-multimodal-gen.yml drives CUDA, pr-test-amd
+    # mirrors the same suite on AMD runners.
+    "server": ("CUDA", "AMD"),
+    "server/musa": ("MUSA",),
+    "server/ascend": ("NPU",),
+    "layers": ("CUDA",),
+    # unit/ are portable CPU-style unit tests. pr-test-amd now runs the `unit`
+    # suite on ROCm (multimodal-gen-unit-test-amd, both 7.0.0 and 7.2.0), so
+    # they are AMD-covered too, not CUDA-only.
+    "unit": ("CUDA", "AMD"),
+    "cli": ("CUDA",),
+    "manual": ("CUDA",),
+    # Standalone server/CLI single-file tests (restructured out of server/).
+    # Run on CUDA CI; AMD parity for these standalone files is TBD, so
+    # CUDA-only for now (previously matched no rule and were dropped entirely).
+    "single_test_file": ("CUDA",),
+    "single_test_file/component_accuracy": ("CUDA",),
+    # Nested unit suites run only on the CUDA lane today (they are not part of
+    # the AMD `unit` suite that multimodal-gen-unit-test-amd executes).
+    "unit/realtime": ("CUDA",),
+    "unit/sana_wm": ("CUDA",),
+    "unit/progressive_resolution": ("CUDA",),
+    # musa-named unit layer kernels.
+    "unit/musa/layers": ("MUSA",),
+}
+
+# Filenames that match `test_*.py` by convention but contain no real tests
+# (utility / fixture modules). Skipped before classification.
+_MM_GEN_HELPER_FILENAMES = frozenset({"test_utils.py"})
+
+# Filename token -> backend override. Tokenization is `stem.split("_")`, so a
+# file like `test_server_1_gpu_musa.py` yields tokens
+# {test, server, 1, gpu, musa} and matches `musa`. This correctly catches
+# both `test_musa_*.py` (musa-named kernels) and `test_*_musa*.py` (musa
+# variants of generic server tests). `nightly` is detected the same way and
+# flips the nightly flag without changing the backend.
+_MM_GEN_FILENAME_BACKEND_TOKENS = {
+    "musa": ("MUSA",),
+    "npu": ("NPU",),
+}
+
+
+def collect_all_tests(registered_dir: str) -> list[CIRegistry]:
+    """Collect all CI registrations from registered directory."""
+    files = glob.glob(f"{registered_dir}/**/*.py", recursive=True)
+    all_tests = []
+
+    for file in sorted(files):
+        try:
+            registries, _ = ut_parse_one_file(file)
+            all_tests.extend(registries)
+        except Exception as e:
+            print(f"Warning: Failed to parse {file}: {e}", file=sys.stderr)
+
+    return all_tests
+
+
+def collect_multimodal_gen_tests(
+    mm_gen_dir: str = MULTIMODAL_GEN_TEST_DIR,
+) -> list[CIRegistry]:
+    """Synthesize CIRegistry records for multimodal_gen tests.
+
+    multimodal_gen doesn't use register_*_ci(); see the module-level comment
+    above MULTIMODAL_GEN_TEST_DIR for the rules. Returns one record per
+    (file, backend) pair, matching the convention used for registered tests
+    that target multiple backends.
+    """
+    mm_gen_path = Path(mm_gen_dir)
+    if not mm_gen_path.is_dir():
+        return []
+
+    # Discover test files: anything matching test_*.py anywhere under the
+    # tree. The csrc/ and apps/ subtrees aren't part of the test/ directory
+    # so we don't need to exclude them.
+    test_files = sorted(mm_gen_path.glob("**/test_*.py"))
+    records: list[CIRegistry] = []
+
+    for file in test_files:
+        rel = file.relative_to(mm_gen_path)
+        subdir = "/".join(rel.parts[:-1])  # "" for top-level
+        filename_only = rel.parts[-1]
+
+        if filename_only in _MM_GEN_HELPER_FILENAMES:
+            continue  # test_utils.py and similar -- helper modules, not tests
+
+        # Tokenize stem to look for explicit backend / nightly markers.
+        stem_tokens = set(filename_only[:-3].split("_"))
+        nightly = "nightly" in stem_tokens
+
+        backends: tuple[str, ...] = ()
+        for token, override in _MM_GEN_FILENAME_BACKEND_TOKENS.items():
+            if token in stem_tokens:
+                backends = override
+                break
+        if not backends:
+            backends = _MM_GEN_SUBDIR_BACKENDS.get(subdir, ())
+
+        if not backends:
+            print(
+                f"Warning: multimodal_gen file {file} matches no backend "
+                f"rule (subdir={subdir!r}, tokens={sorted(stem_tokens)}); "
+                f"add a rule to _MM_GEN_SUBDIR_BACKENDS.",
+                file=sys.stderr,
+            )
+            continue
+
+        for backend_name in backends:
+            records.append(
+                CIRegistry(
+                    backend=HWBackend[backend_name],
+                    filename=str(file),
+                    # mm_gen does its own per-case partitioning -- file-level
+                    # estimates aren't available. Surfaced as 0 in the
+                    # by-suite section, which is correct (we don't know).
+                    est_time=0.0,
+                    suite=f"mm-gen-{backend_name.lower()}",
+                    nightly=nightly,
+                    disabled=None,
+                )
+            )
+
+    return records
+
+
+def get_folder_name(filename: str) -> str:
+    """Extract folder name from test filename.
+
+    Registered tests use test/registered/<folder>/test_*.py and map to
+    <folder>. multimodal_gen tests live outside that tree and get a virtual
+    `mm_gen/<subdir>` folder so they show up as their own rows in the
+    Folder Summary table.
+    """
+    parts = Path(filename).parts
+    if "multimodal_gen" in parts:
+        try:
+            mg_idx = parts.index("multimodal_gen")
+            test_idx = parts.index("test", mg_idx)
+            sub_parts = parts[test_idx + 1 : -1]  # strip 'test' anchor + file
+            if sub_parts:
+                return "mm_gen/" + "/".join(sub_parts)
+            return "mm_gen"
+        except ValueError:
+            pass  # fall through to default
+    if "registered" in parts:
+        idx = parts.index("registered")
+        if idx + 1 < len(parts) - 1:  # Has subfolder
+            return parts[idx + 1]
+    return "root"
+
+
+def get_test_basename(filename: str) -> str:
+    """Extract just the test file name from the path."""
+    return Path(filename).name
+
+
+def organize_test_data(tests: list[CIRegistry]) -> dict:
+    """Organize tests into various groupings."""
+    by_backend = defaultdict(list)
+    by_folder = defaultdict(list)
+    disabled_tests = []
+
+    for t in tests:
+        by_backend[t.backend.name].append(t)
+        by_folder[get_folder_name(t.filename)].append(t)
+        if t.disabled:
+            disabled_tests.append(t)
+
+    # Count unique test files (a file may be registered for multiple backends)
+    unique_files = set(t.filename for t in tests)
+    unique_enabled_files = set(t.filename for t in tests if not t.disabled)
+    unique_disabled_files = set(t.filename for t in tests if t.disabled)
+
+    return {
+        "total": len(tests),
+        "total_unique_files": len(unique_files),
+        "enabled": len(tests) - len(disabled_tests),
+        "enabled_unique_files": len(unique_enabled_files),
+        "disabled_count": len(disabled_tests),
+        "disabled_unique_files": len(unique_disabled_files),
+        "by_backend": by_backend,
+        "by_folder": by_folder,
+        "disabled_tests": disabled_tests,
+    }
+
+
+def generate_summary_section(data: dict) -> str:
+    """Generate the summary/overview section."""
+    lines = []
+    lines.append("# CI Coverage Overview\n")
+    lines.append(
+        f"**Unique Test Files:** {data['total_unique_files']} ({data['enabled_unique_files']} enabled, {data['disabled_unique_files']} disabled)\n"
+    )
+    lines.append(
+        f"**Total Registrations:** {data['total']} ({data['enabled']} enabled, {data['disabled_count']} disabled)\n"
+    )
+    lines.append(
+        "*Note: A test file may be registered for multiple backends (e.g., CUDA + AMD), so total registrations > unique files.*\n"
+    )
+
+    by_backend = data["by_backend"]
+    by_folder = data["by_folder"]
+    disabled_tests = data["disabled_tests"]
+
+    # Backend summary (collapsible)
+    lines.append("<details>")
+    lines.append("<summary><h2>Backend Summary</h2></summary>\n")
+    lines.append("| Backend | Total | Enabled | Disabled | Per-Commit | Nightly |")
+    lines.append("|---------|-------|---------|----------|------------|---------|")
+
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+        b_total = len(backend_tests)
+        b_disabled = sum(1 for t in backend_tests if t.disabled)
+        b_enabled = b_total - b_disabled
+        b_per_commit = sum(1 for t in backend_tests if not t.nightly and not t.disabled)
+        b_nightly = sum(1 for t in backend_tests if t.nightly and not t.disabled)
+        lines.append(
+            f"| {backend} | {b_total} | {b_enabled} | {b_disabled} | {b_per_commit} | {b_nightly} |"
+        )
+
+    lines.append("\n</details>\n")
+
+    # Folder summary (collapsible). Only show columns for backends that
+    # have at least one registered test across the whole report -- otherwise
+    # adding scaffolding for an unused backend would widen every row with a
+    # column of zeros.
+    active_backends = [b for b in BACKEND_DISPLAY_ORDER if by_backend.get(b)]
+    lines.append("<details>")
+    lines.append("<summary><h2>Folder Summary</h2></summary>\n")
+    header_cells = ["Folder", *active_backends, "Total"]
+    lines.append("| " + " | ".join(header_cells) + " |")
+    lines.append("|" + "|".join(["-" * max(len(c), 3) for c in header_cells]) + "|")
+
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        backend_counts = {b.name: 0 for b in HWBackend}
+        for t in folder_tests:
+            backend_counts[t.backend.name] += 1
+        row = [folder] + [str(backend_counts[b]) for b in active_backends]
+        row.append(str(len(folder_tests)))
+        lines.append("| " + " | ".join(row) + " |")
+
+    lines.append("\n</details>\n")
+
+    # Disabled tests section (collapsible)
+    if disabled_tests:
+        lines.append("<details>")
+        lines.append("<summary><h2>Disabled Tests</h2></summary>\n")
+        lines.append("| File | Backend | Suite | Reason |")
+        lines.append("|------|---------|-------|--------|")
+        for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+            test_name = get_test_basename(t.filename)
+            reason = t.disabled[:50] + "..." if len(t.disabled) > 50 else t.disabled
+            lines.append(
+                f"| `{test_name}` | {t.backend.name} | {t.effective_suite} | {reason} |"
+            )
+        lines.append("\n</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_by_folder_section(data: dict) -> str:
+    """Generate the 'All Tests by Folder' section."""
+    lines = []
+    by_folder = data["by_folder"]
+
+    lines.append("# All Tests by Folder\n")
+
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        lines.append("<details>")
+        lines.append(
+            f"<summary><h2>{folder}/ ({len(folder_tests)} tests)</h2></summary>\n"
+        )
+
+        # Group by backend within folder
+        folder_by_backend = defaultdict(list)
+        for t in folder_tests:
+            folder_by_backend[t.backend.name].append(t)
+
+        for backend in BACKEND_DISPLAY_ORDER:
+            backend_tests = folder_by_backend.get(backend, [])
+            if not backend_tests:
+                continue
+
+            lines.append(f"### {backend} ({len(backend_tests)} tests)\n")
+            lines.append("| Test File | Suite | Est. Time | Status |")
+            lines.append("|-----------|-------|-----------|--------|")
+
+            for t in sorted(backend_tests, key=lambda x: x.filename):
+                test_name = get_test_basename(t.filename)
+                status = (
+                    "Disabled"
+                    if t.disabled
+                    else ("Nightly" if t.nightly else "Per-Commit")
+                )
+                lines.append(
+                    f"| `{test_name}` | {t.effective_suite} | {t.est_time:.0f}s | {status} |"
+                )
+
+            lines.append("")
+
+        lines.append("</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_by_suite_section(data: dict) -> str:
+    """Generate the 'All Tests by Test Suite' section."""
+    lines = []
+    by_backend = data["by_backend"]
+
+    lines.append("# All Tests by Test Suite\n")
+
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+
+        b_total = len(backend_tests)
+        b_disabled = sum(1 for t in backend_tests if t.disabled)
+        b_enabled = b_total - b_disabled
+
+        lines.append("<details>")
+        lines.append(
+            f"<summary><h2>{backend} Backend ({b_enabled} enabled, {b_disabled} disabled)</h2></summary>\n"
+        )
+
+        # Group by suite within backend
+        backend_suites = defaultdict(list)
+        for t in backend_tests:
+            backend_suites[t.effective_suite].append(t)
+
+        for suite in sorted(backend_suites.keys()):
+            suite_tests = backend_suites[suite]
+            s_enabled = sum(1 for t in suite_tests if not t.disabled)
+            s_disabled = sum(1 for t in suite_tests if t.disabled)
+            s_est_time = sum(t.est_time for t in suite_tests if not t.disabled)
+            is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+            suite_type = "Nightly" if is_nightly else "Per-Commit"
+            lines.append("<details>")
+            lines.append(
+                f"<summary><h3>{suite} ({s_enabled} enabled, {s_disabled} disabled) - {suite_type}</h3></summary>\n"
+            )
+            lines.append(f"*Estimated total time: {s_est_time:.0f}s*\n")
+
+            lines.append("| Test File | Folder | Est. Time | Status |")
+            lines.append("|-----------|--------|-----------|--------|")
+
+            for t in sorted(suite_tests, key=lambda x: x.filename):
+                test_name = get_test_basename(t.filename)
+                folder = get_folder_name(t.filename)
+                if t.disabled:
+                    status = (
+                        f"Disabled: {t.disabled[:30]}..."
+                        if len(t.disabled) > 30
+                        else f"Disabled: {t.disabled}"
+                    )
+                else:
+                    status = "Nightly" if t.nightly else "Per-Commit"
+                lines.append(
+                    f"| `{test_name}` | {folder} | {t.est_time:.0f}s | {status} |"
+                )
+
+            lines.append("\n</details>\n")
+
+        lines.append("</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_markdown_report(tests: list[CIRegistry], section: str = "all") -> str:
+    """Generate markdown report for GitHub step summary."""
+    data = organize_test_data(tests)
+
+    if section == "summary":
+        return generate_summary_section(data)
+    elif section == "by-folder":
+        return generate_by_folder_section(data)
+    elif section == "by-suite":
+        return generate_by_suite_section(data)
+    else:  # "all"
+        parts = [
+            generate_summary_section(data),
+            "---",
+            generate_by_folder_section(data),
+            "---",
+            generate_by_suite_section(data),
+        ]
+        return "\n".join(parts)
+
+
+def generate_json_report(tests: list[CIRegistry]) -> str:
+    """Generate JSON report with detailed test listings."""
+    by_backend = defaultdict(list)
+    by_folder = defaultdict(list)
+
+    for t in tests:
+        by_backend[t.backend.name].append(t)
+        by_folder[get_folder_name(t.filename)].append(t)
+
+    disabled_tests = [t for t in tests if t.disabled]
+
+    # Build structured data
+    data = {
+        "summary": {
+            "total": len(tests),
+            "enabled": len(tests) - len(disabled_tests),
+            "disabled": len(disabled_tests),
+        },
+        "tests_by_folder": {},
+        "tests_by_suite": {},
+        "backend_summary": {},
+        "folder_summary": {},
+        "disabled_tests": [],
+    }
+
+    # Section 1: Tests by Folder
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        folder_by_backend = defaultdict(list)
+        for t in folder_tests:
+            folder_by_backend[t.backend.name].append(t)
+
+        data["tests_by_folder"][folder] = {
+            "total": len(folder_tests),
+            "backends": {},
+        }
+
+        for backend in BACKEND_DISPLAY_ORDER:
+            backend_tests = folder_by_backend.get(backend, [])
+            if backend_tests:
+                data["tests_by_folder"][folder]["backends"][backend] = [
+                    {
+                        "filename": get_test_basename(t.filename),
+                        "suite": t.effective_suite,
+                        "est_time": t.est_time,
+                        "status": (
+                            "disabled"
+                            if t.disabled
+                            else ("nightly" if t.nightly else "per-commit")
+                        ),
+                    }
+                    for t in sorted(backend_tests, key=lambda x: x.filename)
+                ]
+
+    # Section 2: Tests by Suite (Backend -> Suite)
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+
+        backend_suites = defaultdict(list)
+        for t in backend_tests:
+            backend_suites[t.effective_suite].append(t)
+
+        data["tests_by_suite"][backend] = {
+            "total": len(backend_tests),
+            "enabled": sum(1 for t in backend_tests if not t.disabled),
+            "disabled": sum(1 for t in backend_tests if t.disabled),
+            "suites": {},
+        }
+
+        for suite in sorted(backend_suites.keys()):
+            suite_tests = backend_suites[suite]
+            is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+            data["tests_by_suite"][backend]["suites"][suite] = {
+                "total": len(suite_tests),
+                "enabled": sum(1 for t in suite_tests if not t.disabled),
+                "disabled": sum(1 for t in suite_tests if t.disabled),
+                "est_time": sum(t.est_time for t in suite_tests if not t.disabled),
+                "type": "nightly" if is_nightly else "per-commit",
+                "tests": [
+                    {
+                        "filename": get_test_basename(t.filename),
+                        "folder": get_folder_name(t.filename),
+                        "est_time": t.est_time,
+                        "status": (
+                            "disabled"
+                            if t.disabled
+                            else ("nightly" if t.nightly else "per-commit")
+                        ),
+                        "disabled_reason": t.disabled if t.disabled else None,
+                    }
+                    for t in sorted(suite_tests, key=lambda x: x.filename)
+                ],
+            }
+
+    # Backend summary
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if backend_tests:
+            data["backend_summary"][backend] = {
+                "total": len(backend_tests),
+                "enabled": sum(1 for t in backend_tests if not t.disabled),
+                "disabled": sum(1 for t in backend_tests if t.disabled),
+                "per_commit": sum(
+                    1 for t in backend_tests if not t.nightly and not t.disabled
+                ),
+                "nightly": sum(
+                    1 for t in backend_tests if t.nightly and not t.disabled
+                ),
+            }
+
+    # Folder summary -- one count per backend in HWBackend, in display
+    # order, so every registered backend shows up regardless of whether
+    # this folder has tests for it.
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        backend_counts = {b: 0 for b in BACKEND_DISPLAY_ORDER}
+        for t in folder_tests:
+            backend_counts[t.backend.name] += 1
+        data["folder_summary"][folder] = {
+            **backend_counts,
+            "total": len(folder_tests),
+        }
+
+    # Disabled tests
+    for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+        data["disabled_tests"].append(
+            {
+                "filename": get_test_basename(t.filename),
+                "backend": t.backend.name,
+                "suite": t.effective_suite,
+                "reason": t.disabled,
+            }
+        )
+
+    return json.dumps(data, indent=2)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate CI coverage report")
+    parser.add_argument(
+        "--output-format",
+        choices=["markdown", "json"],
+        default="markdown",
+        help="Output format (default: markdown)",
+    )
+    parser.add_argument(
+        "--section",
+        choices=["all", "summary", "by-folder", "by-suite"],
+        default="all",
+        help="Which section to output (default: all). Only applies to markdown format.",
+    )
+    parser.add_argument(
+        "--registered-dir",
+        default="test/registered",
+        help="Path to registered test directory",
+    )
+    parser.add_argument(
+        "--multimodal-gen-dir",
+        default=MULTIMODAL_GEN_TEST_DIR,
+        help=(
+            "Path to multimodal_gen test directory. Pass empty string to "
+            "skip multimodal_gen tests entirely."
+        ),
+    )
+    args = parser.parse_args()
+
+    # Change to repo root if needed
+    script_dir = Path(__file__).parent.parent
+    repo_root = script_dir.parent.parent
+    os.chdir(repo_root)
+
+    tests = collect_all_tests(args.registered_dir)
+    if args.multimodal_gen_dir:
+        tests.extend(collect_multimodal_gen_tests(args.multimodal_gen_dir))
+
+    if args.output_format == "markdown":
+        report = generate_markdown_report(tests, section=args.section)
+    else:
+        report = generate_json_report(tests)
+
+    print(report)
+
+    # Write to GITHUB_STEP_SUMMARY if available
+    summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file and args.output_format == "markdown":
+        with open(summary_file, "a") as f:
+            f.write(report)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/cleanup_hf_cache.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+Clean up stale HuggingFace cache artifacts from previous failed downloads.
+
+This script removes incomplete marker files, temporary files, and lock files
+from the HuggingFace cache directory. These artifacts can accumulate from
+interrupted or failed downloads and may interfere with future downloads.
+"""
+
+import os
+import sys
+from pathlib import Path
+from typing import List
+
+try:
+    from huggingface_hub import constants
+
+    HF_HUB_AVAILABLE = True
+except ImportError:
+    print("Warning: huggingface_hub not available")
+    HF_HUB_AVAILABLE = False
+
+
+def get_hf_cache_dir() -> str:
+    """Get the HuggingFace cache directory."""
+    if HF_HUB_AVAILABLE:
+        return constants.HF_HUB_CACHE
+
+    # Fallback to environment variable or default
+    hf_home = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface"))
+    return os.path.join(hf_home, "hub")
+
+
+def find_stale_artifacts(cache_dir: str) -> List[Path]:
+    """
+    Find stale artifact files in the HuggingFace cache.
+
+    Args:
+        cache_dir: HuggingFace cache directory
+
+    Returns:
+        List of paths to stale artifact files
+    """
+    cache_path = Path(cache_dir)
+
+    if not cache_path.exists():
+        return []
+
+    # Patterns for stale files to clean up
+    patterns = [
+        "**/*.incomplete",  # Incomplete download markers
+        "**/*.tmp",  # Temporary files
+        "**/*.lock",  # Lock files from interrupted downloads
+    ]
+
+    stale_files = []
+    for pattern in patterns:
+        stale_files.extend(cache_path.glob(pattern))
+
+    return stale_files
+
+
+def cleanup_artifacts(artifacts: List[Path]) -> tuple[int, int]:
+    """
+    Remove stale artifact files.
+
+    Args:
+        artifacts: List of file paths to remove
+
+    Returns:
+        Tuple of (successful_removals, failed_removals)
+    """
+    successful = 0
+    failed = 0
+
+    for file_path in artifacts:
+        try:
+            file_path.unlink()
+            print(f"  Removed: {file_path}")
+            successful += 1
+        except Exception as e:
+            print(f"  Warning: Could not remove {file_path}: {e}")
+            failed += 1
+
+    return successful, failed
+
+
+def main() -> int:
+    """
+    Main cleanup logic.
+
+    Returns:
+        Always returns 0 (cleanup is best-effort and should not fail CI)
+    """
+    print("=" * 70)
+    print("HuggingFace Cache Cleanup")
+    print("=" * 70)
+
+    # Get cache directory
+    cache_dir = get_hf_cache_dir()
+    print(f"Cache directory: {cache_dir}")
+
+    if not os.path.exists(cache_dir):
+        print("Cache directory does not exist - nothing to clean")
+        return 0
+
+    print("-" * 70)
+
+    # Find stale artifacts
+    print("Scanning for stale artifacts...")
+    stale_artifacts = find_stale_artifacts(cache_dir)
+
+    if not stale_artifacts:
+        print("✓ No stale cache artifacts found")
+        return 0
+
+    # Clean up artifacts
+    print(f"Found {len(stale_artifacts)} stale artifact(s) to remove:")
+    successful, failed = cleanup_artifacts(stale_artifacts)
+
+    print("-" * 70)
+
+    # Summary
+    if failed > 0:
+        print(f"⚠ Cleaned up {successful} file(s), {failed} removal(s) failed")
+    else:
+        print(f"✓ Successfully cleaned up {successful} stale file(s)")
+
+    # Always return 0 - cleanup failures should not fail CI
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        exit_code = main()
+        sys.exit(exit_code)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(0)
+    except Exception as e:
+        print(f"ERROR: Unexpected error during cleanup: {e}")
+        import traceback
+
+        traceback.print_exc()
+        # Still return 0 - cleanup failures should not fail CI
+        sys.exit(0)

scripts/ci/utils/compute_partitions.py

@@ -0,0 +1,267 @@
+"""Sum est_time per per-commit suite and emit one $GITHUB_OUTPUT line
+keyed by suite name. Consumed by pr-test.yml stage jobs as
+`fromJson(needs.check-changes.outputs.partitions)['<suite>']`.
+
+    partitions={"base-b-test-1-gpu-small": {"size": 8, "arr": [0,...,7], "max_parallel": 2}, ...}
+"""
+
+import argparse
+import glob
+import importlib.util
+import json
+import math
+import os
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+import yaml  # PyYAML; preinstalled on ubuntu-latest GHA runners.
+
+REPO_ROOT = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+
+# Load ci_register.py directly: `import sglang.test...` pulls torch/orjson via
+# sglang.__init__ but check-changes runs on bare ubuntu-latest. ci_register
+# itself is stdlib-only (AST).
+_CI_REGISTER_PATH = os.path.join(
+    REPO_ROOT, "python", "sglang", "test", "ci", "ci_register.py"
+)
+_spec = importlib.util.spec_from_file_location("ci_register", _CI_REGISTER_PATH)
+_ci_register = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_ci_register)
+collect_tests = _ci_register.collect_tests
+HWBackend = _ci_register.HWBackend
+
+# pr-test-amd.yml / pr-test-npu.yml have their own dispatch.
+_TARGET_BACKENDS = {HWBackend.CUDA, HWBackend.CPU}
+
+# base-a is the critical-path entry gate; pin its fanout to sanity-coverage
+# defaults instead of est_time. max_parallel = size (no throttle).
+_BASE_A_OVERRIDES = {
+    "base-a-test-cpu": 8,
+    "base-a-test-1-gpu-small": 1,
+}
+
+_REUSABLE_STAGE_USES = "./.github/workflows/_pr-test-stage.yml"
+
+
+def load_run_timeouts(pr_test_yml_path: str) -> dict:
+    """Map `self_name -> run_timeout_minutes` from one pr-test*.yml. The input
+    is required in `_pr-test-stage.yml` -- KeyError surfaces missing.
+    Inline base-a-test-cpu is skipped (uses `_BASE_A_OVERRIDES`)."""
+    with open(pr_test_yml_path) as f:
+        wf = yaml.safe_load(f)
+    timeouts = {}
+    for job_id, job in (wf.get("jobs") or {}).items():
+        if not isinstance(job, dict) or job.get("uses") != _REUSABLE_STAGE_USES:
+            continue
+        with_ = job.get("with") or {}
+        suite = with_.get("self_name", job_id)
+        timeouts[suite] = int(with_["run_timeout_minutes"])
+    if not timeouts:
+        raise RuntimeError(
+            f"load_run_timeouts: no jobs matched uses={_REUSABLE_STAGE_USES!r} "
+            f"in {pr_test_yml_path}. The reusable workflow path likely "
+            "changed -- update _REUSABLE_STAGE_USES."
+        )
+    return timeouts
+
+
+def per_shard_target_seconds(suite: str, run_timeouts: dict) -> float:
+    """Per-shard wall budget = 0.75 * stage timeout. 0.75 is the inverse
+    of LPT's 4/3 worst-case approximation ratio, so the most imbalanced
+    LPT shard fills exactly the timeout."""
+    return 0.75 * run_timeouts[suite] * 60
+
+
+def discover_files(repo_root: str) -> list[str]:
+    test_dir = os.path.join(repo_root, "test")
+    files = [
+        f
+        for f in glob.glob(
+            os.path.join(test_dir, "registered", "**", "*.py"), recursive=True
+        )
+        if not f.endswith("/conftest.py") and not f.endswith("/__init__.py")
+    ]
+    jit_kernel_dir = os.path.join(repo_root, "python", "sglang", "jit_kernel")
+    files += glob.glob(
+        os.path.join(jit_kernel_dir, "tests", "**", "test_*.py"), recursive=True
+    )
+    files += glob.glob(
+        os.path.join(jit_kernel_dir, "benchmark", "**", "bench_*.py"), recursive=True
+    )
+    return files
+
+
+def load_partition_model(path):
+    """Read sglang-ci-stats' model.json; None on missing/unparsable.
+    Cross-repo schema -- guard against non-dict top-level."""
+    if not path or not os.path.exists(path):
+        return None
+    try:
+        with open(path) as f:
+            data = json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def compute_max_parallel(size: int) -> int:
+    return max(size // 3, 1)
+
+
+def compute_partitions(
+    tests, repo_root, run_timeouts, partition_model=None, full_parallel=False
+):
+    """Group per-commit tests by suite and emit partition metadata.
+
+    `run_timeouts`: `suite -> minutes` from `load_run_timeouts`.
+    `partition_model`: optional sglang-ci-stats `model.json`; per-file
+    `est` and per-suite `(coeff, bias)` each fall back independently to
+    in-source `est_time` / `(1.0, 0.0)`.
+    `full_parallel=True` lifts the matrix-fanout throttle.
+    """
+    # Allowlist: stages pr-test.yml dispatches. Stress / weekly /
+    # nightly-* live in test/registered/ but pr-test doesn't run them.
+    dispatched_suites = set(run_timeouts) | set(_BASE_A_OVERRIDES)
+    suite_tests = defaultdict(list)
+    for t in tests:
+        if t.backend not in _TARGET_BACKENDS:
+            continue
+        if t.nightly or t.disabled is not None:
+            continue
+        if t.effective_suite not in dispatched_suites:
+            continue
+        suite_tests[t.effective_suite].append(t)
+
+    est_table = (partition_model or {}).get("est", {})
+    fit_table = (partition_model or {}).get("fit", {})
+
+    result = {}
+    for suite, group in suite_tests.items():
+        live_est = est_table.get(suite, {})
+        total = 0.0
+        for t in group:
+            relpath = os.path.relpath(t.filename, repo_root)
+            total += live_est.get(relpath, t.est_time)
+
+        fit = fit_table.get(suite) or {}
+        coeff = fit.get("coeff", 1.0)
+        bias = fit.get("bias", 0.0)
+
+        # Each shard pays `bias` once, so size >= coeff*total / (target-bias).
+        if suite in _BASE_A_OVERRIDES:
+            size = _BASE_A_OVERRIDES[suite]
+            max_parallel = size
+        else:
+            target = per_shard_target_seconds(suite, run_timeouts)
+            budget = target - bias
+            if budget <= 0:
+                raise RuntimeError(
+                    f"Suite {suite!r}: fit bias={bias}s >= target={target}s. "
+                    "Investigate the fit or raise the stage's run_timeout_minutes."
+                )
+            ideal_size = math.ceil(coeff * total / budget)
+            # ideal_size > len(group) -> slowest single file alone exceeds
+            # the per-shard budget; surface via raise instead of empty shard.
+            if ideal_size > len(group):
+                raise RuntimeError(
+                    f"Suite {suite!r}: needs {ideal_size} shards but has only "
+                    f"{len(group)} test file(s). target={target:.0f}s, "
+                    f"coeff={coeff}, bias={bias}s, total_est={total:.0f}s."
+                )
+            size = max(1, ideal_size)
+            max_parallel = size if full_parallel else compute_max_parallel(size)
+        result[suite] = {
+            "size": size,
+            "arr": list(range(size)),
+            "max_parallel": max_parallel,
+        }
+    return result
+
+
+def format_fit_window(model: dict) -> str:
+    """Render the model's fit window as `[start, end)` for the step summary.
+
+    Surfacing the whole span keeps the lower-bound date from reading as a
+    staleness marker (it trails today by fit_window_days)."""
+    start = model.get("fit_window_start")
+    days = model.get("fit_window_days")
+    if not start or not isinstance(days, int):
+        return f"fit_window_start={start}"
+    end = (datetime.strptime(start[:10], "%Y-%m-%d") + timedelta(days=days)).strftime(
+        "%Y-%m-%d"
+    )
+    return f"fit over [{start[:10]}, {end}) ({days}d window)"
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo-root", default=REPO_ROOT)
+    parser.add_argument(
+        "--output-format",
+        choices=("gha", "json"),
+        default="gha",
+        help="`gha` emits `partitions=<json>` for $GITHUB_OUTPUT; `json` is raw",
+    )
+    parser.add_argument(
+        "--full-parallel",
+        choices=("true", "false"),
+        default="false",
+        help="Lift the max_parallel throttle (set by schedule / `high priority`)",
+    )
+    parser.add_argument(
+        "--partition-model-file",
+        default=None,
+        help="Path to sglang-ci-stats model.json (omit/missing -> static fallback)",
+    )
+    parser.add_argument(
+        "--pr-test-yml",
+        default=os.path.join(REPO_ROOT, ".github", "workflows", "pr-test.yml"),
+        help="Path to pr-test*.yml; per-stage `run_timeout_minutes` is read from here.",
+    )
+    args = parser.parse_args()
+
+    files = discover_files(args.repo_root)
+    # Warn-not-fail on unregistered files: run_suite.py catches this at
+    # test-execution time with sanity_check=True; dispatch should keep going.
+    all_tests = collect_tests(files, sanity_check=False)
+    partition_model = load_partition_model(args.partition_model_file)
+    run_timeouts = load_run_timeouts(args.pr_test_yml)
+
+    result = compute_partitions(
+        all_tests,
+        repo_root=args.repo_root,
+        run_timeouts=run_timeouts,
+        partition_model=partition_model,
+        full_parallel=(args.full_parallel == "true"),
+    )
+    payload = json.dumps(result, separators=(",", ":"), sort_keys=True)
+    if args.output_format == "gha":
+        print(f"partitions={payload}")
+    else:
+        print(payload)
+
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_path:
+        with open(summary_path, "a") as f:
+            f.write("## Partitions\n\n")
+            if partition_model is None:
+                src_note = "no live model -- static est_time + (coeff=1, bias=0)"
+            else:
+                src_note = (
+                    f"live model: {format_fit_window(partition_model)}, "
+                    f"`n_runs={partition_model.get('n_runs')}`"
+                )
+            f.write(
+                f"`full_parallel={args.full_parallel}` "
+                f"(`size//3` throttle is lifted when true); {src_note}\n\n"
+            )
+            f.write("| Suite | size | max_parallel |\n")
+            f.write("|---|---:|---:|\n")
+            for suite, info in sorted(result.items()):
+                f.write(f"| `{suite}` | {info['size']} | {info['max_parallel']} |\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/comparison_configs.json

@@ -0,0 +1,191 @@
+{
+    "_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
+    "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
+    "cases": [
+        {
+            "id": "flux1_dev_t2i_1024",
+            "model": "black-forest-labs/FLUX.1-dev",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --dit-layerwise-offload false --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "flux2_dev_t2i_1024",
+            "model": "black-forest-labs/FLUX.2-dev",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --dit-layerwise-offload false --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "qwen_image_2512_t2i_1024",
+            "model": "Qwen/Qwen-Image-2512",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "qwen_image_edit_2511",
+            "model": "Qwen/Qwen-Image-Edit-2511",
+            "task": "image-edit",
+            "prompt": "Make the cat wear a red hat",
+            "reference_image": true,
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "zimage_turbo_t2i_1024",
+            "model": "Tongyi-MAI/Z-Image-Turbo",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "wan22_t2v_a14b_720p",
+            "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 4,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "wan22_ti2v_5b_720p",
+            "model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+            "task": "text-image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "ltx2.3_twostage_ti2v_2gpus",
+            "model": "Lightricks/LTX-2.3",
+            "task": "text-image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 768,
+            "height": 512,
+            "num_frames": 121,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --pipeline-class-name LTX2TwoStagePipeline --cfg-parallel-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "ideogram4_fp8_t2i_2gpu",
+            "model": "ideogram-ai/ideogram-4-fp8",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2 --attention-backend fa",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "cosmos3_super_t2v_2gpu",
+            "model": "nvidia/Cosmos3-Super",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {"SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1"}
+                }
+            }
+        },
+        {
+            "id": "wan22_i2v_a14b_720p",
+            "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+            "task": "image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 4,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory",
+                    "extra_env": {}
+                }
+            }
+        }
+    ]
+}

scripts/ci/utils/diffusion/compute_diffusion_partitions.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""
+Compute dynamic partitions for diffusion CI tests.
+
+This script runs on lightweight CI runners without sglang dependencies and uses
+AST parsing to extract parametrized cases plus standalone files from source.
+"""
+
+import argparse
+import importlib.util
+import json
+import math
+import os
+import sys
+from pathlib import Path
+
+from diffusion_case_parser import (
+    BASELINE_REL_PATH,
+    RUN_SUITE_REL_PATH,
+    DiffusionSuiteInfo,
+    collect_diffusion_suites,
+    resolve_case_config_path,
+)
+
+
+def _load_partitioning_helpers():
+    repo_root = Path(__file__).resolve().parents[4]
+    helper_path = repo_root / "python/sglang/multimodal_gen/test/partitioning.py"
+    spec = importlib.util.spec_from_file_location(
+        "diffusion_test_partitioning", helper_path
+    )
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module.PartitionItem, module.partition_items_by_lpt
+
+
+PartitionItem, partition_items_by_lpt = _load_partitioning_helpers()
+
+SUITE_OUTPUT_NAMES = {"1-gpu": "1gpu", "2-gpu": "2gpu", "1-gpu-b200": "b200"}
+
+USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true")
+
+if USE_NPU_CONFIGS:
+    SUITE_OUTPUT_NAMES = {"1-npu": "1npu", "2-npu": "2npu"}
+
+DEFAULT_STANDALONE_EST_TIME_SECONDS = 300.0
+
+
+def validate_suite_case_coverage(suites: dict[str, DiffusionSuiteInfo]) -> None:
+    """
+    Guardrail: dynamic diffusion suites must contain parametrized cases.
+    """
+    suites_with_no_cases = []
+    for suite_name in SUITE_OUTPUT_NAMES:
+        suite_info = suites.get(suite_name)
+        if suite_info is None:
+            print(f"Error: Required suite '{suite_name}' not found in parsed suites.")
+            sys.exit(1)
+        if len(suite_info.cases) == 0:
+            suites_with_no_cases.append(suite_name)
+
+    if suites_with_no_cases:
+        joined = ", ".join(suites_with_no_cases)
+        print(
+            "Error: Parsed zero parametrized cases for diffusion suites: "
+            f"{joined}. This usually means run_suite case imports changed but "
+            "diffusion parser logic was not updated."
+        )
+        sys.exit(1)
+
+
+def compute_partition_count(
+    total_time_seconds: float,
+    min_time_seconds: float,
+    target_time_seconds: float,
+    max_time_seconds: float,
+    max_partitions: int,
+) -> int:
+    if total_time_seconds <= 0:
+        return 0
+
+    min_partition_count = max(1, math.ceil(total_time_seconds / max_time_seconds))
+    max_partition_count = max(1, math.floor(total_time_seconds / min_time_seconds))
+
+    min_partition_count = min(min_partition_count, max_partitions)
+    max_partition_count = min(max_partition_count, max_partitions)
+
+    if max_partition_count < min_partition_count:
+        fallback_count = math.ceil(total_time_seconds / target_time_seconds)
+        return max(1, min(fallback_count, max_partitions))
+
+    preferred_count = math.ceil(total_time_seconds / target_time_seconds)
+    preferred_count = max(1, min(preferred_count, max_partitions))
+    return max(min_partition_count, min(preferred_count, max_partition_count))
+
+
+def build_partition_items(
+    suite_info: DiffusionSuiteInfo, include_standalone: bool = True
+) -> list[PartitionItem]:
+    items = [
+        PartitionItem(kind="case", item_id=case.case_id, est_time=case.est_time)
+        for case in suite_info.cases
+    ]
+    if not include_standalone:
+        return items
+
+    items.extend(
+        PartitionItem(
+            kind="standalone",
+            item_id=standalone_file,
+            est_time=suite_info.standalone_est_times.get(
+                standalone_file, DEFAULT_STANDALONE_EST_TIME_SECONDS
+            ),
+            used_fallback_estimate=(
+                standalone_file in suite_info.missing_standalone_estimates
+            ),
+        )
+        for standalone_file in suite_info.standalone_files
+    )
+    return items
+
+
+def build_matrix(partition_count: int) -> dict:
+    if partition_count <= 0:
+        return {"include": []}
+    return {"include": [{"part": i} for i in range(partition_count)]}
+
+
+def build_partition_plan(
+    suite_name: str,
+    partitions: list[list[PartitionItem]],
+) -> dict:
+    return {
+        "suite": suite_name,
+        "partition_count": len(partitions),
+        "partitions": [
+            {
+                "part": idx,
+                "case_ids": [item.item_id for item in partition if item.kind == "case"],
+                "standalone_files": [
+                    item.item_id for item in partition if item.kind == "standalone"
+                ],
+                "missing_standalone_estimates": [
+                    item.item_id
+                    for item in partition
+                    if item.kind == "standalone" and item.used_fallback_estimate
+                ],
+                "estimated_time": round(sum(item.est_time for item in partition), 1),
+            }
+            for idx, partition in enumerate(partitions)
+        ],
+    }
+
+
+def output_github_value(name: str, value: dict) -> None:
+    value_json = json.dumps(value, separators=(",", ":"))
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as f:
+            f.write(f"{name}={value_json}\n")
+    print(f"{name}={value_json}")
+
+
+def output_github_scalar(name: str, value: str) -> None:
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as f:
+            f.write(f"{name}={value}\n")
+    print(f"{name}={value}")
+
+
+def print_suite_summary(
+    suite_name: str,
+    suite_info: DiffusionSuiteInfo,
+    partitions: list[list[PartitionItem]],
+    include_standalone: bool = True,
+) -> None:
+    total_time = sum(
+        item.est_time
+        for item in build_partition_items(
+            suite_info, include_standalone=include_standalone
+        )
+    )
+    print(f"{suite_name.upper()} suite:")
+    print(f"  Cases: {len(suite_info.cases)}")
+    standalone_label = "Standalone files"
+    if not include_standalone:
+        standalone_label = "Standalone files ignored"
+    print(f"  {standalone_label}: {len(suite_info.standalone_files)}")
+    print(
+        f"  Missing standalone estimates: {len(suite_info.missing_standalone_estimates)}"
+    )
+    if suite_info.missing_standalone_estimates:
+        print(
+            f"  Fallback standalone estimate: "
+            f"{DEFAULT_STANDALONE_EST_TIME_SECONDS:.1f}s"
+        )
+        for standalone_file in suite_info.missing_standalone_estimates:
+            print(f"    - {standalone_file}")
+    print(f"  Total estimated time: {total_time:.1f}s ({total_time/60:.1f} min)")
+    print(f"  Selected partitions: {len(partitions)}")
+    print()
+
+    print("  Partition assignments:")
+    for idx, partition in enumerate(partitions):
+        partition_time = sum(item.est_time for item in partition)
+        print(f"    Partition {idx}:")
+        print(
+            f"      Estimated time: {partition_time:.1f}s ({partition_time/60:.1f} min)"
+        )
+        for item in partition:
+            fallback_suffix = (
+                ", fallback estimate"
+                if item.kind == "standalone" and item.used_fallback_estimate
+                else ""
+            )
+            print(
+                f"      - {item.kind}: {item.item_id} "
+                f"({item.est_time:.1f}s{fallback_suffix})"
+            )
+    print()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Compute diffusion test partitions for CI"
+    )
+    parser.add_argument(
+        "--min-time",
+        type=float,
+        default=1200.0,
+        help="Minimum desired partition time in seconds (default: 1200 = 20 minutes)",
+    )
+    parser.add_argument(
+        "--target-time",
+        type=float,
+        default=1800.0,
+        help="Preferred partition time in seconds (default: 1800 = 30 minutes)",
+    )
+    parser.add_argument(
+        "--max-time",
+        type=float,
+        default=2400.0,
+        help="Maximum desired partition time in seconds (default: 2400 = 40 minutes)",
+    )
+    parser.add_argument(
+        "--max-partitions",
+        type=int,
+        default=10,
+        help="Maximum number of partitions (default: 10)",
+    )
+    parser.add_argument(
+        "--parametrized-only",
+        action="store_true",
+        help="Only partition DiffusionTestCase parametrized cases.",
+    )
+    args = parser.parse_args()
+
+    script_dir = Path(__file__).resolve().parent
+    repo_root = script_dir.parent.parent.parent.parent
+
+    baseline_path = repo_root / BASELINE_REL_PATH
+    run_suite_path = repo_root / RUN_SUITE_REL_PATH
+
+    if not run_suite_path.exists():
+        print(f"Error: Run suite not found: {run_suite_path}")
+        sys.exit(1)
+    try:
+        if USE_NPU_CONFIGS:
+            case_config_path = (
+                repo_root
+                / "python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py"
+            )
+        else:
+            case_config_path = resolve_case_config_path(repo_root, run_suite_path)
+    except (RuntimeError, FileNotFoundError) as exc:
+        print(f"Error: {exc}")
+        sys.exit(1)
+
+    suites = collect_diffusion_suites(
+        case_config_path,
+        run_suite_path,
+        baseline_path,
+    )
+    validate_suite_case_coverage(suites)
+
+    print("=== Diffusion Partition Computation ===")
+    print(f"Min partition time: {args.min_time}s ({args.min_time/60:.1f} min)")
+    print(f"Target partition time: {args.target_time}s ({args.target_time/60:.1f} min)")
+    print(f"Max partition time: {args.max_time}s ({args.max_time/60:.1f} min)")
+    print()
+
+    for suite_name, suite_info in suites.items():
+        if suite_name not in SUITE_OUTPUT_NAMES:
+            continue
+
+        items = build_partition_items(
+            suite_info, include_standalone=not args.parametrized_only
+        )
+        total_time = sum(item.est_time for item in items)
+        partition_count = compute_partition_count(
+            total_time_seconds=total_time,
+            min_time_seconds=args.min_time,
+            target_time_seconds=args.target_time,
+            max_time_seconds=args.max_time,
+            max_partitions=args.max_partitions,
+        )
+        partitions = partition_items_by_lpt(items, partition_count)
+
+        print_suite_summary(
+            suite_name,
+            suite_info,
+            partitions,
+            include_standalone=not args.parametrized_only,
+        )
+
+        output_name = SUITE_OUTPUT_NAMES[suite_name]
+        output_github_value(f"matrix-{output_name}", build_matrix(partition_count))
+        output_github_scalar(f"partition-count-{output_name}", str(partition_count))
+        output_github_value(
+            f"plan-{output_name}", build_partition_plan(suite_name, partitions)
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/diffusion_case_parser.py

@@ -0,0 +1,517 @@
+#!/usr/bin/env python3
+"""
+AST-based parser for diffusion test cases.
+
+This module parses the diffusion case source and run_suite.py using AST to
+extract test case information without requiring sglang dependencies. The case
+source file is discovered from ONE_GPU_CASES/TWO_GPU_CASES imports in
+run_suite.py so CI keeps a single source of truth.
+
+Usage:
+    # From sibling scripts in this directory:
+    from diffusion_case_parser import collect_diffusion_suites, resolve_case_config_path
+    case_config_path = resolve_case_config_path(repo_root, run_suite_path)
+    suites = collect_diffusion_suites(case_config_path, run_suite_path, baseline_path)
+"""
+
+import ast
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional
+
+# Mapping from list variable names to suite names
+CASE_LIST_TO_SUITE = {
+    "ONE_GPU_CASES": "1-gpu",
+    "ONE_GPU_CASES_A": "1-gpu",
+    "ONE_GPU_CASES_B": "1-gpu",
+    "ONE_GPU_CASES_C": "1-gpu-b200",
+    "ONE_GPU_MODELOPT_FP8_CASES": "1-gpu",
+    "ONE_GPU_MODELOPT_CASES": "1-gpu-b200",
+    "ONE_GPU_B200_CASES": "1-gpu-b200",
+    "TWO_GPU_CASES": "2-gpu",
+    "TWO_GPU_CASES_A": "2-gpu",
+    "TWO_GPU_CASES_B": "2-gpu",
+}
+
+# Default estimated time for cases without baseline (5 minutes)
+DEFAULT_EST_TIME_SECONDS = 300.0
+
+# Fixed overhead for server startup when estimated_full_test_time_s is not set
+STARTUP_OVERHEAD_SECONDS = 120.0
+
+# Paths relative to repository root
+BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines"
+BASELINE_PLATFORM_ORDER = ("h100", "b200", "5090")
+RUN_SUITE_REL_PATH = "python/sglang/multimodal_gen/test/run_suite.py"
+
+USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true")
+
+if USE_NPU_CONFIGS:
+    BASELINE_REL_PATH = (
+        "python/sglang/multimodal_gen/test/server/perf_baselines_npu.json"
+    )
+    CASE_LIST_TO_SUITE = {
+        "ONE_NPU_CASES": "1-npu",
+        "TWO_NPU_CASES": "2-npu",
+    }
+
+
+@dataclass
+class DiffusionCaseInfo:
+    """Information about a single diffusion test case."""
+
+    case_id: str  # e.g., "qwen_image_t2i"
+    suite: str  # "1-gpu" or "2-gpu"
+    est_time: float  # estimated time in seconds
+
+
+@dataclass
+class DiffusionSuiteInfo:
+    """Complete information for a test suite."""
+
+    suite: str  # "1-gpu" or "2-gpu"
+    cases: List[DiffusionCaseInfo]  # parametrized test cases
+    standalone_files: List[str]  # standalone test files
+    standalone_est_times: Dict[str, float]  # standalone file -> estimated seconds
+    missing_standalone_estimates: List[
+        str
+    ]  # standalone files without configured estimate
+
+
+class DiffusionTestCaseVisitor(ast.NodeVisitor):
+    """
+    AST visitor to extract DiffusionTestCase definitions from the case config.
+
+    Parses assignments like:
+        ONE_GPU_CASES_A: list[DiffusionTestCase] = [
+            DiffusionTestCase("case_id", ...),
+            ...
+        ]
+    """
+
+    def __init__(self):
+        self.cases: Dict[str, List[str]] = {}  # list_name -> [case_id, ...]
+        self.factory_case_ids: Dict[str, str] = {}
+
+    def visit_Module(self, node: ast.Module):
+        for stmt in node.body:
+            if not isinstance(stmt, ast.FunctionDef):
+                continue
+            case_id = self._extract_factory_case_id(stmt)
+            if case_id:
+                self.factory_case_ids[stmt.name] = case_id
+
+        self.generic_visit(node)
+
+    def visit_Expr(self, node: ast.Expr):
+        """Handle ``LIST.append(...)`` mutations at any nesting level.
+
+        Previously only module-top-level ``ast.Expr`` statements were scanned for
+        ``.append()`` calls, so cases registered under a platform guard such as
+        ``if not current_platform.is_hip(): ONE_GPU_CASES.append(...)`` (used by
+        ``hunyuan3d_shape_gen`` and ``turbo_wan2_1_t2v_1.3b``) were invisible to
+        the partition planner and therefore never scheduled in CI. Visiting every
+        ``Expr`` lets ``generic_visit`` reach appends inside ``if``/``else`` blocks.
+        """
+        self._process_expr(node.value)
+        self.generic_visit(node)
+
+    def visit_Assign(self, node: ast.Assign):
+        self._process_assignment(node.targets, node.value)
+        self.generic_visit(node)
+
+    def visit_AnnAssign(self, node: ast.AnnAssign):
+        if node.target and node.value:
+            self._process_assignment([node.target], node.value)
+        self.generic_visit(node)
+
+    def visit_AugAssign(self, node: ast.AugAssign):
+        self._process_aug_assignment(node.target, node.op, node.value)
+        self.generic_visit(node)
+
+    def _process_assignment(self, targets: List[ast.AST], value: ast.AST):
+        """Process an assignment to extract case IDs."""
+        for target in targets:
+            if isinstance(target, ast.Name):
+                list_name = target.id
+                case_ids = self._extract_case_ids(value)
+                if case_ids is not None:
+                    self.cases[list_name] = case_ids
+
+    def _process_aug_assignment(self, target: ast.AST, op: ast.AST, value: ast.AST):
+        """Process `+=` style assignment to merge case lists."""
+        if not isinstance(target, ast.Name) or not isinstance(op, ast.Add):
+            return
+
+        if isinstance(value, ast.Name):
+            target_suite = CASE_LIST_TO_SUITE.get(target.id)
+            value_suite = CASE_LIST_TO_SUITE.get(value.id)
+            if target_suite and value_suite and target_suite != value_suite:
+                return
+
+        rhs_case_ids = self._extract_case_ids(value)
+        if rhs_case_ids is None:
+            return
+
+        lhs_case_ids = self.cases.get(target.id, [])
+        self.cases[target.id] = [*lhs_case_ids, *rhs_case_ids]
+
+    def _process_expr(self, node: ast.AST):
+        """Process list mutation calls such as `ONE_GPU_CASES.append(...)`."""
+        if not isinstance(node, ast.Call):
+            return
+        if not isinstance(node.func, ast.Attribute):
+            return
+        if node.func.attr != "append":
+            return
+        if not isinstance(node.func.value, ast.Name):
+            return
+        list_name = node.func.value.id
+        if list_name not in CASE_LIST_TO_SUITE:
+            return
+        if len(node.args) != 1:
+            return
+
+        case_id = self._extract_case_id_from_call(node.args[0])
+        if case_id:
+            self.cases.setdefault(list_name, []).append(case_id)
+
+    def _extract_case_ids(self, node: ast.AST) -> Optional[List[str]]:
+        """Extract case IDs from a supported expression."""
+        if isinstance(node, ast.List):
+            return self._extract_case_ids_from_list(node)
+
+        if isinstance(node, ast.Name):
+            # Reference to a previously parsed list variable.
+            if node.id not in self.cases:
+                return None
+            return list(self.cases[node.id])
+
+        if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+            left_ids = self._extract_case_ids(node.left)
+            right_ids = self._extract_case_ids(node.right)
+            if left_ids is None or right_ids is None:
+                return None
+            return [*left_ids, *right_ids]
+
+        return None
+
+    def _extract_case_ids_from_list(self, node: ast.List) -> List[str]:
+        """Extract case IDs from a literal list of DiffusionTestCase calls."""
+        case_ids = []
+        for elt in node.elts:
+            if isinstance(elt, ast.Starred):
+                starred_case_ids = self._extract_case_ids(elt.value)
+                if starred_case_ids:
+                    case_ids.extend(starred_case_ids)
+                continue
+            case_id = self._extract_case_id_from_call(elt)
+            if case_id:
+                case_ids.append(case_id)
+        return case_ids
+
+    def _extract_case_id_from_call(self, node: ast.AST) -> Optional[str]:
+        """Extract case_id from DiffusionTestCase(...) call."""
+        if not isinstance(node, ast.Call):
+            return None
+
+        # First positional argument is the case_id.
+        if isinstance(node.func, ast.Name) and node.func.id in {
+            "DiffusionTestCase",
+            "_make_modelopt_ci_case",
+        }:
+            if node.args and isinstance(node.args[0], ast.Constant):
+                return node.args[0].value
+        if isinstance(node.func, ast.Name) and not node.args:
+            return self.factory_case_ids.get(node.func.id)
+
+        return None
+
+    def _extract_factory_case_id(self, node: ast.FunctionDef) -> Optional[str]:
+        for child in ast.walk(node):
+            if not isinstance(child, ast.Return) or child.value is None:
+                continue
+            case_id = self._extract_case_id_from_call(child.value)
+            if case_id:
+                return case_id
+        return None
+
+
+def resolve_case_config_path(repo_root: Path, run_suite_path: Path) -> Path:
+    """
+    Resolve the diffusion case config path from run_suite imports.
+
+    run_suite.py must import BOTH ONE_GPU_CASES and TWO_GPU_CASES from the same
+    module. That imported module is treated as the single source of truth.
+    """
+    with open(run_suite_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    tree = ast.parse(content, filename=str(run_suite_path))
+    one_gpu_module: Optional[str] = None
+    two_gpu_module: Optional[str] = None
+
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.ImportFrom) or not node.module:
+            continue
+        imported_names = {alias.name for alias in node.names}
+        if "ONE_GPU_CASES" in imported_names:
+            one_gpu_module = node.module
+        if "TWO_GPU_CASES" in imported_names:
+            two_gpu_module = node.module
+
+    if one_gpu_module is None or two_gpu_module is None:
+        raise RuntimeError(
+            "run_suite.py must import BOTH ONE_GPU_CASES and TWO_GPU_CASES."
+        )
+    if one_gpu_module != two_gpu_module:
+        raise RuntimeError(
+            "run_suite.py imports ONE_GPU_CASES and TWO_GPU_CASES from different "
+            f"modules: {one_gpu_module} vs {two_gpu_module}"
+        )
+
+    rel_path = Path(*one_gpu_module.split(".")).with_suffix(".py")
+    candidates = [repo_root / rel_path, repo_root / "python" / rel_path]
+    case_config_path = next((path for path in candidates if path.exists()), None)
+    if case_config_path is None:
+        raise FileNotFoundError(
+            "Resolved case config from run_suite does not exist. Checked: "
+            + ", ".join(str(path) for path in candidates)
+        )
+    return case_config_path
+
+
+class RunSuiteVisitor(ast.NodeVisitor):
+    """
+    AST visitor to extract standalone metadata from run_suite.py.
+
+    Parses:
+        STANDALONE_FILES = {
+            "1-gpu": ["test_lora_format_adapter.py"],
+            "2-gpu": [],
+        }
+    """
+
+    def __init__(self):
+        self.standalone_files: Dict[str, List[str]] = {}
+        self.standalone_est_times: Dict[str, Dict[str, float]] = {}
+
+    def visit_Assign(self, node: ast.Assign):
+        for target in node.targets:
+            if isinstance(target, ast.Name) and target.id == "STANDALONE_FILES":
+                self.standalone_files = self._extract_file_dict(node.value)
+            if (
+                isinstance(target, ast.Name)
+                and target.id == "STANDALONE_FILE_EST_TIMES"
+            ):
+                self.standalone_est_times = self._extract_est_time_dict(node.value)
+        self.generic_visit(node)
+
+    def _extract_file_dict(self, node: ast.AST) -> Dict[str, List[str]]:
+        """Extract dictionary of suite -> file list."""
+        result = {}
+        if isinstance(node, ast.Dict):
+            for key, value in zip(node.keys, node.values):
+                if isinstance(key, ast.Constant) and isinstance(value, ast.List):
+                    suite = key.value
+                    files = [
+                        elt.value for elt in value.elts if isinstance(elt, ast.Constant)
+                    ]
+                    result[suite] = files
+        return result
+
+    def _extract_est_time_dict(self, node: ast.AST) -> Dict[str, Dict[str, float]]:
+        """Extract dictionary of suite -> standalone file -> estimated seconds."""
+        result = {}
+        if not isinstance(node, ast.Dict):
+            return result
+
+        for key, value in zip(node.keys, node.values):
+            if not isinstance(key, ast.Constant) or not isinstance(value, ast.Dict):
+                continue
+
+            suite = key.value
+            suite_est_times = {}
+            for inner_key, inner_value in zip(value.keys, value.values):
+                if not (
+                    isinstance(inner_key, ast.Constant)
+                    and isinstance(inner_value, ast.Constant)
+                ):
+                    continue
+                suite_est_times[inner_key.value] = float(inner_value.value)
+            result[suite] = suite_est_times
+
+        return result
+
+
+def _iter_baseline_paths(baseline_path: Path) -> List[Path]:
+    if baseline_path.is_file():
+        return [baseline_path]
+    if not baseline_path.is_dir():
+        return []
+
+    ordered_paths = [
+        baseline_path / f"{platform}.json" for platform in BASELINE_PLATFORM_ORDER
+    ]
+    ordered_paths.extend(
+        path
+        for path in sorted(baseline_path.glob("*.json"))
+        if path not in ordered_paths
+    )
+    return [path for path in ordered_paths if path.exists()]
+
+
+def load_baselines(baseline_path: Path) -> Dict[str, float]:
+    """
+    Load performance baselines from a JSON file or platform baseline directory.
+
+    Returns:
+        Dictionary mapping case_id to estimated time in seconds.
+    """
+    baselines = {}
+    for path in _iter_baseline_paths(baseline_path):
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        scenarios = data.get("scenarios", {})
+        for case_id, scenario in scenarios.items():
+            if scenario.get("estimated_full_test_time_s") is not None:
+                est_time = scenario["estimated_full_test_time_s"]
+            else:
+                expected_e2e_ms = scenario.get("expected_e2e_ms", 0)
+                est_time = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS
+            baselines.setdefault(case_id, est_time)
+
+    return baselines
+
+
+def get_case_est_time(case_id: str, baselines: Dict[str, float]) -> float:
+    """Get estimated time for a case, with fallback to default."""
+    return baselines.get(case_id, DEFAULT_EST_TIME_SECONDS)
+
+
+def parse_testcase_configs(config_path: Path) -> Dict[str, List[str]]:
+    """
+    Parse a diffusion case config file to extract case IDs.
+
+    Returns:
+        Dictionary mapping list name to case IDs.
+        e.g., {"ONE_GPU_CASES_A": ["qwen_image_t2i", ...], ...}
+    """
+    with open(config_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    tree = ast.parse(content, filename=str(config_path))
+    visitor = DiffusionTestCaseVisitor()
+    visitor.visit(tree)
+
+    return visitor.cases
+
+
+def parse_run_suite_standalone_data(
+    run_suite_path: Path,
+) -> tuple[Dict[str, List[str]], Dict[str, Dict[str, float]]]:
+    """
+    Parse run_suite.py to extract standalone file metadata.
+
+    Returns:
+        Tuple of:
+          - suite -> standalone file list
+          - suite -> standalone file -> estimated seconds
+    """
+    with open(run_suite_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    tree = ast.parse(content, filename=str(run_suite_path))
+    visitor = RunSuiteVisitor()
+    visitor.visit(tree)
+
+    return visitor.standalone_files, visitor.standalone_est_times
+
+
+def validate_standalone_est_times(
+    standalone_files: Dict[str, List[str]],
+    standalone_est_times: Dict[str, Dict[str, float]],
+) -> Dict[str, List[str]]:
+    missing_by_suite = {}
+    for suite, files in standalone_files.items():
+        suite_est_times = standalone_est_times.get(suite, {})
+        missing = [
+            standalone_file
+            for standalone_file in files
+            if standalone_file not in suite_est_times
+        ]
+        if missing:
+            missing_by_suite[suite] = missing
+    return missing_by_suite
+
+
+def collect_diffusion_suites(
+    case_config_path: Path,
+    run_suite_path: Path,
+    baseline_path: Path,
+) -> Dict[str, DiffusionSuiteInfo]:
+    """
+    Collect all diffusion test suite information using AST parsing.
+
+    Args:
+        case_config_path: Path to case config (resolved from run_suite.py)
+        run_suite_path: Path to run_suite.py
+        baseline_path: Path to perf_baselines/ or a single baseline JSON file
+
+    Returns:
+        Dictionary mapping suite name to DiffusionSuiteInfo.
+    """
+    # Parse case IDs from the single source case config.
+    case_lists = parse_testcase_configs(case_config_path)
+
+    # Parse standalone files from run_suite.py
+    standalone_files, standalone_est_times = parse_run_suite_standalone_data(
+        run_suite_path
+    )
+    missing_standalone_estimates = validate_standalone_est_times(
+        standalone_files, standalone_est_times
+    )
+
+    # Load baselines for time estimation
+    baselines = load_baselines(baseline_path)
+
+    # Build suite info
+    suites = {}
+    for list_name, suite in CASE_LIST_TO_SUITE.items():
+        case_ids = case_lists.get(list_name, [])
+        cases = [
+            DiffusionCaseInfo(
+                case_id=cid,
+                suite=suite,
+                est_time=get_case_est_time(cid, baselines),
+            )
+            for cid in case_ids
+        ]
+
+        if suite not in suites:
+            suites[suite] = DiffusionSuiteInfo(
+                suite=suite,
+                cases=[],
+                standalone_files=standalone_files.get(suite, []),
+                standalone_est_times=dict(standalone_est_times.get(suite, {})),
+                missing_standalone_estimates=list(
+                    missing_standalone_estimates.get(suite, [])
+                ),
+            )
+        suites[suite].cases.extend(cases)
+
+    # Dedupe duplicated case IDs while preserving first-seen order.
+    for suite_info in suites.values():
+        seen_case_ids = set()
+        deduped_cases = []
+        for case in suite_info.cases:
+            if case.case_id in seen_case_ids:
+                continue
+            seen_case_ids.add(case.case_id)
+            deduped_cases.append(case)
+        suite_info.cases = deduped_cases
+
+    return suites

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -0,0 +1,835 @@
+"""Generate a Markdown dashboard for SGLang-Diffusion nightly benchmarks.
+
+Reads current comparison results + historical data from sgl-project/ci-data repo
+and produces a Markdown report with tables and trend charts saved as PNG files.
+
+Usage:
+    python3 scripts/ci/utils/diffusion/generate_diffusion_dashboard.py \
+        --results comparison-results.json \
+        --output dashboard.md \
+        --charts-dir comparison-charts/ \
+        --history-dir history/           # optional, local history JSONs
+        --fetch-history                  # fetch from GitHub API instead
+"""
+
+import argparse
+import json
+import os
+from datetime import datetime, timezone
+
+# ---------------------------------------------------------------------------
+# History fetching (from sgl-project/ci-data repo via GitHub API)
+# ---------------------------------------------------------------------------
+
+CI_DATA_REPO_OWNER = "sgl-project"
+CI_DATA_REPO_NAME = "ci-data"
+CI_DATA_BRANCH = "main"
+HISTORY_PREFIX = "diffusion-comparisons"
+MAX_HISTORY_RUNS = 29
+
+# Base URL for chart images pushed to sgl-project/ci-data
+CHARTS_RAW_BASE_URL = (
+    f"https://raw.githubusercontent.com/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
+    f"/{CI_DATA_BRANCH}/{HISTORY_PREFIX}/charts"
+)
+
+
+def _github_get(url: str, token: str) -> dict | list | None:
+    """Simple GET to GitHub API."""
+    from urllib.error import HTTPError
+    from urllib.request import Request, urlopen
+
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    req = Request(url, headers=headers)
+    try:
+        with urlopen(req) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except HTTPError as e:
+        print(f"  Warning: GitHub API request failed ({e.code}): {url}")
+        return None
+    except Exception as e:
+        print(f"  Warning: GitHub API request error: {e}")
+        return None
+
+
+def fetch_history_from_github(token: str) -> list[dict]:
+    """Fetch recent comparison result JSONs from sgl-project/ci-data repo."""
+    print("Fetching historical comparison data from GitHub...")
+    url = (
+        f"https://api.github.com/repos/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
+        f"/contents/{HISTORY_PREFIX}?ref={CI_DATA_BRANCH}"
+    )
+    listing = _github_get(url, token)
+    if not listing or not isinstance(listing, list):
+        print("  No historical data found.")
+        return []
+
+    # Filter JSON files and sort by name (date prefix) descending
+    json_files = sorted(
+        [f for f in listing if f["name"].endswith(".json")],
+        key=lambda f: f["name"],
+        reverse=True,
+    )[:MAX_HISTORY_RUNS]
+
+    history = []
+    for entry in json_files:
+        raw_url = entry.get("download_url")
+        if not raw_url:
+            continue
+        data = _github_get(raw_url, token)
+        if data and isinstance(data, dict):
+            history.append(data)
+    print(f"  Loaded {len(history)} historical run(s).")
+    return history
+
+
+def load_history_from_dir(history_dir: str) -> list[dict]:
+    """Load historical JSONs from a local directory."""
+    if not os.path.isdir(history_dir):
+        return []
+    files = sorted(
+        [f for f in os.listdir(history_dir) if f.endswith(".json")],
+        reverse=True,
+    )[:MAX_HISTORY_RUNS]
+    history = []
+    for fname in files:
+        try:
+            with open(os.path.join(history_dir, fname)) as f:
+                history.append(json.load(f))
+        except Exception:
+            pass
+    return history
+
+
+# ---------------------------------------------------------------------------
+# Dashboard generation
+# ---------------------------------------------------------------------------
+
+
+def _fmt_latency(val: float | None) -> str:
+    if val is None:
+        return "N/A"
+    return f"{val:.2f}"
+
+
+def _fmt_speedup(sglang_lat: float | None, other_lat: float | None) -> str:
+    if sglang_lat is None or other_lat is None or sglang_lat <= 0:
+        return "N/A"
+    ratio = other_lat / sglang_lat
+    return f"{ratio:.2f}x"
+
+
+def _short_date(ts: str) -> str:
+    """Extract short date from ISO timestamp."""
+    try:
+        dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+        return dt.strftime("%b %d")
+    except Exception:
+        return ts[:10]
+
+
+def _short_sha(sha: str) -> str:
+    return sha[:7] if sha and sha != "unknown" else "?"
+
+
+def _assess_risk(
+    cid: str,
+    current_cases: dict[str, dict[str, float | None]],
+    history: list[dict],
+    other_frameworks: list[str],
+) -> tuple[str, str]:
+    """Assess risk for a given case, returning (emoji, reason).
+
+    Rules (checked in order):
+    - N/A latency → ❌ broken
+    - History exists: SGLang latency >5% vs avg of last 3 runs → ⚠️ regression
+    - Competitor exists & SGLang slower → 🔴 competitive risk
+    - SGLang faster than all competitors by >20% → 🟢 strong advantage
+    - SGLang faster than all competitors by ≤20% → 🟡 moderate advantage
+    - Default → ✅ stable
+    """
+    sg_lat = current_cases.get(cid, {}).get("sglang")
+
+    # Broken: sglang latency is N/A
+    if sg_lat is None:
+        return "❌", f"{cid}: SGLang latency is N/A (broken)"
+
+    # Check regression against 3-run historical average
+    if history:
+        hist_lats: list[float] = []
+        for run in history[:3]:
+            run_cases = _extract_case_results(run)
+            h_lat = run_cases.get(cid, {}).get("sglang")
+            if h_lat is not None:
+                hist_lats.append(h_lat)
+        if hist_lats:
+            avg_3 = sum(hist_lats) / len(hist_lats)
+            if avg_3 > 0 and (sg_lat - avg_3) / avg_3 > 0.05:
+                pct = (sg_lat - avg_3) / avg_3 * 100
+                return (
+                    "⚠️",
+                    f"{cid}: SGLang regression +{pct:.1f}% vs 3-run avg "
+                    f"({sg_lat:.2f}s vs {avg_3:.2f}s)",
+                )
+
+    # Check competitive risk
+    if other_frameworks:
+        competitor_lats: dict[str, float] = {}
+        for ofw in other_frameworks:
+            olat = current_cases.get(cid, {}).get(ofw)
+            if olat is not None:
+                competitor_lats[ofw] = olat
+
+        if competitor_lats:
+            # SGLang slower than any competitor?
+            for ofw, olat in competitor_lats.items():
+                if sg_lat > olat:
+                    return (
+                        "🔴",
+                        f"{cid}: SGLang slower than {ofw} "
+                        f"({sg_lat:.2f}s vs {olat:.2f}s)",
+                    )
+
+            # SGLang faster — check margin
+            min_competitor = min(competitor_lats.values())
+            advantage = (min_competitor - sg_lat) / min_competitor
+            if advantage > 0.20:
+                return "🟢", ""
+            else:
+                return "🟡", ""
+
+    # Default: stable
+    return "✅", ""
+
+
+def _trend_emoji(current: float | None, previous: float | None) -> str:
+    if current is None or previous is None:
+        return ""
+    diff_pct = (current - previous) / previous * 100
+    if diff_pct < -2:
+        return " :arrow_down:"  # faster (good)
+    elif diff_pct > 2:
+        return " :arrow_up:"  # slower (bad)
+    return " :left_right_arrow:"
+
+
+def _extract_case_results(run_data: dict) -> dict[str, dict[str, float | None]]:
+    """Extract {case_id: {framework: latency}} from a run."""
+    mapping: dict[str, dict[str, float | None]] = {}
+    for r in run_data.get("results", []):
+        cid = r["case_id"]
+        fw = r["framework"]
+        if cid not in mapping:
+            mapping[cid] = {}
+        mapping[cid][fw] = r.get("latency_s")
+    return mapping
+
+
+def _sanitize_filename(name: str) -> str:
+    """Sanitize a case ID to be a safe filename."""
+    return name.replace("/", "_").replace(" ", "_").replace(":", "_")
+
+
+def generate_dashboard(
+    current: dict,
+    history: list[dict],
+    charts_dir: str | None = None,
+) -> tuple[str, list[str]]:
+    """Generate full markdown dashboard.
+
+    Returns (markdown_string, alert_reasons) where alert_reasons is a list of
+    human-readable strings for cases that need attention (empty if all is well).
+
+    If charts_dir is provided, saves chart PNGs as files to that directory
+    and references them via raw.githubusercontent URLs. Otherwise, charts
+    are omitted.
+
+    Returns the markdown string.
+    """
+    lines: list[str] = []
+    lines.append("# SGLang-Diffusion Nightly Performance Dashboard\n")
+    ts = current.get("timestamp", datetime.now(timezone.utc).isoformat())
+    sha = current.get("commit_sha", "unknown")
+    lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n")
+
+    current_cases = _extract_case_results(current)
+    case_ids = list(current_cases.keys())
+
+    # ---- Regression detection ----
+    REGRESSION_THRESHOLD = 0.05  # 5%
+    regressions: list[str] = []
+    if history:
+        prev_cases = _extract_case_results(history[0])
+        for cid in case_ids:
+            for fw in ("sglang", "vllm-omni"):
+                cur = current_cases.get(cid, {}).get(fw)
+                prev = prev_cases.get(cid, {}).get(fw)
+                if cur and prev and prev > 0:
+                    pct = (cur - prev) / prev
+                    if pct > REGRESSION_THRESHOLD:
+                        regressions.append(
+                            f"**{cid}** ({fw}): {prev:.2f}s -> {cur:.2f}s "
+                            f"(+{pct*100:.1f}%)"
+                        )
+
+    if regressions:
+        lines.append("> [!WARNING]\n> **Performance Regression Detected**\n>")
+        for reg in regressions:
+            lines.append(f"> - {reg}")
+        lines.append("\n")
+
+    # Discover all frameworks present in results
+    all_frameworks = []
+    seen_fw = set()
+    for r in current.get("results", []):
+        fw = r["framework"]
+        if fw not in seen_fw:
+            all_frameworks.append(fw)
+            seen_fw.add(fw)
+    # Ensure sglang is first
+    if "sglang" in all_frameworks:
+        all_frameworks.remove("sglang")
+        all_frameworks.insert(0, "sglang")
+    other_frameworks = [fw for fw in all_frameworks if fw != "sglang"]
+
+    # ---- Section 1: SGLang-Diffusion performance (current run) ----
+    lines.append("## SGLang-Diffusion Performance\n")
+
+    # Compute risk assessments for all cases
+    risk_map: dict[str, tuple[str, str]] = {}
+    for cid in case_ids:
+        risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks)
+
+    # Dynamic header
+    header = "| Model | Risk |"
+    sep = "|-------|------|"
+    for fw in all_frameworks:
+        header += f" {fw} (s) |"
+        sep += "---------|"
+    for ofw in other_frameworks:
+        header += f" vs {ofw} |"
+        sep += "---------|"
+    lines.append(header)
+    lines.append(sep)
+
+    # One row per case (deduplicated by case_id)
+    seen_cases = set()
+    for r in current.get("results", []):
+        cid = r["case_id"]
+        if cid in seen_cases:
+            continue
+        seen_cases.add(cid)
+
+        case_fws = current_cases.get(cid, {})
+        sg_lat = case_fws.get("sglang")
+
+        risk_emoji, _ = risk_map.get(cid, ("✅", ""))
+        row = f"| {r['model'].split('/')[-1]} | {risk_emoji} |"
+        # Latency columns -- bold the fastest
+        lats = {fw: case_fws.get(fw) for fw in all_frameworks}
+        valid_lats = [v for v in lats.values() if v is not None]
+        min_lat = min(valid_lats) if valid_lats else None
+        for fw in all_frameworks:
+            lat = lats[fw]
+            if lat is not None and min_lat is not None and lat == min_lat:
+                row += f" **{_fmt_latency(lat)}** |"
+            else:
+                row += f" {_fmt_latency(lat)} |"
+        # Speedup columns
+        for ofw in other_frameworks:
+            row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
+        lines.append(row)
+
+    # ---- Section 2: Speedup-over-time vs. other frameworks (rendered only when present) ----
+    if history and other_frameworks:
+        lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
+
+        header = "| Date |"
+        sep = "|------|"
+        for cid in case_ids:
+            header += f" {cid} |"
+            sep += "---------|"
+        lines.append(header)
+        lines.append(sep)
+
+        all_runs = [current] + history
+        for run in all_runs:
+            run_cases = _extract_case_results(run)
+            date = _short_date(run.get("timestamp", ""))
+            row = f"| {date} |"
+            for cid in case_ids:
+                sg = run_cases.get(cid, {}).get("sglang")
+                vl = run_cases.get(cid, {}).get("vllm-omni")
+                row += f" {_fmt_speedup(sg, vl)} |"
+            lines.append(row)
+
+    # ---- Section 4: Matplotlib Trend Charts (saved as PNG files) ----
+    if history and charts_dir:
+        all_runs = list(reversed([current] + history))  # chronological order
+
+        def _chart_label(run: dict) -> str:
+            d = _short_date(run.get("timestamp", ""))
+            s = _short_sha(run.get("commit_sha", ""))
+            return f"{d}\n({s})"
+
+        try:
+            import matplotlib
+
+            matplotlib.use("Agg")
+            import matplotlib.pyplot as plt
+
+            os.makedirs(charts_dir, exist_ok=True)
+
+            # Per-case latency trend charts
+            for cid in case_ids:
+                labels = []
+                sg_vals = []
+                vl_vals = []
+                for run in all_runs:
+                    run_cases = _extract_case_results(run)
+                    sg = run_cases.get(cid, {}).get("sglang")
+                    vl = run_cases.get(cid, {}).get("vllm-omni")
+                    if sg is None:
+                        continue
+                    labels.append(_chart_label(run))
+                    sg_vals.append(sg)
+                    vl_vals.append(vl)
+
+                if not sg_vals:
+                    continue
+
+                has_vl = any(v is not None for v in vl_vals)
+                fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.2), 4))
+
+                # SGLang line
+                ax.plot(
+                    range(len(sg_vals)),
+                    sg_vals,
+                    "o-",
+                    color="#2563eb",
+                    linewidth=2,
+                    markersize=6,
+                    label="SGLang",
+                )
+                for i, v in enumerate(sg_vals):
+                    ax.annotate(
+                        f"{v:.2f}s",
+                        (i, v),
+                        textcoords="offset points",
+                        xytext=(0, 10),
+                        ha="center",
+                        fontsize=8,
+                        fontweight="bold",
+                        color="#2563eb",
+                    )
+
+                # vLLM-Omni line (if data exists)
+                if has_vl:
+                    vl_clean = [v if v is not None else float("nan") for v in vl_vals]
+                    ax.plot(
+                        range(len(vl_clean)),
+                        vl_clean,
+                        "s--",
+                        color="#dc2626",
+                        linewidth=2,
+                        markersize=5,
+                        label="vLLM-Omni",
+                    )
+                    for i, v in enumerate(vl_vals):
+                        if v is not None:
+                            ax.annotate(
+                                f"{v:.2f}s",
+                                (i, v),
+                                textcoords="offset points",
+                                xytext=(0, -14),
+                                ha="center",
+                                fontsize=8,
+                                color="#dc2626",
+                            )
+
+                ax.set_xticks(range(len(labels)))
+                ax.set_xticklabels(labels, fontsize=7)
+                ax.set_ylabel("Latency (s)")
+                ax.set_title(f"Latency Trend -- {cid}", fontsize=11, fontweight="bold")
+                ax.legend(loc="lower right", fontsize=8, framealpha=0.8)
+                ax.grid(True, alpha=0.3)
+                all_vals = sg_vals + [v for v in vl_vals if v is not None]
+                y_min = min(all_vals)
+                y_max = max(all_vals)
+                y_range = y_max - y_min if y_max > y_min else max(y_max * 0.1, 0.1)
+                ax.set_ylim(
+                    bottom=max(0, y_min - y_range * 0.3),
+                    top=y_max + y_range * 0.3,
+                )
+
+                filename = f"latency_{_sanitize_filename(cid)}.png"
+                chart_path = os.path.join(charts_dir, filename)
+                fig.savefig(chart_path, format="png", dpi=120, bbox_inches="tight")
+                plt.close(fig)
+                print(f"  Saved chart: {chart_path}")
+
+                chart_url = f"{CHARTS_RAW_BASE_URL}/{filename}"
+                lines.append(f"\n### Latency Trend: {cid}\n")
+                lines.append(f"![Latency Trend {cid}]({chart_url})\n")
+
+            # Speedup trend chart (only if multiple frameworks)
+            if other_frameworks:
+                fig, ax = plt.subplots(figsize=(max(6, len(all_runs) * 1.2), 4))
+                colors = ["#2563eb", "#dc2626", "#16a34a", "#ea580c"]
+                for ci_idx, cid in enumerate(case_ids):
+                    speedups = []
+                    run_labels = []
+                    for run in all_runs:
+                        run_cases = _extract_case_results(run)
+                        sg = run_cases.get(cid, {}).get("sglang")
+                        vl = run_cases.get(cid, {}).get("vllm-omni")
+                        if sg and vl and sg > 0:
+                            speedups.append(vl / sg)
+                        else:
+                            speedups.append(None)
+                        run_labels.append(_chart_label(run))
+                    clean = [v if v is not None else float("nan") for v in speedups]
+                    ax.plot(
+                        range(len(clean)),
+                        clean,
+                        "o-",
+                        color=colors[ci_idx % len(colors)],
+                        linewidth=2,
+                        markersize=5,
+                        label=cid,
+                    )
+
+                ax.set_xticks(range(len(run_labels)))
+                ax.set_xticklabels(run_labels, fontsize=7)
+                ax.set_ylabel("Speedup (x)")
+                ax.set_title(
+                    "SGLang Speedup Over vLLM-Omni", fontsize=11, fontweight="bold"
+                )
+                ax.axhline(y=1.0, color="gray", linestyle=":", alpha=0.5)
+                ax.legend(loc="upper left", fontsize=7)
+                ax.grid(True, alpha=0.3)
+
+                filename = "speedup_trend.png"
+                chart_path = os.path.join(charts_dir, filename)
+                fig.savefig(chart_path, format="png", dpi=120, bbox_inches="tight")
+                plt.close(fig)
+                print(f"  Saved chart: {chart_path}")
+
+                chart_url = f"{CHARTS_RAW_BASE_URL}/{filename}"
+                lines.append("\n### Speedup Trend (SGLang vs vLLM-Omni)\n")
+                lines.append(f"![Speedup Trend]({chart_url})\n")
+
+        except ImportError:
+            lines.append("\n*Charts unavailable (matplotlib not installed)*\n")
+
+    # ---- SGLang Performance Trend (raw data table, at the end) ----
+    if history:
+        lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
+
+        header = "| Date | Commit |"
+        sep = "|------|--------|"
+        for cid in case_ids:
+            header += f" {cid} (s) |"
+            sep += "---------|"
+        header += " Trend |"
+        sep += "-------|"
+        lines.append(header)
+        lines.append(sep)
+
+        all_runs = [current] + history
+        for i, run in enumerate(all_runs):
+            run_cases = _extract_case_results(run)
+            date = _short_date(run.get("timestamp", ""))
+            sha_s = _short_sha(run.get("commit_sha", ""))
+            row = f"| {date} | `{sha_s}` |"
+            for cid in case_ids:
+                lat = run_cases.get(cid, {}).get("sglang")
+                row += f" {_fmt_latency(lat)} |"
+            if i + 1 < len(all_runs):
+                prev_cases = _extract_case_results(all_runs[i + 1])
+                emojis = []
+                for cid in case_ids:
+                    cur = run_cases.get(cid, {}).get("sglang")
+                    prev = prev_cases.get(cid, {}).get("sglang")
+                    emojis.append(_trend_emoji(cur, prev))
+                row += " ".join(emojis) + " |"
+            else:
+                row += " -- |"
+            lines.append(row)
+
+    # ---- Risk Notification ----
+    alert_cases = [
+        (cid, emoji, reason)
+        for cid, (emoji, reason) in risk_map.items()
+        if emoji in ("⚠️", "🔴", "❌")
+    ]
+    if alert_cases:
+        lines.append("\n> [!CAUTION]")
+        lines.append("> **Action Required — Performance Alert**")
+        lines.append(">")
+        lines.append("> The following cases need attention:")
+        for _cid, _emoji, reason in alert_cases:
+            lines.append(f"> - {reason}")
+        lines.append("")
+
+    # Footer
+    lines.append("\n---")
+    lines.append(
+        "*Generated by `generate_diffusion_dashboard.py` in SGLang nightly CI.*"
+    )
+
+    alert_reasons = [reason for _, _, reason in alert_cases]
+    return "\n".join(lines) + "\n", alert_reasons
+
+
+ALERT_ASSIGNEES = ["mickqian", "bbuf", "yhyang201"]
+ALERT_LABEL = "perf-regression"
+
+
+ALERT_ISSUE_TITLE = "[Diffusion CI] Performance regression tracker"
+
+
+def _find_alert_issue(repo: str) -> tuple[str | None, bool]:
+    """Find the perf-regression tracker issue (open OR closed).
+
+    Returns (issue_number, is_open).  Prefers an open issue; if none,
+    returns the most recent closed one so it can be reopened.
+    """
+    import subprocess
+
+    for state in ("open", "closed"):
+        result = subprocess.run(
+            [
+                "gh",
+                "issue",
+                "list",
+                "--repo",
+                repo,
+                "--label",
+                ALERT_LABEL,
+                "--state",
+                state,
+                "--json",
+                "number",
+                "--limit",
+                "1",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if result.returncode != 0 or not result.stdout.strip():
+            continue
+        issues = json.loads(result.stdout)
+        if issues:
+            return str(issues[0]["number"]), state == "open"
+    return None, False
+
+
+def _create_alert_issue(alert_reasons: list[str]) -> None:
+    """Create or update the single perf-regression tracker issue.
+
+    Logic:
+    - If an open issue exists  → add a comment with the new alert.
+    - If a closed issue exists → reopen it, then add a comment.
+    - If no issue exists       → create one.
+
+    This guarantees at most one tracker issue ever exists.
+
+    Uses `gh` (GitHub CLI) which is available in all GitHub Actions runners.
+    Falls back silently outside CI.
+    """
+    import subprocess
+
+    run_url = ""
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
+    server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+    if run_id:
+        run_url = f"{server_url}/{repo}/actions/runs/{run_id}"
+
+    date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    body_lines = [
+        f"## Performance Alert — {date}",
+        "",
+        "The nightly diffusion benchmark detected the following issue(s):",
+        "",
+    ]
+    for reason in alert_reasons:
+        body_lines.append(f"- {reason}")
+    if run_url:
+        body_lines += ["", f"**CI Run:** {run_url}"]
+    body = "\n".join(body_lines)
+
+    try:
+        existing, is_open = _find_alert_issue(repo)
+
+        if existing:
+            # Reopen if closed
+            if not is_open:
+                subprocess.run(
+                    [
+                        "gh",
+                        "issue",
+                        "reopen",
+                        existing,
+                        "--repo",
+                        repo,
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=30,
+                )
+                print(f"Reopened alert issue #{existing}")
+
+            # Add comment
+            result = subprocess.run(
+                [
+                    "gh",
+                    "issue",
+                    "comment",
+                    existing,
+                    "--repo",
+                    repo,
+                    "--body",
+                    body,
+                ],
+                capture_output=True,
+                text=True,
+                timeout=30,
+            )
+            if result.returncode == 0:
+                print(f"Commented on alert issue #{existing}")
+            else:
+                print(
+                    f"Warning: failed to comment on issue #{existing} "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+        else:
+            # Create a new issue
+            cmd = [
+                "gh",
+                "issue",
+                "create",
+                "--repo",
+                repo,
+                "--title",
+                ALERT_ISSUE_TITLE,
+                "--body",
+                body,
+                "--label",
+                ALERT_LABEL,
+            ]
+            for user in ALERT_ASSIGNEES:
+                cmd += ["--assignee", user]
+
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+            if result.returncode == 0:
+                print(f"Created alert issue: {result.stdout.strip()}")
+            else:
+                print(
+                    f"Warning: failed to create alert issue "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+    except FileNotFoundError:
+        print("Warning: `gh` CLI not found — skipping alert issue creation")
+    except Exception as e:
+        print(f"Warning: failed to create/update alert issue: {e}")
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate SGLang-Diffusion nightly benchmark dashboard"
+    )
+    parser.add_argument(
+        "--results",
+        required=True,
+        help="Path to comparison-results.json from current run",
+    )
+    parser.add_argument(
+        "--output",
+        default="dashboard.md",
+        help="Output markdown file path",
+    )
+    parser.add_argument(
+        "--charts-dir",
+        default="comparison-charts",
+        help="Directory to save chart PNG files (default: comparison-charts/)",
+    )
+    parser.add_argument(
+        "--history-dir",
+        default=None,
+        help="Local directory containing historical comparison JSONs",
+    )
+    parser.add_argument(
+        "--fetch-history",
+        action="store_true",
+        help="Fetch history from ci-data GitHub repo",
+    )
+    parser.add_argument(
+        "--step-summary",
+        action="store_true",
+        help="Also write to $GITHUB_STEP_SUMMARY",
+    )
+
+    args = parser.parse_args()
+
+    # Load current results
+    with open(args.results) as f:
+        current = json.load(f)
+    print(f"Loaded current results: {len(current.get('results', []))} entries")
+
+    # Load history
+    history: list[dict] = []
+    if args.fetch_history:
+        token = os.environ.get("GH_PAT_FOR_NIGHTLY_CI_DATA") or os.environ.get(
+            "GITHUB_TOKEN"
+        )
+        if token:
+            history = fetch_history_from_github(token)
+        else:
+            print("Warning: No GitHub token available, skipping history fetch")
+    elif args.history_dir:
+        history = load_history_from_dir(args.history_dir)
+        print(f"Loaded {len(history)} historical run(s) from {args.history_dir}")
+
+    # Generate dashboard
+    markdown, alert_reasons = generate_dashboard(
+        current, history, charts_dir=args.charts_dir
+    )
+
+    # Write output
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    with open(args.output, "w") as f:
+        f.write(markdown)
+    print(f"Dashboard written to {args.output}")
+
+    # Write to GitHub Step Summary
+    if args.step_summary:
+        summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+        if summary_file:
+            with open(summary_file, "a") as f:
+                f.write(markdown)
+            print("Dashboard appended to $GITHUB_STEP_SUMMARY")
+        else:
+            print("Warning: $GITHUB_STEP_SUMMARY not set, skipping")
+
+    # Create GitHub Issue for performance alerts (so assignees get notified)
+    if alert_reasons:
+        _create_alert_issue(alert_reasons)
+    else:
+        print("No performance alerts — skipping issue creation.")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/publish_comparison_results.py

@@ -0,0 +1,231 @@
+"""Publish SGLang-Diffusion nightly benchmark results to sgl-project/ci-data repo.
+
+Pushes comparison-results.json, dashboard.md, and chart PNG files to the
+ci-data repository for historical tracking. Chart PNGs are stored under
+diffusion-comparisons/charts/ so they can be referenced via
+raw.githubusercontent URLs in the dashboard markdown (GitHub Step Summary
+blocks data: URIs).
+
+Usage:
+    python3 scripts/ci/utils/diffusion/publish_comparison_results.py \
+        --results comparison-results.json \
+        --dashboard dashboard.md \
+        --charts-dir comparison-charts/
+"""
+
+import argparse
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Reuse GitHub API helpers from publish_traces.
+# Support both direct script execution and package-style imports.
+if __package__:
+    from ..publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+else:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    from publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+
+# Repository configuration
+REPO_OWNER = "sgl-project"
+REPO_NAME = "ci-data"
+BRANCH = "main"
+STORAGE_PREFIX = "diffusion-comparisons"
+
+
+def _collect_chart_files(charts_dir: str) -> list[tuple[str, bytes]]:
+    """Collect PNG chart files from directory for upload."""
+    files: list[tuple[str, bytes]] = []
+    if not charts_dir or not os.path.isdir(charts_dir):
+        return files
+
+    for entry in sorted(os.listdir(charts_dir)):
+        if not entry.lower().endswith(".png"):
+            continue
+        full_path = os.path.join(charts_dir, entry)
+        if not os.path.isfile(full_path):
+            continue
+        with open(full_path, "rb") as f:
+            content = f.read()
+        # Store charts under diffusion-comparisons/charts/
+        repo_path = f"{STORAGE_PREFIX}/charts/{entry}"
+        files.append((repo_path, content))
+
+    return files
+
+
+def publish_comparison(
+    results_path: str,
+    dashboard_path: str | None = None,
+    charts_dir: str | None = None,
+) -> None:
+    """Publish comparison results, dashboard, and charts to ci-data repo."""
+    token = os.environ.get("GH_PAT_FOR_NIGHTLY_CI_DATA") or os.environ.get(
+        "GITHUB_TOKEN"
+    )
+    if not token:
+        print("Error: GH_PAT_FOR_NIGHTLY_CI_DATA or GITHUB_TOKEN not set")
+        sys.exit(1)
+
+    run_id = os.environ.get("GITHUB_RUN_ID", "local")
+    run_number = os.environ.get("GITHUB_RUN_NUMBER", "0")
+
+    # Verify permissions
+    perm = verify_token_permissions(REPO_OWNER, REPO_NAME, token)
+    if perm == "rate_limited":
+        print("Warning: Rate limited, skipping publish")
+        return
+    elif not perm:
+        print("Error: Token permission verification failed")
+        sys.exit(1)
+
+    # Prepare files to upload
+    files_to_upload: list[tuple[str, bytes]] = []
+
+    # Results JSON: stored with date prefix for chronological ordering
+    date_prefix = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    results_target = f"{STORAGE_PREFIX}/{date_prefix}_{run_id}.json"
+    with open(results_path, "rb") as f:
+        files_to_upload.append((results_target, f.read()))
+
+    # Dashboard markdown: always overwrite latest
+    if dashboard_path and os.path.exists(dashboard_path):
+        dashboard_target = f"{STORAGE_PREFIX}/dashboard.md"
+        with open(dashboard_path, "rb") as f:
+            files_to_upload.append((dashboard_target, f.read()))
+
+    # Chart PNG files
+    chart_files = _collect_chart_files(charts_dir)
+    if chart_files:
+        print(f"Found {len(chart_files)} chart PNG(s) to upload")
+        files_to_upload.extend(chart_files)
+
+    print(f"Publishing {len(files_to_upload)} file(s) to {REPO_OWNER}/{REPO_NAME}")
+
+    # Create blobs
+    try:
+        tree_items = create_blobs(REPO_OWNER, REPO_NAME, files_to_upload, token)
+    except Exception as e:
+        if is_rate_limit_error(e):
+            print("Warning: Rate limited during blob creation, skipping")
+            return
+        if is_permission_error(e):
+            print(f"Error: No write permission to {REPO_OWNER}/{REPO_NAME}")
+            sys.exit(1)
+        raise
+
+    # Commit with retry (handle concurrent writes)
+    max_retries = 5
+    retry_delay = 5
+
+    for attempt in range(max_retries):
+        try:
+            branch_sha = get_branch_sha(REPO_OWNER, REPO_NAME, BRANCH, token)
+            tree_sha = get_tree_sha(REPO_OWNER, REPO_NAME, branch_sha, token)
+
+            new_tree_sha = create_tree(
+                REPO_OWNER, REPO_NAME, tree_sha, tree_items, token
+            )
+
+            commit_msg = (
+                f"Diffusion comparison results for run {run_id} (#{run_number})"
+            )
+            commit_sha = create_commit(
+                REPO_OWNER, REPO_NAME, new_tree_sha, branch_sha, commit_msg, token
+            )
+
+            update_branch_ref(REPO_OWNER, REPO_NAME, BRANCH, commit_sha, token)
+            print(
+                f"Successfully published comparison results (commit {commit_sha[:7]})"
+            )
+            return
+
+        except Exception as e:
+            is_retryable = False
+            if hasattr(e, "error_body"):
+                body = getattr(e, "error_body", "")
+                if "Update is not a fast forward" in body:
+                    is_retryable = True
+                elif "Object does not exist" in body:
+                    is_retryable = True
+
+            from urllib.error import HTTPError
+
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                is_retryable = True
+
+            if is_rate_limit_error(e):
+                print("Warning: Rate limited, skipping publish")
+                return
+
+            if is_permission_error(e):
+                print(f"Error: No write permission to {REPO_OWNER}/{REPO_NAME}")
+                sys.exit(1)
+
+            if is_retryable and attempt < max_retries - 1:
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed, retrying in {retry_delay}s..."
+                )
+                time.sleep(retry_delay)
+            else:
+                print(f"Failed to publish after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish SGLang-Diffusion nightly benchmark results to ci-data"
+    )
+    parser.add_argument(
+        "--results",
+        required=True,
+        help="Path to comparison-results.json",
+    )
+    parser.add_argument(
+        "--dashboard",
+        default=None,
+        help="Path to dashboard.md (optional)",
+    )
+    parser.add_argument(
+        "--charts-dir",
+        default=None,
+        help="Directory containing chart PNG files to upload (optional)",
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.results):
+        print(f"Error: Results file not found: {args.results}")
+        sys.exit(1)
+
+    publish_comparison(
+        results_path=args.results,
+        dashboard_path=args.dashboard,
+        charts_dir=args.charts_dir,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/publish_diffusion_gt.py

@@ -0,0 +1,507 @@
+"""
+Publish diffusion CI ground-truth images to sgl-project/ci-data
+via the GitHub API (same pattern as publish_traces.py).
+"""
+
+import argparse
+import base64
+import hashlib
+import io
+import json
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.error import HTTPError
+
+import numpy as np
+from PIL import Image, ImageFilter
+
+# Reuse GitHub API helpers from publish_traces.
+# Support both direct script execution and package-style imports.
+if __package__:
+    from ..publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        make_github_request,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+else:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    from publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        make_github_request,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+
+REPO_OWNER = "sgl-project"
+REPO_NAME = "ci-data"
+BRANCH = "main"
+DEFAULT_TARGET_DIR = "diffusion-ci/consistency_gt/sglang_generated"
+
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
+QUALITY_MAX_SIDE = 256
+LOW_DETAIL_STD_THRESHOLD = 0.075
+LOW_DETAIL_ENTROPY_THRESHOLD = 0.55
+LOW_DETAIL_BLUR_RESIDUAL_THRESHOLD = 0.035
+LOW_DETAIL_GRADIENT_P95_THRESHOLD = 0.045
+RANDOM_NOISE_CORRELATION_THRESHOLD = 0.55
+RANDOM_NOISE_LOW_FREQUENCY_THRESHOLD = 0.20
+RANDOM_NOISE_BLUR_RESIDUAL_THRESHOLD = 0.045
+OLD_NEW_MIN_SSIM = 0.20
+OLD_NEW_MAX_MEAN_ABS_DIFF = 45.0
+
+
+@dataclass(frozen=True)
+class ImageQualityMetrics:
+    luminance_std: float
+    entropy: float
+    blur_residual: float
+    gradient_p95: float
+    neighbor_correlation: float
+    low_frequency_ratio: float
+
+
+@dataclass(frozen=True)
+class OldNewMetrics:
+    ssim: float
+    mean_abs_diff: float
+
+
+def collect_images(source_dir, target_dir):
+    """Collect image files from source_dir and return list of (repo_path, content) tuples."""
+    files = []
+    for entry in sorted(os.listdir(source_dir)):
+        ext = os.path.splitext(entry)[1].lower()
+        if ext not in IMAGE_EXTENSIONS:
+            continue
+        full_path = os.path.join(source_dir, entry)
+        if not os.path.isfile(full_path):
+            continue
+        with open(full_path, "rb") as f:
+            content = f.read()
+        repo_path = f"{target_dir}/{entry}"
+        files.append((repo_path, content))
+    return files
+
+
+def git_blob_sha(content):
+    header = f"blob {len(content)}\0".encode()
+    return hashlib.sha1(header + content).hexdigest()
+
+
+def get_remote_blob_shas(repo_owner, repo_name, target_dir, token):
+    return {
+        path: item["sha"]
+        for path, item in get_remote_image_entries(
+            repo_owner, repo_name, target_dir, token
+        ).items()
+    }
+
+
+def get_remote_image_entries(repo_owner, repo_name, target_dir, token):
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents/"
+        f"{target_dir}?ref={BRANCH}"
+    )
+    try:
+        response = make_github_request(url, token)
+    except HTTPError as e:
+        if e.code == 404:
+            return {}
+        raise
+    entries = json.loads(response)
+    return {
+        item["path"]: item
+        for item in entries
+        if item.get("type") == "file"
+        and "sha" in item
+        and os.path.splitext(item["path"])[1].lower() in IMAGE_EXTENSIONS
+    }
+
+
+def filter_changed_files(files, remote_blob_shas):
+    return [
+        (path, content)
+        for path, content in files
+        if remote_blob_shas.get(path) != git_blob_sha(content)
+    ]
+
+
+def get_remote_blob_content(repo_owner, repo_name, blob_sha, token):
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs/{blob_sha}"
+    response = make_github_request(url, token)
+    blob = json.loads(response)
+    if blob.get("encoding") != "base64":
+        raise ValueError(
+            f"Unexpected blob encoding for {blob_sha}: {blob.get('encoding')}"
+        )
+    return base64.b64decode(blob["content"])
+
+
+def _load_quality_image(content):
+    with Image.open(io.BytesIO(content)) as image:
+        image = image.convert("RGB")
+        image.thumbnail((QUALITY_MAX_SIDE, QUALITY_MAX_SIDE), Image.Resampling.BICUBIC)
+        return image.copy()
+
+
+def _image_to_rgb_array(image):
+    return np.asarray(image, dtype=np.float32)
+
+
+def _luminance(rgb):
+    return 0.299 * rgb[..., 0] + 0.587 * rgb[..., 1] + 0.114 * rgb[..., 2]
+
+
+def _neighbor_correlation(luma):
+    def corr(a, b):
+        a = a.ravel()
+        b = b.ravel()
+        if a.std() < 1e-6 or b.std() < 1e-6:
+            return 1.0
+        return float(np.corrcoef(a, b)[0, 1])
+
+    return (corr(luma[:, 1:], luma[:, :-1]) + corr(luma[1:, :], luma[:-1, :])) / 2
+
+
+def _low_frequency_ratio(luma):
+    centered = luma - luma.mean()
+    power = np.abs(np.fft.fftshift(np.fft.fft2(centered))) ** 2
+    total_power = power.sum()
+    if total_power < 1e-12:
+        return 0.0
+
+    height, width = luma.shape
+    y, x = np.ogrid[:height, :width]
+    center_y = height // 2
+    center_x = width // 2
+    radius = np.sqrt((y - center_y) ** 2 + (x - center_x) ** 2)
+    low_frequency_radius = min(height, width) * 0.08
+    return float(power[radius <= low_frequency_radius].sum() / total_power)
+
+
+def compute_image_quality_metrics(content):
+    image = _load_quality_image(content)
+    rgb = _image_to_rgb_array(image)
+    luma = _luminance(rgb) / 255.0
+
+    gradients = np.concatenate(
+        [
+            np.abs(np.diff(luma, axis=1)).ravel(),
+            np.abs(np.diff(luma, axis=0)).ravel(),
+        ]
+    )
+    histogram, _ = np.histogram(luma, bins=32, range=(0, 1))
+    probabilities = histogram / histogram.sum()
+    nonzero_probabilities = probabilities[probabilities > 0]
+    entropy = float(
+        -(nonzero_probabilities * np.log2(nonzero_probabilities)).sum() / 5.0
+    )
+    blurred = _image_to_rgb_array(image.filter(ImageFilter.GaussianBlur(radius=3)))
+
+    return ImageQualityMetrics(
+        luminance_std=float(luma.std()),
+        entropy=entropy,
+        blur_residual=float(np.mean(np.abs(rgb - blurred)) / 255.0),
+        gradient_p95=float(np.percentile(gradients, 95)),
+        neighbor_correlation=_neighbor_correlation(luma),
+        low_frequency_ratio=_low_frequency_ratio(luma),
+    )
+
+
+def get_quality_failure_reasons(metrics):
+    reasons = []
+    low_detail_static = (
+        metrics.luminance_std < LOW_DETAIL_STD_THRESHOLD
+        and metrics.entropy < LOW_DETAIL_ENTROPY_THRESHOLD
+        and (
+            metrics.blur_residual < LOW_DETAIL_BLUR_RESIDUAL_THRESHOLD
+            or metrics.gradient_p95 < LOW_DETAIL_GRADIENT_P95_THRESHOLD
+        )
+    )
+    high_frequency_noise = (
+        metrics.neighbor_correlation < RANDOM_NOISE_CORRELATION_THRESHOLD
+        and metrics.low_frequency_ratio < RANDOM_NOISE_LOW_FREQUENCY_THRESHOLD
+        and metrics.blur_residual > RANDOM_NOISE_BLUR_RESIDUAL_THRESHOLD
+    )
+    if low_detail_static:
+        reasons.append("low-contrast low-detail output")
+    if high_frequency_noise:
+        reasons.append("high-frequency random noise")
+    return reasons
+
+
+def _resize_for_old_new_compare(content, size=None):
+    with Image.open(io.BytesIO(content)) as image:
+        image = image.convert("RGB")
+        if size is None:
+            image.thumbnail(
+                (QUALITY_MAX_SIDE, QUALITY_MAX_SIDE), Image.Resampling.BICUBIC
+            )
+        else:
+            image = image.resize(size, Image.Resampling.BICUBIC)
+        return _image_to_rgb_array(image)
+
+
+def compute_old_new_metrics(old_content, new_content):
+    old_rgb = _resize_for_old_new_compare(old_content)
+    new_rgb = _resize_for_old_new_compare(
+        new_content, size=(old_rgb.shape[1], old_rgb.shape[0])
+    )
+    old_luma = _luminance(old_rgb) / 255.0
+    new_luma = _luminance(new_rgb) / 255.0
+
+    old_mean = old_luma.mean()
+    new_mean = new_luma.mean()
+    old_variance = old_luma.var()
+    new_variance = new_luma.var()
+    covariance = ((old_luma - old_mean) * (new_luma - new_mean)).mean()
+    c1 = 0.01**2
+    c2 = 0.03**2
+    ssim = (
+        (2 * old_mean * new_mean + c1)
+        * (2 * covariance + c2)
+        / ((old_mean**2 + new_mean**2 + c1) * (old_variance + new_variance + c2))
+    )
+
+    return OldNewMetrics(
+        ssim=float(ssim),
+        mean_abs_diff=float(np.abs(old_rgb - new_rgb).mean()),
+    )
+
+
+def _format_quality_metrics(metrics):
+    return (
+        f"std={metrics.luminance_std:.4f}, entropy={metrics.entropy:.4f}, "
+        f"blur_residual={metrics.blur_residual:.4f}, "
+        f"gradient_p95={metrics.gradient_p95:.4f}, "
+        f"neighbor_corr={metrics.neighbor_correlation:.4f}, "
+        f"low_freq={metrics.low_frequency_ratio:.4f}"
+    )
+
+
+def _format_old_new_metrics(metrics):
+    return f"ssim={metrics.ssim:.4f}, mean_abs_diff={metrics.mean_abs_diff:.2f}"
+
+
+def validate_gt_files(files_to_upload, changed_files, remote_image_entries, token):
+    failures = []
+    for path, content in files_to_upload:
+        quality_metrics = compute_image_quality_metrics(content)
+        quality_reasons = get_quality_failure_reasons(quality_metrics)
+        if quality_reasons:
+            failures.append(
+                f"{path}: {', '.join(quality_reasons)} "
+                f"({_format_quality_metrics(quality_metrics)})"
+            )
+
+    for path, content in changed_files:
+        remote_entry = remote_image_entries.get(path)
+        if not remote_entry:
+            continue
+
+        old_content = get_remote_blob_content(
+            REPO_OWNER, REPO_NAME, remote_entry["sha"], token
+        )
+        old_quality_metrics = compute_image_quality_metrics(old_content)
+        old_quality_reasons = get_quality_failure_reasons(old_quality_metrics)
+        if old_quality_reasons:
+            print(
+                f"Skipping old/new drift check for {path} because existing GT is "
+                f"already suspicious: {', '.join(old_quality_reasons)} "
+                f"({_format_quality_metrics(old_quality_metrics)})"
+            )
+            continue
+
+        old_new_metrics = compute_old_new_metrics(old_content, content)
+        if (
+            old_new_metrics.ssim < OLD_NEW_MIN_SSIM
+            and old_new_metrics.mean_abs_diff > OLD_NEW_MAX_MEAN_ABS_DIFF
+        ):
+            failures.append(
+                f"{path}: changed too far from existing GT "
+                f"({_format_old_new_metrics(old_new_metrics)})"
+            )
+
+    if not failures:
+        print(
+            f"GT quality gate passed for {len(files_to_upload)} generated image(s) "
+            f"and {len(changed_files)} changed image(s)."
+        )
+        return
+
+    print("GT quality gate failed; refusing to publish suspicious image updates:")
+    for failure in failures:
+        print(f"  - {failure}")
+    sys.exit(1)
+
+
+def check_quality(source_dir, target_dir=None):
+    target_dir = target_dir or DEFAULT_TARGET_DIR
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    files_to_upload = collect_images(source_dir, target_dir)
+    if not files_to_upload:
+        print(f"No image files found in {source_dir}")
+        return
+
+    remote_image_entries = get_remote_image_entries(
+        REPO_OWNER, REPO_NAME, target_dir, token
+    )
+    remote_blob_shas = {
+        path: item["sha"] for path, item in remote_image_entries.items()
+    }
+    changed_files = filter_changed_files(files_to_upload, remote_blob_shas)
+    validate_gt_files(files_to_upload, changed_files, remote_image_entries, token)
+
+
+def publish(source_dir, target_dir=None):
+    target_dir = target_dir or DEFAULT_TARGET_DIR
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    files_to_upload = collect_images(source_dir, target_dir)
+    if not files_to_upload:
+        print(f"No image files found in {source_dir}")
+        return
+
+    print(
+        f"Found {len(files_to_upload)} image(s) to upload to {REPO_OWNER}/{REPO_NAME}/{target_dir}"
+    )
+
+    # Verify token
+    perm = verify_token_permissions(REPO_OWNER, REPO_NAME, token)
+    if perm == "rate_limited":
+        print("GitHub API rate-limited, skipping upload.")
+        return
+    if not perm:
+        print("Token permission verification failed.")
+        sys.exit(1)
+
+    # Commit with retry (handle concurrent pushes)
+    max_retries = 5
+    for attempt in range(max_retries):
+        try:
+            branch_sha = get_branch_sha(REPO_OWNER, REPO_NAME, BRANCH, token)
+            tree_sha = get_tree_sha(REPO_OWNER, REPO_NAME, branch_sha, token)
+            remote_image_entries = get_remote_image_entries(
+                REPO_OWNER, REPO_NAME, target_dir, token
+            )
+            remote_blob_shas = {
+                path: item["sha"] for path, item in remote_image_entries.items()
+            }
+            changed_files = filter_changed_files(files_to_upload, remote_blob_shas)
+            validate_gt_files(
+                files_to_upload, changed_files, remote_image_entries, token
+            )
+            if not changed_files:
+                print("No image changes to publish.")
+                return
+
+            try:
+                tree_items = create_blobs(REPO_OWNER, REPO_NAME, changed_files, token)
+            except Exception as e:
+                if is_rate_limit_error(e):
+                    print("Rate-limited during blob creation, skipping.")
+                    return
+                if is_permission_error(e):
+                    print(
+                        f"ERROR: Token lacks write permission to {REPO_OWNER}/{REPO_NAME}. "
+                        "Update GH_PAT_FOR_NIGHTLY_CI_DATA with a token that has contents:write."
+                    )
+                    sys.exit(1)
+                raise
+
+            new_tree_sha = create_tree(
+                REPO_OWNER, REPO_NAME, tree_sha, tree_items, token
+            )
+            if new_tree_sha == tree_sha:
+                print("No tree changes to publish.")
+                return
+
+            commit_msg = f"diffusion-ci: update images in {target_dir} ({len(changed_files)} files) [automated]"
+            commit_sha = create_commit(
+                REPO_OWNER, REPO_NAME, new_tree_sha, branch_sha, commit_msg, token
+            )
+            update_branch_ref(REPO_OWNER, REPO_NAME, BRANCH, commit_sha, token)
+            print(
+                f"Successfully pushed {len(changed_files)} changed images (commit {commit_sha[:10]})"
+            )
+            return
+        except Exception as e:
+            if is_rate_limit_error(e):
+                print("Rate-limited, skipping.")
+                return
+            if is_permission_error(e):
+                print(f"ERROR: permission denied to {REPO_OWNER}/{REPO_NAME}")
+                sys.exit(1)
+
+            retryable = False
+            if hasattr(e, "error_body"):
+                if "Update is not a fast forward" in e.error_body:
+                    retryable = True
+                elif "Object does not exist" in e.error_body:
+                    retryable = True
+
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                retryable = True
+
+            if retryable and attempt < max_retries - 1:
+                import time
+
+                wait = 2**attempt
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed, retrying in {wait}s..."
+                )
+                time.sleep(wait)
+            else:
+                print(f"Failed after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish diffusion GT images to GitHub"
+    )
+    parser.add_argument(
+        "--source-dir", required=True, help="Directory containing GT images"
+    )
+    parser.add_argument(
+        "--target-dir",
+        required=False,
+        default=None,
+        help=f"Target directory in the remote repo (default: {DEFAULT_TARGET_DIR})",
+    )
+    parser.add_argument(
+        "--check-only",
+        action="store_true",
+        help="Validate generated GT images without publishing them",
+    )
+    args = parser.parse_args()
+    if args.check_only:
+        check_quality(args.source_dir, args.target_dir)
+    else:
+        publish(args.source_dir, args.target_dir)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/save_diffusion_metrics.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""Collect and save diffusion performance metrics for artifact collection in CI.
+
+This script reads diffusion test results from the pytest stash and saves them
+with metadata for the performance dashboard.
+
+Usage:
+    python3 scripts/ci/utils/diffusion/save_diffusion_metrics.py \
+        --gpu-config 1-gpu-h100 \
+        --run-id 12345678 \
+        --output test/diffusion-metrics-1gpu.json \
+        --results-json test/diffusion-results.json
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def load_diffusion_results(results_file: str) -> list[dict]:
+    """Load diffusion performance results from JSON file."""
+    if not os.path.exists(results_file):
+        print(f"Warning: Results file not found: {results_file}")
+        return []
+
+    try:
+        with open(results_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return data if isinstance(data, list) else [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {results_file}: {e}")
+        return []
+
+
+def transform_diffusion_result(result: dict, gpu_config: str) -> dict:
+    """Transform a diffusion result to match dashboard expectations.
+
+    Dashboard expects:
+    - Separate test_name, class_name
+    - Numeric metrics in consistent units
+    - Optional modality field
+    """
+    return {
+        "test_name": result.get("test_name"),
+        "class_name": result.get("class_name"),
+        "modality": result.get("modality", "image"),
+        "e2e_ms": result.get("e2e_ms"),
+        "avg_denoise_ms": result.get("avg_denoise_ms"),
+        "median_denoise_ms": result.get("median_denoise_ms"),
+        "stage_metrics": result.get("stage_metrics", {}),
+        "sampled_steps": result.get("sampled_steps", {}),
+        # Video-specific metrics (if present)
+        "frames_per_second": result.get("frames_per_second"),
+        "total_frames": result.get("total_frames"),
+        "avg_frame_time_ms": result.get("avg_frame_time_ms"),
+    }
+
+
+def group_results_by_class(results: list[dict], gpu_config: str) -> list[dict]:
+    """Group diffusion results by test class (suite).
+
+    Returns list with one entry per test class, containing all tests in that class.
+    """
+    groups = {}
+
+    for result in results:
+        class_name = result.get("class_name", "unknown")
+
+        if class_name not in groups:
+            groups[class_name] = {
+                "gpu_config": gpu_config,
+                "test_suite": class_name,
+                "tests": [],
+            }
+
+        transformed = transform_diffusion_result(result, gpu_config)
+        groups[class_name]["tests"].append(transformed)
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    run_id: str,
+    output_file: str,
+    results_file: str,
+) -> bool:
+    """Collect diffusion metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Load diffusion results
+    raw_results = load_diffusion_results(results_file)
+    print(f"Loaded {len(raw_results)} diffusion test result(s)")
+
+    # Group by test class
+    grouped = group_results_by_class(raw_results, gpu_config)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "test_type": "diffusion",
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not raw_results:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved diffusion metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect diffusion performance metrics from test results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 1-gpu-h100, 2-gpu-h100)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--results-json",
+        required=True,
+        help="Path to diffusion results JSON file",
+    )
+
+    args = parser.parse_args()
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        run_id=args.run_id,
+        output_file=args.output,
+        results_file=args.results_json,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/verify_diffusion_coverage.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python3
+"""
+Verify 100% coverage of diffusion test cases.
+
+This script checks that all expected test cases were executed across all partitions.
+Designed to run in the CI summary job after all partition jobs complete.
+
+Usage:
+    python scripts/ci/utils/diffusion/verify_diffusion_coverage.py --reports-dir <path>
+
+Exit codes:
+    0 - All cases executed (100% coverage)
+    1 - Missing cases detected (coverage < 100%)
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from diffusion_case_parser import (
+    BASELINE_REL_PATH,
+    RUN_SUITE_REL_PATH,
+    collect_diffusion_suites,
+    resolve_case_config_path,
+)
+
+DYNAMIC_SUITES = {"1-gpu", "2-gpu"}
+
+
+def load_execution_reports(reports_dir: Path) -> list[dict]:
+    """Load all execution report JSON files from the given directory."""
+    reports = []
+    for json_file in reports_dir.glob("**/execution_report_*.json"):
+        with open(json_file, "r", encoding="utf-8") as f:
+            reports.append(json.load(f))
+    return reports
+
+
+def get_expected_cases(repo_root: Path) -> dict[str, set[str]]:
+    """
+    Get all expected cases from case config and run_suite.py.
+
+    Returns:
+        Dictionary mapping suite name to set of expected case IDs.
+        Standalone files are represented as "standalone:<filename>".
+    """
+    baseline_path = repo_root / BASELINE_REL_PATH
+    run_suite_path = repo_root / RUN_SUITE_REL_PATH
+    case_config_path = resolve_case_config_path(repo_root, run_suite_path)
+
+    suites = collect_diffusion_suites(
+        case_config_path,
+        run_suite_path,
+        baseline_path,
+    )
+
+    expected = {}
+    for suite_name, suite_info in suites.items():
+        if suite_name not in DYNAMIC_SUITES:
+            continue
+        case_ids = set(case.case_id for case in suite_info.cases)
+        # Add standalone files as special case IDs
+        for standalone_file in suite_info.standalone_files:
+            case_ids.add(f"standalone:{standalone_file}")
+        expected[suite_name] = case_ids
+
+    empty_dynamic_suites = [
+        suite_name
+        for suite_name in DYNAMIC_SUITES
+        if suite_name in expected
+        and not any(
+            not case_id.startswith("standalone:") for case_id in expected[suite_name]
+        )
+    ]
+    if empty_dynamic_suites:
+        raise RuntimeError(
+            "Parsed zero parametrized cases for diffusion suites: "
+            + ", ".join(sorted(empty_dynamic_suites))
+            + ". Refuse to pass coverage verification."
+        )
+
+    return expected
+
+
+def collect_executed_cases(reports: list[dict]) -> dict[str, set[str]]:
+    """
+    Collect all executed cases from execution reports.
+
+    Returns:
+        Dictionary mapping suite name to set of executed case IDs.
+    """
+    executed = {}
+    for report in reports:
+        suite = report["suite"]
+        if suite not in executed:
+            executed[suite] = set()
+
+        executed_cases = report.get("executed_cases", [])
+        if executed_cases:
+            executed[suite].update(executed_cases)
+        elif report["is_standalone"]:
+            standalone_file = report["standalone_file"]
+            executed[suite].add(f"standalone:{standalone_file}")
+
+    return executed
+
+
+def collect_case_results(reports: list[dict]) -> dict[str, dict[str, str]]:
+    """
+    Collect case results (pass/fail/error status) from execution reports.
+
+    Returns:
+        Dictionary mapping suite name to {case_id: status} dictionary.
+    """
+    results = {}
+    for report in reports:
+        suite = report["suite"]
+        if suite not in results:
+            results[suite] = {}
+
+        # Get case_results from report (empty dict for legacy reports)
+        case_results = report.get("case_results", {})
+        results[suite].update(case_results)
+
+    return results
+
+
+def collect_missing_standalone_estimates(reports: list[dict]) -> dict[str, set[str]]:
+    missing_by_suite: dict[str, set[str]] = {}
+    for report in reports:
+        suite = report["suite"]
+        missing = report.get("missing_standalone_estimates", [])
+        if not missing:
+            continue
+        missing_by_suite.setdefault(suite, set()).update(missing)
+    return missing_by_suite
+
+
+def collect_standalone_measurements(reports: list[dict]) -> dict[tuple[str, str], dict]:
+    measurements: dict[tuple[str, str], dict] = {}
+    for report in reports:
+        for measurement in report.get("standalone_measurements", []):
+            key = (measurement["suite"], measurement["standalone_file"])
+            measurements[key] = measurement
+    return measurements
+
+
+def print_missing_standalone_estimates_summary(
+    missing_by_suite: dict[str, set[str]],
+    measurements: dict[tuple[str, str], dict],
+) -> None:
+    if not missing_by_suite:
+        return
+
+    print("\n" + "=" * 60)
+    print(
+        "Add standalone estimate(s) to "
+        "python/sglang/multimodal_gen/test/run_suite.py"
+    )
+    print("=" * 60)
+    print("The following standalone file(s) used fallback estimate 300.0s.")
+    print("Update STANDALONE_FILE_EST_TIMES with the measured runtime below:\n")
+
+    for suite in sorted(missing_by_suite):
+        print(f'"{suite}": {{')
+        for standalone_file in sorted(missing_by_suite[suite]):
+            measurement = measurements.get((suite, standalone_file))
+            measured_time = (
+                measurement["measured_full_test_time_s"] if measurement else 300.0
+            )
+            print(f'    "{standalone_file}": {measured_time:.1f},')
+        print("}\n")
+
+
+def verify_coverage(
+    expected: dict[str, set[str]],
+    executed: dict[str, set[str]],
+) -> tuple[bool, dict[str, set[str]]]:
+    """
+    Verify that all expected cases were executed.
+
+    Returns:
+        Tuple of (is_complete, missing_cases_by_suite)
+    """
+    missing = {}
+    for suite, expected_cases in expected.items():
+        executed_cases = executed.get(suite, set())
+        suite_missing = expected_cases - executed_cases
+        if suite_missing:
+            missing[suite] = suite_missing
+
+    return len(missing) == 0, missing
+
+
+def print_results_summary(
+    case_results: dict[str, dict[str, str]],
+) -> tuple[int, int, int]:
+    """
+    Print test results summary and return counts.
+
+    Returns:
+        Tuple of (passed_count, failed_count, error_count)
+    """
+    # Check if we have any results data
+    total_results = sum(len(results) for results in case_results.values())
+    if total_results == 0:
+        print("\nTest Results: No results data available (legacy reports)")
+        return (0, 0, 0)
+
+    # Count by status
+    passed_count = 0
+    failed_count = 0
+    error_count = 0
+    failed_cases: dict[str, list[str]] = {}
+
+    for suite, results in case_results.items():
+        for case_id, status in results.items():
+            if status == "pass":
+                passed_count += 1
+            elif status == "fail":
+                failed_count += 1
+                if suite not in failed_cases:
+                    failed_cases[suite] = []
+                failed_cases[suite].append(case_id)
+            elif status == "error":
+                error_count += 1
+                if suite not in failed_cases:
+                    failed_cases[suite] = []
+                failed_cases[suite].append(f"{case_id} (error)")
+
+    # Print summary
+    total = passed_count + failed_count + error_count
+    print("\n" + "=" * 60)
+    print("Test Results Summary")
+    print("=" * 60)
+    print(f"  Total executed: {total}")
+    print(f"  ✅ Passed: {passed_count}")
+    print(f"  ❌ Failed: {failed_count}")
+    if error_count > 0:
+        print(f"  ⚠️  Errors: {error_count}")
+
+    # Print failed cases if any
+    if failed_cases:
+        print("\nFailed cases:")
+        for suite, cases in sorted(failed_cases.items()):
+            print(f"  {suite}:")
+            for case_id in sorted(cases):
+                print(f"    - {case_id}")
+
+    return (passed_count, failed_count, error_count)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Verify 100% coverage of diffusion test cases"
+    )
+    parser.add_argument(
+        "--reports-dir",
+        type=str,
+        required=True,
+        help="Directory containing execution report JSON files",
+    )
+    args = parser.parse_args()
+
+    # Determine repository root
+    script_dir = Path(__file__).resolve().parent
+    repo_root = script_dir.parent.parent.parent.parent
+
+    reports_dir = Path(args.reports_dir)
+
+    print("=" * 60)
+    print("Diffusion CI Coverage Verification")
+    print("=" * 60)
+
+    # Load execution reports
+    reports = load_execution_reports(reports_dir)
+    print(f"\nLoaded {len(reports)} execution reports")
+
+    if not reports:
+        print("\nERROR: No execution reports found!")
+        print(f"Expected reports in: {reports_dir}")
+        sys.exit(1)
+
+    # Get expected cases
+    try:
+        expected = get_expected_cases(repo_root)
+    except (RuntimeError, FileNotFoundError) as exc:
+        print(f"\nERROR: {exc}")
+        sys.exit(1)
+    print("\nExpected cases by suite:")
+    for suite, cases in expected.items():
+        print(f"  {suite}: {len(cases)} cases")
+
+    # Collect executed cases
+    executed = collect_executed_cases(reports)
+    print("\nExecuted cases by suite:")
+    for suite, cases in executed.items():
+        print(f"  {suite}: {len(cases)} cases")
+
+    # Collect case results
+    case_results = collect_case_results(reports)
+    missing_standalone_estimates = collect_missing_standalone_estimates(reports)
+    standalone_measurements = collect_standalone_measurements(reports)
+
+    # Verify coverage
+    is_complete, missing = verify_coverage(expected, executed)
+
+    if is_complete:
+        print("\n" + "=" * 60)
+        print("✅ COVERAGE: 100% - All test cases executed")
+        print("=" * 60)
+    else:
+        print("\n" + "=" * 60)
+        print("❌ COVERAGE FAILURE: Missing test cases detected")
+        print("=" * 60)
+        for suite, cases in missing.items():
+            print(f"\n{suite.upper()} suite - Missing {len(cases)} case(s):")
+            for case_id in sorted(cases):
+                print(f"  - {case_id}")
+
+    # Print test results summary
+    passed_count, failed_count, error_count = print_results_summary(case_results)
+    print_missing_standalone_estimates_summary(
+        missing_standalone_estimates, standalone_measurements
+    )
+
+    # Exit with appropriate code
+    if not is_complete:
+        sys.exit(1)
+    elif missing_standalone_estimates:
+        sys.exit(1)
+    elif failed_count > 0 or error_count > 0:
+        print("\n" + "=" * 60)
+        print("⚠️  WARNING: Some tests failed but coverage is complete")
+        print("=" * 60)
+        sys.exit(0)  # Coverage is complete, failures are visible in results
+    else:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/docker_build_metadata_args.py

@@ -0,0 +1,119 @@
+import argparse
+import datetime
+import json
+import sys
+
+MOVING_TAGS = {"dev", "dev-cu12", "dev-cu13", "latest"}
+
+
+def render_tag_template(tag: str, version: str, date: str, short_sha: str) -> str:
+    return (
+        tag.replace("{version}", version)
+        .replace("{date}", date)
+        .replace("{short_sha}", short_sha)
+    )
+
+
+def is_moving_tag(tag: str) -> bool:
+    return tag in MOVING_TAGS or tag.startswith("latest-")
+
+
+def select_tag(
+    tag_config: str, cuda: str, version: str, date: str, short_sha: str
+) -> str:
+    entries = json.loads(tag_config)
+    for entry in entries:
+        if entry.get("cuda") != cuda:
+            continue
+
+        tags = [
+            render_tag_template(tag, version, date, short_sha)
+            for tag in entry.get("tags", [])
+        ]
+        if not tags:
+            raise ValueError(f"No tags configured for CUDA variant {cuda}")
+
+        for tag in tags:
+            if not is_moving_tag(tag):
+                return tag
+        return tags[0]
+
+    raise ValueError(f"CUDA variant {cuda} not found in tag_config")
+
+
+def build_arg_tokens(
+    *,
+    cuda: str,
+    tag_config: str,
+    image_repo: str,
+    version: str,
+    build_commit: str,
+    build_url: str,
+    date: str,
+    short_sha: str,
+) -> list[str]:
+    image_tag = select_tag(tag_config, cuda, version, date, short_sha)
+    build_args = {
+        "SGLANG_BUILD_COMMIT": build_commit,
+        "SGLANG_BUILD_URL": build_url,
+        "SGLANG_IMAGE_TAG": f"{image_repo}:{image_tag}",
+    }
+
+    tokens = []
+    for key, value in build_args.items():
+        tokens.extend(["--build-arg", f"{key}={value}"])
+    return tokens
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Emit docker build arguments for SGLang image metadata."
+    )
+    parser.add_argument("--cuda", required=True, help="CUDA variant from tag_config.")
+    parser.add_argument("--tag-config", required=True, help="Docker tag JSON config.")
+    parser.add_argument("--image-repo", required=True, help="Docker image repository.")
+    parser.add_argument("--sgl-version", default="", help="SGLang release version.")
+    parser.add_argument(
+        "--build-commit",
+        required=True,
+        help="Commit checked out for the Docker build.",
+    )
+    parser.add_argument("--build-url", default="", help="CI run URL.")
+    parser.add_argument(
+        "--date",
+        default=datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d"),
+        help="Date used for {date} tag templates.",
+    )
+    parser.add_argument(
+        "--short-sha",
+        default="",
+        help="Short SHA used for {short_sha}; defaults to build commit prefix.",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    short_sha = args.short_sha or args.build_commit[:8]
+
+    try:
+        tokens = build_arg_tokens(
+            cuda=args.cuda,
+            tag_config=args.tag_config,
+            image_repo=args.image_repo,
+            version=args.sgl_version,
+            build_commit=args.build_commit,
+            build_url=args.build_url,
+            date=args.date,
+            short_sha=short_sha,
+        )
+    except (json.JSONDecodeError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    print("\n".join(tokens))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/utils/install_protoc.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Ensure protoc is installed for router build (gRPC protobuf compilation).
+set -euxo pipefail
+
+if command -v protoc >/dev/null 2>&1 && protoc --version >/dev/null 2>&1; then
+    echo "protoc already installed: $(protoc --version)"
+    exit 0
+fi
+
+if command -v protoc >/dev/null 2>&1; then
+    echo "protoc found but not runnable, reinstalling..."
+else
+    echo "protoc not found, installing..."
+fi
+
+ARCH=$(uname -m)
+
+if command -v apt-get &> /dev/null; then
+    # Ubuntu/Debian
+    apt-get update || true # May fail due to unrelated broken packages
+    PROTOC_APT_PACKAGES=(wget unzip)
+    apt-get install -y --no-install-recommends "${PROTOC_APT_PACKAGES[@]}" || {
+        echo "Warning: apt-get install failed, checking if required packages are available..."
+        for pkg in "${PROTOC_APT_PACKAGES[@]}"; do
+            if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+                echo "ERROR: Required package $pkg is not installed and apt-get failed"
+                exit 1
+            fi
+        done
+        echo "All required packages are already installed, continuing..."
+    }
+elif command -v yum &> /dev/null; then
+    # RHEL/CentOS
+    yum update -y
+    yum install -y wget unzip
+else
+    echo "ERROR: Neither apt-get nor yum found; cannot install protoc"
+    exit 1
+fi
+
+if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+    PROTOC_ARCH="aarch_64"
+else
+    PROTOC_ARCH="x86_64"
+fi
+PROTOC_ZIP="protoc-32.0-linux-${PROTOC_ARCH}.zip"
+(
+    cd /tmp
+    wget "https://github.com/protocolbuffers/protobuf/releases/download/v32.0/${PROTOC_ZIP}"
+    unzip -o "${PROTOC_ZIP}" -d /usr/local
+    rm -f "${PROTOC_ZIP}"
+)
+protoc --version

scripts/ci/utils/install_rust_protoc.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Install protoc and a Rust toolchain (rustup/cargo). Required by setuptools-rust
+# to build the bundled native gRPC extension (rust/sglang-grpc) when installing
+# the main `sglang` wheel from source. Idempotent — both helpers no-op if
+# already installed.
+#
+# protoc installs system-wide (/usr/local) and apt deps, so it needs root.
+# rustup installs per-user under $HOME/.cargo, so it must run as the calling
+# user (running it under sudo would put cargo in /root/.cargo and the rest of
+# the job wouldn't find it).
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [ "$(id -u)" = "0" ]; then
+    SUDO=""
+elif command -v sudo >/dev/null 2>&1; then
+    SUDO="sudo"
+else
+    SUDO=""
+fi
+
+${SUDO} bash "${SCRIPT_DIR}/install_protoc.sh"
+bash "${SCRIPT_DIR}/install_rustup.sh"

scripts/ci/utils/install_rustup.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Ensure a Rust toolchain (rustc/cargo) is installed for crates built from
+# source, e.g. the native gRPC extension bundled into the sglang wheel via
+# setuptools-rust. Minimum supported version is 1.85 (edition 2024).
+set -euxo pipefail
+
+# Make cargo/rustc visible to the rest of this shell and to subsequent
+# GitHub Actions steps in the same job.
+export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}"
+if [ -n "${GITHUB_PATH:-}" ]; then
+    # Self-heal if _runner_file_commands/ disappears mid-job on some self-hosted
+    # runners; the runner reads this file by its registered UUID at step end, so
+    # recreating the path keeps PATH propagation working for subsequent steps.
+    mkdir -p "$(dirname "${GITHUB_PATH}")" 2>/dev/null || true
+    echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> "${GITHUB_PATH}" || true
+fi
+
+if command -v cargo >/dev/null 2>&1 && command -v rustc >/dev/null 2>&1; then
+    echo "rust already installed: $(rustc --version), $(cargo --version)"
+    exit 0
+fi
+
+echo "rust not found, installing via rustup..."
+
+# rustup.rs requires curl — make sure it's present.
+if ! command -v curl >/dev/null 2>&1; then
+    if command -v apt-get &> /dev/null; then
+        apt-get update || true
+        apt-get install -y --no-install-recommends curl ca-certificates
+    elif command -v yum &> /dev/null; then
+        yum install -y curl ca-certificates
+    else
+        echo "ERROR: curl is required to install rustup, but no supported package manager was found"
+        exit 1
+    fi
+fi
+
+if [ -n "${RUSTUP_CACHE_URL:-}" ]; then
+    # An in-cluster HTTP mirror is available (e.g. on NPU runners).
+    export RUSTUP_DIST_SERVER="${RUSTUP_CACHE_URL}/rustup"
+    export RUSTUP_UPDATE_ROOT="${RUSTUP_CACHE_URL}/rustup/rustup"
+    case "$(uname -m)" in
+        x86_64)  RUSTUP_ARCH="x86_64-unknown-linux-gnu" ;;
+        aarch64) RUSTUP_ARCH="aarch64-unknown-linux-gnu" ;;
+        *) echo "ERROR: unsupported arch $(uname -m)"; exit 1 ;;
+    esac
+    RUSTUP_TMP="$(mktemp -d)"
+    trap 'rm -rf "${RUSTUP_TMP}"' EXIT
+    curl --retry 3 --retry-delay 2 -sSfL \
+        "${RUSTUP_UPDATE_ROOT}/dist/${RUSTUP_ARCH}/rustup-init" \
+        -o "${RUSTUP_TMP}/rustup-init"
+    chmod +x "${RUSTUP_TMP}/rustup-init"
+    "${RUSTUP_TMP}/rustup-init" -y --no-modify-path
+else
+    curl --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 -sSf https://sh.rustup.rs \
+        | sh -s -- -y --no-modify-path
+fi
+
+rustc --version
+cargo --version

scripts/ci/utils/merge_metrics.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Merge per-partition metrics into a consolidated metrics file.
+
+This script reads all per-partition metric JSON files and consolidates them
+into a single JSON file with run-level metadata.
+
+Usage:
+    python3 scripts/ci/utils/merge_metrics.py \
+        --input-dir metrics/ \
+        --output consolidated-metrics-12345678.json \
+        --run-id 12345678 \
+        --commit-sha abc123def456
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_partition_files(input_dir: str) -> list[str]:
+    """Find all partition metric files in the input directory."""
+    patterns = [
+        os.path.join(input_dir, "**/metrics-*.json"),
+        os.path.join(input_dir, "**/diffusion-metrics-*.json"),
+        os.path.join(input_dir, "**/comparison-metrics-*.json"),
+    ]
+    files = set()
+    for pattern in patterns:
+        files.update(glob.glob(pattern, recursive=True))
+    return list(files)
+
+
+def load_partition_metrics(filepath: str) -> dict | None:
+    """Load a partition metrics file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to load {filepath}: {e}")
+        return None
+
+
+def merge_metrics(
+    input_dir: str,
+    output_file: str,
+    run_id: str,
+    commit_sha: str,
+    branch: str | None = None,
+) -> bool:
+    """Merge all partition metrics into a consolidated file."""
+    run_date = datetime.now(timezone.utc).isoformat()
+
+    # Find all partition files
+    partition_files = find_partition_files(input_dir)
+    print(f"Found {len(partition_files)} partition file(s)")
+
+    all_results = []
+    if not partition_files:
+        print("No partition metrics files found")
+    else:
+        # Load all partition files
+        for filepath in sorted(partition_files):
+            print(f"  Reading: {filepath}")
+            metrics = load_partition_metrics(filepath)
+            if metrics and "results" in metrics:
+                all_results.extend(metrics["results"])
+        print(f"Total results collected: {len(all_results)}")
+
+    # Create consolidated structure
+    consolidated = {
+        "run_id": run_id,
+        "run_date": run_date,
+        "commit_sha": commit_sha,
+        "branch": branch,
+        "results": all_results,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(consolidated, f, indent=2)
+
+        if not partition_files:
+            print(f"Created empty consolidated file: {output_file}")
+        else:
+            print(f"Saved consolidated metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing consolidated file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Merge per-partition metrics into consolidated file"
+    )
+    parser.add_argument(
+        "--input-dir",
+        required=True,
+        help="Directory containing partition metric files",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for consolidated metrics JSON",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--commit-sha",
+        required=True,
+        help="Git commit SHA",
+    )
+    parser.add_argument(
+        "--branch",
+        default=None,
+        help="Git branch name (optional)",
+    )
+
+    args = parser.parse_args()
+
+    success = merge_metrics(
+        input_dir=args.input_dir,
+        output_file=args.output,
+        run_id=args.run_id,
+        commit_sha=args.commit_sha,
+        branch=args.branch,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/prevalidate_cached_models.py

@@ -0,0 +1,407 @@
+#!/usr/bin/env python3
+"""
+Pre-validate all cached HuggingFace models to provide detailed feedback.
+
+This script runs once during CI initialization (in prepare_runner.sh) to:
+1. Scan snapshots in ~/.cache/huggingface/hub/ (with time/quantity limits)
+2. Validate completeness (config/tokenizer/weights)
+3. Output detailed failure reasons for debugging
+
+NOTE: This script no longer writes shared validation markers. Each test run
+independently validates its cache using per-run markers to avoid cross-runner
+cache state pollution.
+"""
+
+import glob
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Add python directory to path to import sglang modules
+REPO_ROOT = Path(__file__).parent.parent.parent.parent
+sys.path.insert(0, str(REPO_ROOT / "python"))
+
+from sglang.srt.model_loader.ci_weight_validation import (  # noqa: E402
+    _validate_diffusion_model,
+    validate_cache_with_detailed_reason,
+)
+
+# Limits to avoid spending too much time on validation
+MAX_VALIDATION_TIME_SECONDS = 300  # Max 5 minutes total
+
+
+def find_all_hf_snapshots():
+    """
+    Find all HuggingFace snapshots in cache.
+
+    Returns:
+        List of (model_name, snapshot_dir) tuples, sorted by mtime (newest first)
+    """
+    hf_home = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface"))
+    hub_dir = os.path.join(hf_home, "hub")
+
+    if not os.path.isdir(hub_dir):
+        print(f"HF hub directory not found: {hub_dir}")
+        return []
+
+    snapshots = []
+
+    # Pattern: models--org--model/snapshots/hash
+    for model_dir in glob.glob(os.path.join(hub_dir, "models--*")):
+        # Extract model name from directory (models--org--model -> org/model)
+        dir_name = os.path.basename(model_dir)
+        if not dir_name.startswith("models--"):
+            continue
+
+        # models--meta-llama--Llama-2-7b-hf -> meta-llama/Llama-2-7b-hf
+        # Handle multi-part names: models--a--b--c -> a/b-c (join parts 1+ with /)
+        parts = dir_name.split("--")
+        if len(parts) < 3 or parts[0] != "models":
+            # Invalid format, skip
+            continue
+        # Standard format: models--org--repo -> org/repo
+        # Extended format: models--org--repo--extra -> org/repo-extra (join with -)
+        model_name = parts[1] + "/" + "-".join(parts[2:])
+
+        snapshots_dir = os.path.join(model_dir, "snapshots")
+        if not os.path.isdir(snapshots_dir):
+            continue
+
+        # Find all snapshot hashes
+        for snapshot_hash_dir in os.listdir(snapshots_dir):
+            snapshot_path = os.path.join(snapshots_dir, snapshot_hash_dir)
+            if os.path.isdir(snapshot_path):
+                try:
+                    mtime = os.path.getmtime(snapshot_path)
+                    snapshots.append((model_name, snapshot_path, mtime))
+                except OSError:
+                    continue
+
+    # Sort by mtime (newest first) - prioritize recently used models
+    snapshots.sort(key=lambda x: x[2], reverse=True)
+
+    # Return without mtime
+    return [(name, path) for name, path, _ in snapshots]
+
+
+def is_transformers_text_model(snapshot_dir):
+    """
+    Check if a snapshot is a transformers text model.
+
+    Only excludes (returns False) for models with STRONG evidence of being
+    diffusers/generation pipelines. Uses conservative heuristics to avoid
+    false negatives on multimodal LLMs with tokenizers.
+
+    Args:
+        snapshot_dir: Path to snapshot directory
+
+    Returns:
+        True if this looks like a transformers text model, False otherwise (N/A)
+    """
+    # Check for diffusers pipeline markers (strong evidence)
+    diffusers_markers = [
+        "model_index.json",  # Diffusers pipeline config
+        "scheduler",  # Scheduler directory (diffusers)
+    ]
+    if any(
+        os.path.exists(os.path.join(snapshot_dir, marker))
+        for marker in diffusers_markers
+    ):
+        return False
+
+    config_path = os.path.join(snapshot_dir, "config.json")
+    if not os.path.exists(config_path):
+        # No config.json - likely not a transformers model
+        return False
+
+    try:
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = json.load(f)
+
+        # Check for explicit diffusers/generation model types (conservative keywords)
+        model_type = config.get("_class_name") or config.get("model_type")
+        if model_type:
+            model_type_lower = str(model_type).lower()
+            # Only exclude clear diffusion/generation models
+            if any(
+                keyword in model_type_lower
+                for keyword in [
+                    "diffusion",
+                    "unet",
+                    "vae",
+                    "controlnet",
+                    "stable-diffusion",
+                    "latent-diffusion",
+                ]
+            ):
+                return False
+
+        # Check architectures for explicit generation/diffusion classes
+        architectures = config.get("architectures", [])
+        if architectures:
+            arch_str = " ".join(architectures).lower()
+            # Conservative: only exclude obvious diffusion/generation architectures
+            # Use word boundaries to avoid false positives (e.g., "dit" in "conditional")
+            for keyword in [
+                "diffusion",
+                "unet2d",
+                "unet3d",
+                "vaedecoder",  # More specific than "vae"
+                "vaeencoder",
+                "controlnet",
+                "autoencoder",
+                "ditmodel",  # Diffusion Transformer - use more specific pattern
+                "pixart",  # PixArt diffusion model
+            ]:
+                if keyword in arch_str:
+                    return False
+
+        # Check for standalone vision encoder/image processor (no text component)
+        # Only if model name explicitly indicates non-text usage
+        model_name = config.get("_name_or_path", "").lower()
+
+        if any(
+            keyword in model_name
+            for keyword in [
+                "image-edit-",  # Pure image editing (e.g., Qwen-Image-Edit)
+                "-image-editing",
+                "dit-",  # DiT generation models
+                "pixart-",  # PixArt generation models
+            ]
+        ):
+            # Additional check: does it have tokenizer? If yes, might be multimodal LLM
+            has_tokenizer = any(
+                os.path.exists(os.path.join(snapshot_dir, fname))
+                for fname in ["tokenizer.json", "tokenizer.model", "tiktoken.model"]
+            )
+            if not has_tokenizer:
+                # Image-edit model without tokenizer -> likely pure vision pipeline
+                return False
+
+        # Default: assume it's a transformers text/multimodal model
+        # Even if it lacks tokenizer, let validation report the actual error
+        # (better false positive than false negative for text models)
+        return True
+
+    except (json.JSONDecodeError, OSError, KeyError):
+        # Can't parse config - assume it's transformers and let validation report failure
+        return True
+
+
+def scan_weight_files(snapshot_dir):
+    """
+    Scan for weight files in a snapshot.
+
+    Returns:
+        List of weight file paths, or empty list if scan fails
+    """
+    weight_files = []
+
+    # First, look for index files
+    index_patterns = ["*.safetensors.index.json", "pytorch_model.bin.index.json"]
+    index_files = []
+    for pattern in index_patterns:
+        index_files.extend(glob.glob(os.path.join(snapshot_dir, pattern)))
+
+    # If we have safetensors index, collect shards from it
+    for index_file in index_files:
+        if index_file.endswith(".safetensors.index.json"):
+            try:
+                with open(index_file, "r", encoding="utf-8") as f:
+                    index_data = json.load(f)
+                weight_map = index_data.get("weight_map", {})
+                for weight_file in set(weight_map.values()):
+                    weight_path = os.path.join(snapshot_dir, weight_file)
+                    if os.path.exists(weight_path):
+                        weight_files.append(weight_path)
+            except Exception as e:
+                print(
+                    f"  Warning: Failed to parse index {os.path.basename(index_file)}: {e}"
+                )
+
+    # If no index found or no shards from index, do recursive glob
+    if not weight_files:
+        matched = glob.glob(
+            os.path.join(snapshot_dir, "**/*.safetensors"), recursive=True
+        )
+        MAX_WEIGHT_FILES = 1000
+        if len(matched) > MAX_WEIGHT_FILES:
+            print(
+                f"  Warning: Too many safetensors files ({len(matched)} > {MAX_WEIGHT_FILES})"
+            )
+            return []
+
+        for f in matched:
+            if os.path.exists(f):  # Filter out broken symlinks
+                weight_files.append(f)
+
+    return weight_files
+
+
+def validate_snapshot(model_name, snapshot_dir, weight_files, validated_cache):
+    """
+    Validate a snapshot and return detailed status.
+
+    Uses in-process cache to avoid duplicate validation within the same run.
+
+    Args:
+        model_name: Model identifier
+        snapshot_dir: Path to snapshot directory
+        weight_files: List of weight files to validate
+        validated_cache: Dict to track already-validated snapshots in this run
+
+    Returns:
+        Tuple of (result, reason):
+        - (True, None) if validation passed
+        - (False, reason_str) if validation failed
+        - (None, None) if skipped (already validated in this run)
+    """
+    # Fast path: check in-process cache first
+    if snapshot_dir in validated_cache:
+        return None, None  # Already validated in this run, skip
+
+    try:
+        # Perform validation with detailed reason
+        is_complete, reason = validate_cache_with_detailed_reason(
+            snapshot_dir=snapshot_dir,
+            weight_files=weight_files,
+            model_name_or_path=model_name,
+        )
+
+        # Cache result to avoid re-validation in this run
+        validated_cache[snapshot_dir] = (is_complete, reason)
+
+        return is_complete, reason
+
+    except Exception as e:
+        error_msg = f"Validation raised exception: {e}"
+        return False, error_msg
+
+
+def main():
+    start_time = time.time()
+
+    print("=" * 70)
+    print("CI_OFFLINE: Pre-validating cached HuggingFace models")
+    print("=" * 70)
+    print(f"Max time: {MAX_VALIDATION_TIME_SECONDS}s")
+    print()
+
+    print("Scanning HuggingFace cache for models...")
+    snapshots = find_all_hf_snapshots()
+
+    if not snapshots:
+        print("No cached models found, skipping validation")
+        print("=" * 70)
+        return
+
+    print(f"Found {len(snapshots)} snapshot(s) in cache")
+    print()
+
+    validated_count = 0
+    failed_count = 0
+    skipped_count = 0
+    processed_count = 0
+
+    # In-process cache to avoid re-validating same snapshot in this run
+    validated_cache = {}
+
+    for model_name, snapshot_dir in snapshots:
+        # Check time limit
+        elapsed = time.time() - start_time
+        if elapsed > MAX_VALIDATION_TIME_SECONDS:
+            print()
+            print(
+                f"Time limit reached ({elapsed:.1f}s > {MAX_VALIDATION_TIME_SECONDS}s)"
+            )
+            print(
+                f"Stopping validation, {len(snapshots) - processed_count} snapshots remaining"
+            )
+            break
+
+        snapshot_hash = os.path.basename(snapshot_dir)
+        print(
+            f"[{processed_count + 1}/{len(snapshots)}] {model_name} ({snapshot_hash[:8]}...)"
+        )
+        processed_count += 1
+
+        # Determine model type by checking for model_index.json (diffusers pipeline marker)
+        model_index_path = os.path.join(snapshot_dir, "model_index.json")
+        is_diffusion_model = os.path.exists(model_index_path)
+
+        if is_diffusion_model:
+            # This is a diffusers pipeline - use diffusion validation
+            try:
+                is_valid, reason = _validate_diffusion_model(snapshot_dir)
+
+                if is_valid:
+                    print("  PASS (diffusion) - Cache complete & valid")
+                    validated_count += 1
+                else:
+                    print(f"  FAIL (diffusion) - {reason}")
+                    failed_count += 1
+
+            except Exception as e:
+                print(f"  FAIL (diffusion) - Validation raised exception: {e}")
+                failed_count += 1
+
+            continue
+
+        # Transformers model - use standard validation
+        # First check if this looks like a transformers text model
+        if not is_transformers_text_model(snapshot_dir):
+            # Not a recognized model type, skip
+            print(
+                "  SKIP (unknown type) - Not a diffusers pipeline or transformers model"
+            )
+            skipped_count += 1
+            continue
+
+        # Scan weight files
+        weight_files = scan_weight_files(snapshot_dir)
+
+        if not weight_files:
+            print("  SKIP (no weights) - empty or incomplete download")
+            skipped_count += 1
+            continue
+
+        # Validate
+        try:
+            result, reason = validate_snapshot(
+                model_name, snapshot_dir, weight_files, validated_cache
+            )
+
+            if result is True:
+                print("  PASS - Cache complete & valid")
+                validated_count += 1
+            elif result is False:
+                # Print detailed failure reason
+                if reason:
+                    print(f"  FAIL (incomplete) - {reason}")
+                else:
+                    print("  FAIL (incomplete) - cache validation failed")
+                failed_count += 1
+            else:  # None (skipped)
+                print("  SKIP (already validated in this run)")
+                skipped_count += 1
+
+        except Exception as e:
+            print(f"  FAIL (error) - Validation raised exception: {e}")
+            failed_count += 1
+
+    elapsed_total = time.time() - start_time
+
+    print()
+    print("=" * 70)
+    print(f"Validation summary (completed in {elapsed_total:.1f}s):")
+    print(f"  PASS (complete & valid):      {validated_count}")
+    print(f"  FAIL (incomplete/corrupted):  {failed_count}")
+    print(f"  SKIP (no weights/duplicate):  {skipped_count}")
+    print(f"  Total processed:              {processed_count}/{len(snapshots)}")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/publish_traces.py

@@ -0,0 +1,517 @@
+"""
+Publish performance traces to GitHub repository
+"""
+
+import argparse
+import base64
+import json
+import os
+import sys
+import time
+import warnings
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+
+def is_rate_limit_error(e):
+    """Check if an exception is a GitHub rate limit error (not permission error)"""
+    if not isinstance(e, HTTPError):
+        return False
+    if e.code == 429:
+        return True
+    if e.code == 403:
+        # 403 can be rate limit OR permission error - check the message
+        error_body = getattr(e, "error_body", "")
+        if isinstance(error_body, str):
+            # Rate limit errors contain specific phrases
+            rate_limit_phrases = [
+                "rate limit",
+                "abuse detection",
+                "secondary rate limit",
+            ]
+            return any(phrase in error_body.lower() for phrase in rate_limit_phrases)
+    return False
+
+
+def is_permission_error(e):
+    """Check if an exception is a GitHub permission error"""
+    if not isinstance(e, HTTPError) or e.code != 403:
+        return False
+    error_body = getattr(e, "error_body", "")
+    if isinstance(error_body, str):
+        permission_phrases = [
+            "resource not accessible",
+            "must have push access",
+            "permission",
+            "denied",
+        ]
+        return any(phrase in error_body.lower() for phrase in permission_phrases)
+    return False
+
+
+def make_github_request(url, token, method="GET", data=None):
+    """Make authenticated request to GitHub API"""
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        # "User-Agent": "sglang-ci",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+
+    if data:
+        headers["Content-Type"] = "application/json"
+        data = json.dumps(data).encode("utf-8")
+
+    req = Request(url, data=data, headers=headers, method=method)
+
+    try:
+        with urlopen(req) as response:
+            return response.read().decode("utf-8")
+    except HTTPError as e:
+        print(f"GitHub API request failed: {e}")
+        try:
+            error_body = e.read().decode("utf-8")
+            print(f"Error response body: {error_body}")
+            e.error_body = error_body  # Attach for later inspection
+        except Exception:
+            e.error_body = ""
+        raise
+    except Exception as e:
+        print(f"GitHub API request failed with a non-HTTP error: {e}")
+        raise
+
+
+def verify_token_permissions(repo_owner, repo_name, token):
+    """Verify that the token has necessary permissions for the repository"""
+    print("Verifying token permissions...")
+
+    checks = [
+        (
+            f"https://api.github.com/repos/{repo_owner}/{repo_name}",  # Check if we can access the repository
+            "Repository access verified",
+        ),
+        (
+            f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents",  # Check if we can read the repository contents
+            "Repository contents access verified",
+        ),
+    ]
+
+    for url, success_message in checks:
+        try:
+            response = make_github_request(url, token)
+            if success_message == "Repository access verified":
+                repo_data = json.loads(response)
+                print(f"{success_message}: {repo_data['full_name']}")
+            else:
+                print(success_message)
+        except Exception as e:
+            if is_rate_limit_error(e):
+                warnings.warn(
+                    "GitHub API rate limit exceeded during token verification."
+                )
+                return "rate_limited"
+            print(f"Failed to verify permissions for {url}: {e}")
+            return False
+
+    return True
+
+
+def get_branch_sha(repo_owner, repo_name, branch, token):
+    """Get SHA of the branch head"""
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}"
+    )
+    response = make_github_request(url, token)
+    data = json.loads(response)
+    return data["object"]["sha"]
+
+
+def get_tree_sha(repo_owner, repo_name, commit_sha, token):
+    """Get tree SHA from commit"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}"
+    response = make_github_request(url, token)
+    data = json.loads(response)
+    return data["tree"]["sha"]
+
+
+def create_blob(repo_owner, repo_name, content, token, max_retries=3):
+    """Create a blob with file content"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs"
+
+    # Encode content as base64 for GitHub API
+    content_b64 = base64.b64encode(content).decode("utf-8")
+
+    data = {"content": content_b64, "encoding": "base64"}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            return json.loads(response)["sha"]
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt  # Exponential backoff: 1s, 2s, 4s
+                print(
+                    f"Blob creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def create_blobs(repo_owner, repo_name, files, token):
+    """Create blobs for all files and return tree items with blob SHAs"""
+    tree_items = []
+    for i, (file_path, content) in enumerate(files):
+        # Create blob first to get SHA
+        blob_sha = create_blob(repo_owner, repo_name, content, token)
+        tree_items.append(
+            {
+                "path": file_path,
+                "mode": "100644",
+                "type": "blob",
+                "sha": blob_sha,
+            }
+        )
+        # Progress indicator for large uploads
+        if (i + 1) % 10 == 0 or (i + 1) == len(files):
+            print(f"Created {i + 1}/{len(files)} blobs...")
+    return tree_items
+
+
+def create_tree(repo_owner, repo_name, base_tree_sha, tree_items, token, max_retries=3):
+    """Create a new tree from pre-created blob SHAs"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/trees"
+
+    data = {"base_tree": base_tree_sha, "tree": tree_items}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            return json.loads(response)["sha"]
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Tree creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def create_commit(
+    repo_owner, repo_name, tree_sha, parent_sha, message, token, max_retries=3
+):
+    """Create a new commit"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits"
+
+    data = {"tree": tree_sha, "parents": [parent_sha], "message": message}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            commit_sha = json.loads(response)["sha"]
+
+            # Verify the commit was actually created
+            verify_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}"
+            verify_response = make_github_request(verify_url, token)
+            verify_data = json.loads(verify_response)
+            if verify_data["sha"] != commit_sha:
+                raise Exception(
+                    f"Commit verification failed: expected {commit_sha}, got {verify_data['sha']}"
+                )
+
+            return commit_sha
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Commit creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retries=3):
+    """Update branch reference to point to new commit"""
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}"
+    )
+
+    data = {"sha": commit_sha}
+
+    for attempt in range(max_retries):
+        try:
+            make_github_request(url, token, method="PATCH", data=data)
+            return
+        except HTTPError as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            # Check if this is an "Object does not exist" error
+            is_object_not_exist = False
+            if hasattr(e, "error_body"):
+                try:
+                    error_data = json.loads(e.error_body)
+                    if "Object does not exist" in error_data.get("message", ""):
+                        is_object_not_exist = True
+                except Exception:
+                    pass
+
+            if is_object_not_exist and attempt < max_retries - 1:
+                # This might be a transient consistency issue - wait and retry
+                wait_time = 2**attempt
+                print(
+                    f"Branch update failed with 'Object does not exist' (attempt {attempt + 1}/{max_retries}), waiting {wait_time}s for consistency..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Branch update failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def copy_trace_files(source_dir, target_base_path):
+    """Copy trace files and return list of files to upload.
+
+    Only uploads traces from TP rank 0 to avoid duplicated data across tensor parallel ranks.
+    """
+    files_to_upload = []
+
+    if not os.path.exists(source_dir):
+        print(f"Warning: Traces directory {source_dir} does not exist")
+        return files_to_upload
+
+    # Walk through source directory and find .json.gz files
+    for root, dirs, files in os.walk(source_dir):
+        for file in files:
+            if file.endswith(".json.gz"):
+
+                # Only upload TP rank 0 traces to avoid duplicates across tensor parallel ranks
+                if "TP-" in file and "TP-0" not in file:
+                    continue
+
+                source_file = os.path.join(root, file)
+                # Calculate relative path from source_dir
+                rel_path = os.path.relpath(source_file, source_dir)
+                target_path = f"{target_base_path}/{rel_path}"
+
+                # Read file content
+                with open(source_file, "rb") as f:
+                    content = f.read()
+
+                files_to_upload.append((target_path, content))
+
+    return files_to_upload
+
+
+def publish_traces(traces_dir, run_id, run_number):
+    """Publish traces from a single directory to GitHub repository in a single commit"""
+    target_base_path = f"traces/{run_id}"
+    files_to_upload = copy_trace_files(traces_dir, target_base_path)
+
+    if not files_to_upload:
+        print("No trace files found to upload")
+        return
+
+    print(f"Found {len(files_to_upload)} files to upload")
+    publish_traces_from_files(files_to_upload, run_id, run_number)
+
+
+def publish_traces_from_files(files_to_upload, run_id, run_number):
+    """Publish pre-collected trace files to GitHub repository in a single commit"""
+    # Get environment variables
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    # Repository configuration
+    repo_owner = "sgl-project"
+    repo_name = "ci-data"
+    branch = "main"
+
+    # Verify token permissions before proceeding
+    permission_check = verify_token_permissions(repo_owner, repo_name, token)
+    if permission_check == "rate_limited":
+        warnings.warn(
+            "Skipping trace upload due to GitHub API rate limit. "
+            "This is expected during high CI activity and does not indicate a test failure."
+        )
+        return
+    elif not permission_check:
+        print(
+            "Token permission verification failed. Please check the token permissions."
+        )
+        sys.exit(1)
+
+    max_retries = 5
+    retry_delay = 5  # seconds
+
+    # Create blobs once before retry loop to avoid re-uploading on failures
+    try:
+        tree_items = create_blobs(repo_owner, repo_name, files_to_upload, token)
+    except Exception as e:
+        # Check for rate limit errors during blob creation
+        if is_rate_limit_error(e):
+            warnings.warn(
+                "GitHub API rate limit exceeded during blob creation. Skipping trace upload."
+            )
+            return
+        # Check for permission errors - these should fail loudly
+        if is_permission_error(e):
+            print(
+                f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                "'contents: write' permission for the repository."
+            )
+            sys.exit(1)
+        print(f"Failed to create blobs: {e}")
+        raise
+
+    for attempt in range(max_retries):
+        try:
+            # Get current branch head
+            branch_sha = get_branch_sha(repo_owner, repo_name, branch, token)
+            print(f"Current branch head: {branch_sha}")
+
+            # Get current tree
+            tree_sha = get_tree_sha(repo_owner, repo_name, branch_sha, token)
+            print(f"Current tree SHA: {tree_sha}")
+
+            # Create new tree with pre-created blobs
+            new_tree_sha = create_tree(
+                repo_owner, repo_name, tree_sha, tree_items, token
+            )
+            print(f"Created new tree: {new_tree_sha}")
+
+            # Create commit
+            commit_message = f"Nightly traces for run {run_id} at {run_number} ({len(files_to_upload)} files)"
+            commit_sha = create_commit(
+                repo_owner,
+                repo_name,
+                new_tree_sha,
+                branch_sha,
+                commit_message,
+                token,
+            )
+            print(f"Created commit: {commit_sha}")
+
+            # Update branch reference
+            update_branch_ref(repo_owner, repo_name, branch, commit_sha, token)
+            print("Updated branch reference")
+
+            print("Successfully published all traces in a single commit")
+            return
+
+        except Exception as e:
+            # Check for retryable errors
+            is_retryable = False
+            error_type = "unknown"
+
+            if hasattr(e, "error_body"):
+                if "Update is not a fast forward" in e.error_body:
+                    is_retryable = True
+                    error_type = "fast-forward conflict"
+                elif "Object does not exist" in e.error_body:
+                    is_retryable = True
+                    error_type = "object consistency"
+
+            # Also retry on HTTP errors that might be transient
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                is_retryable = True
+                error_type = f"HTTP {e.code}"
+
+            # Check for rate limit errors (non-fatal - just warn and skip)
+            if is_rate_limit_error(e):
+                warnings.warn("GitHub API rate limit exceeded. Skipping trace upload.")
+                return
+
+            # Check for permission errors - these should fail loudly
+            if is_permission_error(e):
+                print(
+                    f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                    "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                    "'contents: write' permission for the repository."
+                )
+                sys.exit(1)
+
+            if is_retryable and attempt < max_retries - 1:
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed ({error_type}). Retrying in {retry_delay} seconds..."
+                )
+                time.sleep(retry_delay)
+            else:
+                print(f"Failed to publish traces after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish performance traces to GitHub repository"
+    )
+    parser.add_argument(
+        "--traces-dir",
+        type=str,
+        action="append",
+        dest="traces_dirs",
+        required=True,
+        help="Traces directory to publish (can be specified multiple times)",
+    )
+    args = parser.parse_args()
+
+    # Get environment variables
+    run_id = os.getenv("GITHUB_RUN_ID", "test")
+    run_number = os.getenv("GITHUB_RUN_NUMBER", "12345")
+
+    if not run_id or not run_number:
+        print(
+            "Error: GITHUB_RUN_ID and GITHUB_RUN_NUMBER environment variables must be set"
+        )
+        sys.exit(1)
+
+    # Collect trace files from all directories
+    target_base_path = f"traces/{run_id}"
+    all_files = []
+    for traces_dir in args.traces_dirs:
+        print(f"Processing traces from directory: {traces_dir}")
+        files = copy_trace_files(traces_dir, target_base_path)
+        all_files.extend(files)
+
+    if not all_files:
+        print("No trace files found to upload across all directories")
+        return
+
+    print(f"Found {len(all_files)} total files to upload")
+
+    # Publish all collected traces in a single commit
+    publish_traces_from_files(all_files, run_id, run_number)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/runner_utilization_report.py

@@ -0,0 +1,763 @@
+#!/usr/bin/env python3
+"""
+Runner Utilization Report
+
+Analyzes GitHub Actions job data to calculate runner utilization metrics.
+Reports idle time, active time, and utilization percentage per runner label.
+"""
+
+import argparse
+import json
+import os
+import random
+import subprocess
+import time
+from collections import Counter, defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timedelta, timezone
+
+# Labels to skip when grouping runners (GitHub default labels)
+DEFAULT_LABELS_TO_IGNORE = {"self-hosted", "Linux", "X64", "ARM64"}
+GITHUB_HOSTED_LABELS = {"ubuntu-latest", "ubuntu-22.04", "ubuntu-24.04"}
+
+# Human-facing job outcome buckets, in display order, with emoji.
+STATUS_ORDER = ("pass", "fail", "cancel", "running", "queued")
+STATUS_EMOJI = {
+    "pass": "✅",
+    "fail": "❌",
+    "cancel": "🚫",
+    "running": "🔄",
+    "queued": "⏳",
+}
+
+
+def format_status_counts(counts: dict) -> str:
+    """Compact per-label outcome summary, e.g. '✅120 ❌3 🔄2 ⏳4'."""
+    parts = [f"{STATUS_EMOJI[s]}{counts[s]}" for s in STATUS_ORDER if counts.get(s)]
+    return " ".join(parts) if parts else "—"
+
+
+def run_gh_command(args: list[str], max_retries: int = 10) -> dict:
+    """Run gh CLI command and return JSON result.
+
+    Retries on transient failures (5xx, secondary rate limits, network
+    blips) with exponential backoff. The previous fail-fast behavior
+    combined with `except Exception: return None` in the threadpool
+    callers caused entire workflow runs to be silently dropped from
+    the utilization numerator whenever GH API hiccuped, severely
+    undercounting busy time on busy days.
+    """
+    last_err = ""
+    for attempt in range(max_retries):
+        result = subprocess.run(
+            ["gh", "api"] + args,
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode == 0:
+            return json.loads(result.stdout)
+        last_err = result.stderr or "(no stderr)"
+        # Detect retryable conditions: HTTP 5xx, secondary rate limit, abuse
+        # detection, network resets. 4xx other than 429 are non-retryable.
+        retryable = any(
+            s in last_err
+            for s in (
+                "rate limit",
+                "abuse",
+                "Internal Server Error",
+                "502",
+                "503",
+                "504",
+                "Bad Gateway",
+                "Gateway Time-out",
+                "connection reset",
+                "Connection reset",
+                "EOF",
+                "timeout",
+            )
+        )
+        if not retryable:
+            break
+        # Exponential backoff with jitter, capped at 60s.
+        delay = min(60, (2**attempt) + random.uniform(0, 1))
+        time.sleep(delay)
+    raise Exception(f"gh api failed after {max_retries} attempts: {last_err[:300]}")
+
+
+def get_workflow_runs(repo: str, hours: int = 24) -> list[dict]:
+    """Get workflow runs from the last N hours."""
+    since = datetime.now(timezone.utc) - timedelta(hours=hours)
+
+    runs = []
+    page = 1
+    while True:
+        data = run_gh_command(
+            [
+                f"repos/{repo}/actions/runs?per_page=100&page={page}",
+            ]
+        )
+        page_runs = data.get("workflow_runs", [])
+
+        # Filter by time
+        for run in page_runs:
+            created_at = parse_time(run.get("created_at"))
+            if created_at and created_at >= since:
+                runs.append(run)
+            elif created_at and created_at < since:
+                # Runs are ordered by created_at desc, so we can stop
+                return runs
+
+        if len(page_runs) < 100:
+            break
+        page += 1
+        if page > 50:  # Safety limit (5000 runs)
+            break
+    return runs
+
+
+def get_jobs_for_run(repo: str, run_id: int) -> list[dict]:
+    """Get all jobs for a workflow run, including all retry attempts.
+
+    `filter=all` is required so that re-run attempts of the same job
+    appear separately. Each attempt consumed host time on the runner
+    pool, so for utilization we want them all summed in. The default
+    (`filter=latest`) only returns the most recent attempt and silently
+    hides time spent on prior retries.
+    """
+    jobs = []
+    page = 1
+    while True:
+        data = run_gh_command(
+            [
+                f"repos/{repo}/actions/runs/{run_id}/jobs"
+                f"?per_page=100&page={page}&filter=all",
+            ]
+        )
+        jobs.extend(data.get("jobs", []))
+        if len(data.get("jobs", [])) < 100:
+            break
+        page += 1
+        if page > 20:  # Safety limit (2000 jobs per run)
+            break
+    return jobs
+
+
+def get_runners(repo: str, online_only: bool = True) -> list[dict]:
+    """Get all self-hosted runners with pagination. Returns empty if no permission."""
+    try:
+        all_runners = []
+        page = 1
+        while True:
+            data = run_gh_command(
+                [f"repos/{repo}/actions/runners?per_page=100&page={page}"]
+            )
+            runners = data.get("runners", [])
+            all_runners.extend(runners)
+            if len(runners) < 100:
+                break
+            page += 1
+            if page > 10:  # Safety limit
+                break
+        if online_only:
+            all_runners = [r for r in all_runners if r.get("status") == "online"]
+        return all_runners
+    except Exception as e:
+        print(f"Warning: Cannot access runners API (need admin): {e}")
+        return []
+
+
+def parse_time(time_str: str) -> datetime:
+    """Parse ISO timestamp to datetime."""
+    if not time_str:
+        return None
+    return datetime.fromisoformat(time_str.replace("Z", "+00:00"))
+
+
+def classify_job(job: dict, now: datetime):
+    """Derive the queue-wait and busy interval for a single job.
+
+    Returns a job_info dict, or None when the job neither waited for nor
+    occupied a runner (skipped / cancelled-before-start / missing data).
+
+    The queue wait runs from when the job entered the runner queue
+    (`created_at`) until a runner picked it up (`started_at`) — or until
+    `now` if it is still waiting.
+
+    GitHub API gotcha this exists to handle: a still-queued job reports
+    status="queued", runner_name="" and `started_at` set to a PLACEHOLDER
+    equal to `created_at` (not null). The previous code required both a
+    runner_name and a `completed_at`, so every in-flight wait — the
+    multi-hour 8-gpu jobs still sitting in the queue, i.e. the worst cases —
+    was dropped, undercounting max/avg queue time. We therefore measure a
+    queued job's wait against `now` rather than its bogus `started_at`, and
+    don't require completion.
+    """
+    status = job.get("status")
+    runner_name = job.get("runner_name") or ""
+    created_at = parse_time(job.get("created_at"))
+    started_at = parse_time(job.get("started_at"))
+    completed_at = parse_time(job.get("completed_at"))
+
+    if status == "queued":
+        # Still waiting for a runner; ignore the placeholder started_at.
+        queue_end, start, end = now, None, None
+    elif status == "in_progress" and started_at is not None:
+        # Running now: the wait is final and it still occupies the runner.
+        queue_end, start, end = started_at, started_at, now
+    elif (
+        status == "completed"
+        and started_at is not None
+        and completed_at is not None
+        and runner_name
+    ):
+        queue_end, start, end = started_at, started_at, completed_at
+    else:
+        # Skipped, cancelled before start, or missing timestamps: never
+        # waited for or occupied a runner.
+        return None
+
+    if created_at is None:
+        return None
+
+    queue_time = max(0.0, (queue_end - created_at).total_seconds())
+    duration = (end - start).total_seconds() if start is not None else 0.0
+    labels = [
+        label
+        for label in job.get("labels", [])
+        if label not in DEFAULT_LABELS_TO_IGNORE | GITHUB_HOSTED_LABELS
+    ]
+
+    # Human-facing outcome bucket used by the report's status breakdown.
+    if status == "queued":
+        outcome = "queued"
+    elif status == "in_progress":
+        outcome = "running"
+    else:  # completed and actually ran
+        outcome = {"success": "pass", "cancelled": "cancel"}.get(
+            job.get("conclusion"), "fail"
+        )
+
+    return {
+        "start": start,
+        "end": end,
+        "created_at": created_at,
+        "queue_end": queue_end,
+        "duration": duration,
+        "queue_time": queue_time,
+        "job_name": job.get("name", ""),
+        "runner_name": runner_name,
+        "labels": labels,
+        "status": outcome,
+        "html_url": job.get("html_url", ""),
+    }
+
+
+def calculate_concurrency_metrics(
+    jobs: list,
+    window_start: datetime,
+    window_end: datetime,
+    num_runners: int,
+) -> dict:
+    """Sweep-line algorithm: peak/avg concurrent, saturation time, peak queue."""
+    if not jobs:
+        return {
+            "peak_concurrent": 0,
+            "avg_concurrent": 0.0,
+            "saturation_seconds": 0,
+            "saturation_pct": 0.0,
+            "peak_queue": 0,
+        }
+    window_seconds = (window_end - window_start).total_seconds()
+    if window_seconds <= 0:
+        return {
+            "peak_concurrent": 0,
+            "avg_concurrent": 0.0,
+            "saturation_seconds": 0,
+            "saturation_pct": 0.0,
+            "peak_queue": 0,
+        }
+    running_events = []
+    for job in jobs:
+        start, end = job["start"], job["end"]
+        # Still-queued jobs have no running interval yet (start/end are None).
+        if start is None or end is None:
+            continue
+        if end < window_start or start > window_end:
+            continue
+        running_events.append((max(start, window_start), 1))
+        running_events.append((min(end, window_end), -1))
+    queue_events = []
+    for job in jobs:
+        created_at = job.get("created_at")
+        # The wait ends when a runner picks the job up, or `now` if it is
+        # still queued (queue_end was set to now upstream). Counting the
+        # still-open waits is what makes peak_queue reflect the real backlog.
+        queue_end = job.get("queue_end") or job["start"]
+        if created_at and queue_end and created_at < queue_end:
+            if queue_end < window_start or created_at > window_end:
+                continue
+            queue_events.append((max(created_at, window_start), 1))
+            queue_events.append((min(queue_end, window_end), -1))
+    running_events.sort(key=lambda e: (e[0], e[1] == 1))
+    current_running = 0
+    peak_running = 0
+    prev_time = window_start
+    total_running_seconds = 0.0
+    saturation_seconds = 0.0
+    for event_time, delta in running_events:
+        td = (event_time - prev_time).total_seconds()
+        if td > 0:
+            total_running_seconds += current_running * td
+            if current_running >= num_runners:
+                saturation_seconds += td
+        current_running += delta
+        peak_running = max(peak_running, current_running)
+        prev_time = event_time
+    if prev_time < window_end:
+        td = (window_end - prev_time).total_seconds()
+        total_running_seconds += current_running * td
+        if current_running >= num_runners:
+            saturation_seconds += td
+    queue_events.sort(key=lambda e: (e[0], e[1] == 1))
+    current_queued = 0
+    peak_queue = 0
+    for _, delta in queue_events:
+        current_queued += delta
+        peak_queue = max(peak_queue, current_queued)
+    avg_concurrent = total_running_seconds / window_seconds if window_seconds > 0 else 0
+    return {
+        "peak_concurrent": peak_running,
+        "avg_concurrent": avg_concurrent,
+        "saturation_seconds": saturation_seconds,
+        "saturation_pct": (
+            (saturation_seconds / window_seconds * 100) if window_seconds > 0 else 0
+        ),
+        "peak_queue": peak_queue,
+    }
+
+
+_NON_GPU_WORKFLOW_HINTS = (
+    "lint",
+    "deploy",
+    "release",
+    "publish",
+    "docs",
+    "doc",
+    "mintlify",
+    "runner utilization",  # this very script
+    "tag-and-rerun",
+    "auto",  # auto-merge etc.
+    "label",
+    "stale",
+    "dependabot",
+    "codeql",
+)
+
+
+def _likely_no_gpu_jobs(workflow_name: str) -> bool:
+    """Heuristic: skip per-run job-fetch for workflows that don't dispatch
+    to self-hosted GPU runners. The GH API rate limit (~5000 req/hr per
+    token) is the bottleneck on busy 24h windows where ~4000 workflow
+    runs fire — but only a fraction of those (pr-test, nightly-test,
+    pr-test-*kernel, etc.) actually run on GPU runners. Skipping the
+    docs/lint/release runs cuts the API call budget by 2-4x.
+    """
+    if not workflow_name:
+        return False
+    n = workflow_name.lower()
+    return any(h in n for h in _NON_GPU_WORKFLOW_HINTS)
+
+
+def calculate_utilization(repo: str, hours: int = 24, runner_filter: str = None):
+    """Calculate runner utilization metrics."""
+
+    print(f"Fetching workflow runs from last {hours} hours...")
+    all_runs = get_workflow_runs(repo, hours)
+    runs = [r for r in all_runs if not _likely_no_gpu_jobs(r.get("name", ""))]
+    skipped = len(all_runs) - len(runs)
+    print(
+        f"Found {len(all_runs)} workflow runs "
+        f"({skipped} skipped as non-GPU: docs/lint/release/etc.)"
+    )
+
+    # Try to get online runners from API
+    print("Fetching online runners...")
+    runners = get_runners(repo, online_only=True)
+
+    # Build label -> set of online runner names from API
+    api_label_runners = defaultdict(set)
+    if runners:
+        for runner in runners:
+            for label in runner.get("labels", []):
+                label_name = label.get("name", "")
+                if label_name not in DEFAULT_LABELS_TO_IGNORE:
+                    api_label_runners[label_name].add(runner["name"])
+        print(f"Got {len(runners)} online runners from API")
+    else:
+        print("No runner API access, will use observed runners from job data")
+
+    # Track runners seen in jobs (for labels not in API or when API unavailable)
+    job_label_runners = defaultdict(set)
+    label_jobs = defaultdict(list)  # label -> list of job_info
+    # Per-host accumulation: each physical machine appears once regardless of
+    # how many overlapping labels it advertises. This is what we use for the
+    # "Per Host Utilization" section (the source-of-truth view).
+    host_jobs = defaultdict(list)  # runner_name -> list of job_info
+    host_labels = defaultdict(set)  # runner_name -> set of labels it ran jobs under
+
+    # Fetch jobs for all runs in parallel. Cap concurrency lower than the
+    # GH API secondary rate-limit threshold to avoid bursts that silently
+    # drop runs even with retries.
+    total_runs = len(runs)
+    print(f"Fetching jobs for {total_runs} runs in parallel...")
+
+    def fetch_jobs_for_run(run):
+        """Fetch jobs for a single run.
+
+        Returns (run_id, jobs, error_msg). `error_msg` is None on success.
+        We surface failures rather than silently dropping the run so the
+        caller can report how many runs' jobs are missing — silently
+        dropping previously caused 4-gpu-b200 (and every other label) to
+        report wildly different numbers depending on transient API hiccups.
+        """
+        try:
+            return (run["id"], get_jobs_for_run(repo, run["id"]), None)
+        except Exception as e:
+            return (run["id"], None, str(e)[:200])
+
+    all_jobs = []
+    failed_runs = []
+    # Concurrency=4 with longer retry budget keeps us well below the GH
+    # API secondary rate-limit threshold (~10 req/s). On a 24h window
+    # with ~1500 GPU-relevant runs (post-filter), this completes in ~5
+    # min and almost never hits the rate limit.
+    with ThreadPoolExecutor(max_workers=4) as executor:
+        futures = [executor.submit(fetch_jobs_for_run, run) for run in runs]
+        completed = 0
+        for future in as_completed(futures):
+            completed += 1
+            if completed % 100 == 0:
+                print(
+                    f"Fetched jobs for {completed}/{total_runs} runs "
+                    f"({len(failed_runs)} failed so far)..."
+                )
+            run_id, jobs, err = future.result()
+            if err:
+                failed_runs.append((run_id, err))
+            elif jobs:
+                all_jobs.extend(jobs)
+
+    print(f"Processing {len(all_jobs)} jobs...")
+    if failed_runs:
+        print(
+            f"WARNING: {len(failed_runs)}/{total_runs} runs failed to fetch "
+            f"after retries. Utilization will be undercounted. "
+            f"First few errors:"
+        )
+        for rid, err in failed_runs[:5]:
+            print(f"  run {rid}: {err}")
+    fetch_failure_pct = len(failed_runs) / total_runs * 100 if total_runs > 0 else 0
+
+    # `now` anchors the wait of jobs that are still queued or running. It is
+    # captured once so every in-flight job is measured against a single
+    # reference (matches window_end below to within processing time).
+    now = datetime.now(timezone.utc)
+    all_job_infos = []  # one entry per job (deduped across labels) for detail views
+    for job in all_jobs:
+        job_info = classify_job(job, now)
+        if job_info is None:
+            continue
+        all_job_infos.append(job_info)
+        runner_name = job_info["runner_name"]
+
+        # Per-host busy time only applies to jobs that actually occupied a
+        # runner (ran or still running); a still-queued job has no host yet.
+        if job_info["start"] is not None and runner_name:
+            host_jobs[runner_name].append(job_info)
+
+        for label in job_info["labels"]:
+            if runner_name:
+                job_label_runners[label].add(runner_name)
+                host_labels[runner_name].add(label)
+            label_jobs[label].append(job_info)
+
+    # Merge API runners and job-observed runners
+    # Prefer API count (online runners) when available
+    # Include labels seen only on still-queued jobs (no online runner, no
+    # completed job under them yet) so a fully-backed-up pool still reports.
+    all_labels = (
+        set(api_label_runners.keys())
+        | set(job_label_runners.keys())
+        | set(label_jobs.keys())
+    )
+
+    # Filter labels if specified
+    if runner_filter:
+        all_labels = {lbl for lbl in all_labels if runner_filter in lbl}
+
+    print(f"Tracking {len(all_labels)} runner labels: {sorted(all_labels)}")
+
+    window_seconds = hours * 3600
+    window_end = datetime.now(timezone.utc)
+    window_start = window_end - timedelta(hours=hours)
+
+    # Per-host window-clamped busy time (each physical machine counted once).
+    # This is the source of truth for how loaded each host actually is.
+    host_busy_seconds = {}
+    for host, jobs in host_jobs.items():
+        busy = 0.0
+        for j in jobs:
+            cs = max(j["start"], window_start)
+            ce = min(j["end"], window_end)
+            if ce > cs:
+                busy += (ce - cs).total_seconds()
+        host_busy_seconds[host] = busy
+
+    results = []
+    for label in sorted(all_labels):
+        # Hosts to attribute to this label = union of currently-online
+        # runners advertising the label PLUS hosts that actually ran a
+        # job under it during the window. The union catches hosts that
+        # went offline mid-window (their busy time is still real
+        # capacity consumed) and hosts that came online late.
+        hosts = api_label_runners.get(label, set()) | job_label_runners.get(
+            label, set()
+        )
+        num_runners = len(hosts) if hosts else 1
+
+        # Pool busy time: sum of busy time across the hosts that could
+        # serve this label, regardless of which sibling label actually
+        # dispatched the job. This is the right denominator/numerator for
+        # asking "how saturated is the underlying hardware that this
+        # label depends on?" — sibling labels (e.g. `4-gpu-b200` and
+        # `4-gpu-b200-low-disk`) compete for the same physical machines,
+        # so their busy time should not be double-counted into separate
+        # capacity buckets.
+        active_seconds = sum(host_busy_seconds.get(h, 0.0) for h in hosts)
+        capacity_seconds = num_runners * window_seconds
+        utilization = (
+            (active_seconds / capacity_seconds * 100) if capacity_seconds > 0 else 0
+        )
+
+        # Job count + queue stats stay label-specific (only jobs that
+        # were dispatched under THIS label).
+        jobs = label_jobs.get(label, [])
+        queue_times = [j["queue_time"] for j in jobs if j["queue_time"] > 0]
+        avg_queue = sum(queue_times) / len(queue_times) if queue_times else 0
+        max_queue = max(queue_times) if queue_times else 0
+        # Outcome breakdown for this label (pass/fail/cancel/running/queued).
+        status_counts = dict(Counter(j["status"] for j in jobs))
+
+        # Concurrency / saturation / queue-depth metrics. Use observed
+        # peak as effective capacity if it's lower than the API count
+        # (e.g. for autoscaling pools where most listeners sit idle).
+        conc_initial = calculate_concurrency_metrics(
+            jobs, window_start, window_end, num_runners
+        )
+        effective_runners = (
+            min(num_runners, conc_initial["peak_concurrent"]) or num_runners
+        )
+        if effective_runners < num_runners and effective_runners > 0:
+            conc = calculate_concurrency_metrics(
+                jobs, window_start, window_end, effective_runners
+            )
+        else:
+            conc = conc_initial
+
+        results.append(
+            {
+                "label": label,
+                "num_runners": num_runners,
+                "effective_runners": effective_runners,
+                "num_jobs": len(jobs),
+                "total_active_hours": active_seconds / 3600,
+                "utilization_pct": utilization,
+                "avg_queue_min": avg_queue / 60,
+                "max_queue_min": max_queue / 60,
+                "peak_concurrent": conc_initial["peak_concurrent"],
+                "avg_concurrent": conc["avg_concurrent"],
+                "saturation_hours": conc["saturation_seconds"] / 3600,
+                "saturation_pct": conc["saturation_pct"],
+                "peak_queue": conc["peak_queue"],
+                "status_counts": status_counts,
+            }
+        )
+
+    # Per-job detail (deduped across labels), longest waits first, for the
+    # links + status section of the report.
+    longest_waits = sorted(all_job_infos, key=lambda j: j["queue_time"], reverse=True)
+    return results, fetch_failure_pct, longest_waits
+
+
+def format_report(
+    results: list[dict],
+    hours: int,
+    fetch_failure_pct: float = 0.0,
+    longest_waits: list = None,
+    top_n: int = 20,
+) -> str:
+    """One compact summary table — original schema, fixed columns.
+
+    Active (hrs) and Utilization now reflect the actual host pool's
+    busy time (sum across all jobs on the hosts that advertise this
+    label, regardless of which sibling label dispatched them). This
+    makes the column meaningful for shared host pools — e.g.
+    `4-gpu-b200` and `4-gpu-b200-low-disk` both consume the same
+    physical hosts, so their utilization now reflects real hardware
+    saturation instead of being divided across labels.
+    """
+    lines = [
+        "# Runner Utilization Report",
+        "",
+        f"**Time window:** Last {hours} hours · "
+        f"**Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
+        "",
+    ]
+    if fetch_failure_pct > 1.0:
+        lines.append(
+            f"⚠️ **Data completeness warning**: {fetch_failure_pct:.0f}% of "
+            f"GPU-relevant workflow runs failed to fetch jobs after retries "
+            f"(GH API rate limit). Active hours and utilization below are "
+            f"under-counted by approximately this fraction."
+        )
+        lines.append("")
+    lines.extend(
+        [
+            "| Label | Runners | Jobs | Active (hrs) | Utilization | Avg Queue | Max Queue | Status |",
+            "|-------|---------|------|--------------|-------------|-----------|-----------|--------|",
+        ]
+    )
+    for r in results:
+        bar = "█" * int(r["utilization_pct"] / 10) + "░" * (
+            10 - int(r["utilization_pct"] / 10)
+        )
+        lines.append(
+            f"| {r['label']} | {r['num_runners']} | {r['num_jobs']} | "
+            f"{r['total_active_hours']:.1f} | "
+            f"{r['utilization_pct']:.1f}% {bar} | "
+            f"{r['avg_queue_min']:.1f}m | {r['max_queue_min']:.1f}m | "
+            f"{format_status_counts(r.get('status_counts', {}))} |"
+        )
+
+    # Longest queue waits — links to the actual jobs, with live status, so the
+    # worst waits (including jobs still queued/running right now) are one click
+    # away. This is the detail behind the Max Queue column.
+    waits = [j for j in (longest_waits or []) if j.get("queue_time", 0) > 0][:top_n]
+    if waits:
+        lines.extend(
+            [
+                "",
+                f"## Longest Queue Waits (top {len(waits)})",
+                "",
+                "| Wait | Status | Label | Job |",
+                "|------|--------|-------|-----|",
+            ]
+        )
+        for j in waits:
+            status = j.get("status", "")
+            emoji = STATUS_EMOJI.get(status, "")
+            label = ", ".join(j.get("labels", [])) or "—"
+            name = j.get("job_name", "job")
+            url = j.get("html_url", "")
+            job_cell = f"[{name}]({url})" if url else name
+            lines.append(
+                f"| {j['queue_time'] / 60:.0f}m | {emoji} {status} | "
+                f"{label} | {job_cell} |"
+            )
+
+    # Concurrency Analysis section
+    lines.extend(
+        [
+            "",
+            "## Concurrency Analysis",
+            "",
+            "| Label | Runners (API/Effective) | Peak Concurrent | Avg Concurrent | Saturation Time | Peak Queue |",
+            "|-------|-------------------------|-----------------|----------------|-----------------|------------|",
+        ]
+    )
+    for r in results:
+        effective = r["effective_runners"]
+        avg_pct = (r["avg_concurrent"] / effective * 100) if effective > 0 else 0
+        runner_str = (
+            f"{r['num_runners']}/{effective}"
+            if effective != r["num_runners"]
+            else str(r["num_runners"])
+        )
+        lines.append(
+            f"| {r['label']} | {runner_str} | "
+            f"{r['peak_concurrent']} | "
+            f"{r['avg_concurrent']:.1f} ({avg_pct:.0f}%) | "
+            f"{r['saturation_hours']:.1f}h ({r['saturation_pct']:.0f}%) | "
+            f"{r['peak_queue']} jobs |"
+        )
+
+    # Recommendations
+    lines.extend(["", "## Recommendations", ""])
+    has_recs = False
+    for r in results:
+        label = r["label"]
+        sat_pct = r["saturation_pct"]
+        peak_q = r["peak_queue"]
+        effective = r["effective_runners"]
+        avg_pct = (r["avg_concurrent"] / effective * 100) if effective > 0 else 0
+        if sat_pct > 50 or peak_q > 5:
+            lines.append(
+                f"⚠️ **{label}**: High saturation ({sat_pct:.0f}%) "
+                f"with queue buildup ({peak_q} jobs). Consider adding runners."
+            )
+            has_recs = True
+        elif sat_pct > 20 or peak_q > 0:
+            lines.append(
+                f"📊 **{label}**: Moderate saturation ({sat_pct:.0f}%), "
+                f"peak queue {peak_q} jobs. Monitor for trends."
+            )
+            has_recs = True
+        elif avg_pct < 30 and r["num_jobs"] > 0:
+            lines.append(
+                f"💡 **{label}**: Low average utilization ({avg_pct:.0f}%). "
+                f"Runner pool may be oversized."
+            )
+            has_recs = True
+        else:
+            lines.append(f"✓ **{label}**: Healthy utilization with minimal queueing.")
+    if not has_recs and results:
+        lines.append("All runner pools have healthy utilization.")
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate runner utilization report")
+    parser.add_argument("--repo", default="sgl-project/sglang", help="GitHub repo")
+    parser.add_argument(
+        "--hours", type=float, default=24, help="Time window in hours (fractional ok)"
+    )
+    parser.add_argument(
+        "--filter", type=str, help="Filter runner labels (e.g., '5090', 'h200')"
+    )
+    parser.add_argument("--output", type=str, help="Output file (default: stdout)")
+    args = parser.parse_args()
+
+    results, fetch_failure_pct, longest_waits = calculate_utilization(
+        args.repo, args.hours, args.filter
+    )
+    report = format_report(
+        results, args.hours, fetch_failure_pct, longest_waits=longest_waits
+    )
+
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(report)
+        print(f"Report written to {args.output}")
+    else:
+        print(report)
+
+    # Also write to GITHUB_STEP_SUMMARY if available
+    summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file:
+        with open(summary_file, "a") as f:
+            f.write(report)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/save_metrics.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""Collect and save performance metrics from nightly benchmark results.
+
+This script reads benchmark result JSON files from performance profile directories
+and saves them with metadata for artifact collection in CI.
+
+Usage:
+    python3 scripts/ci/utils/save_metrics.py \
+        --gpu-config 8-gpu-h200 \
+        --partition 0 \
+        --run-id 12345678 \
+        --output test/metrics-8gpu-h200-partition-0.json
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_result_files(search_dirs: list[str]) -> list[str]:
+    """Find all results_*.json files in the given directories."""
+    result_files = set()
+    for search_dir in search_dirs:
+        if os.path.exists(search_dir):
+            pattern = os.path.join(search_dir, "**/results_*.json")
+            result_files.update(glob.glob(pattern, recursive=True))
+    return list(result_files)
+
+
+def parse_result_file(filepath: str) -> list[dict]:
+    """Parse a benchmark result JSON file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, list):
+            return data
+        return [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {filepath}: {e}")
+        return []
+
+
+def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict:
+    """Transform a benchmark result to the metrics schema.
+
+    Note: input_len and output_len are preserved here for the flat benchmarks list,
+    but are also used as grouping keys in benchmarks_by_io_len.
+    """
+    # Handle None values safely for numeric conversions
+    latency = result.get("latency")
+    last_ttft = result.get("last_ttft")
+
+    return {
+        "batch_size": result.get("batch_size"),
+        "input_len": result.get("input_len"),
+        "output_len": result.get("output_len"),
+        "latency_ms": latency * 1000 if latency is not None else None,
+        "input_throughput": result.get("input_throughput"),
+        "output_throughput": result.get("output_throughput"),
+        "overall_throughput": result.get("overall_throughput"),
+        "ttft_ms": last_ttft * 1000 if last_ttft is not None else None,
+        "acc_length": result.get("acc_length"),
+    }
+
+
+def get_io_len_key(input_len: int, output_len: int) -> str:
+    """Generate a key for input/output length combination."""
+    return f"{input_len}_{output_len}"
+
+
+def group_results_by_model(
+    results: list[dict], gpu_config: str, partition: int
+) -> list[dict]:
+    """Group benchmark results by model, variant, and server_args.
+
+    Results are organized with two benchmark structures:
+    - benchmarks: flat list of all benchmarks (for backward compatibility)
+    - benchmarks_by_io_len: nested structure grouped by input/output length combinations
+    """
+    groups = {}
+
+    for result in results:
+        model_path = result.get("model_path", "unknown")
+        run_name = result.get("run_name", "default")
+        variant = run_name if run_name != "default" else None
+        server_args = result.get("server_args")
+        # Convert server_args list to tuple for use as dict key (lists are not hashable)
+        server_args_key = tuple(server_args) if server_args else None
+
+        key = (model_path, variant, server_args_key)
+        if key not in groups:
+            groups[key] = {
+                "gpu_config": gpu_config,
+                "partition": partition,
+                "model": model_path,
+                "variant": variant,
+                "server_args": server_args,
+                "benchmarks": [],
+                "benchmarks_by_io_len": {},
+            }
+
+        transformed = transform_benchmark_result(result, gpu_config, partition)
+
+        # Add to flat benchmarks list (backward compatibility)
+        groups[key]["benchmarks"].append(transformed)
+
+        # Add to nested benchmarks_by_io_len structure
+        input_len = result.get("input_len")
+        output_len = result.get("output_len")
+        if input_len is not None and output_len is not None:
+            io_key = get_io_len_key(input_len, output_len)
+            if io_key not in groups[key]["benchmarks_by_io_len"]:
+                groups[key]["benchmarks_by_io_len"][io_key] = {
+                    "input_len": input_len,
+                    "output_len": output_len,
+                    "benchmarks": [],
+                }
+            # For the nested structure, exclude input_len and output_len from individual benchmarks
+            # since they're already in the parent
+            nested_benchmark = {
+                k: v
+                for k, v in transformed.items()
+                if k not in ("input_len", "output_len")
+            }
+            groups[key]["benchmarks_by_io_len"][io_key]["benchmarks"].append(
+                nested_benchmark
+            )
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    partition: int,
+    run_id: str,
+    output_file: str,
+    search_dirs: list[str],
+) -> bool:
+    """Collect metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Find all result files
+    result_files = find_result_files(search_dirs)
+    print(f"Found {len(result_files)} result file(s)")
+
+    grouped = []
+    if not result_files:
+        print("No benchmark result files found")
+    else:
+        # Parse all result files
+        all_results = []
+        for filepath in sorted(result_files):
+            print(f"  Reading: {filepath}")
+            results = parse_result_file(filepath)
+            all_results.extend(results)
+        print(f"Total benchmark results: {len(all_results)}")
+
+        # Group by model/variant
+        grouped = group_results_by_model(all_results, gpu_config, partition)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "partition": partition,
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not result_files:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect performance metrics from benchmark results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 8-gpu-h200, 8-gpu-b200)",
+    )
+    parser.add_argument(
+        "--partition",
+        type=int,
+        required=True,
+        help="Partition number (0, 1, 2, etc.)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--search-dir",
+        action="append",
+        default=[],
+        dest="search_dirs",
+        help="Directory to search for result files (can be specified multiple times)",
+    )
+
+    args = parser.parse_args()
+
+    # Default search directories if none specified
+    search_dirs = args.search_dirs or [
+        "test/performance_profiles_8_gpu",
+        "test/performance_profiles_text_models",
+        "test/performance_profiles_vlms",
+        "test",
+        ".",
+    ]
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        partition=args.partition,
+        run_id=args.run_id,
+        output_file=args.output,
+        search_dirs=search_dirs,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/test_runner_utilization_report.py

@@ -0,0 +1,195 @@
+"""Unit tests for runner_utilization_report.classify_job.
+
+Pure-logic tests (no GitHub API, stdlib only) so they run in the
+runner-utilization workflow without installing dependencies:
+
+    python -m unittest discover -s scripts/ci/utils -p 'test_runner_utilization_report.py'
+
+Regression guard for the queue-time underestimation bug: jobs still
+waiting in the runner queue (or still running) used to be dropped because
+the old code required a runner_name and a completed_at, so multi-hour
+8-gpu waits never showed up in max/avg queue time.
+"""
+
+import os
+import sys
+import unittest
+from datetime import datetime, timedelta, timezone
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import runner_utilization_report as rur  # noqa: E402
+
+NOW = datetime(2026, 5, 27, 21, 50, 56, tzinfo=timezone.utc)
+CREATED = NOW - timedelta(hours=4)  # entered the queue 4h ago
+
+
+def _job(**kw):
+    base = {
+        "name": "base-c-test-8-gpu-h200 / base-c-test-8-gpu-h200 (3)",
+        "status": "completed",
+        "conclusion": "success",
+        "runner_name": "h200-wk03",
+        "labels": ["self-hosted", "X64", "8-gpu-h200"],
+        "created_at": CREATED.isoformat().replace("+00:00", "Z"),
+        "started_at": None,
+        "completed_at": None,
+        "html_url": "https://github.com/o/r/actions/runs/1/job/2",
+    }
+    base.update(kw)
+    return base
+
+
+def _iso(dt):
+    return dt.isoformat().replace("+00:00", "Z")
+
+
+class TestClassifyJob(unittest.TestCase):
+    def test_queued_job_counts_ongoing_wait(self):
+        """The core bug: a still-queued job reports started_at == created_at
+        (placeholder) and no completed_at. Its wait must be now - created_at,
+        not 0, and it must not be dropped."""
+        job = _job(
+            status="queued",
+            runner_name="",
+            started_at=_iso(CREATED),  # GitHub placeholder == created_at
+            completed_at=None,
+        )
+        info = rur.classify_job(job, NOW)
+        self.assertIsNotNone(info)
+        self.assertAlmostEqual(info["queue_time"], 4 * 3600, delta=1)
+        self.assertIsNone(info["start"])  # no runner occupied yet
+        self.assertEqual(info["labels"], ["8-gpu-h200"])  # generic labels dropped
+
+    def test_in_progress_job_counts_final_wait(self):
+        """A running job's wait is final (started - created); old code dropped
+        it for lacking completed_at."""
+        started = CREATED + timedelta(hours=3)
+        job = _job(status="in_progress", started_at=_iso(started), completed_at=None)
+        info = rur.classify_job(job, NOW)
+        self.assertIsNotNone(info)
+        self.assertAlmostEqual(info["queue_time"], 3 * 3600, delta=1)
+        self.assertEqual(info["start"], started)
+        self.assertEqual(info["end"], NOW)  # still occupying the runner
+
+    def test_completed_job_unchanged(self):
+        started = CREATED + timedelta(minutes=30)
+        completed = CREATED + timedelta(minutes=90)
+        job = _job(started_at=_iso(started), completed_at=_iso(completed))
+        info = rur.classify_job(job, NOW)
+        self.assertAlmostEqual(info["queue_time"], 30 * 60, delta=1)
+        self.assertAlmostEqual(info["duration"], 60 * 60, delta=1)
+        self.assertEqual(info["end"], completed)
+
+    def test_skipped_job_dropped(self):
+        """Skipped / cancelled-before-start jobs never waited for a runner."""
+        job = _job(status="completed", runner_name="", started_at=None)
+        self.assertIsNone(rur.classify_job(job, NOW))
+
+    def test_queued_without_created_dropped(self):
+        job = _job(status="queued", runner_name="", created_at=None, started_at=None)
+        self.assertIsNone(rur.classify_job(job, NOW))
+
+
+class TestConcurrencyHandlesQueuedJobs(unittest.TestCase):
+    def test_queued_job_does_not_crash_and_counts_in_peak_queue(self):
+        window_start = NOW - timedelta(hours=24)
+        queued = rur.classify_job(
+            _job(status="queued", runner_name="", started_at=_iso(CREATED)), NOW
+        )
+        ran = rur.classify_job(
+            _job(
+                started_at=_iso(CREATED + timedelta(hours=1)),
+                completed_at=_iso(CREATED + timedelta(hours=2)),
+            ),
+            NOW,
+        )
+        conc = rur.calculate_concurrency_metrics(
+            [queued, ran], window_start, NOW, num_runners=2
+        )
+        # Both jobs were waiting at CREATED before either started -> peak 2.
+        self.assertEqual(conc["peak_queue"], 2)
+
+
+class TestStatusAndFormatting(unittest.TestCase):
+    def test_status_mapping_and_url(self):
+        for conclusion, expected in (
+            ("success", "pass"),
+            ("failure", "fail"),
+            ("timed_out", "fail"),
+            ("cancelled", "cancel"),
+        ):
+            info = rur.classify_job(
+                _job(
+                    conclusion=conclusion,
+                    started_at=_iso(CREATED + timedelta(minutes=5)),
+                    completed_at=_iso(CREATED + timedelta(minutes=10)),
+                ),
+                NOW,
+            )
+            self.assertEqual(info["status"], expected)
+            self.assertEqual(
+                info["html_url"], "https://github.com/o/r/actions/runs/1/job/2"
+            )
+
+        self.assertEqual(
+            rur.classify_job(
+                _job(status="queued", runner_name="", started_at=_iso(CREATED)), NOW
+            )["status"],
+            "queued",
+        )
+        self.assertEqual(
+            rur.classify_job(
+                _job(
+                    status="in_progress", started_at=_iso(CREATED + timedelta(hours=1))
+                ),
+                NOW,
+            )["status"],
+            "running",
+        )
+
+    def test_format_status_counts(self):
+        self.assertEqual(rur.format_status_counts({}), "—")
+        cell = rur.format_status_counts({"pass": 5, "queued": 2, "fail": 0})
+        self.assertIn("✅5", cell)
+        self.assertIn("⏳2", cell)
+        self.assertNotIn("❌", cell)  # zero counts omitted
+
+    def test_format_report_has_links_and_status(self):
+        results = [
+            {
+                "label": "8-gpu-h200",
+                "num_runners": 4,
+                "effective_runners": 4,
+                "num_jobs": 2,
+                "total_active_hours": 1.0,
+                "utilization_pct": 50.0,
+                "avg_queue_min": 100.0,
+                "max_queue_min": 264.0,
+                "peak_concurrent": 1,
+                "avg_concurrent": 0.5,
+                "saturation_hours": 0.0,
+                "saturation_pct": 0.0,
+                "peak_queue": 2,
+                "status_counts": {"pass": 1, "queued": 1},
+            }
+        ]
+        url = "https://github.com/sgl-project/sglang/actions/runs/1/job/2"
+        waits = [
+            {
+                "queue_time": 264 * 60,
+                "status": "queued",
+                "labels": ["8-gpu-h200"],
+                "job_name": "base-c-test-8-gpu-h200 (3)",
+                "html_url": url,
+            }
+        ]
+        report = rur.format_report(results, 24, 0.0, longest_waits=waits)
+        self.assertIn("| Status |", report)  # new main-table column
+        self.assertIn("Longest Queue Waits", report)
+        self.assertIn(f"]({url})", report)  # clickable job link
+        self.assertIn("264m", report)
+        self.assertIn("⏳", report)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/ci/utils/update_rerun_test_status.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""
+Update the per-batch status icon in a /rerun-test reply comment.
+
+State machine for one batch line (anchored by a unique HTML-comment marker
+written by the slash-command handler):
+
+  dispatched   ⏳ ... <!--rrt:i-->          (handler, on dispatch)
+  running      🔄 ... <!--rrt:i-->          (start-beacon, on the test runner)
+  done         ✅/❌ ... <!--rrt:i:done-->  (finalizer, after the test job)
+
+The leading 🚀 on the line is the visual anchor that this came from a
+slash-command trigger and is preserved across all states.
+
+Idempotency:
+- :done marker present  -> no-op (covers reruns and start-after-finalizer race)
+- running and line already has 🔄  -> no-op
+- marker not found after retries  -> warn and exit 0 so a single placeholder
+  glitch does not amplify into N noisy job failures.
+
+Concurrent updates against the same comment from different batches can race
+on the body (read-modify-write of the same field). The finalizer
+serializes itself via job-level concurrency; the beacon does not, because
+splitting it into its own job would re-queue the GPU runner. Worst case
+for the beacon race is one missed 🔄 flicker - comment stays consistent.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+RETRY_DELAYS_SEC = [0, 5, 15]
+
+STATUS_ICONS = {
+    "running": "🔄",
+    "success": "✅",
+    "failure": "❌",
+}
+
+TERMINAL_STATUSES = {"success", "failure"}
+
+
+def gh_request(method, url, token, body=None):
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, method=method, headers=headers)
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            return resp.status, resp.read().decode()
+    except urllib.error.HTTPError as e:
+        return e.code, e.read().decode()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--comment-id", required=True, type=int)
+    ap.add_argument(
+        "--marker", required=True, help="Per-batch marker, e.g. <!--rrt:0-->"
+    )
+    ap.add_argument(
+        "--status",
+        required=True,
+        choices=list(STATUS_ICONS.keys()),
+    )
+    ap.add_argument("--repo", required=True, help="owner/repo")
+    args = ap.parse_args()
+
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        print("ERROR: GITHUB_TOKEN not set")
+        return 1
+
+    icon = STATUS_ICONS[args.status]
+    is_terminal = args.status in TERMINAL_STATUSES
+    done_marker = args.marker.replace("-->", ":done-->")
+
+    url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}"
+
+    body = None
+    for attempt, delay in enumerate(RETRY_DELAYS_SEC):
+        if delay:
+            time.sleep(delay)
+        status, text = gh_request("GET", url, token)
+        if status != 200:
+            print(f"GET failed: {status} {text}")
+            return 1
+        body = json.loads(text).get("body") or ""
+        if done_marker in body:
+            print(f"Marker {done_marker} already present; nothing to do.")
+            return 0
+        if args.marker in body:
+            break
+        print(
+            f"Marker {args.marker} not found "
+            f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry."
+        )
+    else:
+        print(
+            f"WARNING: marker {args.marker} not found after "
+            f"{len(RETRY_DELAYS_SEC)} attempts; skipping. "
+            f"The handler may have failed to edit the placeholder comment."
+        )
+        return 0
+
+    new_lines = []
+    for line in body.splitlines(keepends=True):
+        if args.marker in line:
+            if not is_terminal and icon in line:
+                print(f"Line already has {icon}; nothing to do.")
+                return 0
+            for prior in ("⏳", "🔄"):
+                if prior in line:
+                    line = line.replace(prior, icon, 1)
+                    break
+            if is_terminal:
+                line = line.replace(args.marker, done_marker)
+        new_lines.append(line)
+
+    new_body = "".join(new_lines)
+    status, text = gh_request("PATCH", url, token, body={"body": new_body})
+    if status != 200:
+        print(f"PATCH failed: {status} {text}")
+        return 1
+    print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/xpu/xpu_ci_start_container.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+set -euo pipefail
+
+# Start the Intel XPU CI container (ci_sglang_xpu) using the intel/sglang-dev:latest
+# image published by .github/workflows/release-docker-intel-xpu-nightly.yml.
+#
+# Pulls the :latest tag and starts a long-running container that subsequent
+# steps `docker exec` into.
+
+CONTAINER_NAME="ci_sglang_xpu"
+IMAGE_REPO="intel/sglang-dev"
+IMAGE_TAG="latest"
+CUSTOM_IMAGE=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --custom-image) CUSTOM_IMAGE="$2"; shift 2;;
+    --container-name) CONTAINER_NAME="$2"; shift 2;;
+    --image-tag) IMAGE_TAG="$2"; shift 2;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS]"
+      echo "Options:"
+      echo "  --custom-image IMAGE     Use a specific Docker image directly"
+      echo "  --container-name NAME    Override container name (default: ${CONTAINER_NAME})"
+      echo "  --image-tag TAG          Tag of ${IMAGE_REPO} to pull (default: ${IMAGE_TAG})"
+      exit 0
+      ;;
+    *) echo "Unknown option $1"; exit 1;;
+  esac
+done
+
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s..." >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub when credentials are present (avoids anonymous pull
+# rate limits). Both vars are optional; falls back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_INTEL_USERNAME:-}" && -n "${DOCKERHUB_INTEL_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub..."
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_INTEL_TOKEN}" | docker login -u "${DOCKERHUB_INTEL_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
+
+if [[ -n "${CUSTOM_IMAGE}" ]]; then
+  IMAGE="${CUSTOM_IMAGE}"
+  echo "Using custom image: ${IMAGE}"
+else
+  IMAGE="${IMAGE_REPO}:${IMAGE_TAG}"
+  echo "Using image: ${IMAGE}"
+fi
+# Always pull so each stage runs the registry's current image; the cleanup
+# step removes the image after the stage so the runner doesn't accumulate
+# stale layers across runs.
+retry_with_backoff 6 docker pull "${IMAGE}"
+
+# Export the resolved image so the cleanup step can rmi the exact tag used.
+if [[ -n "${GITHUB_ENV:-}" ]]; then
+  echo "CI_SGLANG_XPU_IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
+fi
+
+# Remove any stale container of the same name so re-runs are idempotent.
+if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then
+  echo "Removing existing container: ${CONTAINER_NAME}"
+  docker rm -f "${CONTAINER_NAME}" >/dev/null
+fi
+
+VIDEO_GID=$(getent group video | cut -d: -f3)
+RENDER_GID=$(getent group render | cut -d: -f3)
+
+HF_TOKEN_FILE="${HOME}/huggingface_token.txt"
+HF_TOKEN_VALUE=""
+if [[ -n "${HF_TOKEN:-}" ]]; then
+  HF_TOKEN_VALUE="${HF_TOKEN}"
+elif [[ -r "${HF_TOKEN_FILE}" ]]; then
+  HF_TOKEN_VALUE=$(cat "${HF_TOKEN_FILE}")
+fi
+
+echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}"
+docker run -dt \
+  --shm-size 8g \
+  --group-add 992 \
+  ${VIDEO_GID:+--group-add "${VIDEO_GID}"} \
+  ${RENDER_GID:+--group-add "${RENDER_GID}"} \
+  --device /dev/dri \
+  -v /dev/dri/by-path:/dev/dri/by-path \
+  -v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
+  -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
+  -e HF_TOKEN="${HF_TOKEN_VALUE}" \
+  --name "${CONTAINER_NAME}" \
+  "${IMAGE}"
+
+# Mark the workspace mount as a safe directory so git operations as root
+# inside the container don't trip the cross-user repo guard.
+docker exec "${CONTAINER_NAME}" git config --global --add safe.directory /sglang-checkout || true