chore: import upstream snapshot with attribution

scripts/build_sgl_deep_gemm.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+# Build sgl-deep-gemm wheel inside a CUDA-versioned container.
+#
+# Usage: build_sgl_deep_gemm.sh <PYTHON_VERSION> <CUDA_VERSION> <DEEPGEMM_SRC> [ARCH]
+#   PYTHON_VERSION: e.g. 3.10
+#   CUDA_VERSION:   e.g. 12.9 or 13.0
+#   DEEPGEMM_SRC:   path to a checkout of sgl-project/DeepGEMM
+#   ARCH:           x86_64 (default) or aarch64
+#
+# Writes:
+#   <DEEPGEMM_SRC>/dist/      — wheel(s) tagged +cu129 / +cu130 and manylinux
+#   <DEEPGEMM_SRC>/dist-pypi/ — cu130 only: same wheel(s) with +cu130 stripped
+#                              (PyPI rejects local-version segments)
+set -ex
+
+if [ $# -lt 3 ]; then
+  echo "Usage: $0 <PYTHON_VERSION> <CUDA_VERSION> <DEEPGEMM_SRC> [ARCH]"
+  exit 1
+fi
+
+PYTHON_VERSION="$1"
+CUDA_VERSION="$2"
+DEEPGEMM_SRC="$(cd "$3" && pwd)"
+ARCH="${4:-$(uname -i)}"
+
+case "${CUDA_VERSION}" in
+  13.0) CU_TAG=cu130 ;;
+  12.9) CU_TAG=cu129 ;;
+  *)
+    echo "Unsupported CUDA_VERSION: ${CUDA_VERSION}" >&2
+    exit 1
+    ;;
+esac
+
+if [ "${ARCH}" = "aarch64" ]; then
+  BASE_IMG="pytorch/manylinuxaarch64-builder"
+else
+  BASE_IMG="pytorch/manylinux2_28-builder"
+fi
+
+PY_TAG="cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+DOCKERFILE="${REPO_ROOT}/docker/sgl-deep-gemm.Dockerfile"
+RENAME_SCRIPT="${SCRIPT_DIR}/rename_sgl_deep_gemm_whl.sh"
+
+DEPS_TAG="sgl-deep-gemm-deps:cuda${CUDA_VERSION}-${PY_TAG}-${ARCH}"
+
+echo "----------------------------------------"
+echo "PYTHON_VERSION: ${PYTHON_VERSION}"
+echo "CUDA_VERSION:   ${CUDA_VERSION}"
+echo "CU_TAG:         ${CU_TAG}"
+echo "ARCH:           ${ARCH}"
+echo "BASE_IMG:       ${BASE_IMG}"
+echo "DEEPGEMM_SRC:   ${DEEPGEMM_SRC}"
+echo "DEPS_TAG:       ${DEPS_TAG}"
+echo "----------------------------------------"
+
+docker build \
+  -f "${DOCKERFILE}" "$(dirname "${DOCKERFILE}")" \
+  --build-arg BASE_IMG="${BASE_IMG}" \
+  --build-arg CUDA_VERSION="${CUDA_VERSION}" \
+  --build-arg ARCH="${ARCH}" \
+  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
+  --build-arg PYTHON_TAG="${PY_TAG}" \
+  -t "${DEPS_TAG}" \
+  --network=host
+
+mkdir -p "${DEEPGEMM_SRC}/dist"
+
+# 1) Build the wheel inside the deps container.
+docker run --rm \
+  --network=host \
+  -v "${DEEPGEMM_SRC}:/deepgemm" \
+  -w /deepgemm \
+  "${DEPS_TAG}" \
+  bash build_sgl_deep_gemm.sh
+
+# 2) Rename inside the same image so we have a working pip / wheel CLI and can
+#    rewrite the root-owned wheel files written by the build container above.
+docker run --rm \
+  -v "${DEEPGEMM_SRC}:/deepgemm" \
+  -v "${RENAME_SCRIPT}:/rename_sgl_deep_gemm_whl.sh:ro" \
+  -w /deepgemm \
+  "${DEPS_TAG}" \
+  bash /rename_sgl_deep_gemm_whl.sh dist "${CU_TAG}" "${ARCH}"
+
+# 3) cu130 only: produce a sibling dist-pypi/ with the +cu130 local-version
+#    stripped (PyPI rejects local versions).
+if [ "${CU_TAG}" = "cu130" ]; then
+  docker run --rm \
+    -v "${DEEPGEMM_SRC}:/deepgemm" \
+    -w /deepgemm \
+    "${DEPS_TAG}" \
+    bash -c '
+set -eux
+mkdir -p dist-pypi
+for w in dist/*.whl; do
+  tmp=$(mktemp -d)
+  python3 -m wheel unpack "$w" --dest "$tmp"
+  unpacked=$(find "$tmp" -mindepth 1 -maxdepth 1 -type d | head -1)
+  info=$(find "$unpacked" -maxdepth 1 -type d -name "*.dist-info" | head -1)
+  meta="$info/METADATA"
+  orig=$(grep "^Version:" "$meta" | head -1 | sed "s/^Version:[[:space:]]*//")
+  new=$(echo "$orig" | sed "s/+cu[0-9]\+$//")
+  if [ "$orig" != "$new" ]; then
+    sed -i "s/^Version:.*/Version: ${new}/" "$meta"
+    old_base=$(basename "$info")
+    new_base="${old_base/${orig}/${new}}"
+    mv "$info" "$(dirname "$info")/${new_base}"
+  fi
+  python3 -m wheel pack "$unpacked" --dest-dir dist-pypi
+  rm -rf "$tmp"
+done
+ls -lh dist-pypi/
+'
+fi
+
+echo "Wheels in ${DEEPGEMM_SRC}/dist:"
+ls -lh "${DEEPGEMM_SRC}/dist"/*.whl 2>/dev/null || true
+if [ "${CU_TAG}" = "cu130" ]; then
+  echo "PyPI-ready wheels in ${DEEPGEMM_SRC}/dist-pypi:"
+  ls -lh "${DEEPGEMM_SRC}/dist-pypi"/*.whl 2>/dev/null || true
+fi

scripts/ci/amd/amd_ci_exec.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -euo pipefail
+
+# Detect GPU family from hostname (e.g., linux-mi35x-gpu-1-xxxxx-runner-zzzzz)
+HOSTNAME_VALUE=$(hostname)
+GPU_FAMILY=""
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_FAMILY="${BASH_REMATCH[1]}"
+  echo "Detected GPU family from hostname: ${GPU_FAMILY}"
+else
+  echo "Warning: could not parse GPU family from '${HOSTNAME_VALUE}'"
+fi
+
+WORKDIR="/sglang-checkout/test"
+declare -A ENV_MAP=(
+  [SGLANG_IS_IN_CI_AMD]=1
+  [SGLANG_IS_IN_CI]=1
+  # Disabled on AMD: the async-assert probes (#27461) fire torch._assert_async in
+  # the MXFP4 EAGLE-MTP decode path and abort the queue with an HSA hardware
+  # exception.
+  [SGLANG_ENABLE_ASYNC_ASSERT]=0
+  [SGLANG_USE_AITER]=1
+)
+
+# Conditionally add GPU_ARCHS only for mi35x
+if [[ "${GPU_FAMILY}" == "mi35x" ]]; then
+  ENV_MAP[GPU_ARCHS]="gfx950"
+fi
+
+# Parse -w/--workdir and -e ENV=VAL
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -w|--workdir)
+      WORKDIR="$2"
+      shift 2
+      ;;
+    -e)
+      IFS="=" read -r key val <<< "$2"
+      ENV_MAP["$key"]="$val"
+      shift 2
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      break
+      ;;
+  esac
+done
+
+# Build final ENV_ARGS
+ENV_ARGS=()
+for key in "${!ENV_MAP[@]}"; do
+  ENV_ARGS+=("-e" "$key=${ENV_MAP[$key]}")
+done
+
+# Run docker exec with retry logic for HuggingFace network/download issues
+# When HF model downloads fail due to network timeouts or rate limits,
+# retrying with HF_HUB_OFFLINE=1 uses cached models from previous downloads.
+#
+# First attempt: normal mode (allows HF downloads)
+if docker exec \
+  -w "$WORKDIR" \
+  "${ENV_ARGS[@]}" \
+  ci_sglang "$@"; then
+  exit 0
+else
+  FIRST_EXIT_CODE=$?
+fi
+
+echo "First attempt failed with exit code $FIRST_EXIT_CODE"
+
+# Skip retry for test failures that won't be fixed by offline mode:
+#   - Exit 1: Test assertion failures (accuracy below threshold)
+#   - Exit 137 (128+9): Process killed by OOM
+#   - Exit 255: Test suite completed with test errors
+# Only retry for other exit codes (e.g., network timeouts, HF download failures)
+if [[ "$FIRST_EXIT_CODE" -eq 1 || "$FIRST_EXIT_CODE" -eq 137 || "$FIRST_EXIT_CODE" -eq 255 ]]; then
+  echo "Exit code $FIRST_EXIT_CODE indicates test failure (not network issue), not retrying"
+  exit $FIRST_EXIT_CODE
+fi
+
+echo "Retrying with HF_HUB_OFFLINE=1 (offline mode to use cached models)..."
+
+# Second attempt: force HF offline mode to avoid network timeouts
+docker exec \
+  -w "$WORKDIR" \
+  "${ENV_ARGS[@]}" \
+  -e HF_HUB_OFFLINE=1 \
+  ci_sglang "$@"

scripts/ci/amd/amd_ci_install_dependency.sh

@@ -0,0 +1,356 @@
+#!/bin/bash
+set -euo pipefail
+HOSTNAME_VALUE=$(hostname)
+GPU_ARCH="mi30x"   # default
+SKIP_TT_DEPS=""
+SKIP_SGLANG_BUILD=""
+SKIP_AITER_BUILD=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-aiter-build) SKIP_AITER_BUILD="1"; shift;;
+    --skip-sglang-build) SKIP_SGLANG_BUILD="1"; shift;;
+    --skip-test-time-deps) SKIP_TT_DEPS="1"; shift;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS] [OPTIONAL_DEPS]"
+      echo "Options:"
+      echo "  --skip-sglang-build         Don't build checkout sglang, use what was shipped with the image"
+      echo "  --skip-aiter-build          Don't build aiter, use what was shipped with the image"
+      echo "  --skip-test-time-deps       Don't build miscellaneous dependencies"
+      exit 0
+      ;;
+    *) break ;;
+  esac
+done
+
+OPTIONAL_DEPS="${1:-}"
+
+# Build python extras
+EXTRAS="dev_hip,tracing"
+if [ -n "$OPTIONAL_DEPS" ]; then
+    EXTRAS="dev_hip,tracing,${OPTIONAL_DEPS}"
+fi
+echo "Installing python extras: [${EXTRAS}]"
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_ARCH="${BASH_REMATCH[1]}"
+  echo "Detected GPU architecture from hostname: ${GPU_ARCH}"
+else
+  echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}"
+fi
+
+# Install the required dependencies in CI.
+# Fix permissions on pip cache, ignore errors from concurrent access or missing temp files
+docker exec ci_sglang chown -R root:root /sgl-data/pip-cache 2>/dev/null || true
+docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade pip
+
+# Helper function to install with retries and fallback PyPI mirror
+install_with_retry() {
+  local max_attempts=3
+  local cmd="$@"
+
+  for attempt in $(seq 1 $max_attempts); do
+    echo "Attempt $attempt/$max_attempts: $cmd"
+    if eval "$cmd"; then
+      echo "Success!"
+      return 0
+    fi
+
+    if [ $attempt -lt $max_attempts ]; then
+      echo "Failed, retrying in 5 seconds..."
+      sleep 5
+      # Try with alternative PyPI index on retry
+      if [[ "$cmd" =~ "pip install" ]] && [ $attempt -eq 2 ]; then
+        cmd="$cmd --index-url https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com"
+        echo "Using fallback PyPI mirror: $cmd"
+      fi
+    fi
+  done
+
+  echo "Failed after $max_attempts attempts"
+  return 1
+}
+
+# The anthropic SDK passes `socket_options` to httpx.HTTPTransport, which only
+# exists in httpx>=0.25.0. The CI image ships an older httpx, and several deps
+# installed below (lmms-eval, aiter's requirements.txt, etc.) can pull a stale
+# httpx back in, so test_anthropic_server fails with:
+#   TypeError: HTTPTransport.__init__() got an unexpected keyword argument 'socket_options'
+# Call this as the LAST pip operation so nothing can downgrade httpx afterwards.
+ensure_httpx() {
+  install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade 'httpx>=0.25.0'
+}
+
+# Helper function to git clone with retries
+git_clone_with_retry() {
+  local repo_url="$1"
+  local dest_dir="${2:-}"
+  local branch_args="${3:-}"
+  local max_attempts=3
+
+  for attempt in $(seq 1 $max_attempts); do
+    echo "Git clone attempt $attempt/$max_attempts: $repo_url"
+
+    # prevent from partial clone
+    if [ -n "$dest_dir" ] && [ -d "$dest_dir" ]; then
+      rm -rf "$dest_dir"
+    fi
+
+    if git \
+      -c http.lowSpeedLimit=1000 \
+      -c http.lowSpeedTime=30 \
+      clone --depth 1 ${branch_args:+$branch_args} "$repo_url" "$dest_dir"; then
+      echo "Git clone succeeded."
+      return 0
+    fi
+
+    if [ $attempt -lt $max_attempts ]; then
+      echo "Git clone failed, retrying in 5 seconds..."
+      sleep 5
+    fi
+  done
+
+  echo "Git clone failed after $max_attempts attempts: $repo_url"
+  return 1
+}
+
+# Install checkout sglang
+if [ -n "$SKIP_SGLANG_BUILD" ]; then
+  echo "Didn't build checkout SGLang"
+else
+  docker exec ci_sglang pip uninstall sgl-kernel -y || true
+  docker exec ci_sglang pip uninstall sglang-kernel -y || true
+  docker exec ci_sglang pip uninstall sglang -y || true
+  # Clear Python cache to ensure latest code is used
+  docker exec ci_sglang find /opt/venv -name "*.pyc" -delete || true
+  docker exec ci_sglang find /opt/venv -name "__pycache__" -type d -exec rm -rf {} + || true
+  # Also clear cache in sglang-checkout
+  docker exec ci_sglang find /sglang-checkout -name "*.pyc" -delete || true
+  docker exec ci_sglang find /sglang-checkout -name "__pycache__" -type d -exec rm -rf {} + || true
+  docker exec -w /sglang-checkout/sgl-kernel ci_sglang bash -c "rm -f pyproject.toml && mv pyproject_rocm.toml pyproject.toml && python3 setup_rocm.py install"
+
+  docker exec ci_sglang bash -c 'rm -rf python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml'
+  install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]"
+fi
+
+if [[ -n "${SKIP_TT_DEPS}" ]]; then
+  echo "Didn't build lmms_eval, human-eval, and others"
+else
+  # For lmms_evals evaluating MMMU
+  # Clone on host (with retry), then copy into the container. The checkout is
+  # owned by the runner (non-root); mark it safe so setuptools_scm /
+  # vcs_versioning can run `git` introspection during pip install.
+  git_clone_with_retry https://github.com/EvolvingLMMs-Lab/lmms-eval.git lmms-eval "--branch v0.4.1"
+  docker cp lmms-eval ci_sglang:/
+  docker exec ci_sglang git config --global --add safe.directory /lmms-eval
+  install_with_retry docker exec -w /lmms-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e .
+
+  git_clone_with_retry https://github.com/akao-amd/human-eval.git human-eval
+  docker cp human-eval ci_sglang:/
+  install_with_retry docker exec -w /human-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e .
+
+  mkdir -p dummy-grok
+  cat > dummy-grok/config.json << 'EOF'
+  {
+    "architectures": [
+      "Grok1ModelForCausalLM"
+    ],
+    "embedding_multiplier_scale": 78.38367176906169,
+    "output_multiplier_scale": 0.5773502691896257,
+    "vocab_size": 131072,
+    "hidden_size": 6144,
+    "intermediate_size": 32768,
+    "max_position_embeddings": 8192,
+    "num_experts_per_tok": 2,
+    "num_local_experts": 8,
+    "num_attention_heads": 48,
+    "num_hidden_layers": 64,
+    "num_key_value_heads": 8,
+    "head_dim": 128,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 10000.0,
+    "model_type": "mixtral",
+    "torch_dtype": "bfloat16"
+  }
+EOF
+  docker exec -w / ci_sglang mkdir -p /dummy-grok
+  docker cp ./dummy-grok/config.json ci_sglang:/dummy-grok/config.json
+
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache huggingface_hub[hf_xet]
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache pytest
+
+  # Install cache-dit for qwen_image_t2i_cache_dit_enabled test (added in PR 16204)
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache --upgrade 'cache-dit==1.3.0' || echo "cache-dit installation failed"
+
+  # Install accelerate for distributed training and inference support
+  docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache accelerate || echo "accelerate installation failed"
+fi
+
+# -----------------------
+# MORI
+# The CI image bakes MORI at the docker/rocm.Dockerfile-pinned commit; when a PR
+# bumps MORI_COMMIT the image is not rebuilt, so reinstall MORI here the same way
+# the Dockerfile does. Only ENABLE_MORI=1 images ship /sgl-workspace/mori.
+if docker exec ci_sglang test -d /sgl-workspace/mori; then
+  MORI_REPO=$(grep -E '^[[:space:]]*ARG[[:space:]]+MORI_REPO=' docker/rocm.Dockerfile | head -n1 | sed 's/.*MORI_REPO="\([^"]*\)".*/\1/')
+  MORI_COMMIT=$(grep -E '^[[:space:]]*ARG[[:space:]]+MORI_COMMIT=' docker/rocm.Dockerfile | head -n1 | sed 's/.*MORI_COMMIT="\([^"]*\)".*/\1/')
+
+  if [[ "${GPU_ARCH}" == "mi35x" ]]; then
+    MORI_GPU_ARCHS="gfx950"
+  else
+    MORI_GPU_ARCHS="gfx942"
+  fi
+
+  echo "[MORI] Reinstalling MORI ${MORI_COMMIT} (MORI_GPU_ARCHS=${MORI_GPU_ARCHS})"
+  docker exec ci_sglang bash -c "
+    set -euo pipefail
+    export MORI_GPU_ARCHS='${MORI_GPU_ARCHS}'
+    rm -rf /sgl-workspace/mori
+    git clone '${MORI_REPO}' /sgl-workspace/mori
+    cd /sgl-workspace/mori
+    git checkout '${MORI_COMMIT}'
+    git submodule update --init --recursive
+    python3 setup.py develop
+    python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf
+    ldconfig
+  "
+  echo "[MORI] Done."
+fi
+
+if [[ -n "${SKIP_AITER_BUILD}" ]]; then
+  ensure_httpx
+  exit 0
+fi
+
+# Detect AITER version
+#############################################
+# Detect correct AITER_COMMIT for this runner
+# + Check mismatch
+# + Rebuild AITER if needed
+#############################################
+
+echo "[CI-AITER-CHECK] === AITER VERSION CHECK START ==="
+
+DOCKERFILE="docker/rocm.Dockerfile"
+
+# GPU_ARCH
+GPU_ARCH="${GPU_ARCH:-mi30x}"
+echo "[CI-AITER-CHECK] Runner GPU_ARCH=${GPU_ARCH}"
+
+#############################################
+# 1. Extract AITER_COMMIT from correct Dockerfile block
+#############################################
+if [[ "${GPU_ARCH}" == "mi35x" ]]; then
+    echo "[CI-AITER-CHECK] Using gfx950 block from Dockerfile..."
+    REPO_AITER_COMMIT=$(grep -F -A20 'FROM $BASE_IMAGE_950 AS gfx950' docker/rocm.Dockerfile \
+                        | grep 'AITER_COMMIT_DEFAULT=' \
+                        | head -n1 \
+                        | sed 's/.*AITER_COMMIT_DEFAULT="\([^"]*\)".*/\1/')
+else
+    echo "[CI-AITER-CHECK] Using gfx942 block from Dockerfile..."
+    REPO_AITER_COMMIT=$(grep -F -A20 'FROM $BASE_IMAGE_942 AS gfx942' docker/rocm.Dockerfile \
+                        | grep 'AITER_COMMIT_DEFAULT=' \
+                        | head -n1 \
+                        | sed 's/.*AITER_COMMIT_DEFAULT="\([^"]*\)".*/\1/')
+fi
+
+
+if [[ -z "${REPO_AITER_COMMIT}" ]]; then
+    echo "[CI-AITER-CHECK] ERROR: Failed to extract AITER_COMMIT from Dockerfile."
+    exit 1
+fi
+
+echo "[CI-AITER-CHECK] Dockerfile expects AITER_COMMIT=${REPO_AITER_COMMIT}"
+
+#############################################
+# 2. Check container pre-installed AITER version
+#############################################
+IMAGE_AITER_VERSION=$(docker exec ci_sglang bash -c "pip show amd-aiter 2>/dev/null | grep '^Version:' | awk '{print \$2}'" || echo "none")
+IMAGE_AITER_VERSION="v${IMAGE_AITER_VERSION}"
+echo "[CI-AITER-CHECK] AITER version inside CI image: ${IMAGE_AITER_VERSION}"
+
+#############################################
+# 3. Decide rebuild
+#############################################
+NEED_REBUILD="false"
+
+if [[ -n "${AITER_COMMIT_OVERRIDE:-}" ]]; then
+    echo "[CI-AITER-CHECK] AITER_COMMIT_OVERRIDE=${AITER_COMMIT_OVERRIDE} → forcing rebuild"
+    REPO_AITER_COMMIT="${AITER_COMMIT_OVERRIDE}"
+    NEED_REBUILD="true"
+elif [[ "${IMAGE_AITER_VERSION}" == "vnone" || "${IMAGE_AITER_VERSION}" == "v" ]]; then
+    echo "[CI-AITER-CHECK] No AITER found in image → rebuild needed"
+    NEED_REBUILD="true"
+elif [[ "${IMAGE_AITER_VERSION}" == "${REPO_AITER_COMMIT}" ]]; then
+    echo "[CI-AITER-CHECK] AITER version matches"
+elif [[ "${IMAGE_AITER_VERSION}" =~ (dev|\+g[0-9a-f]+) ]]; then
+    # Dev/patched version (contains 'dev' or git hash) → preserve it
+    echo "[CI-AITER-CHECK] Dev/patched version detected: ${IMAGE_AITER_VERSION} → skipping rebuild"
+else
+    echo "[CI-AITER-CHECK] Version mismatch: image=${IMAGE_AITER_VERSION}, repo=${REPO_AITER_COMMIT}"
+    NEED_REBUILD="true"
+fi
+
+
+#############################################
+# 4. Rebuild AITER if needed
+#############################################
+if [[ "${NEED_REBUILD}" == "true" ]]; then
+    echo "[CI-AITER-CHECK] === AITER REBUILD START ==="
+
+    # uninstall existing aiter
+    docker exec ci_sglang pip uninstall -y amd-aiter || true
+
+    # delete old aiter directory
+    docker exec ci_sglang rm -rf /sgl-workspace/aiter
+
+    # clone a fresh copy to /sgl-workspace/aiter
+    docker exec ci_sglang git clone https://github.com/ROCm/aiter.git /sgl-workspace/aiter
+
+    # checkout correct version and install requirements
+    # Use `checkout -f` so the smudge-filter-induced "dirty" working tree from
+    # AITER's .gitattributes (*.csv text eol=lf, added in ROCm/aiter#3370) does
+    # not block switching to commits that predate that rule. The working tree
+    # was just produced by `rm -rf` + fresh `git clone` above, so there are no
+    # real user changes to preserve.
+    docker exec ci_sglang bash -c "
+        cd /sgl-workspace/aiter && \
+        git fetch --all && \
+        git checkout -f ${REPO_AITER_COMMIT} && \
+        git submodule update --init --recursive && \
+        pip install -r requirements.txt
+    "
+
+    if [[ "${GPU_ARCH}" == "mi35x" ]]; then
+        GPU_ARCH_LIST="gfx950"
+    else
+        GPU_ARCH_LIST="gfx942"
+    fi
+    echo "[CI-AITER-CHECK] GPU_ARCH_LIST=${GPU_ARCH_LIST}"
+
+    # build AITER
+    docker exec ci_sglang bash -c "
+        cd /sgl-workspace/aiter && \
+        AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${GPU_ARCH_LIST} python3 setup.py develop
+    "
+
+    echo "[CI-AITER-CHECK] === AITER REBUILD COMPLETE ==="
+fi
+
+echo "[CI-AITER-CHECK] === AITER VERSION CHECK END ==="
+
+# Must be the final pip operation: force httpx>=0.25.0 so the anthropic SDK can
+# construct its httpx transport (see ensure_httpx definition above).
+ensure_httpx
+
+
+# # Clear pre-built AITER kernels from Docker image to avoid segfaults
+# # The Docker image may contain pre-compiled kernels incompatible with the current environment
+# echo "Clearing pre-built AITER kernels from Docker image..."
+# docker exec ci_sglang find /sgl-workspace/aiter/aiter/jit -name "*.so" -delete 2>/dev/null || true
+# docker exec ci_sglang ls -la /sgl-workspace/aiter/aiter/jit/ 2>/dev/null || echo "jit dir empty or not found"
+
+# # Pre-build AITER kernels to avoid timeout during tests
+# echo "Warming up AITER JIT kernels..."
+# docker exec -e SGLANG_USE_AITER=1 ci_sglang python3 /sglang-checkout/scripts/ci/amd/amd_ci_warmup_aiter.py || echo "AITER warmup completed (some kernels may not be available)"

scripts/ci/amd/amd_ci_start_container.sh

@@ -0,0 +1,341 @@
+#!/bin/bash
+set -euo pipefail
+
+# Get version from git tags
+SGLANG_VERSION="v0.5.5"   # Default version, will be overridden if git tags are found
+
+# Fetch tags from origin to ensure we have the latest
+if git fetch --tags origin; then
+  # Use the shared helper so stable/post releases sort above rc tags.
+  VERSION_FROM_TAG=$(python3 python/tools/get_version_tag.py --tag-only || true)
+  if [ -n "$VERSION_FROM_TAG" ]; then
+    SGLANG_VERSION="$VERSION_FROM_TAG"
+    echo "Using SGLang version from git tags: $SGLANG_VERSION"
+  else
+    echo "Warning: No version tags found; using default $SGLANG_VERSION" >&2
+  fi
+else
+  echo "Warning: Failed to fetch tags from origin; using default $SGLANG_VERSION" >&2
+fi
+
+
+# Default base tags (can be overridden by command line arguments)
+ROCM_VERSION="rocm700"
+DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
+
+# Parse command line arguments
+MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
+MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}"
+CUSTOM_IMAGE=""
+BUILD_FROM_DOCKERFILE=""
+GPU_ARCH_BUILD=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --mi30x-base-tag) MI30X_BASE_TAG="$2"; shift 2;;
+    --mi35x-base-tag) MI35X_BASE_TAG="$2"; shift 2;;
+    --custom-image) CUSTOM_IMAGE="$2"; shift 2;;
+    --build-from-dockerfile) BUILD_FROM_DOCKERFILE="1"; shift;;
+    --gpu-arch) GPU_ARCH_BUILD="$2"; shift 2;;
+    --rocm-version)
+      ROCM_VERSION="$2"
+      MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+      MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+      echo "Using ROCm version override: ${ROCM_VERSION}"
+      shift 2;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS]"
+      echo "Options:"
+      echo "  --mi30x-base-tag TAG       Override MI30x base image tag"
+      echo "  --mi35x-base-tag TAG       Override MI35x base image tag"
+      echo "  --custom-image IMAGE       Use a specific Docker image directly"
+      echo "  --build-from-dockerfile    Build image from docker/rocm.Dockerfile"
+      echo "  --gpu-arch ARCH            GPU architecture for Dockerfile build (e.g., gfx950-rocm720)"
+      echo "  --rocm-version VERSION     Override ROCm version for image lookup (e.g., rocm720)"
+      echo ""
+      echo "Environment:"
+      echo "  ENABLE_CACHE_HOST=1|0"
+      echo "      Mount /home/runner/sglang-data to /sgl-data. Defaults to 0."
+      exit 0
+      ;;
+    *) echo "Unknown option $1"; exit 1;;
+  esac
+done
+
+
+
+# Detect GPU architecture from the Kubernetes runner hostname
+HOSTNAME_VALUE=$(hostname)
+GPU_ARCH="mi30x"   # default
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_ARCH="${BASH_REMATCH[1]}"
+  echo "Detected GPU architecture from hostname: ${GPU_ARCH}"
+else
+  echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}"
+fi
+
+# Normalise / collapse architectures we don't yet build specifically for
+case "${GPU_ARCH}" in
+  mi35x)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi35x image."
+    ;;
+  mi30x|mi300|mi325)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi30x image."
+    GPU_ARCH="mi30x"
+    ;;
+  *)
+    echo "Runner architecture '${GPU_ARCH}' unrecognised; defaulting to mi30x image." >&2
+    GPU_ARCH="mi30x"
+    ;;
+esac
+
+
+# Set up DEVICE_FLAG based on Kubernetes pod info
+if [[ -f /etc/podinfo/gha-render-devices ]]; then
+  DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
+else
+  DEVICE_FLAG="--device /dev/dri"
+fi
+
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub to avoid anonymous pull rate limits.
+# Credentials are optional; when absent we fall back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub…"
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
+
+# Find the latest image
+find_latest_image() {
+  local gpu_arch=$1
+  local base_tag days_back image_tag image_id remote_tags
+
+  case "${gpu_arch}" in
+      mi30x) base_tag="${MI30X_BASE_TAG}" ;;
+      mi35x) base_tag="${MI35X_BASE_TAG}" ;;
+      *)     echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
+  esac
+
+  # First, check local cache on the runner.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
+    if [[ -n "$image_id" ]]; then
+      echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # If not found locally, fall back to pulling from public registry.
+  # We intentionally do not probe ${LOCAL_DOCKER_REGISTRY} here with
+  # `docker manifest inspect --insecure` because that command runs in the
+  # runner pod's network namespace, which on every observed AMD scale set
+  # cannot reach 10.44.14.109:5000 (every probe either fast-fails with TLS
+  # reject or hits a 30s TCP timeout, multiplied across 7 daily candidates).
+  # The actual local-registry pull still happens in the call site below via
+  # `docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}"`, which goes through the
+  # docker daemon on the host and inherits its insecure-registries config.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
+    if docker manifest inspect "rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
+      echo "Found available image: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # Docker Hub's `name=` filter is fuzzy; only accept official version tags.
+  echo "Exact version not found. Searching remote registry for versioned ${ROCM_VERSION}-${gpu_arch} images…" >&2
+  for days_back in {0..6}; do
+    local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
+    local sgl_tag_regex="^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-${target_date}$"
+    remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | while read -r tag; do
+      if [[ "${tag}" =~ ${sgl_tag_regex} ]]; then
+        echo "${tag}"
+        break
+      fi
+    done || true)
+    if [[ -n "$remote_tags" ]]; then
+      echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
+      echo "rocm/sgl-dev:${remote_tags}"
+      return 0
+    fi
+  done
+
+  echo "No recent images found. Searching cached local versioned images matching ROCm+arch…" >&2
+  local any_local
+  any_local=$(docker images --format '{{.Repository}}:{{.Tag}}' --filter "reference=rocm/sgl-dev:v*-${ROCM_VERSION}-${gpu_arch}-*" | while read -r image; do
+    local tag="${image#rocm/sgl-dev:}"
+    if [[ "${tag}" =~ ^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-[0-9]{8}$ ]]; then
+      echo "${image}"
+    fi
+  done | sort -r | head -n 1)
+  if [[ -n "$any_local" ]]; then
+      echo "Using cached fallback image: ${any_local}" >&2
+      echo "${any_local}"
+      return 0
+  fi
+
+  echo "Error: no ${gpu_arch} image found in the last 7 days for base ${base_tag}" >&2
+  echo "Using hard-coded fallback for ${ROCM_VERSION}…" >&2
+  case "${ROCM_VERSION}" in
+    rocm720)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260211-preview"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
+      fi
+      ;;
+    rocm700)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
+      fi
+      ;;
+    *)
+      echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
+      return 1
+      ;;
+  esac
+}
+
+# Determine which image to use
+if [[ -n "${CUSTOM_IMAGE}" ]]; then
+  # Use explicitly provided custom image
+  IMAGE="${CUSTOM_IMAGE}"
+  echo "Using custom image: ${IMAGE}"
+  if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
+    docker pull "${IMAGE}"
+  else
+    retry_with_backoff 6 docker pull "${IMAGE}"
+  fi
+elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
+  # Build image from Dockerfile
+  if [[ -z "${GPU_ARCH_BUILD}" ]]; then
+    echo "Error: --gpu-arch is required when using --build-from-dockerfile" >&2
+    exit 1
+  fi
+
+  DOCKERFILE_DIR="${GITHUB_WORKSPACE:-$PWD}/docker"
+  DOCKERFILE="${DOCKERFILE_DIR}/rocm.Dockerfile"
+
+  if [[ ! -f "${DOCKERFILE}" ]]; then
+    echo "Error: Dockerfile not found at ${DOCKERFILE}" >&2
+    exit 1
+  fi
+
+  IMAGE="sglang-ci:${GPU_ARCH_BUILD}-$(date +%Y%m%d)"
+  echo "Building Docker image from ${DOCKERFILE} with GPU_ARCH=${GPU_ARCH_BUILD}..."
+
+  # Pass full GPU_ARCH (e.g., gfx950-rocm720) - Dockerfile handles stripping suffix
+  docker build \
+    --build-arg GPU_ARCH="${GPU_ARCH_BUILD}" \
+    --build-arg SGL_BRANCH="main" \
+    -t "${IMAGE}" \
+    -f "${DOCKERFILE}" \
+    "${DOCKERFILE_DIR}"
+  echo "Successfully built image: ${IMAGE}"
+else
+  # Find the latest pre-built image
+  IMAGE=$(find_latest_image "${GPU_ARCH}")
+  # Try the local docker registry first (avoids Docker Hub rate limits and is
+  # faster on the LAN); if that fails for any reason, fall back to the
+  # public registry with exponential-backoff retries. Capture stderr so the
+  # real failure reason (TLS handshake, 404, connection refused, etc.) is
+  # visible in the job log instead of being silently swallowed.
+  if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
+    echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
+    docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
+  else
+    echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
+    printf '%s\n' "${local_pull_output}" | sed 's/^/  [local-pull] /' >&2
+    retry_with_backoff 6 docker pull "${IMAGE}"
+  fi
+fi
+
+CACHE_HOST=/home/runner/sglang-data
+ENABLE_CACHE_HOST="${ENABLE_CACHE_HOST:-0}"
+case "${ENABLE_CACHE_HOST,,}" in
+  1|true|yes|on|pvc|persistent)
+    if [[ ! -d "$CACHE_HOST" ]]; then
+      echo "Error: ENABLE_CACHE_HOST=1 but ${CACHE_HOST} does not exist." >&2
+      exit 1
+    fi
+    CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
+    echo "Mounting persistent CI data: ${CACHE_HOST} -> /sgl-data"
+    ;;
+  0|false|no|off|"")
+    CACHE_VOLUME=""
+    echo "Not mounting ${CACHE_HOST}; /sgl-data will be container-local."
+    ;;
+  *)
+    echo "Error: unsupported ENABLE_CACHE_HOST='${ENABLE_CACHE_HOST}'" >&2
+    echo "Use 1/true/pvc/persistent or 0/false/off." >&2
+    exit 1
+    ;;
+esac
+
+echo "Launching container: ci_sglang"
+docker run -dt --user root --device=/dev/kfd ${DEVICE_FLAG} \
+  --ulimit nofile=65536:65536 \
+  -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
+  $CACHE_VOLUME \
+  --group-add video \
+  --shm-size 32g \
+  --cap-add=SYS_PTRACE \
+  -e HF_TOKEN="${HF_TOKEN:-}" \
+  -e HF_HOME=/sgl-data/hf-cache \
+  -e HF_HUB_ETAG_TIMEOUT=300 \
+  -e HF_HUB_DOWNLOAD_TIMEOUT=300 \
+  -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
+  -e MIOPEN_CUSTOM_CACHE_DIR=/sgl-data/miopen-cache \
+  -e PYTHONPATH="/opt/tilelang:${PYTHONPATH:-}" \
+  --security-opt seccomp=unconfined \
+  -w /sglang-checkout \
+  --name ci_sglang \
+  "${IMAGE}"
+
+docker exec ci_sglang mkdir -p \
+  /sgl-data/hf-cache/hub \
+  /sgl-data/pip-cache \
+  /sgl-data/miopen-cache \
+  /sgl-data/aiter-kernels
+
+# The checkout is owned by the runner (non-root) but the container runs as
+# root.  Git >= 2.35.2 rejects cross-user repos; mark the mount as safe so
+# setuptools-scm / vcs_versioning can resolve the package version.
+docker exec ci_sglang git config --global --add safe.directory /sglang-checkout

scripts/ci/amd/amd_ci_start_container_disagg.sh

@@ -0,0 +1,331 @@
+#!/bin/bash
+set -euo pipefail
+
+# Get version from git tags
+SGLANG_VERSION="v0.5.5"   # Default version, will be overridden if git tags are found
+
+# Fetch tags from origin to ensure we have the latest
+if git fetch --tags origin; then
+  # Use the shared helper so stable/post releases sort above rc tags.
+  VERSION_FROM_TAG=$(python3 python/tools/get_version_tag.py --tag-only || true)
+  if [ -n "$VERSION_FROM_TAG" ]; then
+    SGLANG_VERSION="$VERSION_FROM_TAG"
+    echo "Using SGLang version from git tags: $SGLANG_VERSION"
+  else
+    echo "Warning: No version tags found; using default $SGLANG_VERSION" >&2
+  fi
+else
+  echo "Warning: Failed to fetch tags from origin; using default $SGLANG_VERSION" >&2
+fi
+
+
+# Default base tags (can be overridden by command line arguments)
+ROCM_VERSION="rocm700"
+DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+LOCAL_DOCKER_REGISTRY="10.44.14.109:5000"
+
+# Parse command line arguments
+MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
+MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --mi30x-base-tag) MI30X_BASE_TAG="$2"; shift 2;;
+    --mi35x-base-tag) MI35X_BASE_TAG="$2"; shift 2;;
+    --rocm-version)
+      ROCM_VERSION="$2"
+      MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
+      MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
+      echo "Using ROCm version override: ${ROCM_VERSION}"
+      shift 2;;
+    -h|--help)
+      echo "Usage: $0 [--mi30x-base-tag TAG] [--mi35x-base-tag TAG] [--rocm-version VERSION]"
+      exit 0
+      ;;
+    *) echo "Unknown option $1"; exit 1;;
+  esac
+done
+
+
+
+# Detect GPU architecture from the Kubernetes runner hostname
+HOSTNAME_VALUE=$(hostname)
+GPU_ARCH="mi30x"   # default
+
+# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz
+if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then
+  GPU_ARCH="${BASH_REMATCH[1]}"
+  echo "Detected GPU architecture from hostname: ${GPU_ARCH}"
+else
+  echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}"
+fi
+
+# Normalise / collapse architectures we don’t yet build specifically for
+case "${GPU_ARCH}" in
+  mi35x)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi35x image."
+    ;;
+  mi30x|mi300|mi325)
+    echo "Runner uses ${GPU_ARCH}; will fetch mi30x image."
+    GPU_ARCH="mi30x"
+    ;;
+  *)
+    echo "Runner architecture '${GPU_ARCH}' unrecognised; defaulting to mi30x image." >&2
+    GPU_ARCH="mi30x"
+    ;;
+esac
+
+
+# Set up DEVICE_FLAG based on Kubernetes pod info
+if [[ -f /etc/podinfo/gha-render-devices ]]; then
+  DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
+else
+  DEVICE_FLAG="--device /dev/dri"
+fi
+
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  # Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub to avoid anonymous pull rate limits.
+# Credentials are optional; when absent we fall back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub…"
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
+
+# Find the latest image
+find_latest_image() {
+  local gpu_arch=$1
+  local base_tag days_back image_tag image_id remote_tags
+
+  case "${gpu_arch}" in
+      mi30x) base_tag="${MI30X_BASE_TAG}" ;;
+      mi35x) base_tag="${MI35X_BASE_TAG}" ;;
+      *)     echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
+  esac
+
+  # First, check local cache on the runner.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
+    if [[ -n "$image_id" ]]; then
+      echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # If not found locally, fall back to pulling from public registry.
+  # See amd_ci_start_container.sh for why we don't probe
+  # ${LOCAL_DOCKER_REGISTRY} with `docker manifest inspect --insecure` from
+  # the runner pod's network namespace; the actual local-registry pull
+  # happens at the call site below via the docker daemon on the host.
+  for days_back in {0..6}; do
+    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
+    echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
+    if docker manifest inspect "rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
+      echo "Found available image: rocm/sgl-dev:${image_tag}" >&2
+      echo "rocm/sgl-dev:${image_tag}"
+      return 0
+    fi
+  done
+
+  # Docker Hub's `name=` filter is fuzzy; only accept official version tags.
+  echo "Exact version not found. Searching remote registry for versioned ${ROCM_VERSION}-${gpu_arch} images…" >&2
+  for days_back in {0..6}; do
+    local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
+    local sgl_tag_regex="^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-${target_date}$"
+    remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | while read -r tag; do
+      if [[ "${tag}" =~ ${sgl_tag_regex} ]]; then
+        echo "${tag}"
+        break
+      fi
+    done || true)
+    if [[ -n "$remote_tags" ]]; then
+      echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
+      echo "rocm/sgl-dev:${remote_tags}"
+      return 0
+    fi
+  done
+
+  echo "No recent images found. Searching cached local versioned images matching ROCm+arch…" >&2
+  local any_local
+  any_local=$(docker images --format '{{.Repository}}:{{.Tag}}' --filter "reference=rocm/sgl-dev:v*-${ROCM_VERSION}-${gpu_arch}-*" | while read -r image; do
+    local tag="${image#rocm/sgl-dev:}"
+    if [[ "${tag}" =~ ^v[0-9][A-Za-z0-9._-]*-${ROCM_VERSION}-${gpu_arch}-[0-9]{8}$ ]]; then
+      echo "${image}"
+    fi
+  done | sort -r | head -n 1)
+  if [[ -n "$any_local" ]]; then
+      echo "Using cached fallback image: ${any_local}" >&2
+      echo "${any_local}"
+      return 0
+  fi
+
+  echo "Error: no ${gpu_arch} image found in the last 7 days for base ${base_tag}" >&2
+  echo "Using hard-coded fallback for ${ROCM_VERSION}…" >&2
+  case "${ROCM_VERSION}" in
+    rocm720)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi35x-20260211-preview"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm720-mi30x-20260211-preview"
+      fi
+      ;;
+    rocm700)
+      if [[ "${gpu_arch}" == "mi35x" ]]; then
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi35x-20260211"
+      else
+        echo "rocm/sgl-dev:v0.5.8.post1-rocm700-mi30x-20260211"
+      fi
+      ;;
+    *)
+      echo "Error: no hard-coded fallback available for ${ROCM_VERSION}" >&2
+      return 1
+      ;;
+  esac
+}
+
+# Pull and run the latest image
+IMAGE=$(find_latest_image "${GPU_ARCH}")
+# Try the local docker registry first (avoids Docker Hub rate limits and is
+# faster on the LAN); if that fails for any reason, fall back to the
+# public registry with exponential-backoff retries. Capture stderr so the
+# real failure reason (TLS handshake, 404, connection refused, etc.) is
+# visible in the job log instead of being silently swallowed.
+if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
+  echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
+  docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
+else
+  echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
+  printf '%s\n' "${local_pull_output}" | sed 's/^/  [local-pull] /' >&2
+  retry_with_backoff 6 docker pull "${IMAGE}"
+fi
+
+# CACHE_HOST=/home/runner/sgl-data
+CACHE_HOST=/home/runner/temp-sglang-data
+if [[ -d "$CACHE_HOST" ]]; then
+    CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
+else
+    CACHE_VOLUME=""
+fi
+
+# Detect libionic library for RDMA support
+LIBIONIC_MOUNT=""
+IONIC_SYMLINK="/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so"
+if [[ -L "$IONIC_SYMLINK" ]]; then
+    LIBIONIC_LIB=$(readlink -f "$IONIC_SYMLINK" 2>/dev/null)
+    if [[ -f "$LIBIONIC_LIB" ]]; then
+        echo "Found libionic library: $LIBIONIC_LIB (resolved from symlink)"
+        LIBIONIC_MOUNT="-v ${LIBIONIC_LIB}:${LIBIONIC_LIB}:ro"
+    else
+        echo "Warning: libionic symlink exists but target does not: $LIBIONIC_LIB"
+    fi
+else
+    # Fallback: try to find directly
+    LIBIONIC_FOUND=$(find /usr/lib/x86_64-linux-gnu -maxdepth 1 -name "libionic.so.*" 2>/dev/null | head -1)
+    if [[ -n "$LIBIONIC_FOUND" ]]; then
+        LIBIONIC_LIB=$(readlink -f "$LIBIONIC_FOUND" 2>/dev/null)
+        if [[ -f "$LIBIONIC_LIB" ]]; then
+            echo "Found libionic library: $LIBIONIC_LIB"
+            LIBIONIC_MOUNT="-v ${LIBIONIC_LIB}:${LIBIONIC_LIB}:ro"
+        else
+            echo "Warning: libionic found but cannot resolve real path: $LIBIONIC_FOUND"
+        fi
+    else
+        echo "Warning: libionic library not found on host, RDMA may not work"
+    fi
+fi
+
+MOUNT_ARGS=""
+
+add_mount_if_exists() {
+    local name=$1
+    local search_pattern=$2
+    local path=$(find /lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu /lib64 /usr/lib64 -name "$search_pattern" -print -quit 2>/dev/null)
+
+    if [ -n "$path" ]; then
+        echo "Found $name at: $path"
+        MOUNT_ARGS="$MOUNT_ARGS -v $path:$path:ro"
+    else
+        echo "WARNING: Could not find $name on host! (Pattern: $search_pattern)"
+    fi
+}
+
+IONIC_LINK="/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so"
+if [ -L "$IONIC_LINK" ]; then
+    IONIC_REAL=$(readlink -f "$IONIC_LINK")
+    if [ -f "$IONIC_REAL" ]; then
+        echo "Ionic Driver: $IONIC_REAL"
+        MOUNT_ARGS="$MOUNT_ARGS -v $IONIC_REAL:$IONIC_REAL:ro"
+    fi
+fi
+
+add_mount_if_exists "libnl-3" "libnl-3.so*"
+add_mount_if_exists "libmnl" "libmnl.so*"
+
+echo "Mount args: $MOUNT_ARGS"
+
+echo "Launching container: ci_sglang"
+docker run -dt --user root \
+  --device=/dev/kfd \
+  --device=/dev/dri \
+  ${DEVICE_FLAG} \
+  -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
+  -v /sys/class/infiniband:/sys/class/infiniband:ro \
+  -v /sys/class/infiniband_verbs:/sys/class/infiniband_verbs:ro \
+  -v /sys/class/net:/sys/class/net:ro \
+  -v /etc/libibverbs.d:/etc/libibverbs.d:ro \
+  -v /usr/lib/x86_64-linux-gnu/libibverbs:/usr/lib/x86_64-linux-gnu/libibverbs:ro \
+  $MOUNT_ARGS \
+  $CACHE_VOLUME \
+  --privileged \
+  --network=host \
+  --ipc=host \
+  --ulimit memlock=-1 \
+  --cap-add=IPC_LOCK \
+  --cap-add=SYS_PTRACE \
+  --security-opt seccomp=unconfined \
+  --group-add video \
+  --group-add rdma \
+  --shm-size 32g \
+  -e HF_TOKEN="${HF_TOKEN:-}" \
+  -e HF_HOME=/sgl-data/hf-cache \
+  -e HF_HUB_ETAG_TIMEOUT=300 \
+  -e HF_HUB_DOWNLOAD_TIMEOUT=300 \
+  -e MIOPEN_USER_DB_PATH=/sgl-data/miopen-cache \
+  -e MIOPEN_CUSTOM_CACHE_DIR=/sgl-data/miopen-cache \
+  -w /sglang-checkout \
+  --name ci_sglang \
+  "${IMAGE}"
+
+# The checkout is owned by the runner (non-root) but the container runs as
+# root.  Git >= 2.35.2 rejects cross-user repos; mark the mount as safe so
+# setuptools-scm / vcs_versioning can resolve the package version.
+docker exec ci_sglang git config --global --add safe.directory /sglang-checkout

scripts/ci/amd/amd_ci_warmup_aiter.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""
+Warmup script to pre-build AITER JIT kernels.
+
+This script triggers compilation of commonly used AITER kernels by importing
+the relevant modules and calling functions with sample data. This avoids
+timeouts during actual tests when kernels need to be compiled on first use.
+
+Run this after clearing pre-built AITER kernels from the Docker image.
+"""
+
+import os
+import sys
+import time
+
+# Ensure AITER is enabled
+os.environ["SGLANG_USE_AITER"] = "1"
+
+
+def warmup_aiter_kernels():
+    """Trigger AITER JIT kernel compilation."""
+    import torch
+
+    if not torch.cuda.is_available():
+        print("CUDA/ROCm not available, skipping AITER warmup")
+        return
+
+    print("=" * 60)
+    print("AITER JIT Kernel Warmup")
+    print("=" * 60)
+
+    device = torch.device("cuda:0")
+    start_time = time.time()
+
+    # Warmup module_rmsnorm_quant (small module, ~2MB)
+    # Triggered by rmsnorm2d_fwd when hidden_size <= 8192
+    try:
+        print(
+            "\n[1/5] Warming up module_rmsnorm_quant (rmsnorm2d_fwd, hidden<=8192)..."
+        )
+        from aiter import rmsnorm2d_fwd
+
+        hidden_size = 4096
+        batch_size = 512  # Use larger batch to match CUDA graph capture
+        x = torch.randn(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+        weight = torch.ones(hidden_size, dtype=torch.bfloat16, device=device)
+        eps = 1e-6
+
+        # hidden_size=4096 <= 8192 -> takes rmsnorm() path -> compiles module_rmsnorm_quant
+        _ = rmsnorm2d_fwd(x, weight, eps)
+        torch.cuda.synchronize()
+        print("   module_rmsnorm_quant compiled successfully")
+    except Exception as e:
+        print(f"   module_rmsnorm_quant warmup failed: {e}")
+
+    # Warmup module_rmsnorm (large CK module, ~159MB)
+    # Triggered by rmsnorm2d_fwd_with_add (always uses CK path)
+    # NOTE: rmsnorm2d_fwd_with_add signature is:
+    #   rmsnorm2d_fwd_with_add(out, input, residual_in, residual_out, weight, epsilon)
+    try:
+        print("\n[2/5] Warming up module_rmsnorm (rmsnorm2d_fwd_with_add, CK path)...")
+        from aiter import rmsnorm2d_fwd_with_add
+
+        hidden_size = 4096
+        batch_size = 512
+        x = torch.randn(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+        residual_in = torch.randn(
+            batch_size, hidden_size, dtype=torch.bfloat16, device=device
+        )
+        output = torch.empty_like(x)
+        residual_out = torch.empty_like(x)
+        weight = torch.ones(hidden_size, dtype=torch.bfloat16, device=device)
+        eps = 1e-6
+
+        # This triggers JIT compilation of module_rmsnorm (CK kernels)
+        rmsnorm2d_fwd_with_add(output, x, residual_in, residual_out, weight, eps)
+        torch.cuda.synchronize()
+        print("   module_rmsnorm compiled successfully")
+    except Exception as e:
+        print(f"   module_rmsnorm warmup failed: {e}")
+
+    # Warmup module_rmsnorm via rmsnorm2d_fwd with large hidden_size (CK path)
+    # When hidden_size > 8192, rmsnorm2d_fwd takes the rmsnorm2d_fwd_ck path
+    # which also uses module_rmsnorm (already compiled in step 2, but this
+    # ensures the CK rmsnorm2d_fwd path is exercised as well)
+    try:
+        print("\n[3/5] Warming up rmsnorm2d_fwd CK path (hidden>8192)...")
+        from aiter import rmsnorm2d_fwd
+
+        hidden_size = 16384  # > 8192 to trigger rmsnorm2d_fwd_ck (module_rmsnorm)
+        batch_size = 32
+        x = torch.randn(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+        weight = torch.ones(hidden_size, dtype=torch.bfloat16, device=device)
+        eps = 1e-6
+
+        _ = rmsnorm2d_fwd(x, weight, eps)
+        torch.cuda.synchronize()
+        print("   rmsnorm2d_fwd CK path compiled successfully")
+    except Exception as e:
+        print(f"   rmsnorm2d_fwd CK path warmup skipped: {e}")
+
+    # Warmup rotary embedding kernel if available
+    try:
+        print("\n[4/5] Warming up rotary embedding kernel...")
+        from aiter import rotary_embedding
+
+        head_size = 128
+        seq_len = 32
+        num_heads = 32
+        positions = torch.arange(seq_len, device=device)
+        query = torch.randn(
+            seq_len, num_heads, head_size, dtype=torch.bfloat16, device=device
+        )
+        key = torch.randn(
+            seq_len, num_heads, head_size, dtype=torch.bfloat16, device=device
+        )
+        cos = torch.ones(seq_len, head_size // 2, dtype=torch.bfloat16, device=device)
+        sin = torch.zeros(seq_len, head_size // 2, dtype=torch.bfloat16, device=device)
+
+        _ = rotary_embedding(positions, query, key, head_size, cos, sin, True)
+        torch.cuda.synchronize()
+        print("   Rotary embedding kernel compiled successfully")
+    except Exception as e:
+        print(f"   Rotary embedding warmup skipped (may not be available): {e}")
+
+    # Warmup activation kernels if available
+    try:
+        print("\n[5/5] Warming up activation kernels...")
+        from aiter import silu_and_mul
+
+        hidden_size = 4096
+        batch_size = 512
+        x = torch.randn(
+            batch_size, hidden_size * 2, dtype=torch.bfloat16, device=device
+        )
+        out = torch.empty(batch_size, hidden_size, dtype=torch.bfloat16, device=device)
+
+        silu_and_mul(out, x)
+        torch.cuda.synchronize()
+        print("   Activation kernel compiled successfully")
+    except Exception as e:
+        print(f"   Activation warmup skipped (may not be available): {e}")
+
+    elapsed = time.time() - start_time
+    print("\n" + "=" * 60)
+    print(f"AITER warmup completed in {elapsed:.1f}s")
+    print("=" * 60 + "\n")
+
+
+if __name__ == "__main__":
+    warmup_aiter_kernels()

scripts/ci/amd/check_vram_clear.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+check_vram_clear() {
+    local vram_threshold_percent=5  # Allow up to 5% VRAM usage
+    local memory_threshold_mb=500   # Allow up to 500MB memory usage
+
+    if command -v rocm-smi >/dev/null 2>&1; then
+        echo "Checking ROCm GPU VRAM usage..."
+        # Check if any GPU has more than threshold VRAM allocated
+        local high_usage=$(rocm-smi --showmemuse | grep -E "GPU Memory Allocated \(VRAM%\): ([6-9]|[1-9][0-9]|100)")
+        if [ -n "$high_usage" ]; then
+            echo "ERROR: VRAM usage exceeds threshold (${vram_threshold_percent}%) on some GPUs:"
+            echo "$high_usage"
+            rocm-smi --showmemuse
+            return 1
+        else
+            echo "✓ VRAM usage is within acceptable limits on all GPUs"
+            return 0
+        fi
+   fi
+}
+
+# If this script is run directly (not sourced), run the check
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    set -e
+    check_vram_clear
+fi

scripts/ci/amd/ensure_vram_clear.sh

@@ -0,0 +1,269 @@
+#!/bin/bash
+
+# Source the VRAM checking function
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/check_vram_clear.sh"
+
+# Stop and remove every container that holds any /dev/kfd or /dev/dri device.
+# Some failing CI runs leave behind containers other than `ci_sglang` (e.g.
+# from previous AMD jobs that were force-killed mid-run); those still hold
+# VRAM via KFD even though the host pgrep finds nothing.
+stop_all_gpu_containers() {
+    if ! command -v docker >/dev/null 2>&1; then
+        return 0
+    fi
+
+    local all_ids
+    all_ids=$(docker ps -aq 2>/dev/null || true)
+    if [ -z "$all_ids" ]; then
+        echo "No docker containers found on host"
+        return 0
+    fi
+
+    local gpu_ids=""
+    local cid
+    for cid in $all_ids; do
+        # A container is "GPU-attached" if its inspect output mentions any
+        # GPU device or NVIDIA/ROCm GPU capability. Inspecting the raw JSON
+        # (instead of a specific field) survives docker version differences.
+        if docker inspect "$cid" 2>/dev/null \
+            | grep -qE '"PathOnHost":"/dev/(kfd|dri)|"Capabilities":\[\["gpu"\]\]'; then
+            gpu_ids+=" $cid"
+        fi
+    done
+
+    gpu_ids=$(echo "$gpu_ids" | tr ' ' '\n' | grep -E '^[a-f0-9]+$' || true)
+    if [ -z "$gpu_ids" ]; then
+        echo "No GPU-attached docker containers found on host"
+        return 0
+    fi
+
+    echo "Found GPU-attached containers, stopping them:"
+    for cid in $gpu_ids; do
+        docker ps -a --filter "id=$cid" --format '  {{.ID}} {{.Image}} {{.Status}} {{.Names}}' 2>/dev/null || true
+    done
+    echo "$gpu_ids" | xargs -r docker stop --time 5 2>/dev/null || true
+    echo "$gpu_ids" | xargs -r docker rm -f 2>/dev/null || true
+}
+
+# Find and kill any host process that holds an open handle to /dev/kfd or
+# /dev/dri/renderD*. This is far more reliable than `rocm-smi --showpids`,
+# which only sees processes that registered a HSA queue (zombies and
+# processes that crashed mid-init are invisible to it).
+kill_processes_holding_gpu_devices() {
+    local signal=${1:-TERM}
+    local pids=""
+
+    # If neither tool is present we silently degrade to a no-op, which used
+    # to look identical in the log to "no holders found" and made the
+    # script's failures very confusing. Emit a loud warning so the runner
+    # owner knows why GPU device cleanup isn't happening.
+    if ! command -v fuser >/dev/null 2>&1 && ! command -v lsof >/dev/null 2>&1; then
+        echo "WARNING: neither fuser nor lsof installed on the host;" \
+             "cannot detect processes holding /dev/kfd or /dev/dri/renderD*."
+        echo "         Install psmisc (for fuser) or lsof on the runner host" \
+             "to enable device-fd-based cleanup."
+        return 0
+    fi
+
+    if command -v fuser >/dev/null 2>&1; then
+        # `fuser` prints PIDs to stdout, names to stderr; collect everything
+        # that has any handle on KFD or render nodes.
+        pids+=" $(fuser /dev/kfd 2>/dev/null || true)"
+        for dev in /dev/dri/renderD*; do
+            [ -e "$dev" ] || continue
+            pids+=" $(fuser "$dev" 2>/dev/null || true)"
+        done
+    fi
+
+    if command -v lsof >/dev/null 2>&1; then
+        pids+=" $(lsof -t /dev/kfd 2>/dev/null || true)"
+        for dev in /dev/dri/renderD*; do
+            [ -e "$dev" ] || continue
+            pids+=" $(lsof -t "$dev" 2>/dev/null || true)"
+        done
+    fi
+
+    pids=$(echo "$pids" | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -u || true)
+    if [ -z "$pids" ]; then
+        return 0
+    fi
+
+    local self_pid=$$
+    echo "Processes holding /dev/kfd or /dev/dri/renderD*:"
+    for pid in $pids; do
+        # Skip our own PID and any of our ancestors so we don't suicide.
+        if [ "$pid" = "$self_pid" ]; then
+            continue
+        fi
+        local cmd
+        cmd=$(ps -p "$pid" -o pid,ppid,stat,cmd --no-headers 2>/dev/null || true)
+        if [ -z "$cmd" ]; then
+            continue
+        fi
+        echo "  $cmd"
+        # If it's a zombie, kill the parent instead — kill -9 on a zombie is
+        # a no-op, the only way to reap it is to make its parent reap it.
+        local stat
+        stat=$(ps -p "$pid" -o stat= 2>/dev/null | tr -d ' ')
+        if [[ "$stat" == Z* ]]; then
+            local ppid
+            ppid=$(ps -p "$pid" -o ppid= 2>/dev/null | tr -d ' ')
+            if [ -n "$ppid" ] && [ "$ppid" != "1" ] && [ "$ppid" != "$self_pid" ]; then
+                echo "    -> $pid is a zombie, sending SIG$signal to parent $ppid"
+                kill "-$signal" "$ppid" 2>/dev/null || true
+            fi
+            continue
+        fi
+        kill "-$signal" "$pid" 2>/dev/null || true
+    done
+}
+
+# Print rich diagnostics that explain *why* VRAM is still allocated when no
+# obvious owner exists. Helpful when zombies / other namespaces hold memory.
+dump_gpu_diagnostics() {
+    echo "=== GPU device file holders (fuser) ==="
+    if command -v fuser >/dev/null 2>&1; then
+        fuser -v /dev/kfd 2>&1 || true
+        for dev in /dev/dri/renderD* /dev/dri/card*; do
+            [ -e "$dev" ] || continue
+            fuser -v "$dev" 2>&1 || true
+        done
+    else
+        echo "fuser not installed"
+    fi
+
+    echo "=== GPU device file holders (lsof) ==="
+    if command -v lsof >/dev/null 2>&1; then
+        lsof /dev/kfd 2>/dev/null || true
+        lsof /dev/dri/renderD* 2>/dev/null || true
+    else
+        echo "lsof not installed"
+    fi
+
+    echo "=== Zombie processes on host ==="
+    ps -eo pid,ppid,stat,etime,cmd 2>/dev/null | awk 'NR==1 || $3 ~ /^Z/ {print}'
+
+    echo "=== Docker containers on host ==="
+    if command -v docker >/dev/null 2>&1; then
+        docker ps -a --format 'table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Names}}' 2>/dev/null || true
+    fi
+
+    echo "=== rocm-smi --showpids ==="
+    timeout 30 rocm-smi --showpids 2>&1 || echo "rocm-smi --showpids timed out"
+
+    echo "=== rocm-smi --showmemuse ==="
+    timeout 30 rocm-smi --showmemuse 2>&1 || echo "rocm-smi --showmemuse timed out"
+}
+
+ensure_vram_clear() {
+    local max_retries=3
+    local retry_count=0
+
+    # Log host information for debugging
+    echo "=== Host Information ==="
+    echo "Hostname: $(hostname)"
+    echo "Host IP: $(hostname -I 2>/dev/null || echo 'N/A')"
+    echo "Date: $(date)"
+    echo "Mode: rocm"
+    echo "========================"
+    echo "Running in ROCm mode"
+
+    # Always stop the well-known CI container first (best-effort).
+    echo "Stopping any existing ci_sglang container..."
+    docker stop ci_sglang 2>/dev/null || true
+    docker rm -f ci_sglang 2>/dev/null || true
+
+    # Show initial GPU status
+    echo "=== Initial GPU Memory Status ==="
+    rocm-smi --showmemuse
+    echo "=================================="
+
+    # Fast path: if the runner is already clean, skip the cleanup loop
+    # entirely so healthy jobs don't pay the ~35s/attempt cleanup cost.
+    if check_vram_clear; then
+        echo "✓ VRAM is already clear; skipping cleanup."
+        return 0
+    fi
+
+    while [ $retry_count -lt $max_retries ]; do
+        echo "=== Cleanup Attempt $((retry_count + 1))/$max_retries ==="
+
+        # Step 1: kill SGLang-named processes on the host (cheap, fast).
+        # NOTE: host pgrep cannot see PIDs inside a container's PID
+        # namespace, so in CI this almost never matches anything; the
+        # heavy lifting is done by step 2 below. Kept as a fast early
+        # cleanup for the rare case where something runs on the host.
+        echo "Killing SGLang processes..."
+        pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' \
+            | xargs -r kill -9 2>/dev/null || true
+
+        # Step 2: aggressive cleanup. Run on EVERY attempt — the previous
+        # version skipped this on attempt 1, which made attempt 1 a near
+        # no-op for the most common failure mode (a leftover container
+        # holding VRAM, invisible to host pgrep).
+        echo "Performing aggressive cleanup..."
+
+        # 2a. Stop ALL GPU-attached containers, not just ci_sglang. A
+        # leftover container from a previous job will keep VRAM held even
+        # though `pgrep` on the host shows nothing.
+        stop_all_gpu_containers
+
+        # 2b. SIGTERM anything that has /dev/kfd or /dev/dri/renderD* open.
+        # `lsof`/`fuser` see processes that `rocm-smi --showpids` misses
+        # (notably zombies and processes outside our PID namespace).
+        echo "Sending SIGTERM to processes holding GPU device files..."
+        kill_processes_holding_gpu_devices TERM
+        sleep 5
+
+        # 2c. SIGKILL anything still holding GPU device files.
+        echo "Sending SIGKILL to remaining holders..."
+        kill_processes_holding_gpu_devices KILL
+
+        # 2d. Best-effort: also kill anything `rocm-smi --showpids` reports.
+        # Handles both the legacy "PID: <n>" line format and the modern
+        # tabular format (`<pid>\t<name>\t<gpus>\t...`); the previous
+        # `grep 'PID:'` matched nothing on ROCm 5+ tabular output.
+        rocm-smi --showpids 2>/dev/null \
+            | awk '/^PID:[[:space:]]*[0-9]+/ {print $2} /^[0-9]+/ {print $1}' \
+            | xargs -r kill -9 2>/dev/null || true
+
+        echo "Waiting 30 seconds for VRAM to clear..."
+        sleep 30
+
+        # Step 3: re-check.
+        echo "Checking VRAM status..."
+        if check_vram_clear; then
+            echo "✓ VRAM cleanup successful after $((retry_count + 1)) attempts"
+            return 0
+        else
+            echo "✗ VRAM still not clear after attempt $((retry_count + 1))"
+            # Step 4: dump diagnostics on every failed attempt so the next
+            # attempt's logs already explain WHY cleanup didn't work.
+            # Without this we'd only see what's holding the GPU at the very
+            # end, which makes triage much harder.
+            echo "--- Diagnostics for failed attempt $((retry_count + 1)) ---"
+            dump_gpu_diagnostics
+            echo "--- End of diagnostics for attempt $((retry_count + 1)) ---"
+            retry_count=$((retry_count + 1))
+        fi
+    done
+
+    # Failed after all retries — diagnostics for the last cleanup attempt
+    # were already dumped above; just print the actionable hint.
+    echo "=== FAILED: VRAM cleanup unsuccessful after $max_retries attempts ==="
+    echo "(See diagnostics above for the final attempt.)"
+    echo "=================================================================="
+    echo "Hint: if no host process / container holds the GPU but VRAM is"
+    echo "still allocated, this is almost certainly a zombie KFD context"
+    echo "(see ROCm/aiter#2061). The node will need to be rebooted before"
+    echo "subsequent jobs can succeed."
+    echo "=================================================================="
+    return 1
+}
+
+# If this script is run directly (not sourced), run the ensure function
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    set -e
+    ensure_vram_clear "$@"
+fi

scripts/ci/amd/test_rccl_multi_gpu.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Simple RCCL test for multi-GPU communication.
+This test verifies that RCCL can initialize and communicate across multiple GPUs.
+"""
+
+import os
+import sys
+
+import torch
+import torch.distributed as dist
+
+
+def test_rccl_allreduce():
+    """Test basic RCCL allreduce operation across all GPUs."""
+    if not torch.cuda.is_available():
+        print("CUDA not available, skipping test")
+        sys.exit(1)
+
+    # Initialize process group with NCCL (RCCL on AMD)
+    dist.init_process_group(backend="nccl")
+
+    rank = dist.get_rank()
+    world_size = dist.get_world_size()
+
+    print(f"[Rank {rank}/{world_size}] Initialized successfully")
+
+    # Set device
+    device = torch.device(f"cuda:{rank}")
+    torch.cuda.set_device(device)
+
+    print(f"[Rank {rank}] Device: {torch.cuda.get_device_name(device)}")
+    print(
+        f"[Rank {rank}] Device memory: {torch.cuda.get_device_properties(device).total_memory / 1e9:.2f} GB"
+    )
+
+    # Create a tensor and perform allreduce
+    tensor = torch.ones(1000, device=device) * rank
+    print(f"[Rank {rank}] Before allreduce: tensor sum = {tensor.sum().item()}")
+
+    dist.all_reduce(tensor, op=dist.ReduceOp.SUM)
+
+    expected_sum = sum(range(world_size)) * 1000
+    actual_sum = tensor.sum().item()
+
+    print(
+        f"[Rank {rank}] After allreduce: tensor sum = {actual_sum}, expected = {expected_sum}"
+    )
+
+    if abs(actual_sum - expected_sum) < 0.1:
+        print(f"[Rank {rank}] ✓ RCCL allreduce test PASSED")
+        dist.destroy_process_group()
+        sys.exit(0)
+    else:
+        print(f"[Rank {rank}] ✗ RCCL allreduce test FAILED")
+        dist.destroy_process_group()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    test_rccl_allreduce()

scripts/ci/check_no_docs_changes.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Reject staged changes under the legacy docs/ tree."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+ERROR_MESSAGE = """\
+Changes under the legacy docs/ directory are not allowed.
+
+The documentation has been migrated. Please make documentation updates in the
+corresponding location under docs_new/ instead.
+"""
+
+LEGACY_DOCS_ALLOWLIST = {
+    "docs/_static/css/custom_log.css",
+    "docs/_static/js/deprecation_banner.js",
+    "docs/conf.py",
+    # Has relative links into the source tree that the offline lychee check
+    # validates, so it must be updated when the linked source files move.
+    "docs/developer_guide/development_jit_kernel_guide.md",
+}
+
+
+def staged_paths() -> list[str]:
+    result = subprocess.run(
+        [
+            "git",
+            "diff",
+            "--cached",
+            "--name-only",
+            "--diff-filter=ACMRDTUXB",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return [line.strip() for line in result.stdout.splitlines() if line.strip()]
+
+
+def main() -> int:
+    paths = sys.argv[1:] or staged_paths()
+    docs_paths = sorted(
+        path
+        for path in paths
+        if (path == "docs" or path.startswith("docs/"))
+        and path not in LEGACY_DOCS_ALLOWLIST
+    )
+
+    if not docs_paths:
+        return 0
+
+    print(ERROR_MESSAGE, file=sys.stderr)
+    print("Detected legacy docs/ changes:", file=sys.stderr)
+    for path in docs_paths:
+        print(f"  - {path}", file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ci/check_no_registered_tests_in_package.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""
+Pre-commit hook: reject CI-registered tests that live inside the importable
+`sglang` package (python/sglang/).
+
+Registered tests and benchmarks must live under test/registered/ (e.g.
+test/registered/jit/ for JIT kernel tests and test/registered/jit/benchmark/
+for JIT kernel benchmarks) so they are not shipped in the wheel and are
+collected by run_suite.py's registered glob. A registered file placed inside
+the package would be shipped to users AND silently dropped by run_suite.py
+(which no longer globs the package) -- it would never run in CI. This guard
+turns that silent skip into a hard failure.
+
+Reuses ut_parse_one_file() from ci_register.py (AST-based) so the registry
+detection matches run_suite.py's collect_tests() exactly.
+"""
+
+import glob
+import importlib.util
+import os
+import sys
+
+# Markers whose mere presence in the source is worth an AST parse. Anything
+# without one of these strings cannot register a test, so we skip parsing it.
+_MARKERS = (
+    "register_cuda_ci",
+    "register_amd_ci",
+    "register_cpu_ci",
+    "register_npu_ci",
+    "register_xpu_ci",
+    "register_musa_ci",
+)
+
+
+def main() -> int:
+    # Import ci_register directly to avoid pulling in all of sglang.
+    spec = importlib.util.spec_from_file_location(
+        "ci_register",
+        os.path.join("python", "sglang", "test", "ci", "ci_register.py"),
+    )
+    ci_register = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(ci_register)
+
+    offenders = []
+    for f in sorted(glob.glob("python/sglang/**/*.py", recursive=True)):
+        try:
+            with open(f, "r", encoding="utf-8") as fh:
+                source = fh.read()
+        except (OSError, UnicodeDecodeError):
+            continue
+        if not any(marker in source for marker in _MARKERS):
+            continue
+        try:
+            registries, _has_main_entry = ci_register.ut_parse_one_file(f)
+        except Exception:
+            # A malformed register call still indicates a misplaced test.
+            offenders.append(f)
+            continue
+        if registries:
+            offenders.append(f)
+
+    if offenders:
+        print(
+            "ERROR: CI-registered test(s)/benchmark(s) found inside the sglang package:"
+        )
+        print(
+            "  Registered tests and benchmarks must live under test/registered/\n"
+            "  (e.g. test/registered/jit/ for JIT kernel tests and\n"
+            "  test/registered/jit/benchmark/ for JIT kernel benchmarks) so they\n"
+            "  are not shipped in the wheel and are collected by run_suite.py.\n"
+        )
+        for f in offenders:
+            print(f"  {f}")
+        print()
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/check_registered_tests.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+Pre-commit hook: validate CI registry calls under test/registered/.
+
+1. Every test file must contain a CI registry call (register_cuda_ci,
+   register_amd_ci, etc.).
+2. A CUDA test must register its PR-test suite via the modern
+   `stage=`/`runner_config=` form. The legacy single-string `suite=` is reserved
+   for the nightly/stress/weekly families (and for AMD/CPU/NPU suites); any other
+   CUDA `suite=` resolves to a name no PR-test workflow invokes, so the test
+   silently never runs. Two shapes are rejected:
+     a. `{stage}-test-{runner_config}` -- the modern name stuffed back into the
+        legacy form. Reported with the exact stage/runner split to use.
+     b. an older `{stage}-{runner_config}` PR-test name (e.g. the pre-migration
+        `base-b-kernel-unit-1-gpu-large`) -- no longer matches any workflow
+        suite at all.
+   The modern form resolves to the identical suite (CIRegistry.effective_suite
+   is f"{stage}-test-{runner_config}") and is /rerun-test-able.
+
+Reuses ut_parse_one_file() from ci_register.py (AST-based parsing)
+to match the same logic used by run_suite.py's collect_tests().
+"""
+
+import glob
+import importlib.util
+import os
+import re
+import sys
+
+# Suite names of the form `{stage}-test-{runner_config}` are exactly what the
+# modern stage=/runner_config= form produces, so a legacy suite= carrying this
+# shape is always expressible (and should be expressed) the modern way.
+_MODERN_SHAPE = re.compile(r"^(.+)-test-(.+)$")
+
+# The only suite families a CUDA registry may keep on the legacy single-string
+# `suite=` form. Everything else is a PR-test/base stage that must use the
+# modern stage=/runner_config= form (otherwise its effective_suite matches no
+# suite the PR-test workflows invoke, and the test silently never runs).
+_LEGACY_CUDA_PREFIXES = ("nightly", "stress", "weekly")
+
+
+def main() -> int:
+    # Import ci_register directly to avoid pulling in all of sglang
+    spec = importlib.util.spec_from_file_location(
+        "ci_register",
+        os.path.join("python", "sglang", "test", "ci", "ci_register.py"),
+    )
+    ci_register = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(ci_register)
+    cuda = ci_register.HWBackend.CUDA
+
+    # Same filter as run_suite.py: skip conftest.py, __init__.py, and utils.py
+    files = sorted(
+        f
+        for f in glob.glob("test/registered/**/*.py", recursive=True)
+        if os.path.basename(f) not in ("conftest.py", "__init__.py", "utils.py")
+    )
+    if not files:
+        return 0
+
+    missing = []
+    legacy_shape = []  # (file, suite, stage, runner_config) -- has a -test- split
+    non_dispatchable = []  # (file, suite) -- legacy CUDA suite no workflow invokes
+    for f in files:
+        try:
+            registries, _has_main_entry = ci_register.ut_parse_one_file(f)
+        except Exception:
+            # Skip files that can't be parsed (syntax errors, etc.)
+            continue
+        if len(registries) == 0:
+            missing.append(f)
+            continue
+        for r in registries:
+            # Pure legacy form on a CUDA registry: suite set, stage/runner unset.
+            if not (
+                r.backend == cuda
+                and r.suite is not None
+                and r.stage is None
+                and r.runner_config is None
+            ):
+                continue
+            # nightly/stress/weekly are the only CUDA suites allowed to stay on
+            # the legacy single-string form.
+            if r.suite.split("-", 1)[0] in _LEGACY_CUDA_PREFIXES:
+                continue
+            m = _MODERN_SHAPE.match(r.suite)
+            if m:
+                legacy_shape.append((f, r.suite, m.group(1), m.group(2)))
+            else:
+                non_dispatchable.append((f, r.suite))
+
+    exit_code = 0
+    if missing:
+        print("ERROR: Files in test/registered/ missing CI registry call:")
+        print("  Move manual-only tests to test/manual/.\n")
+        for f in missing:
+            print(f"  {f}")
+        print()
+        exit_code = 1
+    if legacy_shape:
+        print(
+            "ERROR: CUDA test(s) register a `{stage}-test-{runner_config}`-shaped "
+            'suite via the legacy `suite="..."` form, which is not dispatchable '
+            "via /rerun-test. Switch to the modern `stage=`/`runner_config=` form "
+            "(same stage, same runner):\n"
+        )
+        for f, suite, stage, runner_config in legacy_shape:
+            print(
+                f"  {f}\n"
+                f'    suite="{suite}"'
+                f'  ->  stage="{stage}", runner_config="{runner_config}"'
+            )
+        print()
+        exit_code = 1
+    if non_dispatchable:
+        print(
+            'ERROR: CUDA test(s) register a legacy `suite="..."` that is neither a '
+            "nightly/stress/weekly suite nor the modern `stage=`/`runner_config=` "
+            "form. This name matches no suite the PR-test workflows invoke, so the "
+            "test silently never runs. Switch to the modern form:\n"
+        )
+        for f, suite in non_dispatchable:
+            print(
+                f"  {f}\n"
+                f'    suite="{suite}"'
+                f'  ->  stage="...", runner_config="..."'
+            )
+        print()
+        exit_code = 1
+
+    return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/check_workflow_job_names.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Check that required status check job names are unique across workflows.
+
+Duplicate job names on the same commit allow a passing job in one workflow
+to satisfy a required status check meant for a different workflow, bypassing
+branch protection.
+
+See: https://github.com/sgl-project/sglang/pull/20208 for an example where
+pr-test-npu.yml's "pr-test-finish" job (which passed) caused GitHub to treat
+the required "pr-test-finish" check (from pr-test.yml, which failed) as met.
+"""
+
+import glob
+import sys
+from collections import defaultdict
+
+import yaml
+
+# Job names used as required status checks in branch protection.
+# These MUST be unique across all workflow files.
+PROTECTED_JOB_NAMES = {
+    "pr-test-finish",
+    "lint",
+}
+
+
+def main() -> int:
+    workflows = sorted(glob.glob(".github/workflows/*.yml"))
+    job_to_files: dict[str, list[str]] = defaultdict(list)
+
+    for wf in workflows:
+        with open(wf, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        if not data or "jobs" not in data:
+            continue
+        for job in data["jobs"]:
+            if job in PROTECTED_JOB_NAMES:
+                job_to_files[job].append(wf)
+
+    duplicates = {job: files for job, files in job_to_files.items() if len(files) > 1}
+
+    if not duplicates:
+        return 0
+
+    print("ERROR: Required status check job names must be unique across workflows.")
+    print("Duplicates allow branch protection bypass via auto-merge.\n")
+    for job, files in sorted(duplicates.items()):
+        print(f"  Job '{job}' appears in:")
+        for f in files:
+            print(f"    - {f}")
+        print()
+
+    print("Fix: rename the job in non-primary workflows to avoid collision.")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/cuda/ci_cleanup_venv.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Remove the per-job uv venv created by ci_install_dependency.sh.
+#
+# Meant to run in a post-job workflow step with `if: always()` so the venv is
+# destroyed even on job failure/cancel. Runner-level safety net: a cron or
+# startup task should also purge stale /tmp/sglang-ci-* directories to catch
+# cancelled or crashed jobs that never reached this cleanup.
+
+# Best-effort cleanup: never fail the job.
+set +e
+set -u
+
+# Bound the persistent uv cache (~/.cache/uv, bind-mounted and shared across all
+# runner containers). Nothing else evicts it, so it grows unbounded — on the 5090
+# hosts it reached ~500 GB and filled the disk, failing jobs with ENOSPC at
+# dependency install. Prune only under real disk pressure so healthy jobs pay
+# nothing (just a df check) and no rebuildable wheel is dropped until it matters:
+# `uv cache prune --ci` keeps downloaded wheels + sdist archives (no re-download)
+# but drops built wheels, so a later install may recompile source-built packages.
+# Best effort; runs regardless of venv mode; never fails the job.
+if command -v uv >/dev/null 2>&1; then
+    cache_dir="$(uv cache dir 2>/dev/null || echo "${HOME:-/root}/.cache/uv")"
+    [ -d "$cache_dir" ] || cache_dir=/
+    used="$(df --output=pcent "$cache_dir" 2>/dev/null | tr -dc '0-9')"
+    if [ "${used:-0}" -ge 85 ]; then
+        echo "uv cache filesystem at ${used}% — pruning"
+        uv cache prune --ci >/dev/null 2>&1 || true
+    fi
+fi
+
+# Skip entirely when venv mode is disabled — no /tmp/sglang-ci-* dir exists
+# and there's nothing to sweep. Matches the USE_VENV parsing in
+# ci_install_dependency.sh (accepts 1/true/yes, case-insensitive).
+USE_VENV_RAW="${USE_VENV:-true}"
+case "$(printf '%s' "$USE_VENV_RAW" | tr '[:upper:]' '[:lower:]')" in
+    1 | true | yes) ;;
+    *)
+        echo "USE_VENV=${USE_VENV_RAW}: skipping venv cleanup"
+        exit 0
+        ;;
+esac
+
+# Prefer the path propagated via GITHUB_ENV. Fallback: glob for any venv from
+# this run+job (covers the case where install crashed before exporting the path).
+if [ -n "${SGLANG_CI_VENV_PATH:-}" ] && [ -d "$SGLANG_CI_VENV_PATH" ]; then
+    if rm -rf "$SGLANG_CI_VENV_PATH"; then
+        echo "Cleaned up venv: $SGLANG_CI_VENV_PATH"
+    else
+        echo "::warning::Failed to remove $SGLANG_CI_VENV_PATH — runner cron should sweep /tmp/sglang-ci-*"
+    fi
+else
+    matched=0
+    for venv in /tmp/sglang-ci-${GITHUB_RUN_ID:-unknownrun}-${GITHUB_JOB:-unknownjob}-*; do
+        [ -d "$venv" ] || continue
+        matched=1
+        if rm -rf "$venv"; then
+            echo "Cleaned up venv (via glob): $venv"
+        else
+            echo "::warning::Failed to remove $venv — runner cron should sweep /tmp/sglang-ci-*"
+        fi
+    done
+    [ "$matched" -eq 0 ] && echo "No venv to clean for run=${GITHUB_RUN_ID:-?} job=${GITHUB_JOB:-?}"
+fi
+
+# Sweep stale venvs from cancelled/crashed jobs that never reached cleanup.
+# Any /tmp/sglang-ci-* dir older than 4 hours is considered orphaned.
+stale_count=0
+for venv in /tmp/sglang-ci-*; do
+    [ -d "$venv" ] || continue
+    if find "$venv" -maxdepth 0 -mmin +240 -print -quit | grep -q .; then
+        rm -rf "$venv" && stale_count=$((stale_count + 1))
+    fi
+done
+[ "$stale_count" -gt 0 ] && echo "Swept $stale_count stale venv(s) older than 4h"
+
+exit 0

scripts/ci/cuda/ci_download_flashinfer_jit_cache.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# Install flashinfer-jit-cache with caching and retry logic (flashinfer.ai can have transient DNS issues).
+# The jit-cache wheel is 1.2+ GB, so we skip the download entirely if already installed.
+#
+# Required environment (caller must export or set):
+#   UNINSTALL_JIT_CACHE          — literal true/false (skip download when false)
+#   FLASHINFER_PYTHON_REQUIRED   — e.g. from python/pyproject.toml (flashinfer_python)
+#   CU_VERSION                   — e.g. cu130
+#   PIP_CMD                      — e.g. "pip" or "uv pip"
+#   PIP_INSTALL_SUFFIX           — extra pip args for this runner
+set -euxo pipefail
+
+: "${UNINSTALL_JIT_CACHE:?must be set}"
+: "${FLASHINFER_PYTHON_REQUIRED:?must be set}"
+: "${CU_VERSION:?must be set}"
+: "${PIP_CMD:?must be set}"
+
+FLASHINFER_JIT_CACHE_INSTALLED=false
+if [ "$UNINSTALL_JIT_CACHE" = false ]; then
+    FLASHINFER_JIT_CACHE_INSTALLED=true
+    echo "flashinfer-jit-cache already at correct version, skipping download"
+fi
+
+if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then
+    FLASHINFER_CACHE_DIR="${HOME}/.cache/flashinfer-wheels"
+    mkdir -p "${FLASHINFER_CACHE_DIR}"
+
+    FLASHINFER_WHEEL_PATTERN="flashinfer_jit_cache-${FLASHINFER_PYTHON_REQUIRED}+${CU_VERSION}*.whl"
+    CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
+
+    if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
+        echo "Found cached flashinfer wheel: $CACHED_WHEEL"
+        if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
+            FLASHINFER_JIT_CACHE_INSTALLED=true
+            echo "Successfully installed flashinfer-jit-cache from cache"
+        else
+            echo "Failed to install from cache, will try downloading..."
+            rm -f "$CACHED_WHEEL"
+        fi
+    fi
+
+    if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then
+        for i in {1..5}; do
+            # Download wheel to cache directory (use pip directly as uv pip doesn't support download)
+            if timeout 600 pip download "flashinfer-jit-cache==${FLASHINFER_PYTHON_REQUIRED}" \
+                --index-url "https://flashinfer.ai/whl/${CU_VERSION}" \
+                -d "${FLASHINFER_CACHE_DIR}"; then
+
+                CACHED_WHEEL=$(find "${FLASHINFER_CACHE_DIR}" -name "${FLASHINFER_WHEEL_PATTERN}" -type f 2>/dev/null | head -n 1)
+                if [ -n "$CACHED_WHEEL" ] && [ -f "$CACHED_WHEEL" ]; then
+                    if $PIP_CMD install "$CACHED_WHEEL" $PIP_INSTALL_SUFFIX; then
+                        FLASHINFER_JIT_CACHE_INSTALLED=true
+                        echo "Successfully downloaded and installed flashinfer-jit-cache"
+                        break
+                    fi
+                else
+                    echo "Warning: Download succeeded but wheel file not found"
+                fi
+            fi
+            echo "Attempt $i to download flashinfer-jit-cache failed, retrying in 10 seconds..."
+            sleep 10
+        done
+    fi
+fi
+
+if [ "$FLASHINFER_JIT_CACHE_INSTALLED" = false ]; then
+    echo "ERROR: Failed to install flashinfer-jit-cache after 5 attempts"
+    exit 1
+fi

scripts/ci/cuda/ci_install_deepep.sh

@@ -0,0 +1,177 @@
+#!/bin/bash
+# Install the dependency in CI.
+set -euxo pipefail
+
+# Source (not bash) so that venv activation, $PIP_CMD, $CU_VERSION, $NVCC_VER, and
+# $PIP_INSTALL_SUFFIX all propagate into this shell. Without sourcing, the subshell
+# exits and this script would fall back to system Python.
+#
+# Note: any `exit N` or `set -e` trip inside the sourced script terminates *this*
+# script too (bash runs sourced commands in the current shell, so `exit` is not
+# caught by `if`/`||`). The real error message appears upstream in the log.
+# shellcheck disable=SC1091
+source scripts/ci/cuda/ci_install_dependency.sh
+
+# In venv mode, PIP_CMD must be set by the sourced script. If it isn't, the
+# source chain is broken and we'd silently fall back to system `pip` below —
+# exactly the split-install bug the migration is meant to prevent.
+if [ -z "${PIP_CMD:-}" ]; then
+    echo "FATAL:PIP_CMD is unset after sourcing ci_install_dependency.sh"
+    exit 1
+fi
+
+export GDRCOPY_HOME=/usr/src/gdrdrv-2.5.1/
+export CUDA_HOME=/usr/local/cuda
+
+GRACE_BLACKWELL=${GRACE_BLACKWELL:-0}
+# Detect architecture
+ARCH=$(uname -m)
+if [ "$ARCH" != "x86_64" ] && [ "$ARCH" != "aarch64" ]; then
+    echo "Unsupported architecture: $ARCH"
+    exit 1
+fi
+
+if [ "${FORCE_REBUILD_DEEPEP:-0}" = "1" ]; then
+    echo "FORCE_REBUILD_DEEPEP=1; uninstalling any cached deep_ep before rebuild."
+    ${PIP_UNINSTALL_CMD:-pip uninstall -y} deep_ep ${PIP_UNINSTALL_SUFFIX:-} || true
+elif python3 -c "import deep_ep" >/dev/null 2>&1; then
+    echo "deep_ep is already installed or importable. Skipping installation."
+    exit 0
+fi
+
+# Install system dependencies
+# Use fallback logic in case apt fails due to unrelated broken packages on the runner
+DEEPEP_SYSTEM_DEPS="curl wget git sudo rdma-core infiniband-diags openssh-server perftest libibumad3 libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils libnl-3-200 libnl-route-3-200 librdmacm1 build-essential cmake"
+apt-get install -y --no-install-recommends $DEEPEP_SYSTEM_DEPS || {
+    echo "Warning: apt-get install failed, checking if required packages are available..."
+    for pkg in $DEEPEP_SYSTEM_DEPS; do
+        if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+            echo "ERROR: Required package $pkg is not installed and apt-get failed"
+            exit 1
+        fi
+    done
+    echo "All required packages are already installed, continuing..."
+}
+
+# Install GDRCopy
+rm -rf /opt/gdrcopy && mkdir -p /opt/gdrcopy
+cd /opt/gdrcopy
+git clone https://github.com/NVIDIA/gdrcopy.git .
+git checkout v2.5.1
+apt-get update || true  # May fail due to unrelated broken packages
+GDRCOPY_DEPS_1="nvidia-dkms-580"
+GDRCOPY_DEPS_2="build-essential devscripts debhelper fakeroot pkg-config dkms"
+GDRCOPY_DEPS_3="check libsubunit0 libsubunit-dev python3-venv"
+for deps_group in "$GDRCOPY_DEPS_1" "$GDRCOPY_DEPS_2" "$GDRCOPY_DEPS_3"; do
+    apt-get install -y --no-install-recommends $deps_group || {
+        echo "Warning: apt-get install failed for '$deps_group', checking if packages are available..."
+        for pkg in $deps_group; do
+            if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+                echo "ERROR: Required package $pkg is not installed and apt-get failed"
+                exit 1
+            fi
+        done
+        echo "All required packages from '$deps_group' are already installed, continuing..."
+    }
+done
+cd packages
+CUDA=/usr/local/cuda ./build-deb-packages.sh
+dpkg -i gdrdrv-dkms_*.deb
+dpkg -i libgdrapi_*.deb
+dpkg -i gdrcopy-tests_*.deb
+dpkg -i gdrcopy_*.deb
+
+# Set up library paths based on architecture
+LIB_PATH="/usr/lib/$ARCH-linux-gnu"
+if [ ! -e "$LIB_PATH/libmlx5.so" ]; then
+    ln -s $LIB_PATH/libmlx5.so.1 $LIB_PATH/libmlx5.so
+fi
+apt-get update || true
+apt-get install -y --no-install-recommends libfabric-dev || {
+    if ! dpkg -l libfabric-dev 2>/dev/null | grep -q "^ii"; then
+        echo "ERROR: Required package libfabric-dev is not installed and apt-get failed"
+        exit 1
+    fi
+    echo "libfabric-dev is already installed, continuing..."
+}
+
+# Install DeepEP
+DEEPEP_DIR=/root/.cache/deepep
+rm -rf ${DEEPEP_DIR}
+if [ "$GRACE_BLACKWELL" = "1" ]; then
+    GRACE_BLACKWELL_DEEPEP_BRANCH=hybrid-ep
+    git clone https://github.com/deepseek-ai/DeepEP.git -b ${GRACE_BLACKWELL_DEEPEP_BRANCH} ${DEEPEP_DIR} && \
+    pushd ${DEEPEP_DIR} && \
+    git checkout d28bd676c2120573c9f1425f0c16c39faa4117e6 && \
+    sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh && \
+    popd
+else
+    git clone https://github.com/deepseek-ai/DeepEP.git ${DEEPEP_DIR} && \
+    pushd ${DEEPEP_DIR} && \
+    git checkout 9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee && \
+    popd
+fi
+
+cd ${DEEPEP_DIR}
+if [ "$GRACE_BLACKWELL" = "1" ]; then
+    # Resolve the toolkit CUDA version. Preference order:
+    #   1. $NVCC_VER inherited from the sourced ci_install_dependency.sh
+    #      (both scripts agree on the detected value, no re-detection cost).
+    #   2. Local `nvcc --version` (authoritative — container toolkit).
+    #   3. `nvidia-smi` (host driver; last resort).
+    if [ -n "${NVCC_VER:-}" ]; then
+        CUDA_VERSION="$NVCC_VER"
+    elif command -v nvcc >/dev/null 2>&1; then
+        CUDA_VERSION=$(nvcc --version | grep -oP 'release \K[0-9]+\.[0-9]+')
+    else
+        CUDA_VERSION=$(nvidia-smi | grep "CUDA Version" | head -n1 | awk '{print $9}' || true)
+    fi
+    if [ -z "${CUDA_VERSION:-}" ]; then
+        echo "FATAL: could not determine CUDA toolkit version (NVCC_VER unset, nvcc missing, nvidia-smi empty)"
+        exit 1
+    fi
+    if [ "$CUDA_VERSION" = "12.8" ]; then
+        CHOSEN_TORCH_CUDA_ARCH_LIST='10.0'
+    elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then
+        # CUDA > 12.8 supports sm_103 (Blackwell)
+        CHOSEN_TORCH_CUDA_ARCH_LIST='10.0;10.3'
+    else
+        echo "Unsupported CUDA version for Grace Blackwell: $CUDA_VERSION" && exit 1
+    fi && \
+    if [ "${CUDA_VERSION%%.*}" = "13" ]; then \
+        sed -i "/^    include_dirs = \['csrc\/'\]/a\    include_dirs.append('${CUDA_HOME}/include/cccl')" setup.py; \
+    fi
+    TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" ${PIP_CMD:-pip} install --no-build-isolation . ${PIP_INSTALL_SUFFIX:-}
+else
+    # CUDA 13.0 puts CCCL headers in /usr/local/cuda/include/cccl/ but nvshmem
+    # includes them as <cuda/__cccl_config> expecting /usr/local/cuda/include/cuda/.
+    # Add the cccl path to setup.py include_dirs so the compiler finds them.
+    NVCC_MAJOR=$(nvcc --version 2>/dev/null | grep -oP 'release \K[0-9]+' || echo "0")
+    if [ "$NVCC_MAJOR" = "13" ]; then
+        sed -i "/^    include_dirs = \['csrc\/'\]/a\    include_dirs.append('${CUDA_HOME:-/usr/local/cuda}/include/cccl')" setup.py
+    fi
+
+    # Build for both Hopper (sm_90) and Blackwell (sm_100) so the same wheel
+    # runs on H200 and B200 runners. Mirrors the CUDA-version-keyed list in
+    # docker/Dockerfile's DeepEP build stage.
+    if [ -n "${NVCC_VER:-}" ]; then
+        CUDA_VERSION="$NVCC_VER"
+    elif command -v nvcc >/dev/null 2>&1; then
+        CUDA_VERSION=$(nvcc --version | grep -oP 'release \K[0-9]+\.[0-9]+')
+    else
+        CUDA_VERSION=$(nvidia-smi | grep "CUDA Version" | head -n1 | awk '{print $9}' || true)
+    fi
+    if [ -z "${CUDA_VERSION:-}" ]; then
+        echo "FATAL: could not determine CUDA toolkit version (NVCC_VER unset, nvcc missing, nvidia-smi empty)"
+        exit 1
+    fi
+    if [ "$CUDA_VERSION" = "12.8" ]; then
+        CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0'
+    elif awk -v ver="$CUDA_VERSION" 'BEGIN {exit !(ver > 12.8)}'; then
+        # CUDA > 12.8 supports sm_103 (Blackwell)
+        CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0;10.3'
+    else
+        CHOSEN_TORCH_CUDA_ARCH_LIST='9.0'
+    fi
+    TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" python3 setup.py install
+fi

scripts/ci/cuda/ci_install_dependency.sh

@@ -0,0 +1,613 @@
+#!/bin/bash
+# Install dependencies for CUDA CI jobs.
+#
+# CU_VERSION (default: cu130) controls PyTorch index URL, FlashInfer JIT cache
+# index, and nvrtc variant selection.
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+
+# ---------------------------------------------------------------------------
+# Timing helper
+# ---------------------------------------------------------------------------
+SECONDS=0
+_CI_MARK_PREV=${SECONDS}
+
+mark_step_done() {
+    local label=$1
+    local now=${SECONDS}
+    local step=$((now - _CI_MARK_PREV))
+    printf '\n[STEP DONE] %s,  step: %ss,  total: %ss,  date: %s\n' \
+        "${label}" "${step}" "${now}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+    _CI_MARK_PREV=${now}
+}
+
+# ---------------------------------------------------------------------------
+# Functions
+# ---------------------------------------------------------------------------
+
+configure_environment() {
+    # CU_VERSION controls PyTorch index URL, FlashInfer JIT cache index, and
+    # nvrtc variant selection (cu12 vs cu13).
+    CU_VERSION="${CU_VERSION:-cu130}"
+    CU_STRIP="${CU_VERSION#cu}"
+    CU_MAJOR="${CU_STRIP:0:2}"
+
+    OPTIONAL_DEPS="${1:-}"
+
+    # Whether to create a uv venv (set USE_VENV=1). Default: 0.
+    USE_VENV="${USE_VENV:-0}"
+    echo "USE_VENV=${USE_VENV}"
+
+    python3 -m pip install --upgrade pip
+    if ! command -v uv >/dev/null 2>&1; then
+        pip install uv
+    fi
+
+    SYS_PYTHON_VER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+
+    if [ "$USE_VENV" = "1" ]; then
+        UV_VENV="/tmp/sglang-ci-${GITHUB_RUN_ID:-norun}-${GITHUB_JOB:-nojob}-$$"
+        uv venv "$UV_VENV" --python "python${SYS_PYTHON_VER}" --seed
+        # shellcheck disable=SC1091
+        source "$UV_VENV/bin/activate"
+        [ "${VIRTUAL_ENV:-}" = "$UV_VENV" ] || { echo "FATAL: venv activation did not set VIRTUAL_ENV correctly"; exit 1; }
+        [ "$(command -v python3)" = "$UV_VENV/bin/python3" ] || { echo "FATAL: python3 still resolves outside venv (got $(command -v python3))"; exit 1; }
+
+        if [ -n "${GITHUB_ENV:-}" ]; then
+            # Self-heal: see install_rustup.sh for context on missing _runner_file_commands/.
+            mkdir -p "$(dirname "$GITHUB_ENV")" 2>/dev/null || true
+            echo "VIRTUAL_ENV=$UV_VENV" >> "$GITHUB_ENV" || true
+            echo "SGLANG_CI_VENV_PATH=$UV_VENV" >> "$GITHUB_ENV" || true
+            echo "BASH_ENV=$UV_VENV/env.sh" >> "$GITHUB_ENV" || true
+            touch "$UV_VENV/env.sh"
+        fi
+        if [ -n "${GITHUB_PATH:-}" ]; then
+            mkdir -p "$(dirname "$GITHUB_PATH")" 2>/dev/null || true
+            echo "$UV_VENV/bin" >> "$GITHUB_PATH" || true
+        fi
+    else
+        echo "USE_VENV=0: skipping uv venv creation, installing into system Python"
+        UV_VENV=""
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+detect_host() {
+    ARCH=$(uname -m)
+    echo "Detected architecture: ${ARCH}"
+
+    if [ "${IS_BLACKWELL+set}" = set ]; then
+        case "$IS_BLACKWELL" in 1 | true | yes) IS_BLACKWELL=1 ;; *) IS_BLACKWELL=0 ;; esac
+        echo "IS_BLACKWELL=${IS_BLACKWELL} (manually set via environment)"
+    else
+        IS_BLACKWELL=0
+        if command -v nvidia-smi >/dev/null 2>&1; then
+            while IFS= read -r cap; do
+                major="${cap%%.*}"
+                if [ "${major:-0}" -ge 10 ] 2>/dev/null; then
+                    IS_BLACKWELL=1
+                    break
+                fi
+            done <<< "$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null || true)"
+        fi
+        echo "IS_BLACKWELL=${IS_BLACKWELL} (auto-detected via nvidia-smi)"
+    fi
+
+    if [ "${USE_UV+set}" != set ]; then
+        if [ "$IS_BLACKWELL" = "1" ]; then
+            USE_UV=false
+        else
+            USE_UV=true
+        fi
+    fi
+    case "$(printf '%s' "$USE_UV" | tr '[:upper:]' '[:lower:]')" in 1 | true | yes) USE_UV=1 ;; *) USE_UV=0 ;; esac
+    echo "USE_UV=${USE_UV}"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+kill_existing_processes() {
+    python3 "${REPO_ROOT}/python/sglang/cli/killall.py"
+    KILLALL_EXIT=$?
+    echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}"
+
+    if [ $KILLALL_EXIT -ne 0 ]; then
+        echo "ERROR: killall.py detected uncleanable GPU memory. Aborting CI."
+        exit 1
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+cleanup_stale_shm() {
+    # Reclaim /dev/shm segments leaked by SIGKILLed processes from earlier
+    # jobs; leaked segments accumulate until the tmpfs fills and scheduler
+    # init dies with SIGBUS. Runs right after killall so every dead creator's
+    # segments are reclaimable. The module is dependency-free and runnable by
+    # path, so this works before sglang is installed.
+    SGLANG_IS_IN_CI=true python3 "${REPO_ROOT}/python/sglang/srt/utils/stale_shm_cleanup.py" || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_apt_packages() {
+    apt-get update || true
+    CI_APT_PACKAGES=(
+        python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config
+        libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils
+        ffmpeg libavcodec-dev libavformat-dev libavutil-dev libswscale-dev
+    )
+    apt-get install -y --no-install-recommends "${CI_APT_PACKAGES[@]}" || {
+        echo "Warning: apt-get install failed, checking if required packages are available..."
+        for pkg in "${CI_APT_PACKAGES[@]}"; do
+            if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+                echo "ERROR: Required package $pkg is not installed and apt-get failed"
+                exit 1
+            fi
+        done
+        echo "All required packages are already installed, continuing..."
+    }
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+clean_site_packages() {
+    # Clear torch compilation cache
+    python3 -c 'import os, shutil, tempfile, getpass; cache_dir = os.environ.get("TORCHINDUCTOR_CACHE_DIR") or os.path.join(tempfile.gettempdir(), "torchinductor_" + getpass.getuser()); shutil.rmtree(cache_dir, ignore_errors=True)'
+
+    # Remove broken dist-info directories (missing METADATA per PEP 376)
+    SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])")
+    if [ -d "$SITE_PACKAGES" ]; then
+        { set +x; } 2>/dev/null
+        find "$SITE_PACKAGES" -maxdepth 1 -name "*.dist-info" -type d | while read -r d; do
+            if [ ! -f "$d/METADATA" ]; then
+                echo "Removing broken dist-info: $d"
+                rm -rf "$d"
+            fi
+        done
+        set -x
+    fi
+
+    # Install protoc + Rust toolchain (needed by setuptools-rust, e.g. the native gRPC extension)
+    bash "${SCRIPT_DIR}/../utils/install_rust_protoc.sh"
+    export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+setup_pip_toolchain() {
+    python3 -m pip install --upgrade pip
+
+    if [ "$USE_VENV" != "1" ]; then
+        export UV_SYSTEM_PYTHON=1
+    fi
+
+    export UV_LINK_MODE=copy
+    PIP_CMD="uv pip"
+    PIP_INSTALL_SUFFIX="--index-strategy unsafe-best-match"
+    PIP_UNINSTALL_CMD="uv pip uninstall"
+    PIP_UNINSTALL_SUFFIX=""
+
+    $PIP_UNINSTALL_CMD sgl-kernel sglang-kernel sglang sgl-fa4 flash-attn-4 $PIP_UNINSTALL_SUFFIX || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+remove_stale_cuda12_nvidia_wheels() {
+    if [ "$CU_MAJOR" != "13" ]; then
+        mark_step_done "${FUNCNAME[0]}"
+        return
+    fi
+
+    mapfile -t STALE_CUDA12_NVIDIA_WHEELS < <(
+        python3 -m pip list --format=freeze | sed -n 's/^\(nvidia-.*-cu12\)==.*/\1/p'
+    )
+    if [ ${#STALE_CUDA12_NVIDIA_WHEELS[@]} -eq 0 ]; then
+        echo "No stale CUDA 12 NVIDIA wheels found for ${CU_VERSION} job"
+        mark_step_done "${FUNCNAME[0]}"
+        return
+    fi
+
+    echo "Removing stale CUDA 12 NVIDIA wheels from ${CU_VERSION} job: ${STALE_CUDA12_NVIDIA_WHEELS[*]}"
+    $PIP_UNINSTALL_CMD "${STALE_CUDA12_NVIDIA_WHEELS[@]}" $PIP_UNINSTALL_SUFFIX
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+uninstall_stale_flashinfer() {
+    # Keep flashinfer packages if version matches to avoid re-downloading:
+    # - flashinfer-cubin: 150+ MB
+    # - flashinfer-jit-cache: 1.2+ GB
+    FLASHINFER_PYTHON_REQUIRED=$(grep -Po -m1 'flashinfer_python(\[[^]]+\])?==\K[0-9A-Za-z\.\-]+' python/pyproject.toml || echo "")
+    # flashinfer-cubin is no longer a pyproject dependency (installed explicitly below), tracks the same version as flashinfer_python
+    FLASHINFER_CUBIN_REQUIRED="$FLASHINFER_PYTHON_REQUIRED"
+    FLASHINFER_CUBIN_INSTALLED=$(pip show flashinfer-cubin 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    FLASHINFER_JIT_INSTALLED=$(pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//' || echo "")
+    FLASHINFER_JIT_CU_VERSION=$(pip show flashinfer-jit-cache 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+//p' || echo "")
+
+    UNINSTALL_CUBIN=true
+    UNINSTALL_JIT_CACHE=true
+
+    if [ "$FLASHINFER_CUBIN_INSTALLED" = "$FLASHINFER_CUBIN_REQUIRED" ] && [ -n "$FLASHINFER_CUBIN_REQUIRED" ]; then
+        echo "flashinfer-cubin==${FLASHINFER_CUBIN_REQUIRED} already installed, keeping it"
+        UNINSTALL_CUBIN=false
+    else
+        echo "flashinfer-cubin version mismatch (installed: ${FLASHINFER_CUBIN_INSTALLED:-none}, required: ${FLASHINFER_CUBIN_REQUIRED}), reinstalling"
+    fi
+
+    if [ "$FLASHINFER_JIT_INSTALLED" = "$FLASHINFER_PYTHON_REQUIRED" ] && [ -n "$FLASHINFER_PYTHON_REQUIRED" ]; then
+        echo "flashinfer-jit-cache==${FLASHINFER_PYTHON_REQUIRED} already installed, keeping it"
+        UNINSTALL_JIT_CACHE=false
+    else
+        echo "flashinfer-jit-cache version mismatch (installed: ${FLASHINFER_JIT_INSTALLED:-none}, required: ${FLASHINFER_PYTHON_REQUIRED}), will reinstall"
+    fi
+
+    if [ "$UNINSTALL_JIT_CACHE" = false ] && [ "$FLASHINFER_JIT_CU_VERSION" != "$CU_VERSION" ]; then
+        echo "flashinfer-jit-cache CUDA version mismatch (installed: ${FLASHINFER_JIT_CU_VERSION:-none}, required: ${CU_VERSION}), will reinstall"
+        UNINSTALL_JIT_CACHE=true
+    fi
+
+    FLASHINFER_UNINSTALL="flashinfer-python"
+    [ "$UNINSTALL_CUBIN" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-cubin"
+    [ "$UNINSTALL_JIT_CACHE" = true ] && FLASHINFER_UNINSTALL="$FLASHINFER_UNINSTALL flashinfer-jit-cache"
+    $PIP_UNINSTALL_CMD $FLASHINFER_UNINSTALL $PIP_UNINSTALL_SUFFIX || true
+    $PIP_UNINSTALL_CMD opencv-python opencv-python-headless $PIP_UNINSTALL_SUFFIX || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_sglang() {
+    EXTRAS="dev,runai,tracing"
+    if [ -n "$OPTIONAL_DEPS" ]; then
+        EXTRAS="dev,runai,tracing,${OPTIONAL_DEPS}"
+    fi
+    echo "Installing python extras: [${EXTRAS}]"
+    $PIP_CMD install -e "python[${EXTRAS}]" $PIP_INSTALL_SUFFIX
+
+    # Defensive: some runners ended up with nvidia-cusparselt-cu13 metadata
+    # present but libcusparseLt.so.0 missing on disk, breaking any torch import.
+    # If the file is missing, force-reinstall the wheel before downstream steps.
+    SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])")
+    if [ ! -f "$SITE_PACKAGES/nvidia/cusparselt/lib/libcusparseLt.so.0" ] \
+       && pip show nvidia-cusparselt-cu13 >/dev/null 2>&1; then
+        echo "WARNING: nvidia-cusparselt-cu13 metadata present but libcusparseLt.so.0 missing — reinstalling"
+        $PIP_CMD install --reinstall nvidia-cusparselt-cu13 $PIP_INSTALL_SUFFIX
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_sglang_kernel() {
+    SGL_KERNEL_VERSION_FROM_KERNEL=$(grep -Po '(?<=^version = ")[^"]*' sgl-kernel/pyproject.toml)
+    SGL_KERNEL_VERSION_FROM_SRT=$(grep -Po -m1 '(?<=sglang-kernel==)[0-9A-Za-z\.\-]+' python/pyproject.toml)
+    echo "SGL_KERNEL_VERSION_FROM_KERNEL=${SGL_KERNEL_VERSION_FROM_KERNEL} SGL_KERNEL_VERSION_FROM_SRT=${SGL_KERNEL_VERSION_FROM_SRT}"
+
+    if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ -d "sgl-kernel/dist" ]; then
+        ls -alh sgl-kernel/dist
+        if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+            WHEEL_ARCH="aarch64"
+        else
+            WHEEL_ARCH="x86_64"
+        fi
+        KERNEL_WHL=$(ls sgl-kernel/dist/sglang_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}+${CU_VERSION}-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl 2>/dev/null | head -1 || true)
+        if [ -z "$KERNEL_WHL" ]; then
+            echo "ERROR: No matching sgl-kernel wheel found in sgl-kernel/dist/ for version ${SGL_KERNEL_VERSION_FROM_KERNEL} arch ${WHEEL_ARCH} cuda ${CU_VERSION}"
+            ls -alh sgl-kernel/dist/
+            exit 1
+        fi
+        echo "Installing sgl-kernel wheel: $KERNEL_WHL"
+        $PIP_CMD install "$KERNEL_WHL" --force-reinstall $PIP_INSTALL_SUFFIX
+    else
+        if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ] && [ ! -d "sgl-kernel/dist" ]; then
+            echo "ERROR: CUSTOM_BUILD_SGL_KERNEL=true but sgl-kernel/dist not found."
+            echo "This usually happens when rerunning a stage without the sgl-kernel-build-wheels job."
+            echo "Please re-run the full workflow using /tag-and-rerun-ci to rebuild the kernel."
+            exit 1
+        fi
+    fi
+
+    # Reinstall torch with matching CUDA version if needed
+    # TODO: Remove after torch 2.11 where cu13 is enabled by default
+    REINSTALL_TORCH=false
+    if TORCH_CUDA_VER=$(python3 -c "import torch; v=torch.version.cuda; parts=v.split('.'); print(f'cu{parts[0]}{parts[1]}')" 2>&1); then
+        echo "Detected torch CUDA version: ${TORCH_CUDA_VER}"
+    else
+        TORCH_IMPORT_ERROR="${TORCH_CUDA_VER}"
+        TORCH_CUDA_VER=""
+        echo "WARNING: importing torch failed while probing CUDA version; force-reinstalling torch packages."
+        printf '%s\n' "${TORCH_IMPORT_ERROR}"
+        REINSTALL_TORCH=true
+    fi
+    TORCHAUDIO_CUDA_VER=$(pip show torchaudio 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+\(cu[0-9][0-9]*\)$/\1/p' || true)
+    TORCHVISION_CUDA_VER=$(pip show torchvision 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed -n 's/.*+\(cu[0-9][0-9]*\)$/\1/p' || true)
+    if [ "${TORCH_CUDA_VER}" != "${CU_VERSION}" ]; then
+        REINSTALL_TORCH=true
+    else
+        for cuda_ver in "${TORCHAUDIO_CUDA_VER}" "${TORCHVISION_CUDA_VER}"; do
+            if [ -n "${cuda_ver}" ] && [ "${cuda_ver}" != "${CU_VERSION}" ]; then
+                REINSTALL_TORCH=true
+                break
+            fi
+        done
+    fi
+    if [ "${REINSTALL_TORCH}" = true ]; then
+        TORCH_VER=$(pip show torch 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//')
+        TORCHAUDIO_VER=$(pip show torchaudio 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//')
+        TORCHVISION_VER=$(pip show torchvision 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//')
+        if [ -z "${TORCH_VER}" ] || [ -z "${TORCHAUDIO_VER}" ] || [ -z "${TORCHVISION_VER}" ]; then
+            echo "ERROR: could not determine installed torch package versions before reinstall."
+            pip show torch torchaudio torchvision || true
+            exit 1
+        fi
+        echo "Reinstalling torch==${TORCH_VER} torchaudio==${TORCHAUDIO_VER} torchvision==${TORCHVISION_VER} from ${CU_VERSION} index to match torch..."
+        $PIP_CMD install "torch==${TORCH_VER}" "torchaudio==${TORCHAUDIO_VER}" "torchvision==${TORCHVISION_VER}" --index-url "https://download.pytorch.org/whl/${CU_VERSION}" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    fi
+
+    if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" != "true" ]; then
+        # install_sglang above pulls sglang-kernel from PyPI, whose default wheel
+        # tracks one CUDA version (currently cu130). Force-reinstall from the
+        # CU_VERSION-matched sglang wheel index so runners on a different CUDA
+        # (e.g. h20 / cu129) get a wheel linked against the right libnvrtc.
+        $PIP_CMD install "sglang-kernel==${SGL_KERNEL_VERSION_FROM_SRT}" --index-url "https://docs.sglang.ai/whl/${CU_VERSION}/" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    else
+        echo "CUSTOM_BUILD_SGL_KERNEL=true: keeping freshly built sgl-kernel wheel."
+    fi
+    SGL_DEEP_GEMM_VERSION=$(grep -Po -m1 '(?<=sgl-deep-gemm==)[0-9A-Za-z\.\-]+' python/pyproject.toml)
+    if [ "$CU_MAJOR" = "13" ]; then
+        $PIP_CMD install "sgl-deep-gemm==${SGL_DEEP_GEMM_VERSION}" --force-reinstall $PIP_INSTALL_SUFFIX
+    else
+        $PIP_CMD install "https://github.com/sgl-project/whl/releases/download/v${SGL_DEEP_GEMM_VERSION}/sgl_deep_gemm-${SGL_DEEP_GEMM_VERSION}+cu129-py3-none-manylinux2014_$(uname -m).whl" --force-reinstall $PIP_INSTALL_SUFFIX
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_sglang_router() {
+    $PIP_CMD install sglang-router $PIP_INSTALL_SUFFIX
+    $PIP_CMD list
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_flashinfer_cubin() {
+    if [ "$UNINSTALL_CUBIN" = false ]; then
+        echo "flashinfer-cubin==${FLASHINFER_CUBIN_REQUIRED} already installed, skipping install"
+    else
+        # flashinfer-cubin is CUDA-version-agnostic, unlike jit-cache, so its index-url has no cu${CU_VERSION} suffix
+        $PIP_CMD install "flashinfer-cubin==${FLASHINFER_CUBIN_REQUIRED}" --index-url https://flashinfer.ai/whl $PIP_INSTALL_SUFFIX
+    fi
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+download_flashinfer_cache() {
+    UNINSTALL_JIT_CACHE="$UNINSTALL_JIT_CACHE" \
+        FLASHINFER_PYTHON_REQUIRED="$FLASHINFER_PYTHON_REQUIRED" \
+        CU_VERSION="$CU_VERSION" \
+        PIP_CMD="$PIP_CMD" \
+        PIP_INSTALL_SUFFIX="$PIP_INSTALL_SUFFIX" \
+        bash "${SCRIPT_DIR}/ci_download_flashinfer_jit_cache.sh"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+force_reinstall_cutlass_dsl_libs_cu13() {
+    # nvidia-cutlass-dsl[cu13] has additive PyPI extras: installing it pulls in
+    # both -libs-base and -libs-cu13. The two wheels ship intentionally-different
+    # content for the same paths (cutlass/_mlir/dialects/_gpu_ops_gen.py and
+    # cutlass/_mlir/_mlir_libs/_cutlass_ir.cpython-*.so) -- each Python wrapper
+    # is paired with a matching pybind11 .so. If install order leaves the .py
+    # from one wheel and the .so from the other, GPUModuleOp.__init__ raises
+    # TypeError: incompatible function arguments at kernel-compile time.
+    #
+    # Force-reinstall -libs-cu13 LAST so both files come from the same wheel
+    # (BOTH-cu13 state), eliminating the mismatch. The version is parsed from
+    # pyproject.toml so this stays in sync with whatever nvidia-cutlass-dsl
+    # version the project pins.
+    if [ "$CU_MAJOR" != "13" ]; then
+        return
+    fi
+
+    CUTLASS_DSL_VERSION=$(grep -Po -m1 'nvidia-cutlass-dsl(\[[^]]+\])?==\K[0-9A-Za-z\.\-]+' "${REPO_ROOT}/python/pyproject.toml" || echo "")
+    if [ -z "$CUTLASS_DSL_VERSION" ]; then
+        echo "WARNING: could not detect nvidia-cutlass-dsl version from pyproject.toml; skipping libs-cu13 force-reinstall"
+        return
+    fi
+
+    $PIP_CMD install --force-reinstall --no-deps "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}" $PIP_INSTALL_SUFFIX
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+stabilize_flashinfer_jit_paths() {
+    # In venv mode, FlashInfer JIT writes build.ninja with hardcoded -isystem
+    # paths. Per-job venvs get unique paths, but the JIT cache is shared on the
+    # host mount. Fix by symlinking venv copies to a stable host-mounted path.
+    if [ "$USE_VENV" != "1" ]; then
+        return
+    fi
+
+    STABLE_FI_DIR="${HOME}/.cache/flashinfer/_stable_src"
+
+    # Clear stale cached_ops (keep valid compiled kernels)
+    if [ -d "${HOME}/.cache/flashinfer" ]; then
+        STALE_COUNT=0
+        while IFS= read -r ninja_file; do
+            STALE_PATH=$(grep -o '/tmp/sglang-ci-[^ ]*\|flashinfer-src' "$ninja_file" 2>/dev/null | head -1 || true)
+            if [ -n "$STALE_PATH" ]; then
+                if echo "$STALE_PATH" | grep -q "flashinfer-src" || [ ! -d "$STALE_PATH" ]; then
+                    rm -rf "$(dirname "$ninja_file")"
+                    STALE_COUNT=$((STALE_COUNT + 1))
+                fi
+            fi
+        done < <(find "${HOME}/.cache/flashinfer" -name "build.ninja" -type f 2>/dev/null)
+        echo "Cleaned $STALE_COUNT stale FlashInfer cached_ops (kept valid ones)"
+    fi
+
+    # Copy source files to stable path and symlink venv copies there
+    FI_DATA=$(python3 -c "import flashinfer, os; print(os.path.join(os.path.dirname(flashinfer.__file__), 'data'))")
+    TVM_INC=$(python3 -c "import tvm_ffi, os; print(os.path.join(os.path.dirname(tvm_ffi.__file__), 'include'))")
+
+    FI_VERSION="${FLASHINFER_PYTHON_REQUIRED}"
+    if [ ! -d "$STABLE_FI_DIR/flashinfer-data" ] || [ "$(cat "$STABLE_FI_DIR/.version" 2>/dev/null)" != "$FI_VERSION" ]; then
+        rm -rf "$STABLE_FI_DIR"
+        mkdir -p "$STABLE_FI_DIR"
+        cp -a "$FI_DATA" "$STABLE_FI_DIR/flashinfer-data"
+        cp -a "$TVM_INC" "$STABLE_FI_DIR/tvm-ffi-include"
+        echo "$FI_VERSION" > "$STABLE_FI_DIR/.version"
+        echo "Copied flashinfer source files to stable path: $STABLE_FI_DIR (version=$FI_VERSION)"
+    else
+        echo "Stable flashinfer source path up to date (version=$FI_VERSION)"
+    fi
+
+    rm -rf "$FI_DATA"
+    ln -s "$STABLE_FI_DIR/flashinfer-data" "$FI_DATA"
+    TVM_INC_PARENT=$(dirname "$TVM_INC")
+    rm -rf "$TVM_INC_PARENT/include"
+    ln -s "$STABLE_FI_DIR/tvm-ffi-include" "$TVM_INC_PARENT/include"
+    echo "Symlinked venv flashinfer/tvm_ffi -> $STABLE_FI_DIR"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_extra_deps() {
+    MOONCAKE_VERSION="0.3.11.post1"
+    NIXL_VERSION="1.3.0"
+    if [ "$CU_MAJOR" = "13" ]; then
+        MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
+        MOONCAKE_STALE_PKG="mooncake-transfer-engine"
+        NIXL_BIN_NAME="nixl-cu13"
+        EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc"
+    else
+        MOONCAKE_PKG="mooncake-transfer-engine==${MOONCAKE_VERSION}"
+        MOONCAKE_STALE_PKG="mooncake-transfer-engine-cuda13"
+        NIXL_BIN_NAME="nixl-cu12"
+        EXTRA_NVIDIA_SPECS="nvidia-cuda-nvrtc-cu12"
+    fi
+    # Both variants own the same mooncake/ package files and bin/ scripts
+    # (mooncake_master, etc.). Uninstalling the stale variant deletes shared
+    # files that the live variant's RECORD still references, so we force a
+    # reinstall to restore them — pip would otherwise see "already satisfied"
+    # and skip.
+    if pip show ${MOONCAKE_STALE_PKG} >/dev/null 2>&1; then
+        $PIP_UNINSTALL_CMD ${MOONCAKE_STALE_PKG} $PIP_UNINSTALL_SUFFIX || true
+        $PIP_CMD install ${MOONCAKE_PKG} --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    fi
+    $PIP_CMD install ${MOONCAKE_PKG} ${EXTRA_NVIDIA_SPECS} py-spy scipy huggingface_hub[hf_xet] pytest $PIP_INSTALL_SUFFIX
+
+    NIXL_INSTALLED=$(pip show nixl 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    NIXL_BIN_INSTALLED=$(pip show "${NIXL_BIN_NAME}" 2>/dev/null | grep "^Version:" | awk '{print $2}' || echo "")
+    if [ "$NIXL_INSTALLED" = "$NIXL_VERSION" ] && [ "$NIXL_BIN_INSTALLED" = "$NIXL_VERSION" ]; then
+        echo "nixl==${NIXL_VERSION} and ${NIXL_BIN_NAME}==${NIXL_VERSION} already installed, keeping them"
+    else
+        echo "nixl mismatch (meta: ${NIXL_INSTALLED:-none}, ${NIXL_BIN_NAME}: ${NIXL_BIN_INSTALLED:-none}, required: ${NIXL_VERSION}); installing"
+        # Meta stub owns the nixl import path; install only the CUDA binary for
+        # this runner's torch CUDA major. --no-deps avoids pulling the other CUDA
+        # variant; leave any other variant already on the runner image untouched.
+        $PIP_CMD install "nixl==${NIXL_VERSION}" "${NIXL_BIN_NAME}==${NIXL_VERSION}" \
+            --no-deps --force-reinstall $PIP_INSTALL_SUFFIX
+    fi
+
+    if [ "$IS_BLACKWELL" != "1" ]; then
+        git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
+        $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX
+        # lmms-eval v0.5 pulls antlr4-python3-runtime==4.7.2, clobbering the
+        # 4.9.3 that sgl-eval's latex2sympy2_extended needs (4.7.2 ImportError
+        # at sgl-eval import). Pin it back so the nightly sgl-eval path works.
+        $PIP_CMD install "antlr4-python3-runtime==4.9.3" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
+    fi
+    $PIP_CMD uninstall xformers || true
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+install_test_tools() {
+    # Download kernels from kernels community
+    kernels download python || true
+    kernels lock python || true
+    [ -e "${HOME}/.cache/sglang" ] && [ ! -d "${HOME}/.cache/sglang" ] && rm -f "${HOME}/.cache/sglang"
+    mkdir -p "${HOME}/.cache/sglang/"
+    mv python/kernels.lock "${HOME}/.cache/sglang/" || true
+
+    # Install human-eval (subshell keeps cd local)
+    $PIP_CMD install "setuptools==70.0.0" $PIP_INSTALL_SUFFIX
+    [ -d human-eval ] || git clone https://github.com/merrymercy/human-eval.git
+    (
+        cd human-eval
+        $PIP_CMD install -e . --no-build-isolation $PIP_INSTALL_SUFFIX
+    )
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+prepare_runner() {
+    bash "${SCRIPT_DIR}/prepare_runner.sh"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+setup_ld_library_path() {
+    # NVIDIA pip packages and torch ship .so files under site-packages that are
+    # not on the default LD_LIBRARY_PATH.
+    SITE_PACKAGES=$(python3 -c "import site, sys; print(site.getsitepackages()[0])")
+    NVIDIA_LIBS=$(find "$SITE_PACKAGES" -path "*/nvidia/*/lib" -type d 2>/dev/null | tr '\n' ':')
+    TORCH_LIB="$SITE_PACKAGES/torch/lib"
+    VENV_LD="${NVIDIA_LIBS}${TORCH_LIB}"
+    export LD_LIBRARY_PATH="${VENV_LD}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+    if [ "$USE_VENV" = "1" ] && [ -n "$UV_VENV" ]; then
+        echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH\"" >> "$UV_VENV/env.sh"
+    fi
+    if [ -n "${GITHUB_ENV:-}" ]; then
+        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$GITHUB_ENV" || echo "WARNING: GITHUB_ENV write failed; LD_LIBRARY_PATH will be set via BASH_ENV instead"
+    fi
+    echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+verify_imports() {
+    $PIP_CMD list
+    python3 -c "import torch; print(torch.version.cuda)"
+    python3 -c "import cutlass; import cutlass.cute;"
+
+    mark_step_done "${FUNCNAME[0]}"
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+main() {
+    configure_environment "$@"
+    detect_host
+    kill_existing_processes
+    cleanup_stale_shm
+    install_apt_packages
+    clean_site_packages
+    setup_pip_toolchain
+    remove_stale_cuda12_nvidia_wheels
+    uninstall_stale_flashinfer
+    install_sglang
+    # Diffusion B200 CI imports torch inside install_sglang_kernel after removing
+    # stale CUDA 12 NVIDIA wheels, so opt into one early LD_LIBRARY_PATH refresh.
+    if [ "${SGLANG_CI_EARLY_LD_LIBRARY_PATH:-0}" = "1" ]; then
+        setup_ld_library_path
+    fi
+    install_sglang_kernel
+    install_sglang_router
+    install_flashinfer_cubin
+    download_flashinfer_cache
+    force_reinstall_cutlass_dsl_libs_cu13
+    stabilize_flashinfer_jit_paths
+    install_extra_deps
+    install_test_tools
+    prepare_runner
+    setup_ld_library_path
+    verify_imports
+}
+
+main "$@"

scripts/ci/cuda/ci_install_gateway_dependencies.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Install dependencies for the sgl-model-gateway CI jobs.
+#
+# Gateway-specific apt deps are installed here; protoc and the Rust toolchain
+# are delegated to the shared installer (the toolchain version is pinned by
+# sgl-model-gateway/rust-toolchain.toml, picked up automatically on first
+# `cargo` invocation).
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+GATEWAY_APT_PACKAGES=(libssl-dev pkg-config redis-server)
+APT_OPTS=(
+    -y
+    -o "Acquire::Retries=5"
+    -o "Acquire::http::Timeout=30"
+    -o "Acquire::https::Timeout=30"
+)
+SUDO=""
+command -v sudo >/dev/null 2>&1 && SUDO="sudo"
+
+# GH-hosted runners' Azure Ubuntu mirrors flake periodically. Retry the
+# whole install with backoff so we don't fail the whole CI on a 1-min
+# DNS hiccup at apt-mirrors.txt → azure.archive.ubuntu.com.
+for attempt in 1 2 3 4 5; do
+    if $SUDO apt-get update "${APT_OPTS[@]}" \
+       && $SUDO apt-get install "${APT_OPTS[@]}" "${GATEWAY_APT_PACKAGES[@]}"; then
+        break
+    fi
+    if [ "$attempt" = 5 ]; then
+        echo "apt-get install failed after 5 attempts; giving up." >&2
+        exit 1
+    fi
+    sleep $((attempt * 15))
+done
+
+bash "${SCRIPT_DIR}/../utils/install_rust_protoc.sh"
+
+# Make cargo/rustc/protoc visible in this shell.
+. "$HOME/.cargo/env"
+
+rustc --version
+cargo --version
+protoc --version

scripts/ci/cuda/ci_start_disaggregation_servers.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+set -euo pipefail
+
+# Optional: set DISAGG_READY_FILE to a filepath; when all servers are healthy, the script will
+# create this file as a readiness signal (useful for CI to proceed to next steps).
+DISAGG_READY_FILE="${DISAGG_READY_FILE:-}"
+
+MODEL_PATH="/raid/models/meta-llama/Llama-3.1-8B-Instruct"
+
+# Function to find the first available active IB device
+find_active_ib_device() {
+    for device in mlx5_{0..11}; do
+        if ibv_devinfo $device >/dev/null 2>&1; then
+            state=$(ibv_devinfo $device | grep "state:" | head -1 | awk '{print $2}')
+            if [[ "$state" == "PORT_ACTIVE" ]]; then
+                echo "$device"
+                return 0
+            fi
+        fi
+    done
+    echo "No active IB device found" >&2
+    return 1
+}
+
+# Get the first available active IB device
+DEVICE=$(find_active_ib_device)
+echo "Using IB device: $DEVICE"
+
+# Launch prefill servers on GPU 0–3
+for i in {0..3}; do
+  PORT=$((30001 + i))
+  BOOTSTRAP_PORT=$((9001 + i))
+  HOST="127.0.0.$((i + 1))"
+  echo "Launching PREFILL server on GPU $i at $HOST:$PORT (bootstrap: $BOOTSTRAP_PORT)"
+  CUDA_VISIBLE_DEVICES=$i \
+  python3 -m sglang.launch_server \
+    --model-path "$MODEL_PATH" \
+    --disaggregation-mode prefill \
+    --host "$HOST" \
+    --port "$PORT" \
+    --disaggregation-ib-device "$DEVICE" \
+    --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" &
+done
+
+# Launch decode servers on GPU 4–7
+for i in {4..7}; do
+  PORT=$((30001 + i))
+  HOST="127.0.0.$((i + 1))"
+  echo "Launching DECODE server on GPU $i at $HOST:$PORT"
+  CUDA_VISIBLE_DEVICES=$i \
+  python3 -m sglang.launch_server \
+    --model-path "$MODEL_PATH" \
+    --disaggregation-mode decode \
+    --host "$HOST" \
+    --port "$PORT" \
+    --disaggregation-ib-device "$DEVICE" \
+    --base-gpu-id 0 &
+done
+
+# Wait for disaggregation servers to initialize
+echo "Waiting for disaggregation servers to initialize..."
+
+# Health check with 5-minute timeout
+TIMEOUT=300
+START_TIME=$(date +%s)
+
+echo "Checking health of all 8 servers..."
+while true; do
+    CURRENT_TIME=$(date +%s)
+    ELAPSED=$((CURRENT_TIME - START_TIME))
+
+    if [ $ELAPSED -ge $TIMEOUT ]; then
+        echo "❌ Timeout: Servers did not become healthy within 5 minutes"
+        exit 1
+    fi
+
+    HEALTHY_COUNT=0
+    # Check all 8 servers (127.0.0.1-8:30001-30008)
+    for i in {1..8}; do
+        if curl -s -f "http://127.0.0.$i:$((30000 + i))/health" >/dev/null 2>&1; then
+            HEALTHY_COUNT=$((HEALTHY_COUNT + 1))
+        fi
+    done
+
+    echo "Healthy servers: $HEALTHY_COUNT/8 (elapsed: ${ELAPSED}s)"
+
+    if [ $HEALTHY_COUNT -eq 8 ]; then
+        echo "✅ All 8 servers are healthy!"
+        # Emit readiness signal file if requested
+        if [ -n "$DISAGG_READY_FILE" ]; then
+            echo "Creating readiness flag: $DISAGG_READY_FILE"
+            # Ensure parent dir exists; ignore errors
+            mkdir -p "$(dirname "$DISAGG_READY_FILE")" 2>/dev/null || true
+            touch "$DISAGG_READY_FILE"
+        fi
+        break
+    else
+        sleep 10  # Wait 10 seconds before next check
+    fi
+done
+
+# Don't launch router here - just keep servers running
+echo "✅ All disaggregation servers are ready and waiting for router connections"
+
+# Keep the script running
+wait

scripts/ci/cuda/prepare_runner.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Prepare the CI runner by cleaning up stale HuggingFace cache artifacts and validating models
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "Preparing CI runner..."
+echo ""
+
+# Clean up stale HuggingFace cache artifacts from previous failed downloads
+python3 "${SCRIPT_DIR}/../utils/cleanup_hf_cache.py"
+echo ""
+
+# Pre-validate cached models and write markers for offline mode
+# This allows tests to run with HF_HUB_OFFLINE=1 for models that are fully cached
+python3 "${SCRIPT_DIR}/../utils/prevalidate_cached_models.py"
+echo ""
+
+echo "CI runner preparation complete!"

scripts/ci/cuda/warmup_deep_gemm.py

@@ -0,0 +1,603 @@
+"""
+Lightweight DeepGEMM JIT compilation warmup without loading model weights.
+
+Reads model config.json from HF cache to derive kernel shapes, then compiles
+DeepGEMM kernels directly. This avoids the expensive model weight loading step
+that the full `sglang.compile_deep_gemm` requires.
+
+Supports DeepSeek V2/V3 family models. Falls back to `sglang.compile_deep_gemm`
+for unsupported architectures.
+
+Usage:
+    python3 scripts/ci/cuda/warmup_deep_gemm.py \
+        deepseek-ai/DeepSeek-V3-0324:8 \
+        deepseek-ai/DeepSeek-V3.2:8
+"""
+
+import hashlib
+import json
+import os
+import signal
+import subprocess
+import sys
+import threading
+import time
+from math import ceil
+from pathlib import Path
+from typing import Dict, List
+
+# Shared with warmup_server.py. Wipe alongside /root/.cache/deep_gemm if you
+# clear the DeepGEMM JIT cache — a stale marker → in-test JIT compile.
+MARKER_DIR = os.path.join(os.path.expanduser("~"), ".cache", "sglang", "warmup_markers")
+
+# Outer cap for stuck fallback subprocesses; CRASH_MARKERS abort sooner.
+FALLBACK_TIMEOUT_SEC = 600
+
+# Per-model launch flags forwarded to `sglang.compile_deep_gemm`. DeepGEMM
+# cache key includes per-rank N/K (depends on tp/dp/ep) — must match each
+# model's `other_args` in test/registered/ or warmed shapes won't be hit.
+FALLBACK_ARGS: Dict[str, List[str]] = {
+    "deepseek-ai/DeepSeek-V3.2": ["--dp", "8", "--enable-dp-attention"],
+    "zai-org/GLM-5-FP8": ["--dp", "8", "--enable-dp-attention"],
+    "XiaomiMiMo/MiMo-V2-Flash": [
+        "--dp",
+        "2",
+        "--enable-dp-attention",
+        "--attention-backend",
+        "fa3",
+    ],
+    # --mm-enable-dp-encoder is required: without it DP0 runs the vision
+    # encoder alone and DP1 deadlocks at the next collective.
+    "XiaomiMiMo/MiMo-V2.5": [
+        "--dp",
+        "2",
+        "--enable-dp-attention",
+        "--mm-enable-dp-encoder",
+        "--attention-backend",
+        "fa3",
+        "--mm-attention-backend",
+        "fa3",
+    ],
+}
+
+# compile_deep_gemm polls /v1/models for the full timeout even after a TP rank
+# dies; the watcher uses these to kill the group within seconds instead.
+CRASH_MARKERS = (
+    "Scheduler hit an exception",
+    "Received sigquit from a child",
+)
+
+# Configure DeepGEMM cache before importing deep_gemm
+os.environ["DG_JIT_CACHE_DIR"] = os.getenv(
+    "SGLANG_DG_CACHE_DIR",
+    os.path.join(os.path.expanduser("~"), ".cache", "deep_gemm"),
+)
+os.environ["DG_JIT_USE_NVRTC"] = os.getenv("SGL_DG_USE_NVRTC", "0")
+
+BLOCK_SIZE = 128
+
+
+def get_config_json(model_name):
+    """Load config.json for a cached model from HF cache."""
+    cache_dir = os.environ.get(
+        "HF_HOME", os.path.join(os.path.expanduser("~"), ".cache", "huggingface")
+    )
+    hub_dir = os.path.join(cache_dir, "hub")
+    safe_name = "models--" + model_name.replace("/", "--")
+    snapshots_dir = os.path.join(hub_dir, safe_name, "snapshots")
+
+    if not os.path.isdir(snapshots_dir):
+        return None
+
+    snapshots = sorted(
+        Path(snapshots_dir).iterdir(), key=lambda p: p.stat().st_mtime, reverse=True
+    )
+    for snapshot in snapshots:
+        config_path = snapshot / "config.json"
+        if config_path.exists():
+            with open(config_path) as f:
+                return json.load(f)
+    return None
+
+
+def is_deepseek_v2v3(config):
+    """Check if a model is from the DeepSeek V2/V3 family."""
+    architectures = config.get("architectures", [])
+    model_type = config.get("model_type", "")
+    return any(
+        "DeepseekV2" in a or "DeepseekV3" in a for a in architectures
+    ) or model_type in ("deepseek_v2", "deepseek_v3")
+
+
+def compute_deepseek_v2v3_shapes(config, tp):
+    """Compute all DeepGEMM (kernel_type, N, K, num_groups) for DeepSeek V2/V3.
+
+    Shape derivation based on:
+    - MoE: python/sglang/srt/layers/moe/fused_moe_triton/layer.py
+    - MLA: python/sglang/srt/models/deepseek_v2.py
+    - FP8: python/sglang/srt/layers/quantization/fp8_kernel.py
+    """
+    shapes = []
+
+    hidden_size = config["hidden_size"]
+    num_attention_heads = config.get("num_attention_heads", 128)
+    kv_lora_rank = config.get("kv_lora_rank", 512)
+    qk_nope_head_dim = config.get("qk_nope_head_dim", 128)
+    v_head_dim = config.get("v_head_dim", 128)
+    n_routed_experts = config.get("n_routed_experts", 0)
+    n_shared_experts = config.get("n_shared_experts", 0)
+    moe_intermediate_size = config.get("moe_intermediate_size", 0)
+
+    num_local_heads = num_attention_heads // tp
+    # Shared expert fusion is enabled by default (disable_shared_experts_fusion=False)
+    # so the FusedMoE weight tensor includes shared experts
+    num_local_experts = n_routed_experts + n_shared_experts
+
+    # --- MoE expert GEMM shapes ---
+    # FusedMoE shards intermediate_size across TP ranks (column parallel for gate/up,
+    # row parallel for down). All experts are replicated on each TP rank.
+    if n_routed_experts > 0 and moe_intermediate_size > 0:
+        moe_inter_per_tp = moe_intermediate_size // tp
+
+        # Gate-Up projection: (tokens, hidden_size) @ (experts, 2*inter_per_tp, hidden_size)^T
+        # Both masked and contiguous paths are used at runtime
+        shapes.append(("MASKED", moe_inter_per_tp * 2, hidden_size, num_local_experts))
+        shapes.append(("CONTIG", moe_inter_per_tp * 2, hidden_size, num_local_experts))
+
+        # Down projection: (tokens, inter_per_tp) @ (experts, hidden_size, inter_per_tp)^T
+        shapes.append(("MASKED", hidden_size, moe_inter_per_tp, num_local_experts))
+        shapes.append(("CONTIG", hidden_size, moe_inter_per_tp, num_local_experts))
+
+    # --- MLA attention GEMM shapes (masked grouped GEMM) ---
+    if kv_lora_rank > 0 and num_local_heads > 0:
+        # Q_nope -> compressed K: (heads, m, qk_nope_head_dim) @ (heads, kv_lora_rank, qk_nope_head_dim)^T
+        shapes.append(("MASKED", kv_lora_rank, qk_nope_head_dim, num_local_heads))
+
+        # Attention output -> V: (heads, m, kv_lora_rank) @ (heads, v_head_dim, kv_lora_rank)^T
+        shapes.append(("MASKED", v_head_dim, kv_lora_rank, num_local_heads))
+
+    # --- kv_b_proj (non-grouped GEMM via FP8 kernel) ---
+    # ColumnParallelLinear(kv_lora_rank, num_heads * (qk_nope + v_head_dim))
+    # Per TP rank: N = num_local_heads * (qk_nope_head_dim + v_head_dim)
+    if kv_lora_rank > 0 and num_local_heads > 0:
+        kv_b_proj_n = num_local_heads * (qk_nope_head_dim + v_head_dim)
+        shapes.append(("NORMAL", kv_b_proj_n, kv_lora_rank, 1))
+
+    return shapes
+
+
+def get_architecture_key(config, tp):
+    """Key for dedup: models with same key share DeepGEMM kernels."""
+    if config is None:
+        return None
+    fields = [
+        config.get("hidden_size", 0),
+        config.get("moe_intermediate_size", 0),
+        config.get("n_routed_experts", 0),
+        config.get("n_shared_experts", 0),
+        config.get("num_attention_heads", 0),
+        config.get("kv_lora_rank", 0),
+        config.get("qk_nope_head_dim", 0),
+        config.get("v_head_dim", 0),
+        tp,
+    ]
+    return tuple(fields)
+
+
+def compute_m_list(fast_warmup=False, chunked_prefill_size=8192):
+    """Compute the list of M values to compile (matches compile_utils.py logic)."""
+    m_list = []
+    if fast_warmup:
+        m_list += list(range(1, 1025))
+        next_m, sample_step = 1024, 2
+        max_prefill_bs = min(chunked_prefill_size, 32 * 1024)
+        while next_m < max_prefill_bs:
+            m_list += list(range(next_m, 2 * next_m, sample_step))
+            next_m *= 2
+            sample_step *= 2
+        m_list.append(max_prefill_bs)
+        m_list = sorted(set(m_list))
+    else:
+        m_max = 16 * 1024
+        if chunked_prefill_size > 8192:
+            m_max = chunked_prefill_size * 2
+        m_max = min(128 * 1024, m_max)
+        m_list = list(range(1, m_max + 1))
+    return m_list
+
+
+def _empty_token_fp8(size):
+    """Create FP8 token tensor + per-block scale tensor."""
+    import torch
+
+    *dims, k = size
+    return (
+        torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
+        torch.empty((*dims, ceil(k / BLOCK_SIZE)), device="cuda", dtype=torch.float32),
+    )
+
+
+def _empty_block_fp8(size):
+    """Create FP8 block tensor + per-block scale tensor."""
+    import torch
+
+    *dims, n, k = size
+    return (
+        torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn),
+        torch.empty(
+            (*dims, ceil(n / BLOCK_SIZE), ceil(k / BLOCK_SIZE)),
+            device="cuda",
+            dtype=torch.float32,
+        ),
+    )
+
+
+def get_memory_requirement(kernel_type, max_m, n, k, num_groups):
+    """Estimate GPU memory needed in GB for compilation buffers."""
+    _GB = 1 << 30
+    if kernel_type == "NORMAL":
+        return (max_m * k + n * k + max_m * n * 2) / _GB
+    elif kernel_type == "CONTIG":
+        return (max_m * k + num_groups * n * k + max_m * 4 + max_m * n * 2) / _GB
+    elif kernel_type == "MASKED":
+        return (
+            num_groups * max_m * k
+            + num_groups * n * k
+            + num_groups * 4
+            + num_groups * max_m * n * 2
+        ) / _GB
+    return 0
+
+
+def compile_one_shape(kernel_type, n, k, num_groups, m_list):
+    """Compile DeepGEMM kernels for one (kernel_type, N, K, num_groups) shape."""
+    import deep_gemm
+    import torch
+    from tqdm import tqdm
+
+    # Filter M list for contiguous layout alignment
+    if kernel_type == "CONTIG":
+        m_alignment = deep_gemm.get_mk_alignment_for_contiguous_layout()
+        m_list = sorted(set(m for m in m_list if m % m_alignment == 0))
+
+    if not m_list:
+        return
+
+    max_m = max(m_list)
+
+    # Reduce max_m if not enough GPU memory
+    mem_free = torch.cuda.mem_get_info()[0] / (1 << 30)
+    mem_required = get_memory_requirement(kernel_type, max_m, n, k, num_groups)
+    if mem_required > mem_free:
+        while (
+            get_memory_requirement(kernel_type, max_m, n, k, num_groups) > mem_free
+            and max_m > 4096
+        ):
+            max_m //= 2
+        print(
+            f"  Memory {mem_free:.1f}GB < required {mem_required:.1f}GB, "
+            f"reducing max_m to {max_m}"
+        )
+        m_list = [m for m in m_list if m <= max_m]
+
+    get_compile_mode = getattr(deep_gemm, "get_compile_mode", None)
+    set_compile_mode = getattr(deep_gemm, "set_compile_mode", None)
+    old_mode = get_compile_mode() if get_compile_mode is not None else None
+    if set_compile_mode is not None:
+        set_compile_mode(1)
+    try:
+        if kernel_type == "NORMAL":
+            lhs_q, lhs_s = _empty_token_fp8((max_m, k))
+            rhs_q, rhs_s = _empty_block_fp8((n, k))
+            out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16)
+            for m in tqdm(m_list, desc=f"  NORMAL N={n} K={k}"):
+                deep_gemm.fp8_gemm_nt((lhs_q[:m], lhs_s[:m]), (rhs_q, rhs_s), out[:m])
+
+        elif kernel_type == "CONTIG":
+            lhs_q, lhs_s = _empty_token_fp8((max_m, k))
+            rhs_q, rhs_s = _empty_block_fp8((num_groups, n, k))
+            m_indices = torch.zeros((max_m,), device="cuda", dtype=torch.int32)
+            out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16)
+            for m in tqdm(m_list, desc=f"  CONTIG N={n} K={k} G={num_groups}"):
+                deep_gemm.m_grouped_fp8_gemm_nt_contiguous(
+                    (lhs_q[:m], lhs_s[:m]),
+                    (rhs_q, rhs_s),
+                    out[:m],
+                    m_indices[:m],
+                )
+
+        elif kernel_type == "MASKED":
+            lhs_q, lhs_s = _empty_token_fp8((num_groups, max_m, k))
+            rhs_q, rhs_s = _empty_block_fp8((num_groups, n, k))
+            masked_m = torch.zeros((num_groups,), device="cuda", dtype=torch.int32)
+            out = torch.empty(
+                (num_groups, max_m, n), device="cuda", dtype=torch.bfloat16
+            )
+            for m in tqdm(m_list, desc=f"  MASKED N={n} K={k} G={num_groups}"):
+                deep_gemm.fp8_m_grouped_gemm_nt_masked(
+                    (lhs_q, lhs_s),
+                    (rhs_q, rhs_s),
+                    out,
+                    masked_m=masked_m,
+                    expected_m=m,
+                )
+    finally:
+        if set_compile_mode is not None and old_mode is not None:
+            set_compile_mode(old_mode)
+
+    torch.cuda.current_stream().synchronize()
+    torch.cuda.empty_cache()
+
+
+def compile_shapes_lightweight(shapes, m_list):
+    """Compile all DeepGEMM shapes directly (no model loading)."""
+    for i, (kernel_type, n, k, num_groups) in enumerate(shapes, 1):
+        print(f"\n[{i}/{len(shapes)}] {kernel_type} N={n} K={k} G={num_groups}")
+        t0 = time.time()
+        compile_one_shape(kernel_type, n, k, num_groups, m_list)
+        elapsed = time.time() - t0
+        print(f"  Done in {elapsed:.1f}s")
+
+
+def _kill_pg_and_wait(proc):
+    """SIGTERM the subprocess's process group, escalate to SIGKILL if needed."""
+    try:
+        os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+    except (ProcessLookupError, OSError):
+        pass
+    try:
+        return proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+        except (ProcessLookupError, OSError):
+            pass
+        try:
+            return proc.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            return -1
+
+
+def get_version_key():
+    """Hash of Python + Triton + PyTorch versions; invalidates markers on upgrade."""
+    parts = [sys.version]
+    try:
+        import triton  # noqa: WPS433
+
+        parts.append(f"triton={triton.__version__}")
+    except ImportError:
+        parts.append("triton=none")
+    try:
+        import torch  # noqa: WPS433
+
+        parts.append(f"torch={torch.__version__}")
+    except ImportError:
+        parts.append("torch=none")
+    return hashlib.sha256("|".join(parts).encode()).hexdigest()[:12]
+
+
+def get_fallback_marker_path(model, tp, extra_args):
+    """Marker path for one (model, tp, extra_args) fallback invocation."""
+    args_blob = json.dumps(list(extra_args))
+    args_hash = hashlib.md5(args_blob.encode()).hexdigest()[:8]
+    safe_model = model.replace("/", "--")
+    return os.path.join(
+        MARKER_DIR,
+        f"deepgemm_fallback_{safe_model}_tp{tp}_{args_hash}_{get_version_key()}.done",
+    )
+
+
+def check_fallback_marker(model, tp, extra_args):
+    return os.path.exists(get_fallback_marker_path(model, tp, extra_args))
+
+
+def write_fallback_marker(model, tp, extra_args):
+    marker = get_fallback_marker_path(model, tp, extra_args)
+    os.makedirs(os.path.dirname(marker), exist_ok=True)
+    Path(marker).write_text(
+        json.dumps(
+            {
+                "model": model,
+                "tp": tp,
+                "extra_args": list(extra_args),
+                "version_key": get_version_key(),
+                "timestamp": time.time(),
+            }
+        )
+    )
+    print(f"  Wrote marker: {marker}")
+
+
+def fallback_compile_deep_gemm(model, tp):
+    """Fall back to full sglang.compile_deep_gemm (loads model weights).
+
+    Runs in its own process group so a hung subprocess (e.g. one TP rank
+    crashes and the rest deadlock on NCCL collectives) can be killed
+    cleanly without leaking children. Watches subprocess output for crash
+    markers so a deterministic failure aborts in seconds rather than burning
+    the full FALLBACK_TIMEOUT_SEC.
+    """
+    extra_args = FALLBACK_ARGS.get(model, [])
+    print(
+        f"Falling back to full compile_deep_gemm for {model} "
+        f"(tp={tp}, extra_args={extra_args})..."
+    )
+    cmd = [
+        sys.executable,
+        "-m",
+        "sglang.compile_deep_gemm",
+        "--model",
+        model,
+        "--tp",
+        str(tp),
+        "--trust-remote-code",
+        "--model-loader-extra-config",
+        '{"enable_multithread_load": true, "num_threads": 64}',
+        # Cap compile_deep_gemm's own /v1/models polling loop so it gives up
+        # before our outer timeout has to SIGTERM it.
+        "--timeout",
+        str(FALLBACK_TIMEOUT_SEC),
+        *extra_args,
+    ]
+
+    crashed = threading.Event()
+    proc = subprocess.Popen(
+        cmd,
+        preexec_fn=os.setsid,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        bufsize=1,
+        text=True,
+    )
+
+    def _watch():
+        # Stream child output to our stdout while scanning for crash markers.
+        for line in proc.stdout:
+            sys.stdout.write(line)
+            sys.stdout.flush()
+            if not crashed.is_set() and any(m in line for m in CRASH_MARKERS):
+                crashed.set()
+
+    watcher = threading.Thread(target=_watch, daemon=True)
+    watcher.start()
+
+    deadline = time.monotonic() + FALLBACK_TIMEOUT_SEC
+    while True:
+        rc = proc.poll()
+        if rc is not None:
+            watcher.join(timeout=2)
+            if rc != 0:
+                print(f"Warning: fallback failed for {model} (exit code {rc})")
+            return rc == 0
+        if crashed.is_set():
+            print(
+                f"Warning: detected crash marker in {model} (tp={tp}) subprocess; "
+                "killing process group and continuing."
+            )
+            _kill_pg_and_wait(proc)
+            return False
+        if time.monotonic() >= deadline:
+            print(
+                f"Warning: fallback timed out after {FALLBACK_TIMEOUT_SEC}s for "
+                f"{model} (tp={tp}); killing process group and continuing."
+            )
+            _kill_pg_and_wait(proc)
+            return False
+        time.sleep(2)
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
+        print("Usage: warmup_deep_gemm.py model1:tp1 [model2:tp2 ...]")
+        print("\nDerives DeepGEMM kernel shapes from config.json without loading model")
+        print(
+            "weights. Falls back to full compile_deep_gemm for unknown architectures."
+        )
+        sys.exit(0)
+
+    # Parse model:tp pairs
+    model_tp_pairs = []
+    for arg in sys.argv[1:]:
+        if ":" not in arg:
+            print(f"Error: expected model:tp format, got '{arg}'")
+            sys.exit(1)
+        model, tp_str = arg.rsplit(":", 1)
+        model_tp_pairs.append((model, int(tp_str)))
+
+    fast_warmup = os.environ.get("SGLANG_JIT_DEEPGEMM_FAST_WARMUP", "0").lower() in (
+        "1",
+        "true",
+    )
+    print(f"=== DeepGEMM Lightweight Warmup ({len(model_tp_pairs)} model(s)) ===")
+    print(f"    Fast warmup: {fast_warmup}")
+    print(
+        f"    Cache dir: {os.environ.get('DG_JIT_CACHE_DIR', '~/.cache/deep_gemm')}\n"
+    )
+
+    # Load configs and deduplicate by architecture
+    seen_keys = {}
+    to_process = []  # (model, tp, config_or_None, shapes_or_None)
+
+    for model, tp in model_tp_pairs:
+        config = get_config_json(model)
+        if config is None:
+            print(f"  SKIP   {model} (tp={tp}): config.json not in HF cache")
+            continue
+
+        # Models with FALLBACK_ARGS launch with extra dp/ep/dp-attention flags
+        # that change per-rank N/K. The lightweight path doesn't model those —
+        # it computes attention shapes assuming TP-only sharding — so dedup'ing
+        # such a model to a no-override DeepSeek V2/V3 lookalike (e.g. V3.2 →
+        # V3-0324) silently picks the wrong attention shapes for the test's
+        # actual launch config. Force these through fallback so the populated
+        # cache matches the real test.
+        has_fallback_override = model in FALLBACK_ARGS
+
+        key = get_architecture_key(config, tp)
+        if key in seen_keys and not has_fallback_override:
+            print(f"  DEDUP  {model} (tp={tp}): same shapes as {seen_keys[key]}")
+            continue
+
+        if is_deepseek_v2v3(config) and not has_fallback_override:
+            shapes = compute_deepseek_v2v3_shapes(config, tp)
+            seen_keys[key] = model
+            to_process.append((model, tp, config, shapes))
+            print(f"  FOUND  {model} (tp={tp}): {len(shapes)} DeepGEMM shape(s)")
+        else:
+            seen_keys[key] = model
+            to_process.append((model, tp, config, None))
+            if has_fallback_override:
+                print(
+                    f"  FOUND  {model} (tp={tp}): forced fallback (extra args "
+                    f"{FALLBACK_ARGS[model]})"
+                )
+            else:
+                arch = config.get("architectures", ["unknown"])
+                print(
+                    f"  FOUND  {model} (tp={tp}): unknown arch {arch}, "
+                    "will use fallback"
+                )
+
+    if not to_process:
+        print("\nNo models to process. Done.")
+        return
+
+    m_list = compute_m_list(fast_warmup=fast_warmup)
+    print(f"\nM list: {len(m_list)} values (range {min(m_list)}-{max(m_list)})")
+
+    for model, tp, config, shapes in to_process:
+        print(f"\n{'=' * 60}")
+        print(f"Model: {model} (tp={tp})")
+        print(f"{'=' * 60}")
+
+        if shapes is None:
+            # Fallback path: full sglang.compile_deep_gemm with the test's launch
+            # flags. Loading model weights inside that subprocess is the dominant
+            # cost (45-170s/model) and dwarfs the actual DeepGEMM compile, so we
+            # skip the whole fallback when the marker says we already populated
+            # this cache. Cache is keyed on (model, tp, extra_args, version_key).
+            extra_args = FALLBACK_ARGS.get(model, [])
+            if check_fallback_marker(model, tp, extra_args):
+                print(
+                    f"  SKIP fallback (warm marker found): {model} (tp={tp}, "
+                    f"extra_args={extra_args})"
+                )
+                continue
+            if fallback_compile_deep_gemm(model, tp):
+                write_fallback_marker(model, tp, extra_args)
+            continue
+
+        # Print shape summary
+        for kernel_type, n, k, num_groups in shapes:
+            print(f"  {kernel_type:8s} N={n:<6d} K={k:<6d} G={num_groups}")
+
+        t0 = time.time()
+        compile_shapes_lightweight(shapes, m_list)
+        elapsed = time.time() - t0
+        print(f"\nCompleted {model} in {elapsed:.1f}s")
+
+    print("\nDeepGEMM lightweight warmup complete.")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/cuda/warmup_server.py

@@ -0,0 +1,335 @@
+"""
+Full server warmup to pre-warm Triton autotuning and CUDA graph capture.
+
+On cold H200 nodes (new nodes or after container recreation), CUDA graph capture
+triggers Triton autotuning which takes ~330s per server launch. This script
+launches actual servers with CUDA graphs enabled to cache the autotuned kernels,
+so subsequent test launches are fast (~30-60s).
+
+Uses marker files to skip warmup on already-warm nodes. Marker files are
+invalidated when Python, Triton, or PyTorch versions change.
+
+Usage:
+    python3 scripts/ci/cuda/warmup_server.py \
+        deepseek-ai/DeepSeek-V3-0324:8
+"""
+
+import hashlib
+import json
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+# Reuse helpers from warmup_deep_gemm (same directory)
+sys.path.insert(0, os.path.dirname(__file__))
+from warmup_deep_gemm import get_architecture_key, get_config_json
+
+MARKER_DIR = os.path.join(os.path.expanduser("~"), ".cache", "sglang", "warmup_markers")
+HEALTH_POLL_INTERVAL = 10  # seconds between health checks
+SERVER_STARTUP_TIMEOUT = 900  # 15 min max to wait for server ready
+DEFAULT_PORT = 39876
+
+
+def get_version_key():
+    """Hash of Python + Triton + PyTorch versions to invalidate markers on upgrades."""
+    parts = [sys.version]
+    try:
+        import triton
+
+        parts.append(f"triton={triton.__version__}")
+    except ImportError:
+        parts.append("triton=none")
+    try:
+        import torch
+
+        parts.append(f"torch={torch.__version__}")
+    except ImportError:
+        parts.append("torch=none")
+    return hashlib.sha256("|".join(parts).encode()).hexdigest()[:12]
+
+
+def get_marker_path(model, tp):
+    """Get the marker file path for a model:tp pair."""
+    version_key = get_version_key()
+    safe_model = model.replace("/", "--")
+    return os.path.join(
+        MARKER_DIR, f"server_warmup_{safe_model}_tp{tp}_{version_key}.done"
+    )
+
+
+def check_marker(model, tp):
+    """Check if warmup marker exists (node already warm)."""
+    marker = get_marker_path(model, tp)
+    return os.path.exists(marker)
+
+
+def write_marker(model, tp):
+    """Write warmup marker after successful warmup."""
+    marker = get_marker_path(model, tp)
+    os.makedirs(os.path.dirname(marker), exist_ok=True)
+    Path(marker).write_text(
+        json.dumps(
+            {
+                "model": model,
+                "tp": tp,
+                "version_key": get_version_key(),
+                "timestamp": time.time(),
+            }
+        )
+    )
+    print(f"  Wrote marker: {marker}")
+
+
+def kill_server(proc):
+    """Kill server process tree."""
+    if proc.poll() is None:
+        try:
+            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+        except (ProcessLookupError, OSError):
+            pass
+        try:
+            proc.wait(timeout=15)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+            except (ProcessLookupError, OSError):
+                pass
+            try:
+                proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                pass
+
+    # sglang's scheduler_TP* and detokenizer workers spawn through
+    # multiprocessing with their own session/process group, so they escape
+    # killpg on launch_server and stay alive holding GPU memory after a
+    # readiness-timeout or unclean exit. Kill any survivors by name so the
+    # next model (or the next CI step) starts with empty GPUs.
+    for pattern in ("sglang::scheduler", "sglang::detokenizer"):
+        try:
+            subprocess.run(
+                ["pkill", "-9", "-f", pattern],
+                timeout=5,
+                check=False,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        except (FileNotFoundError, subprocess.TimeoutExpired):
+            pass
+    # Let the driver release device memory before the caller measures it.
+    time.sleep(2)
+
+
+def wait_for_server(base_url, proc, timeout):
+    """Poll /health_generate until server is ready or timeout."""
+    import requests
+
+    start = time.time()
+    while time.time() - start < timeout:
+        ret = proc.poll()
+        if ret is not None:
+            return False, f"Server exited with code {ret}"
+        try:
+            resp = requests.get(f"{base_url}/health_generate", timeout=5)
+            if resp.status_code == 200:
+                return True, None
+        except requests.RequestException:
+            pass
+        time.sleep(HEALTH_POLL_INTERVAL)
+    return False, "Timed out waiting for server"
+
+
+def send_generate_request(base_url):
+    """Send one /generate request to exercise the full inference path."""
+    import requests
+
+    payload = {
+        "input_ids": [0, 1, 2, 3],
+        "sampling_params": {
+            "max_new_tokens": 8,
+            "temperature": 0,
+        },
+    }
+    try:
+        resp = requests.post(f"{base_url}/generate", json=payload, timeout=120)
+        if resp.status_code == 200:
+            print("  Generate request succeeded")
+        else:
+            print(f"  Warning: generate request returned {resp.status_code}")
+    except requests.RequestException as e:
+        print(f"  Warning: generate request failed: {e}")
+
+
+def warmup_one_model(model, tp, port):
+    """Launch server, wait for ready, send one request, then kill."""
+    base_url = f"http://127.0.0.1:{port}"
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "sglang.launch_server",
+        "--model-path",
+        model,
+        "--tp",
+        str(tp),
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(port),
+        "--trust-remote-code",
+        "--model-loader-extra-config",
+        '{"enable_multithread_load": true, "num_threads": 64}',
+    ]
+
+    # Use a temp file for server output to avoid pipe buffer deadlock
+    # (server logs can exceed the 64KB pipe buffer during CUDA graph capture)
+    log_file = tempfile.NamedTemporaryFile(
+        mode="w", prefix="warmup_server_", suffix=".log", delete=False
+    )
+    log_path = log_file.name
+
+    print(f"  Launching server: {' '.join(cmd)}")
+    print(f"  Server log: {log_path}")
+    proc = subprocess.Popen(
+        cmd,
+        stdout=log_file,
+        stderr=subprocess.STDOUT,
+        preexec_fn=os.setsid,
+    )
+
+    try:
+        # Wait for server to be ready (includes CUDA graph capture)
+        print(
+            f"  Waiting for server (timeout={SERVER_STARTUP_TIMEOUT}s, "
+            f"polling every {HEALTH_POLL_INTERVAL}s)..."
+        )
+        ok, err = wait_for_server(base_url, proc, SERVER_STARTUP_TIMEOUT)
+        if not ok:
+            print(f"  Warning: server not ready: {err}")
+            return False
+
+        print("  Server ready, sending generate request...")
+        send_generate_request(base_url)
+        return True
+
+    finally:
+        # Surface the tail of the server log so CI captures validation
+        # messages, exceptions, and warmup progress (the launch_server
+        # subprocess writes stdout/stderr to the tempfile, not our stdout).
+        try:
+            log_file.flush()
+            with open(log_path) as f:
+                lines = f.readlines()
+            print(f"  --- server log tail ({len(lines)} lines, last 30) ---")
+            for line in lines[-30:]:
+                print(f"    | {line.rstrip()}")
+            print("  --- end server log ---")
+        except Exception:
+            pass
+
+        print("  Killing server...")
+        kill_server(proc)
+        log_file.close()
+        try:
+            os.unlink(log_path)
+        except OSError:
+            pass
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
+        print("Usage: warmup_server.py model1:tp1 [model2:tp2 ...]")
+        print(
+            "\nLaunches full servers with CUDA graphs enabled to pre-warm"
+            " Triton autotuning."
+        )
+        print("Skips instantly on warm nodes (marker file exists).")
+        sys.exit(0)
+
+    # Parse model:tp pairs
+    model_tp_pairs = []
+    for arg in sys.argv[1:]:
+        if ":" not in arg:
+            print(f"Error: expected model:tp format, got '{arg}'")
+            sys.exit(1)
+        model, tp_str = arg.rsplit(":", 1)
+        model_tp_pairs.append((model, int(tp_str)))
+
+    print(f"=== Server CUDA Graph Warmup ({len(model_tp_pairs)} model(s)) ===")
+    print(f"    Marker dir: {MARKER_DIR}")
+    print(f"    Version key: {get_version_key()}\n")
+
+    # Deduplicate by architecture and check markers
+    seen_keys = {}
+    to_warmup = []
+
+    for model, tp in model_tp_pairs:
+        # Check marker first (fast path)
+        if check_marker(model, tp):
+            print(f"  SKIP   {model} (tp={tp}): already warm (marker exists)")
+            continue
+
+        # Architecture dedup
+        config = get_config_json(model)
+        if config is not None:
+            key = get_architecture_key(config, tp)
+            if key in seen_keys:
+                print(
+                    f"  DEDUP  {model} (tp={tp}): same architecture as {seen_keys[key]}"
+                )
+                continue
+            seen_keys[key] = model
+
+        to_warmup.append((model, tp))
+        print(f"  QUEUE  {model} (tp={tp}): needs warmup")
+
+    if not to_warmup:
+        print("\nAll models already warm. Done.")
+        return
+
+    print(f"\n{len(to_warmup)} model(s) to warm up.\n")
+
+    port = DEFAULT_PORT
+    for i, (model, tp) in enumerate(to_warmup, 1):
+        print(f"\n{'=' * 60}")
+        print(f"[{i}/{len(to_warmup)}] {model} (tp={tp})")
+        print(f"{'=' * 60}")
+
+        t0 = time.time()
+        success = warmup_one_model(model, tp, port)
+        elapsed = time.time() - t0
+
+        if success:
+            print(f"  Completed in {elapsed:.0f}s")
+            write_marker(model, tp)
+            # Also write markers for dedup'd models that share this architecture
+            config = get_config_json(model)
+            if config is not None:
+                key = get_architecture_key(config, tp)
+                for other_model, other_tp in model_tp_pairs:
+                    if (other_model, other_tp) == (model, tp):
+                        continue
+                    other_config = get_config_json(other_model)
+                    if other_config is not None:
+                        other_key = get_architecture_key(other_config, other_tp)
+                        if other_key == key and not check_marker(other_model, other_tp):
+                            write_marker(other_model, other_tp)
+                            print(
+                                f"  Also marked {other_model} (tp={other_tp}) as warm (same arch)"
+                            )
+        else:
+            print(
+                f"  Warning: warmup failed after {elapsed:.0f}s (non-fatal, tests will still work)"
+            )
+
+        # Use a different port for the next model to avoid bind conflicts
+        port += 100
+
+    print("\nServer CUDA graph warmup complete.")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/list_stage_models.py

@@ -0,0 +1,688 @@
+#!/usr/bin/env python3
+"""Build a per-stage model inventory for NVIDIA (CUDA) CI.
+
+Emits a mapping `CI suite -> [HuggingFace model ids]` so the models a stage
+exercises can be pre-warmed into a runner cache. The mapping is produced by
+*static analysis* of the registered test files (no GPU, no sglang import), so
+it can run on a plain runner and stays fresh per commit.
+
+How `suite -> files` is resolved
+    Reuses the AST registry parser (`ut_parse_one_file` in ci_register.py, the
+    same one `run_suite.py` uses): registered test files call
+    `register_<backend>_ci(...)`; we group each file under its
+    `effective_suite` for the requested backend (the property falls back to a
+    legacy single-string `suite=` when `stage=`/`runner_config=` are unset).
+
+How `file -> models` is resolved (best effort, recall-favoring)
+    - A constant table built from `python/sglang/test/**/*.py` module-level
+      assignments (`DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/..."`, including
+      tuple/list values) plus each test file's own module-level constants.
+    - Inline HuggingFace-id string literals in the file (f-string fragments are
+      skipped: they are partial/dynamic and would yield truncated ids).
+    - `ast.Name` references that resolve to a known model constant.
+    Anything we cannot resolve is reported per suite as `unresolved_files`, and
+    any file we cannot parse is reported in `parse_failures`, so recall gaps are
+    visible rather than silent. A `--overrides` JSON file supplies models for
+    dynamic cases and trims false positives.
+
+How `runner label -> models` is aggregated
+    Registration/prewarm decisions are made per GH runner *label* (a runner's
+    `runs-on` tag), not per suite. Each suite's runner_config maps to a label
+    via scripts/ci/runner_configs.yml (several configs can share one label,
+    e.g. `4-gpu-h100` and `deepep-4-gpu-h100`), so `runner_labels` carries the
+    per-label UNION -- the set a runner registered under that label must have
+    cached before it takes jobs. Suites without a mappable runner_config are
+    listed in `unmapped_suites`.
+
+Usage:
+    python3 scripts/ci/list_stage_models.py --backend cuda \
+        --commit "$GITHUB_SHA" --output models-per-stage.json --markdown out.md
+"""
+
+import argparse
+import ast
+import glob
+import importlib.util
+import json
+import os
+import re
+import subprocess
+import sys
+from typing import Dict, List, Optional, Set, Tuple
+
+# A HuggingFace repo id is `namespace/name`: exactly one slash, each side
+# starting alphanumeric and made of alnum plus `.`, `_`, `-`. Note `.` is kept
+# because real ids carry it (e.g. `RedHatAI/Llama-3.2-3B-quantized.w8a8`).
+MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$")
+
+# `namespace/...` values that look like model ids but are MIME types or similar.
+_MIME_NAMESPACES = frozenset(
+    {
+        "application",
+        "audio",
+        "example",
+        "font",
+        "image",
+        "message",
+        "model",
+        "multipart",
+        "text",
+        "video",
+    }
+)
+
+# Trailing extensions that mark a file path or weight file, not a model name.
+# (Real ids use suffixes like `-GGUF`/`.w8a8`, not these dotted extensions.)
+_FILE_EXTENSIONS = (
+    ".py",
+    ".json",
+    ".txt",
+    ".md",
+    ".rst",
+    ".yaml",
+    ".yml",
+    ".sh",
+    ".cu",
+    ".cuh",
+    ".cpp",
+    ".cc",
+    ".h",
+    ".hpp",
+    ".so",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".csv",
+    ".log",
+    ".safetensors",
+    ".bin",
+    ".h5",
+    ".gguf",
+    ".pt",
+    ".pth",
+    ".onnx",
+)
+
+# Non-test helper files under test/registered/ (skipped by basename, matching
+# scripts/ci/check_registered_tests.py). run_suite.py skips `cpu/utils.py` by
+# path; excluding every `utils.py` by basename is a superset that drops no
+# CUDA-registered test (the other `utils.py` registers CPU only).
+_NON_TEST_BASENAMES = frozenset({"conftest.py", "__init__.py", "utils.py"})
+
+
+def looks_like_model_id(value: str, deny: Optional[Set[str]] = None) -> bool:
+    """Heuristic: does ``value`` look like a HuggingFace repo id?
+
+    Recall-favoring (a few false positives are cheap for cache-warming) but
+    drops the obvious non-models: MIME types, file paths, numeric ratios.
+    """
+    if deny and value in deny:
+        return False
+    if not MODEL_ID_RE.match(value):
+        return False
+    if not any(c.isalpha() for c in value):  # e.g. "2/3"
+        return False
+    namespace, name = value.split("/", 1)
+    if len(namespace) < 2 or len(name) < 2:  # e.g. "N/A"; real ids have longer parts
+        return False
+    if namespace.lower() in _MIME_NAMESPACES:
+        return False
+    if name.lower().endswith(_FILE_EXTENSIONS):
+        return False
+    return True
+
+
+def _string_values(node: ast.AST) -> List[str]:
+    """String constants directly held by ``node`` (a Constant, Tuple, or List)."""
+    if isinstance(node, ast.Constant):
+        return [node.value] if isinstance(node.value, str) else []
+    if isinstance(node, (ast.Tuple, ast.List)):
+        out: List[str] = []
+        for elt in node.elts:
+            if isinstance(elt, ast.Constant) and isinstance(elt.value, str):
+                out.append(elt.value)
+        return out
+    return []
+
+
+def extract_constants_from_source(
+    source: str, deny: Optional[Set[str]] = None
+) -> Dict[str, Set[str]]:
+    """Module-level ``NAME = "<model id>"`` (and tuple/list) assignments.
+
+    Handles both bare ``Assign`` and annotated ``AnnAssign`` (``NAME: str =
+    ...``). Returns ``{constant_name: {model_id, ...}}``; only model-shaped
+    values are kept, so referencing a non-model constant later contributes
+    nothing.
+    """
+    table: Dict[str, Set[str]] = {}
+    tree = ast.parse(source)
+    for stmt in tree.body:
+        if isinstance(stmt, ast.Assign):
+            targets, value = stmt.targets, stmt.value
+        elif isinstance(stmt, ast.AnnAssign) and stmt.value is not None:
+            targets, value = [stmt.target], stmt.value
+        else:
+            continue
+        models = {v for v in _string_values(value) if looks_like_model_id(v, deny)}
+        if not models:
+            continue
+        for target in targets:
+            if isinstance(target, ast.Name):
+                table.setdefault(target.id, set()).update(models)
+    return table
+
+
+def extract_models_from_source(
+    source: str,
+    const_table: Dict[str, Set[str]],
+    deny: Optional[Set[str]] = None,
+) -> Set[str]:
+    """All model ids reachable from ``source``: inline literals + name refs.
+
+    ``const_table`` is the merged (global + local) constant lookup. Name
+    references resolve against it, so an imported ``DEFAULT_*`` constant is
+    picked up even though its value is defined elsewhere. String fragments
+    inside f-strings are skipped -- they are partial/dynamic and would yield
+    truncated, non-existent ids (e.g. ``f"org/model-{ver}"``).
+    """
+    tree = ast.parse(source)
+    fstring_fragments = {
+        id(part)
+        for node in ast.walk(tree)
+        if isinstance(node, ast.JoinedStr)
+        for part in node.values
+        if isinstance(part, ast.Constant)
+    }
+    found: Set[str] = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Constant) and isinstance(node.value, str):
+            if id(node) in fstring_fragments:
+                continue
+            if looks_like_model_id(node.value, deny):
+                found.add(node.value)
+        elif isinstance(node, ast.Name) and node.id in const_table:
+            found.update(m for m in const_table[node.id] if not (deny and m in deny))
+    return found
+
+
+def build_global_constant_table(
+    repo_root: str, deny: Optional[Set[str]] = None
+) -> Tuple[Dict[str, Set[str]], Dict[str, str]]:
+    """Constant table from every module under ``python/sglang/test/``.
+
+    These shared helpers (e.g. test_utils, lora_utils) define the ``DEFAULT_*``
+    model constants test files reference by name. Returns ``(table, errors)``
+    where ``errors`` maps any unparsable helper to its exception string -- a
+    broken shared helper drops constants across many suites, so the gap must be
+    surfaced rather than swallowed.
+    """
+    table: Dict[str, Set[str]] = {}
+    errors: Dict[str, str] = {}
+    pattern = os.path.join(repo_root, "python", "sglang", "test", "**", "*.py")
+    for path in glob.glob(pattern, recursive=True):
+        try:
+            with open(path, encoding="utf-8") as f:
+                source = f.read()
+            local = extract_constants_from_source(source, deny)
+        except (OSError, SyntaxError) as exc:
+            rel = os.path.relpath(path, repo_root)
+            errors[rel] = f"{type(exc).__name__}: {exc}"
+            print(
+                f"WARNING: could not parse constant source {rel}; its model "
+                f"constants are EXCLUDED: {errors[rel]}",
+                file=sys.stderr,
+            )
+            continue
+        for name, models in local.items():
+            table.setdefault(name, set()).update(models)
+    return table, errors
+
+
+def _load_ci_register(repo_root: str):
+    """Import ci_register.py by path, sidestepping the heavy `sglang` package."""
+    spec = importlib.util.spec_from_file_location(
+        "ci_register",
+        os.path.join(repo_root, "python", "sglang", "test", "ci", "ci_register.py"),
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def collect_suite_files(
+    repo_root: str, backend_name: str, include_disabled: bool = False
+) -> Tuple[
+    Dict[str, List[str]],
+    Dict[str, bool],
+    Dict[str, str],
+    Dict[str, Optional[str]],
+]:
+    """Map ``effective_suite -> [relative test file]`` for one backend.
+
+    By default only enabled (`disabled is None`) registries are grouped, since a
+    disabled suite does not run and thus needs no cache warming. Pass
+    ``include_disabled=True`` to also group disabled registries (useful to see
+    what a suite *would* download once re-enabled). Returns the mapping,
+    ``{suite: is_nightly}``, ``{file: parse error}`` for files whose registry
+    could not be parsed (their models are excluded -- surfaced, not silently
+    dropped), and ``{suite: runner_config}`` (None for legacy single-string
+    ``suite=`` registrations, which carry no runner_config).
+    """
+    ci_register = _load_ci_register(repo_root)
+    backend = getattr(ci_register.HWBackend, backend_name.upper())
+
+    pattern = os.path.join(repo_root, "test", "registered", "**", "*.py")
+    files = sorted(
+        f
+        for f in glob.glob(pattern, recursive=True)
+        if os.path.basename(f) not in _NON_TEST_BASENAMES
+    )
+
+    suite_files: Dict[str, List[str]] = {}
+    suite_nightly: Dict[str, bool] = {}
+    suite_runner_config: Dict[str, Optional[str]] = {}
+    errors: Dict[str, str] = {}
+    for path in files:
+        rel = os.path.relpath(path, repo_root)
+        # Narrow catch: SyntaxError (bad source), ValueError (malformed
+        # registration, raised by RegistryVisitor), OSError (vanished file). A
+        # broader failure (e.g. AttributeError from a parser API drift) should
+        # crash loudly rather than silently empty the inventory.
+        try:
+            registries, _ = ci_register.ut_parse_one_file(path)
+        except (SyntaxError, ValueError, OSError) as exc:
+            errors[rel] = f"{type(exc).__name__}: {exc}"
+            print(
+                f"WARNING: could not parse {rel}; its models are EXCLUDED from "
+                f"the inventory: {errors[rel]}",
+                file=sys.stderr,
+            )
+            continue
+        for r in registries:
+            if r.backend != backend:
+                continue
+            if r.disabled is not None and not include_disabled:
+                continue
+            suite = r.effective_suite
+            if suite is None:
+                continue
+            if rel not in suite_files.setdefault(suite, []):
+                suite_files[suite].append(rel)
+            suite_nightly[suite] = suite_nightly.get(suite, False) or bool(r.nightly)
+            # Modern registrations name the suite `{stage}-test-{runner_config}`,
+            # so every registry in a suite shares one runner_config; legacy
+            # `suite=` registrations have none (stays None).
+            if r.runner_config is not None:
+                suite_runner_config[suite] = r.runner_config
+            else:
+                suite_runner_config.setdefault(suite, None)
+    return suite_files, suite_nightly, errors, suite_runner_config
+
+
+def load_overrides(path: Optional[str]) -> Dict[str, object]:
+    """Read the overrides JSON: ``by_file``, ``by_suite``, ``deny``,
+    ``suite_labels`` (all optional).
+
+    ``suite_labels`` maps a legacy ``suite=`` registration (which carries no
+    runner_config) to the GH runner label(s) its dispatching workflow
+    hardcodes in ``runs-on`` -- a LIST, since one suite can run on several
+    labels. A present-but-null key is treated as its default, so a hand-edit
+    like ``"deny": null`` does not blow up downstream iteration.
+    """
+    overrides: Dict[str, object] = {
+        "by_file": {},
+        "by_suite": {},
+        "deny": [],
+        "suite_labels": {},
+    }
+    if not path:
+        return overrides
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"overrides file not found: {path}")
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    for key in ("by_file", "by_suite", "deny", "suite_labels"):
+        if data.get(key) is not None:
+            overrides[key] = data[key]
+    return overrides
+
+
+# One runner_config entry in runner_configs.yml: a two-space-indented key with
+# a flow-style (inline `{...}`) mapping. This is the file's documented shape;
+# entries are matched line-by-line so the tool stays stdlib-only (the workflow
+# installs nothing, so PyYAML is not available).
+_RUNNER_CONFIG_LINE_RE = re.compile(r"^  ([A-Za-z0-9_-]+):\s*\{(.*)\}\s*$")
+_RUNS_ON_RE = re.compile(r"\bruns_on:\s*([^,}\s]+)")
+
+# runner_configs.yml uses this placeholder for the dynamically-selected b200
+# runner label (resolved at workflow-load time by runner_configs.py --map).
+# The inventory keeps it literal unless --b200-runner substitutes it, so the
+# consumer can see the group is dynamic rather than silently guessing a label.
+B200_SENTINEL = "$b200_runner"
+
+
+def load_runner_labels(path: str) -> Dict[str, str]:
+    """Parse ``{runner_config: runs_on label}`` out of runner_configs.yml.
+
+    The mapping is what turns per-suite model sets into per-runner-LABEL sets:
+    a runner is registered under a `runs_on` label (several runner_configs can
+    share one, e.g. `4-gpu-h100` and `deepep-4-gpu-h100` both run on
+    `4-gpu-h100`), so a runner's cache must cover the union of every suite
+    that can land on its label. Raises ValueError on an entry without
+    `runs_on` or a file with no entries at all -- a format drift must fail
+    the workflow loudly, not silently empty the label aggregation.
+    """
+    labels: Dict[str, str] = {}
+    in_section = False
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("runner_configs:"):
+                in_section = True
+                continue
+            if in_section and line.strip() and not line.startswith(" "):
+                break  # next top-level key
+            if not in_section:
+                continue
+            m = _RUNNER_CONFIG_LINE_RE.match(line)
+            if not m:
+                continue
+            name, body = m.group(1), m.group(2)
+            runs_on = _RUNS_ON_RE.search(body)
+            if not runs_on:
+                raise ValueError(f"{path}: runner_config {name!r} has no runs_on field")
+            labels[name] = runs_on.group(1)
+    if not labels:
+        raise ValueError(
+            f"{path}: no runner_configs entries parsed -- format drift? "
+            f"(expected two-space-indented `name: {{...}}` lines under "
+            f"a `runner_configs:` key)"
+        )
+    return labels
+
+
+def build_inventory(
+    repo_root: str,
+    backend_name: str,
+    overrides: Dict[str, object],
+    commit: str,
+    include_disabled: bool = False,
+    b200_runner: Optional[str] = None,
+) -> Dict[str, object]:
+    deny: Set[str] = set(overrides.get("deny", []))  # type: ignore[arg-type]
+    by_file: Dict[str, List[str]] = overrides.get("by_file", {})  # type: ignore[assignment]
+    by_suite: Dict[str, List[str]] = overrides.get("by_suite", {})  # type: ignore[assignment]
+    suite_labels_override: Dict[str, List[str]] = overrides.get("suite_labels", {})  # type: ignore[assignment]
+
+    global_table, table_errors = build_global_constant_table(repo_root, deny)
+    suite_files, suite_nightly, registry_errors, suite_runner_config = (
+        collect_suite_files(repo_root, backend_name, include_disabled)
+    )
+
+    runner_labels_map: Dict[str, str] = {}
+    runner_configs_path = os.path.join(repo_root, "scripts", "ci", "runner_configs.yml")
+    if os.path.exists(runner_configs_path):
+        runner_labels_map = load_runner_labels(runner_configs_path)
+    else:
+        print(
+            f"WARNING: {runner_configs_path} not found; every suite will be "
+            f"reported as unmapped_suites (no per-runner-label aggregation).",
+            file=sys.stderr,
+        )
+
+    # Resolve models once per file (a file can belong to several suites).
+    file_models: Dict[str, Set[str]] = {}
+    extract_errors: Dict[str, str] = {}
+    for files in suite_files.values():
+        for rel in files:
+            if rel in file_models:
+                continue
+            abs_path = os.path.join(repo_root, rel)
+            try:
+                with open(abs_path, encoding="utf-8") as f:
+                    source = f.read()
+                local_table = extract_constants_from_source(source, deny)
+                merged = dict(global_table)
+                for name, models in local_table.items():
+                    merged.setdefault(name, set()).update(models)
+                resolved = extract_models_from_source(source, merged, deny)
+            except (OSError, SyntaxError) as exc:
+                extract_errors[rel] = f"{type(exc).__name__}: {exc}"
+                print(
+                    f"WARNING: could not extract models from {rel}; treating it "
+                    f"as unresolved: {extract_errors[rel]}",
+                    file=sys.stderr,
+                )
+                resolved = set()
+            resolved.update(by_file.get(rel, []))
+            file_models[rel] = resolved
+
+    suites: Dict[str, object] = {}
+    all_models: Set[str] = set()
+    for suite in sorted(suite_files):
+        models: Set[str] = set(by_suite.get(suite, []))
+        unresolved: List[str] = []
+        for rel in suite_files[suite]:
+            resolved = file_models.get(rel, set())
+            if resolved:
+                models.update(resolved)
+            else:
+                unresolved.append(rel)
+        all_models.update(models)
+        suites[suite] = {
+            "nightly": suite_nightly.get(suite, False),
+            "runner_config": suite_runner_config.get(suite),
+            "models": sorted(models),
+            "test_file_count": len(suite_files[suite]),
+            "unresolved_files": sorted(unresolved),
+        }
+
+    # Per-runner-LABEL aggregation: registration/prewarm decisions are made per
+    # GH runner label (what a runner is registered with), and several suites --
+    # via several runner_configs -- can route to one label. A runner's cache
+    # must cover the UNION of every suite that can land on it. Label
+    # resolution order: an explicit `suite_labels` override (legacy suites
+    # whose runs-on lives hardcoded in their dispatching workflow; may name
+    # several labels), else runner_config -> runner_configs.yml. Suites we
+    # cannot map land in `unmapped_suites` -- visible, never silently
+    # dropped, same contract as unresolved_files.
+    label_models: Dict[str, Set[str]] = {}
+    label_suites: Dict[str, List[str]] = {}
+    unmapped_suites: List[str] = []
+    for suite in sorted(suites):
+        labels = suite_labels_override.get(suite)
+        if labels is None:
+            rc = suite_runner_config.get(suite)
+            label = runner_labels_map.get(rc) if rc is not None else None
+            labels = [label] if label is not None else []
+        if not labels:
+            unmapped_suites.append(suite)
+            continue
+        for label in labels:
+            if label == B200_SENTINEL and b200_runner:
+                label = b200_runner
+            label_models.setdefault(label, set()).update(suites[suite]["models"])
+            label_suites.setdefault(label, []).append(suite)
+    runner_labels: Dict[str, object] = {
+        label: {
+            "models": sorted(label_models[label]),
+            "suites": label_suites[label],
+        }
+        for label in sorted(label_models)
+    }
+
+    parse_failures = {}
+    parse_failures.update(table_errors)
+    parse_failures.update(registry_errors)
+    parse_failures.update(extract_errors)
+
+    return {
+        "generated_at_commit": commit,
+        "backend": backend_name.lower(),
+        "suite_count": len(suites),
+        "model_count": len(all_models),
+        "runner_label_count": len(runner_labels),
+        "all_models": sorted(all_models),
+        "parse_failures": dict(sorted(parse_failures.items())),
+        "runner_labels": runner_labels,
+        "unmapped_suites": unmapped_suites,
+        "suites": suites,
+    }
+
+
+def render_markdown(inventory: Dict[str, object]) -> str:
+    suites: Dict[str, dict] = inventory["suites"]  # type: ignore[assignment]
+    failures = inventory.get("parse_failures") or {}
+    runner_labels: Dict[str, dict] = inventory.get("runner_labels") or {}  # type: ignore[assignment]
+    unmapped = inventory.get("unmapped_suites") or []
+    lines = [
+        f"## NVIDIA CI model inventory (`{inventory['backend']}`)",
+        "",
+        f"- Commit: `{inventory['generated_at_commit']}`",
+        f"- Suites: **{inventory['suite_count']}**, "
+        f"distinct models: **{inventory['model_count']}**, "
+        f"runner labels: **{inventory.get('runner_label_count', 0)}**",
+    ]
+    if failures:
+        lines.append(
+            f"- ⚠️ Unparsable files: **{len(failures)}** (see `parse_failures`)"
+        )
+    if unmapped:
+        lines.append(
+            f"- ⚠️ Suites with no runner label: **{len(unmapped)}** "
+            f"({', '.join(f'`{s}`' for s in unmapped)})"
+        )
+    if runner_labels:
+        lines += [
+            "",
+            "### Per runner label (prewarm a runner's cache with this union)",
+            "",
+            "| Runner label | Suites | Models |",
+            "| --- | ---: | --- |",
+        ]
+        for label in sorted(runner_labels):
+            info = runner_labels[label]
+            models = ", ".join(info["models"]) if info["models"] else "_(none)_"
+            lines.append(f"| `{label}` | {len(info['suites'])} | {models} |")
+    lines += [
+        "",
+        "### Per suite",
+        "",
+        "| Suite | Nightly | Models | Unresolved files |",
+        "| --- | :---: | --- | ---: |",
+    ]
+    for suite in sorted(suites):
+        info = suites[suite]
+        models = ", ".join(info["models"]) if info["models"] else "_(none)_"
+        nightly = "✓" if info["nightly"] else ""
+        lines.append(
+            f"| `{suite}` | {nightly} | {models} | {len(info['unresolved_files'])} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def resolve_commit(arg: Optional[str], repo_root: str) -> str:
+    if arg:
+        return arg
+    env = os.environ.get("GITHUB_SHA")
+    if env:
+        return env
+    try:
+        return subprocess.check_output(
+            ["git", "-C", repo_root, "rev-parse", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (OSError, subprocess.CalledProcessError):
+        return "unknown"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--repo-root",
+        default=".",
+        help="Repository root (default: current directory).",
+    )
+    parser.add_argument(
+        "--backend",
+        default="cuda",
+        help="Hardware backend name (default: cuda).",
+    )
+    parser.add_argument(
+        "--overrides",
+        default=os.path.join("scripts", "ci", "stage_models_overrides.json"),
+        help="Path to the overrides JSON (relative paths resolve under "
+        "--repo-root). A warning is printed if it does not exist.",
+    )
+    parser.add_argument("--commit", default=None, help="Commit sha to record.")
+    parser.add_argument(
+        "--include-disabled",
+        action="store_true",
+        help="Also include suites whose tests are currently disabled.",
+    )
+    parser.add_argument(
+        "--b200-runner",
+        default=None,
+        help="Concrete runner label to substitute for the $b200_runner "
+        "placeholder in runner_configs.yml (default: keep the placeholder "
+        "as the runner_labels key, marking the group as dynamically routed).",
+    )
+    parser.add_argument(
+        "--output", default=None, help="Write JSON here (default: stdout)."
+    )
+    parser.add_argument(
+        "--markdown", default=None, help="Also write a Markdown summary here."
+    )
+    args = parser.parse_args()
+
+    repo_root = os.path.abspath(args.repo_root)
+    # Resolve a relative overrides path against repo_root (not cwd) so the flag
+    # works when invoked from elsewhere with --repo-root.
+    overrides_path = args.overrides
+    if not os.path.isabs(overrides_path):
+        overrides_path = os.path.join(repo_root, overrides_path)
+    if not os.path.exists(overrides_path):
+        print(
+            f"WARNING: overrides file not found at {overrides_path}; "
+            f"proceeding without overrides.",
+            file=sys.stderr,
+        )
+        overrides_path = None
+    overrides = load_overrides(overrides_path)
+    commit = resolve_commit(args.commit, repo_root)
+
+    inventory = build_inventory(
+        repo_root,
+        args.backend,
+        overrides,
+        commit,
+        args.include_disabled,
+        b200_runner=args.b200_runner,
+    )
+    payload = json.dumps(inventory, indent=2, sort_keys=False) + "\n"
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(payload)
+        print(
+            f"Wrote {args.output}: {inventory['suite_count']} suites, "
+            f"{inventory['runner_label_count']} runner labels, "
+            f"{inventory['model_count']} distinct models, "
+            f"{len(inventory['parse_failures'])} parse failures, "
+            f"{len(inventory['unmapped_suites'])} unmapped suites.",
+            file=sys.stderr,
+        )
+    else:
+        sys.stdout.write(payload)
+
+    if args.markdown:
+        with open(args.markdown, "w", encoding="utf-8") as f:
+            f.write(render_markdown(inventory))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/musa/musa_install_dependency.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -euo pipefail
+
+# Parse command line arguments
+OPTIONAL_DEPS=""
+SKIP_SGLANG_BUILD=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --skip-sglang-build) SKIP_SGLANG_BUILD="1"; shift;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS] [OPTIONAL_DEPS]"
+      echo "Options:"
+      echo "  --skip-sglang-build         Don't build checkout sglang, use what was shipped with the image"
+      exit 0
+      ;;
+    *)
+      OPTIONAL_DEPS="$1"
+      shift
+      ;;
+  esac
+done
+
+PIP_INSTALL="python3 -m pip install --no-cache-dir"
+${PIP_INSTALL} --upgrade pip setuptools torchada --user
+
+echo "Checking stale torchada extension locks..."
+active_torchada_builds="$(
+    pgrep -af '(^|[[:space:]/])(mcc|ninja)([[:space:]]|$)|torchada_cpp_ops' 2>/dev/null \
+        | awk -v self="$$" '$1 != self'
+)" || true
+if [ -n "$active_torchada_builds" ]; then
+    echo "$active_torchada_builds"
+    echo "::error::Active torchada extension build detected; refusing to remove lock files"
+    exit 1
+fi
+torch_extensions_dir="${HOME}/.cache/torch_extensions"
+if [ -d "$torch_extensions_dir" ]; then
+    find "$torch_extensions_dir" \
+        -path '*/torchada_cpp_ops/lock' \
+        -type f \
+        -print \
+        -delete
+fi
+
+WHL_DIR="/sglang-checkout/whl"
+if [ -d "$WHL_DIR" ] && compgen -G "${WHL_DIR}"/*.whl > /dev/null; then
+    echo "Uninstall old packages based on wheel METADATA..."
+    PKGS=$(
+      for whl in "${WHL_DIR}"/*.whl; do
+        meta_file=$(zipinfo -1 "$whl" | awk '/\.dist-info\/METADATA$/ {print; exit}')
+        [ -n "$meta_file" ] || continue
+        unzip -p "$whl" "$meta_file" 2>/dev/null | sed -n 's/^Name: //p' | head -n1
+      done | sort -u
+    )
+    for pkg in $PKGS; do
+      echo "Uninstalling $pkg"
+      pip uninstall -y "$pkg" || true
+    done
+    echo "Installing wheel files without dependency resolution..."
+    ${PIP_INSTALL} "${WHL_DIR}"/*.whl --user
+fi
+
+if [ -n "$SKIP_SGLANG_BUILD" ]; then
+    echo "Didn't build checkout SGLang"
+    exit 0
+else
+    pip uninstall sgl-kernel -y || true
+    pip uninstall sglang -y || true
+    # Clear Python cache to ensure latest code is used (works for any env: venv, system, conda)
+    REPO_ROOT="${GITHUB_WORKSPACE:-$(pwd)}"
+    find "$REPO_ROOT" -name "*.pyc" -delete 2>/dev/null || true
+    find "$REPO_ROOT" -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true
+
+    rm -f "${REPO_ROOT}/python/pyproject.toml" && mv "${REPO_ROOT}/python/pyproject_other.toml" "${REPO_ROOT}/python/pyproject.toml"
+    cd "${REPO_ROOT}" && ${PIP_INSTALL} -v -e "python[dev_musa]" --user
+
+    cd "${REPO_ROOT}/sgl-kernel"
+    rm -f pyproject.toml && mv pyproject_musa.toml pyproject.toml && MTGPU_TARGET=mp_31 python3 setup_musa.py install --user
+    echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+fi

scripts/ci/musa/rename_wheels_musa.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# Align MUSA wheel filenames (+musa43/...) with internal METADATA Version and
+# WHEEL tags after build. Two drifts need fixing in lockstep:
+#   - METADATA `Version:` must carry the `+musa<suffix>` local version, or
+#     recent pip versions reject the wheel with "inconsistent version".
+#   - WHEEL `Tag:` must be `manylinux2014_*` when the filename says so;
+#     leaving it as `linux_*` can trip installers that re-derive the platform.
+# Unpack → patch WHEEL/METADATA → wheel pack (RECORD regenerated; no hand-editing).
+#
+# Usage:
+#   rename_wheels_musa.sh <musa_suffix> [wheel_dir]
+# Example:
+#   rename_wheels_musa.sh 43 sgl-kernel/dist
+set -euxo pipefail
+
+if [[ $# -lt 1 || $# -gt 2 ]]; then
+  echo "Usage: $0 <musa_suffix> [wheel_dir]" >&2
+  exit 1
+fi
+
+MUSA_SUFFIX="+musa$1"
+WHEEL_DIR="${2:-dist}"
+
+patch_wheel_platform_tags() {
+    local wheel_file="$1"
+    # Line-end anchors: "linux_x86_64" is a substring of "manylinux2014_x86_64", so
+    # unanchored global replace corrupts tags on a second run.
+    sed -i \
+        -e 's/-linux_x86_64$/-manylinux2014_x86_64/' \
+        -e 's/-linux_aarch64$/-manylinux2014_aarch64/' \
+        "$wheel_file"
+}
+
+wheel_files=("$WHEEL_DIR"/*.whl)
+if [[ ! -e "${wheel_files[0]}" ]]; then
+  echo "No wheel files found in ${WHEEL_DIR}/, nothing to rename."
+  exit 0
+fi
+
+for wheel in "${wheel_files[@]}"; do
+    [[ -f "$wheel" ]] || continue
+
+    intermediate_wheel="$wheel"
+    case "$wheel" in
+        *-linux_x86_64.whl)
+            intermediate_wheel="${wheel%-linux_x86_64.whl}-manylinux2014_x86_64.whl"
+            ;;
+        *-linux_aarch64.whl)
+            intermediate_wheel="${wheel%-linux_aarch64.whl}-manylinux2014_aarch64.whl"
+            ;;
+    esac
+    if [[ "$wheel" != "$intermediate_wheel" ]]; then
+        mv -- "$wheel" "$intermediate_wheel"
+        wheel="$intermediate_wheel"
+    fi
+
+    TMPDIR=$(mktemp -d)
+    trap 'rm -rf -- "$TMPDIR"' ERR
+
+    "${PYTHON:-python3}" -m wheel unpack "$wheel" --dest "$TMPDIR"
+    # `find | head -1` succeeds with empty stdout when there are no matches —
+    # `set -e` won't catch that. Assert each path is real so a malformed wheel
+    # surfaces with a useful message instead of a downstream `sed: /WHEEL` error.
+    UNPACKED=$(find "$TMPDIR" -mindepth 1 -maxdepth 1 -type d | head -1)
+    [[ -d "$UNPACKED" ]] || { echo "ERROR: wheel unpack produced no top-level dir for $wheel" >&2; exit 1; }
+    DIST_INFO=$(find "$UNPACKED" -maxdepth 1 -type d -name "*.dist-info" | head -1)
+    [[ -d "$DIST_INFO" ]] || { echo "ERROR: no *.dist-info under $UNPACKED (malformed wheel?): $wheel" >&2; exit 1; }
+    WHEEL_META="${DIST_INFO}/WHEEL"
+    METADATA_FILE="${DIST_INFO}/METADATA"
+    [[ -f "$WHEEL_META" && -f "$METADATA_FILE" ]] || { echo "ERROR: missing WHEEL or METADATA in $DIST_INFO" >&2; exit 1; }
+
+    patch_wheel_platform_tags "$WHEEL_META"
+
+    ORIG_VERSION=$(grep '^Version:' "$METADATA_FILE" | head -1 | sed 's/^Version:[[:space:]]*//')
+    # Empty ORIG_VERSION would fall through the `+musa` check below and silently
+    # produce `Version: +musa43` — a broken release. Fail loud instead.
+    [[ -n "$ORIG_VERSION" ]] || { echo "ERROR: no 'Version:' line in $METADATA_FILE" >&2; exit 1; }
+    if [[ "$ORIG_VERSION" == *"$MUSA_SUFFIX"* ]]; then
+        echo "Skipping $wheel: version in METADATA is already suffixed."
+        rm -rf "$TMPDIR"
+        trap - ERR
+        continue
+    fi
+    NEW_VERSION="${ORIG_VERSION}${MUSA_SUFFIX}"
+    sed -i "s/^Version:.*/Version: ${NEW_VERSION}/" "$METADATA_FILE"
+    # `sed -i` exits 0 even when the pattern matched zero lines. Verify the
+    # rewrite actually landed before we publish.
+    grep -qx "Version: ${NEW_VERSION}" "$METADATA_FILE" || { echo "ERROR: METADATA Version rewrite did not land in $METADATA_FILE" >&2; exit 1; }
+
+    OLD_BASE=$(basename "$DIST_INFO")
+    NEW_BASE="${OLD_BASE/${ORIG_VERSION}/${NEW_VERSION}}"
+    # `${var/pat/repl}` silently leaves var unchanged if pat is empty or absent.
+    [[ "$NEW_BASE" != "$OLD_BASE" ]] || { echo "ERROR: dist-info dir '$OLD_BASE' did not contain ORIG_VERSION='$ORIG_VERSION'" >&2; exit 1; }
+    mv "$DIST_INFO" "${UNPACKED}/${NEW_BASE}"
+
+    rm -f "$wheel"
+    "${PYTHON:-python3}" -m wheel pack "$UNPACKED" --dest-dir "$WHEEL_DIR"
+    rm -rf "$TMPDIR"
+    trap - ERR
+done
+
+echo "MUSA wheel renaming completed."

scripts/ci/npu/npu_ci_install_dependency.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+set -euo pipefail
+
+PIP_INSTALL="python3 -m pip install --no-cache-dir"
+UV_PIP_INSTALL="uv pip install "
+DEVICE_TYPE=$1
+
+
+# Install the required dependencies in CI.
+apt update -y && apt install -y \
+    unzip \
+    build-essential \
+    cmake \
+    wget \
+    curl \
+    net-tools \
+    zlib1g-dev \
+    lld \
+    clang \
+    locales \
+    ccache \
+    libgl1-mesa-glx \
+    libgl1-mesa-dri \
+    ca-certificates \
+    libgl1 \
+    libglib2.0-0
+update-ca-certificates
+${PIP_INSTALL} --upgrade pip
+${PIP_INSTALL} uv
+export UV_NO_CACHE=true
+export UV_SYSTEM_PYTHON=true
+export UV_INDEX_STRATEGY=unsafe-best-match
+
+# Install Rust toolchain (needed by crates built via setuptools-rust, e.g. the
+# native gRPC extension bundled into the sglang wheel).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+bash "${SCRIPT_DIR}/../utils/install_rustup.sh"
+export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}"
+
+${UV_PIP_INSTALL} pybind11 pyyaml decorator scipy attrs psutil
+
+
+### Install MemFabric
+${UV_PIP_INSTALL} memfabric-hybrid==1.0.8
+
+
+### Install PyTorch and PTA
+PYTORCH_VERSION="2.10.0"
+TORCHVISION_VERSION="0.25.0"
+TORCHAUDIO_VERSION="2.10.0"
+${UV_PIP_INSTALL} torch==${PYTORCH_VERSION} torchvision==${TORCHVISION_VERSION} torchaudio==${TORCHAUDIO_VERSION} --index-url ${TORCH_CACHE_URL:="https://download.pytorch.org/whl/cpu"} --extra-index-url ${PYPI_CACHE_URL:="https://pypi.org/simple/"}
+PTA_URL="https://gitcode.com/Ascend/pytorch/releases/download/v26.0.0-pytorch2.10.0/torch_npu-2.10.0-cp311-cp311-manylinux_2_28_aarch64.whl"
+# GitCode does not allow UV downloads.
+${PIP_INSTALL} ${PTA_URL}
+
+### Install zbal
+${UV_PIP_INSTALL} memfabric-zbal==1.1.1
+
+### Install Triton-Ascend
+${PIP_INSTALL} triton-ascend==3.2.1.dev20260530 --extra-index-url=https://mirrors.huaweicloud.com/ascend/repos/pypi/nightly --trusted-host triton-ascend.osinfra.cn
+
+
+### Install sgl-kernel-npu
+SGLANG_KERNEL_NPU_TAG="2026.6.2"
+mkdir sgl-kernel-npu
+(cd sgl-kernel-npu && wget "${GITHUB_PROXY_URL:=""}https://github.com/sgl-project/sgl-kernel-npu/releases/download/${SGLANG_KERNEL_NPU_TAG}/sgl-kernel-npu-${SGLANG_KERNEL_NPU_TAG}-torch${PYTORCH_VERSION}-py311-cann9.0.0-${DEVICE_TYPE}-$(arch).zip" \
+&& unzip ./sgl-kernel-npu-${SGLANG_KERNEL_NPU_TAG}-torch${PYTORCH_VERSION}-py311-cann9.0.0-${DEVICE_TYPE}-$(arch).zip \
+&& ${UV_PIP_INSTALL} ./deep_ep*.whl ./sgl_kernel_npu*.whl \
+&& (cd "$(python3 -m pip show deep-ep | grep -E '^Location:' | awk '{print $2}')" && ln -s deep_ep/deep_ep_cpp*.so))
+
+
+### Install SGLang
+rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml
+${UV_PIP_INSTALL} -v -e "python[dev_npu]"

scripts/ci/npu/npu_log_print.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+
+# Print log information(sglang version, commit sha, sgl-kernel-npu version, sgl-kernel-npu commit sha, npu-smi info and pip list.
+npu-smi info
+pip list
+get_version() {
+    [ -f "$1" ] && python3 -c 'import re, sys; print(sys.argv[2] + " version: v" + re.search(r"__version__\s*=\s*[\"'"'"'](.*?)[\"'"'"']", open(sys.argv[1]).read()).group(1))' "$1" "$2" 2>/dev/null || echo "$2 version: unknown"
+}
+get_version "./python/sglang/version.py" "sglang"
+get_version "./sgl-kernel/python/sgl_kernel/version.py" "sgl_kernel"
+SGLANG_URL="https://github.com/sgl-project/sglang.git"
+SGL_KERNEL_URL="https://github.com/sgl-project/sgl-kernel-npu.git"
+SGLANG_BRANCH="main"
+SGL_KERNEL_BRANCH="main"
+get_sha() {
+    local name="$1"
+    local url="$2"
+    local branch="$3"
+    local sha
+    sha=$(git ls-remote "$url" "refs/heads/$branch" | cut -f1)
+    echo "$name SHA for branch $branch: ${sha:-"Not Found"}"
+}
+get_sha "sglang" "$SGLANG_URL" "$SGLANG_BRANCH"
+get_sha "sgl-kernel" "$SGL_KERNEL_URL" "$SGL_KERNEL_BRANCH"
+chmod +x scripts/ci/npu/npu_log_print.sh

scripts/ci/runner_configs.py

@@ -0,0 +1,60 @@
+"""Emit runner_config setup for GitHub Actions $GITHUB_OUTPUT.
+
+runner_configs.py <runner_config>
+    Per-field `key=value` lines (install / artifact_version /
+    install_timeout / rdma_devices). `runs_on` is intentionally omitted —
+    it carries the `$b200_runner` sentinel and is resolved via --map.
+    Called per stage by _pr-test-stage.yml.
+
+runner_configs.py --map <b200_runner_label>
+    `runs_on_map={json}` — flat dict {runner_config: runs_on}, with
+    `$b200_runner` substituted. Called once by _pr-test-check-changes.yml.
+"""
+
+import json
+import os
+import sys
+
+import yaml
+
+_YAML_PATH = os.path.join(os.path.dirname(__file__), "runner_configs.yml")
+_B200_SENTINEL = "$b200_runner"
+
+
+def load() -> dict:
+    with open(_YAML_PATH) as f:
+        return yaml.safe_load(f)["runner_configs"]
+
+
+def _emit_single(rc: str) -> None:
+    # runs_on goes through --map (resolves $b200_runner). Suppress it here so a
+    # consumer can't accidentally read the raw sentinel value.
+    cfg = load().get(rc)
+    if cfg is None:
+        sys.exit(f"unknown runner_config: {rc!r}")
+    for key, value in cfg.items():
+        if key == "runs_on":
+            continue
+        print(f"{key}={value}")
+
+
+def _emit_map(b200_runner: str) -> None:
+    runs_on = {
+        name: (b200_runner if cfg.get("runs_on") == _B200_SENTINEL else cfg["runs_on"])
+        for name, cfg in load().items()
+    }
+    print(f"runs_on_map={json.dumps(runs_on, separators=(',', ':'))}")
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    if len(args) == 1:
+        _emit_single(args[0])
+    elif len(args) == 2 and args[0] == "--map":
+        _emit_map(args[1])
+    else:
+        sys.exit(
+            "usage:\n"
+            "  runner_configs.py <runner_config>\n"
+            "  runner_configs.py --map <b200_runner_label>"
+        )

scripts/ci/runner_configs.yml

@@ -0,0 +1,35 @@
+# Per-runner-config CUDA setup details. Single source of truth for the
+# `runner_config` field on `register_cuda_ci(...)` calls. Consumed by
+# scripts/ci/runner_configs.py (CLI wrapper), which is in turn called by
+# .github/workflows/_pr-test-stage.yml.
+#
+# Each runner_config carries:
+#   - install: install script path
+#   - artifact_version: actions/download-artifact major version
+#   - install_timeout: install-step wall-clock cap (minutes), enforced via
+#     `timeout-minutes:` on the install step in _pr-test-stage.yml
+#   - grace_blackwell (optional): exported as GRACE_BLACKWELL for the install
+#     step. Used by GB300 DeePEP setup.
+#   - runs_on: GHA runner label for the stage's `runs-on:`. The literal
+#     `$b200_runner` is substituted at workflow-load time with the dynamic
+#     b200 runner tag from check-changes (see runner_configs.py --map).
+#   - rdma_devices (optional): exported as SGLANG_CI_RDMA_ALL_DEVICES env
+#     to the stage job; absent means unset.
+
+_anchors:
+  default_install: &default scripts/ci/cuda/ci_install_dependency.sh
+  deepep_install:  &deepep  scripts/ci/cuda/ci_install_deepep.sh
+
+runner_configs:
+  1-gpu-small:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-5090 }
+  1-gpu-large:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-h100 }
+  2-gpu-large:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 2-gpu-h100 }
+  4-gpu-b200:        { install: *default, artifact_version: v6, install_timeout: "20", runs_on: $b200_runner }
+  4-gpu-gb300:       { install: *deepep,  artifact_version: v6, install_timeout: "20", grace_blackwell: "1", runs_on: 4-gpu-gb300 }
+  4-gpu-h100:        { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 }
+  8-gpu-h200:        { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 }
+  8-gpu-b200:        { install: *default, artifact_version: v6, install_timeout: "20", runs_on: 8-gpu-b200 }
+  8-gpu-h20:         { install: *deepep,  artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h20, rdma_devices: "mlx5_1,mlx5_2,mlx5_3,mlx5_4" }
+  deepep-4-gpu-h100: { install: *deepep,  artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 }
+  deepep-4-gpu-b200: { install: *deepep,  artifact_version: v6, install_timeout: "20", runs_on: $b200_runner }
+  deepep-8-gpu-h200: { install: *deepep,  artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 }

scripts/ci/slurm/analyze_logs_with_modal.py

@@ -0,0 +1,376 @@
+#!/usr/bin/env python3
+"""Analyze srtslurm logs with opencode inside a Modal sandbox.
+
+This script accepts either:
+- a local log directory
+- a `.tar.gz` bundle such as `multinode_server_logs.tar.gz`
+
+It uploads the logs into an ephemeral Modal sandbox, installs and runs
+opencode with an analysis prompt, and prints the resulting markdown.
+
+Example:
+  uv run --with modal python scripts/ci/slurm/analyze_logs_with_modal.py \
+    --tarball /tmp/multinode_server_logs.tar.gz \
+    --job-id 4645
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import re
+import shlex
+import shutil
+import tarfile
+import tempfile
+from pathlib import Path
+
+try:
+    import modal
+except ImportError:  # pragma: no cover - runtime guard for local usage
+    modal = None
+
+
+logger = logging.getLogger("slurm_log_analysis")
+
+SANDBOX_TIMEOUT = 600
+DEFAULT_MODAL_SECRET_NAME = "or"
+DEFAULT_MODEL = "openrouter/minimax/minimax-m2.7"
+DEFAULT_REPOS = [
+    "https://github.com/sgl-project/sglang.git",
+]
+PROMPT_PATH = Path(__file__).with_name("log_analysis_prompt.md")
+
+# Matches common API key / token patterns (sk-..., ak-..., as-..., key-..., etc.)
+_SECRET_PATTERN = re.compile(
+    r"""(?:"""
+    r"""(?:sk|ak|as|key|token|secret|bearer)[-_][A-Za-z0-9_\-]{16,}"""
+    r"""|"""
+    r"""(?:OPENROUTER_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|ANTHROPIC_API_KEY)"""
+    r"""[=:]\s*\S+"""
+    r""")""",
+    re.IGNORECASE,
+)
+
+
+def sanitize(text: str) -> str:
+    """Redact strings that look like API keys or secrets."""
+    if not text:
+        return text
+    sanitized = _SECRET_PATTERN.sub("[REDACTED]", text)
+    # Also redact any env var values we know are secrets
+    for var in ("OPENROUTER_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"):
+        val = os.environ.get(var)
+        if val and len(val) > 8:
+            sanitized = sanitized.replace(val, "[REDACTED]")
+    return sanitized
+
+
+def configure_logging(verbose: bool) -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(levelname)s: %(message)s",
+    )
+    logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+
+
+def extract_tarball(tarball: Path, destination: Path) -> None:
+    with tarfile.open(tarball, "r:gz") as archive:
+        # Python 3.14 changes the default extraction behavior. Use the
+        # data filter when available so extraction remains explicit.
+        if "data" in tarfile._NAMED_FILTERS:  # type: ignore[attr-defined]
+            archive.extractall(destination, filter="data")
+        else:
+            archive.extractall(destination)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Analyze a srtslurm log bundle with opencode in Modal."
+    )
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument(
+        "--tarball",
+        type=Path,
+        help="Path to a local multinode_server_logs.tar.gz bundle.",
+    )
+    source.add_argument(
+        "--log-dir",
+        type=Path,
+        help="Path to an unpacked log directory.",
+    )
+    parser.add_argument(
+        "--job-id",
+        default="unknown",
+        help="Job identifier used in the report header and logs.",
+    )
+    parser.add_argument(
+        "--model",
+        default=DEFAULT_MODEL,
+        help="Model selector to pass to opencode run.",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="Optional path to write the markdown analysis.",
+    )
+    parser.add_argument(
+        "--repo-url",
+        action="append",
+        dest="repo_urls",
+        help=(
+            "Optional extra repo URL to clone into the sandbox for context. "
+            "Can be specified multiple times."
+        ),
+    )
+    parser.add_argument(
+        "--timeout-seconds",
+        type=int,
+        default=SANDBOX_TIMEOUT,
+        help="Sandbox lifetime in seconds.",
+    )
+    parser.add_argument(
+        "--modal-secret-name",
+        default=DEFAULT_MODAL_SECRET_NAME,
+        help="Modal secret name that provides OPENROUTER_API_KEY to the sandbox.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable debug logging.",
+    )
+    return parser.parse_args()
+
+
+def build_sandbox_image() -> modal.Image:
+    if modal is None:
+        raise RuntimeError(
+            "The 'modal' package is required. Run this script with "
+            "`uv run --with modal python ...` or install modal locally."
+        )
+
+    return (
+        modal.Image.debian_slim(python_version="3.12")
+        .apt_install("bash", "curl", "git", "gh")
+        .run_commands(
+            "curl -fsSL https://opencode.ai/install | bash",
+        )
+        .env(
+            {
+                "PATH": "/root/.opencode/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            }
+        )
+    )
+
+
+def prepare_log_dir(args: argparse.Namespace) -> tuple[Path, Path | None]:
+    if args.log_dir:
+        if not args.log_dir.is_dir():
+            raise FileNotFoundError(f"log directory not found: {args.log_dir}")
+        return args.log_dir.resolve(), None
+
+    assert args.tarball is not None
+    if not args.tarball.is_file():
+        raise FileNotFoundError(f"tarball not found: {args.tarball}")
+
+    temp_dir = Path(tempfile.mkdtemp(prefix="sglang_logs_"))
+    extract_tarball(args.tarball, temp_dir)
+    return temp_dir.resolve(), temp_dir
+
+
+def build_prompt(job_id: str, repo_urls: list[str]) -> str:
+    skill_content = PROMPT_PATH.read_text()
+    repo_lines = []
+    for repo_url in repo_urls:
+        repo_name = repo_url.rsplit("/", 1)[-1].removesuffix(".git")
+        repo_lines.append(f"- **{repo_name} repo**: `/workspace/repos/{repo_name}/`")
+    repo_section = (
+        "\n".join(repo_lines) if repo_lines else "- No extra repos were requested."
+    )
+
+    return f"""{skill_content}
+
+---
+
+## Your Environment
+
+- **Logs**: `/workspace/logs/`
+- **GitHub CLI**: `gh` is installed and authenticated.
+{repo_section}
+
+## Job
+
+You are analyzing job `{job_id}`. Follow Steps 1–5 in the prompt above.
+
+**You MUST write the final markdown report to `/workspace/logs/ai_analysis.md`.**
+This is a hard requirement. Do not just print the report to stdout. Use your
+file-writing tool to create `/workspace/logs/ai_analysis.md` with the full
+analysis. The downstream pipeline reads this file.
+
+**You MUST file GitHub issues when the root cause is clear (Category A or B).**
+Do not skip issue filing. The whole point of this system is automated triage.
+"""
+
+
+def upload_tree(sandbox: modal.Sandbox, log_dir: Path) -> None:
+    log_files = [path for path in log_dir.rglob("*") if path.is_file()]
+    logger.info("Uploading %d log files into the sandbox", len(log_files))
+    for index, log_file in enumerate(log_files, start=1):
+        rel_path = log_file.relative_to(log_dir)
+        remote_path = str(Path("/workspace/logs") / rel_path)
+        sandbox.mkdir(str(Path(remote_path).parent), parents=True)
+        sandbox.filesystem.write_bytes(log_file.read_bytes(), remote_path)
+        if index % 10 == 0 or index == len(log_files):
+            logger.info("Uploaded %d/%d files", index, len(log_files))
+
+
+def clone_context_repos(sandbox: modal.Sandbox, repo_urls: list[str]) -> None:
+    if not repo_urls:
+        return
+
+    sandbox.mkdir("/workspace/repos", parents=True)
+
+    for repo_url in repo_urls:
+        repo_name = repo_url.rsplit("/", 1)[-1].removesuffix(".git")
+        logger.info("Cloning %s into the sandbox", repo_name)
+        sandbox.exec(
+            "git",
+            "clone",
+            "--depth",
+            "100",
+            repo_url,
+            f"/workspace/repos/{repo_name}",
+        ).wait()
+
+
+def read_optional_file(sandbox: modal.Sandbox, path: str) -> str | None:
+    try:
+        return sandbox.filesystem.read_text(path)
+    except Exception:
+        return None
+
+
+def run_opencode_analysis(
+    *,
+    log_dir: Path,
+    job_id: str,
+    model: str,
+    repo_urls: list[str],
+    timeout_seconds: int,
+    modal_secret_name: str,
+) -> str:
+    prompt = build_prompt(job_id, repo_urls)
+    app = modal.App.lookup("sglang-log-analyzer", create_if_missing=True)
+    sandbox = modal.Sandbox.create(
+        app=app,
+        image=build_sandbox_image(),
+        timeout=timeout_seconds,
+        secrets=[modal.Secret.from_name(modal_secret_name)],
+    )
+    logger.info("Created Modal sandbox %s", sandbox.object_id)
+
+    try:
+        sandbox.mkdir("/workspace/logs", parents=True)
+        sandbox.mkdir("/workspace/repos", parents=True)
+
+        clone_context_repos(sandbox, repo_urls)
+        upload_tree(sandbox, log_dir)
+
+        sandbox.filesystem.write_text(prompt, "/workspace/prompt.txt")
+
+        runner_script = f"""#!/bin/bash
+set -uo pipefail
+cd /workspace
+if opencode run \\
+  --dangerously-skip-permissions \\
+  --dir /workspace/logs \\
+  -m {shlex.quote(model)} \\
+  "$(cat /workspace/prompt.txt)" \\
+  < /dev/null \\
+  > /workspace/logs/opencode.stdout \\
+  2> /workspace/logs/opencode.stderr; then
+  echo 0 > /workspace/logs/opencode.exitcode
+else
+  echo $? > /workspace/logs/opencode.exitcode
+fi
+ls -la /workspace/logs > /workspace/logs/log_dir_listing.txt
+"""
+        sandbox.filesystem.write_text(runner_script, "/workspace/run_opencode.sh")
+        sandbox.exec("chmod", "+x", "/workspace/run_opencode.sh").wait()
+
+        logger.info("Running opencode analysis")
+        process = sandbox.exec(
+            "bash",
+            "/workspace/run_opencode.sh",
+        )
+        process.wait()
+
+        stderr = process.stderr.read()
+        if stderr:
+            logger.warning("runner stderr: %s", sanitize(stderr[:500]))
+
+        exitcode = read_optional_file(sandbox, "/workspace/logs/opencode.exitcode")
+        opencode_stdout = (
+            read_optional_file(sandbox, "/workspace/logs/opencode.stdout") or ""
+        )
+        opencode_stderr = read_optional_file(sandbox, "/workspace/logs/opencode.stderr")
+        log_dir_listing = read_optional_file(
+            sandbox, "/workspace/logs/log_dir_listing.txt"
+        )
+        try:
+            ai_analysis = read_optional_file(sandbox, "/workspace/logs/ai_analysis.md")
+            if ai_analysis and ai_analysis.strip():
+                return sanitize(ai_analysis)
+
+            if opencode_stdout.strip():
+                return sanitize(opencode_stdout)
+
+            raise RuntimeError("opencode completed without producing analysis output")
+        except Exception as exc:
+            stdout = process.stdout.read()
+            if stdout:
+                return sanitize(stdout)
+            details = [
+                f"opencode analysis did not produce a usable report: {exc}",
+                f"exitcode={exitcode!r}",
+                f"stdout_preview={sanitize(opencode_stdout[:500])!r}",
+                f"stderr_preview={sanitize((opencode_stderr or '')[:500])!r}",
+                f"log_dir_listing={sanitize((log_dir_listing or '')[:500])!r}",
+            ]
+            raise RuntimeError(" ".join(details)) from exc
+    finally:
+        sandbox.terminate()
+
+
+def main() -> int:
+    args = parse_args()
+    configure_logging(args.verbose)
+
+    repo_urls = list(DEFAULT_REPOS)
+    if args.repo_urls:
+        repo_urls.extend(args.repo_urls)
+
+    log_dir, cleanup_dir = prepare_log_dir(args)
+    try:
+        analysis = run_opencode_analysis(
+            log_dir=log_dir,
+            job_id=args.job_id,
+            model=args.model,
+            repo_urls=repo_urls,
+            timeout_seconds=args.timeout_seconds,
+            modal_secret_name=args.modal_secret_name,
+        )
+    finally:
+        if cleanup_dir is not None:
+            shutil.rmtree(cleanup_dir, ignore_errors=True)
+
+    print(analysis)
+    if args.output:
+        args.output.write_text(analysis)
+        logger.info("Wrote analysis to %s", args.output)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/ci/slurm/generate_matrix.py

@@ -0,0 +1,98 @@
+"""
+Reads nightly-configs.yaml and generates one matrix entry per recipe YAML,
+where each srt-slurm recipe runs its full concurrency sweep as a single Slurm job.
+
+conc-list in the config is documentation only and is not used to split jobs.
+
+Output: JSON array written to stdout, consumed by the workflow setup job as
+a dynamic matrix via fromJson(needs.setup.outputs.matrix).
+
+Usage:
+    python3 generate_matrix.py <path-to-nightly-configs.yaml> --runner <label> [--filter NAMES]
+
+Example:
+    python3 generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200
+    python3 generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200 \\
+        --filter dsr1-fp8-1k1k-max-tpt,dsr1-fp4-1k1k-mid-curve
+"""
+
+import argparse
+import json
+import sys
+
+import yaml
+
+
+def seq_len_str(isl, osl):
+    def fmt(n):
+        return f"{n // 1024}k" if n % 1024 == 0 else str(n)
+
+    return f"{fmt(isl)}{fmt(osl)}"
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("config_file", help="Path to nightly-configs.yaml")
+    parser.add_argument(
+        "--runner",
+        required=True,
+        help="Filter configs by runner label (e.g. gb200, b200)",
+    )
+    parser.add_argument(
+        "--filter",
+        default="",
+        help=(
+            "Optional comma-separated list of matrix entry names to include "
+            "(e.g. 'dsr1-fp8-1k1k-max-tpt'). Names must match exactly."
+        ),
+    )
+    args = parser.parse_args()
+
+    with open(args.config_file) as f:
+        data = yaml.safe_load(f)
+
+    matrix = []
+    for exp_name, exp in data.items():
+        if exp["runner"] != args.runner:
+            continue
+
+        for seq_cfg in exp["seq-len-configs"]:
+            isl, osl = seq_cfg["isl"], seq_cfg["osl"]
+            sl = seq_len_str(isl, osl)
+
+            for entry in seq_cfg["search-space"]:
+                config_file = entry["config_file"]
+                topology = config_file.rsplit("/", 1)[-1].replace(".yaml", "")
+
+                matrix.append(
+                    {
+                        "name": f"{exp['model-prefix']}-{exp['precision']}-{sl}-{topology}",
+                        "exp_name": exp_name,
+                        "model": exp["model"],
+                        "model_prefix": exp["model-prefix"],
+                        "model_path": exp.get("model_path", ""),
+                        "precision": exp["precision"],
+                        "isl": str(isl),
+                        "osl": str(osl),
+                        "config_file": config_file,
+                    }
+                )
+
+    wanted = [n.strip() for n in args.filter.split(",") if n.strip()]
+    if wanted:
+        available = [e["name"] for e in matrix]
+        unknown = [n for n in wanted if n not in available]
+        if unknown:
+            print(
+                f"ERROR: unknown config name(s): {', '.join(unknown)}. "
+                f"Available for runner '{args.runner}': {', '.join(available)}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        matrix = [e for e in matrix if e["name"] in wanted]
+
+    print(json.dumps(matrix))
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/slurm/launch_gb200.sh

@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+# Launch a dynamo-sglang benchmark job on the GB200 cluster via srt-slurm.
+#
+# Required environment variables (set by the GitHub Actions workflow):
+#   FRAMEWORK         - must be "dynamo-sglang"
+#   MODEL             - HuggingFace model ID (used as fallback if no local path)
+#   MODEL_PREFIX      - short prefix: "dsr1"
+#   PRECISION         - "fp8" or "fp4"
+#   ISL               - input sequence length (e.g. "1024")
+#   OSL               - output sequence length (e.g. "1024")
+#   CONFIG_FILE       - path relative to srt-slurm repo root (e.g. recipes/gb200-fp8/1k1k/low-latency.yaml)
+#   RESULT_FILENAME   - prefix for output JSON filenames
+#   RUNNER_NAME       - GitHub Actions runner name (used to tag the Slurm job)
+#   SQUASH_FILE       - path to pre-imported sglang enroot squash file on Lustre
+#   NGINX_SQUASH_FILE - path to pre-imported nginx enroot squash file on Lustre
+#   SLURM_PARTITION   - Slurm partition (default: batch)
+#   SLURM_ACCOUNT     - Slurm account  (default: sglang)
+#   SRT_SLURM_BRANCH  - branch of srt-slurm repo to check out
+#   GITHUB_WORKSPACE  - set automatically by GitHub Actions
+#   MATRIX_CONFIG_NAME- matrix entry name (e.g. dsr1-fp4-1k1k-mid-curve); used in S3 prefix
+#   S3_BUCKET         - MinIO bucket for benchmark log uploads
+#   S3_ENDPOINT_URL   - MinIO endpoint URL (e.g. https://minio.<host>.nip.io)
+#   AWS_ACCESS_KEY_ID - writer access key for S3_BUCKET (via GH secrets)
+#   AWS_SECRET_ACCESS_KEY - writer secret key for S3_BUCKET (via GH secrets)
+
+set -euo pipefail
+set -x
+
+# ---------------------------------------------------------------------------
+# Validate required vars
+# ---------------------------------------------------------------------------
+: "${FRAMEWORK:?}"
+: "${MODEL_PREFIX:?}"
+: "${PRECISION:?}"
+: "${ISL:?}"
+: "${OSL:?}"
+: "${CONFIG_FILE:?}"
+: "${RESULT_FILENAME:?}"
+: "${RUNNER_NAME:?}"
+: "${SQUASH_FILE:?}"
+: "${NGINX_SQUASH_FILE:?}"
+: "${GITHUB_WORKSPACE:?}"
+
+SLURM_PARTITION="${SLURM_PARTITION:-batch}"
+SLURM_ACCOUNT="${SLURM_ACCOUNT:-sglang}"
+SRT_SLURM_BRANCH="${SRT_SLURM_BRANCH:-sglang-nightly-regression}"
+
+# ---------------------------------------------------------------------------
+# Resolve local model paths on Lustre (avoids re-downloading on each run)
+# ---------------------------------------------------------------------------
+if [[ "$MODEL_PREFIX" == "dsr1" && "$PRECISION" == "fp8" ]]; then
+    MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528"
+    SRT_SLURM_MODEL_PREFIX="dsr1-fp8"
+elif [[ "$MODEL_PREFIX" == "dsr1" && "$PRECISION" == "fp4" ]]; then
+    MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528-fp4-v2/"
+    SRT_SLURM_MODEL_PREFIX="dsr1-fp4"
+else
+    MODEL_PATH="$MODEL"
+    SRT_SLURM_MODEL_PREFIX="$MODEL_PREFIX"
+fi
+
+# ---------------------------------------------------------------------------
+# Set up per-runner Lustre workspace (cleaned before each run, accessible
+# to both the runner and compute nodes)
+# ---------------------------------------------------------------------------
+LUSTRE_WORKSPACE="/mnt/lustre01/users-public/sglang-ci/workspace/${RUNNER_NAME}"
+rm -rf "$LUSTRE_WORKSPACE"
+mkdir -p "$LUSTRE_WORKSPACE"
+
+# ---------------------------------------------------------------------------
+# Clone and set up srt-slurm
+# ---------------------------------------------------------------------------
+SRT_REPO_DIR="$LUSTRE_WORKSPACE/srt-slurm"
+
+git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+cd "$SRT_REPO_DIR"
+git checkout "$SRT_SLURM_BRANCH"
+echo "--- srt-slurm last commit ---"
+git log -1 --format="commit %H%nauthor %an%ndate   %ad%nsubject %s" --date=iso
+
+curl -LsSf https://astral.sh/uv/install.sh | sh
+source "$HOME/.local/bin/env"
+
+uv venv
+source .venv/bin/activate
+uv pip install -e .
+
+if ! command -v srtctl &>/dev/null; then
+    echo "ERROR: srtctl installation failed"
+    exit 1
+fi
+
+# ---------------------------------------------------------------------------
+# Generate srtslurm.yaml
+# ---------------------------------------------------------------------------
+SRTCTL_ROOT="$SRT_REPO_DIR"
+
+: "${S3_BUCKET:?S3_BUCKET must be set}"
+: "${S3_ENDPOINT_URL:?S3_ENDPOINT_URL must be set}"
+: "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID must be set}"
+: "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY must be set}"
+: "${MATRIX_CONFIG_NAME:?MATRIX_CONFIG_NAME must be set}"
+: "${GITHUB_RUN_ID:?GITHUB_RUN_ID must be set}"
+: "${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT must be set}"
+
+# Map the GitHub trigger into a friendlier top-level prefix: cron/manual.
+case "${GITHUB_EVENT_NAME:-}" in
+    schedule)          TRIGGER=cron ;;
+    workflow_dispatch) TRIGGER=manual ;;
+    *)                 TRIGGER="${GITHUB_EVENT_NAME:-unknown}" ;;
+esac
+
+# Format ISL/OSL as "1k1k" / "1k8k" / "8k1k" etc. for the S3 prefix, so logs
+# group naturally by sequence-length bucket under each run.
+fmt_seq_len() {
+    local n=$1
+    if (( n % 1024 == 0 )); then echo "$((n / 1024))k"; else echo "$n"; fi
+}
+SEQ_LEN="$(fmt_seq_len "$ISL")$(fmt_seq_len "$OSL")"
+
+S3_PREFIX="${TRIGGER}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${SEQ_LEN}/${MATRIX_CONFIG_NAME}"
+
+cat > srtslurm.yaml <<EOF
+# SRT SLURM configuration for SGLang GB200 nightly CI
+default_account: "${SLURM_ACCOUNT}"
+default_partition: "${SLURM_PARTITION}"
+default_time_limit: "6:00:00"
+
+gpus_per_node: 4
+network_interface: ""
+
+srtctl_root: "${SRTCTL_ROOT}"
+
+model_paths:
+  "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}"
+
+containers:
+  dynamo-sglang: ${SQUASH_FILE}
+  nginx: ${NGINX_SQUASH_FILE}
+  nginx-sqsh: ${NGINX_SQUASH_FILE}
+
+# srt-slurm postprocess uploads /logs to this bucket after each Slurm job.
+# Credentials are read from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars,
+# not written to disk. srt-slurm appends /<date>/<slurm-job-id>/ after prefix.
+reporting:
+  s3:
+    bucket: "${S3_BUCKET}"
+    prefix: "${S3_PREFIX}"
+    endpoint_url: "${S3_ENDPOINT_URL}"
+EOF
+
+echo "--- srtslurm.yaml ---"
+cat srtslurm.yaml
+echo "--- S3 log upload: s3://${S3_BUCKET}/${S3_PREFIX}/ ---"
+
+make setup ARCH=aarch64
+
+# ---------------------------------------------------------------------------
+# Patch job name and submit via srtctl
+# ---------------------------------------------------------------------------
+sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE"
+
+SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \
+    --tags "gb200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},sglang-nightly-$(date +%Y%m%d)" \
+    --setup-script install-torchao.sh 2>&1)
+echo "$SRTCTL_OUTPUT"
+
+JOB_ID=$(echo "$SRTCTL_OUTPUT" | grep -oP '✅ Job \K[0-9]+' || echo "$SRTCTL_OUTPUT" | grep -oP 'Job \K[0-9]+' || true)
+
+if [ -z "$JOB_ID" ]; then
+    echo "ERROR: Could not extract JOB_ID from srtctl output"
+    exit 1
+fi
+
+echo "Submitted Slurm job: $JOB_ID"
+
+set +x
+
+# ---------------------------------------------------------------------------
+# Wait for job and stream logs
+# ---------------------------------------------------------------------------
+LOGS_DIR="outputs/$JOB_ID/logs"
+LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log"
+
+mkdir -p "$LOGS_DIR"
+
+while ! ls "$LOG_FILE" &>/dev/null; do
+    if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then
+        echo "ERROR: Job $JOB_ID failed before creating log file"
+        scontrol show job "$JOB_ID" || true
+        exit 1
+    fi
+    echo "Waiting for job $JOB_ID to start and $LOG_FILE to appear..."
+    sleep 5
+done
+
+(
+    while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do
+        sleep 10
+    done
+) &
+POLL_PID=$!
+
+tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
+
+wait $POLL_PID
+
+set -x
+
+echo "Job $JOB_ID completed. Collecting results..."
+
+# ---------------------------------------------------------------------------
+# Collect results
+# ---------------------------------------------------------------------------
+if [ ! -d "$LOGS_DIR" ]; then
+    echo "WARNING: Logs directory not found at $LOGS_DIR"
+    exit 1
+fi
+
+cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS"
+tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" .
+
+RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null || true)
+
+if [ -z "$RESULT_SUBDIRS" ]; then
+    echo "ERROR: No result subdirectories found in $LOGS_DIR — benchmark did not produce any output"
+    exit 1
+else
+    RESULT_COUNT=0
+    for result_subdir in $RESULT_SUBDIRS; do
+        CONFIG_NAME=$(basename "$result_subdir")
+        RESULT_FILES=$(find "$result_subdir" -name "results_concurrency_*.json" 2>/dev/null || true)
+        for result_file in $RESULT_FILES; do
+            if [ -f "$result_file" ]; then
+                filename=$(basename "$result_file")
+                concurrency=$(echo "$filename" | sed -n 's/results_concurrency_\([0-9]*\)_gpus_.*/\1/p')
+                gpus=$(echo "$filename" | sed -n 's/results_concurrency_[0-9]*_gpus_\([0-9]*\)_ctx_.*/\1/p')
+                ctx=$(echo "$filename" | sed -n 's/.*_ctx_\([0-9]*\)_gen_.*/\1/p')
+                gen=$(echo "$filename" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
+                DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${CONFIG_NAME}_conc${concurrency}_gpus_${gpus}_ctx_${ctx}_gen_${gen}.json"
+                cp "$result_file" "$DEST"
+                echo "Saved: $DEST"
+                RESULT_COUNT=$((RESULT_COUNT + 1))
+            fi
+        done
+    done
+    if [ "$RESULT_COUNT" -eq 0 ]; then
+        echo "ERROR: Result subdirectories found but no result JSON files produced — benchmark failed"
+        exit 1
+    fi
+fi
+
+echo "Done."

scripts/ci/slurm/launch_mi355x.sh

@@ -0,0 +1,777 @@
+#!/usr/bin/env bash
+# Launch a 2-node 1P1D disaggregation benchmark on the AMD MI355X `amd-sglang`
+# Slurm cluster, then emit per-concurrency result JSONs that
+# scripts/ci/slurm/process_result.py aggregates.
+#
+# salloc's (prefill_workers + decode_workers) nodes -- one server per node --
+# and runs the Docker harness: prefill server(s) on the first nodes, decode
+# server(s) on the rest, a standalone load balancer on the prefill node, then an
+# sglang.bench_serving concurrency sweep over MORI. Default recipe is 1P1D (2
+# nodes); see the drive.sh note on reserving 2P2D / 1P3D / 3P1D.
+#
+# Required environment variables (set by the GitHub Actions workflow):
+#   MODEL              - HuggingFace model id (table label / served model)
+#   MODEL_PREFIX       - short prefix, e.g. dsv4flash
+#   PRECISION          - fp8 / fp4
+#   ISL, OSL           - input / output sequence lengths for the sweep
+#   CONFIG_FILE        - path to the recipe YAML (relative to repo root)
+#   RESULT_FILENAME    - prefix for the emitted result JSONs
+#   MATRIX_CONFIG_NAME - matrix entry name (used in filenames/tags)
+#   GITHUB_WORKSPACE   - set by GitHub Actions; where result JSONs are written
+# Optional:
+#   MODEL_PATH         - local snapshot dir (preferred over downloading MODEL)
+#   SLURM_PARTITION    - default: amd-sglang
+#   SLURM_NODELIST     - optional explicit node pin (else scheduler chooses)
+#   SLURM_EXCLUDE      - optional comma-separated nodes to keep the scheduler
+#                        off (e.g. hosts with a broken RDMA driver)
+#   SGLANG_USE_CHECKOUT_RUNTIME
+#                      - default 1. Reinstall this workflow checkout's Python
+#                        sglang package inside each runtime container, and the
+#                        checkout sglang-router package inside the bench
+#                        container, before launching servers/bench. Set 0 to
+#                        use the image's baked-in packages.
+#   RUNNER_NAME        - GitHub runner name (a built-in default env var)
+#   GITHUB_RUN_ID      - GitHub Actions run id (a built-in default env var)
+#                        The allocation is named
+#                          mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>
+#                        so workflow cleanup can scancel exactly this leg's job
+#                        (full name) or this runner's stale jobs (RUNNER_NAME
+#                        prefix) -- never a blanket `squeue --me`. The run id +
+#                        config make the name unique per matrix leg even if two
+#                        runners happen to share a name.
+#   SLURM_EXCLUSIVE    - request whole nodes (default 1); set 0 to disable
+#   TIME_LIMIT         - salloc time limit, default 02:30:00 (covers server
+#                        load + perf sweep + full GSM8K, under the 180m step cap)
+
+set -euo pipefail
+set -x
+
+: "${MODEL_PREFIX:?}"
+: "${PRECISION:?}"
+: "${ISL:?}"
+: "${OSL:?}"
+: "${CONFIG_FILE:?}"
+: "${RESULT_FILENAME:?}"
+: "${MATRIX_CONFIG_NAME:?}"
+: "${GITHUB_WORKSPACE:?}"
+
+SLURM_PARTITION="${SLURM_PARTITION:-amd-sglang}"
+TIME_LIMIT="${TIME_LIMIT:-02:30:00}"
+MODEL_PATH="${MODEL_PATH:-${MODEL:-}}"
+SGLANG_USE_CHECKOUT_RUNTIME="${SGLANG_USE_CHECKOUT_RUNTIME:-1}"
+case "${SGLANG_USE_CHECKOUT_RUNTIME,,}" in
+    0|false|no|off) SGLANG_USE_CHECKOUT_RUNTIME=0 ;;
+    *) SGLANG_USE_CHECKOUT_RUNTIME=1 ;;
+esac
+
+if [[ -z "$MODEL_PATH" ]]; then
+    echo "ERROR: set MODEL_PATH (local snapshot) or MODEL" >&2
+    exit 1
+fi
+
+# Resolve a HuggingFace cache dir (models--org--name) to its live snapshot dir.
+# Lets nightly-configs / recipes point at the shared cache without hardcoding a
+# snapshot hash; a concrete snapshot dir (or plain dir) is returned unchanged.
+# Used for both MODEL_PATH and an optional speculative draft model path.
+resolve_snapshot() {
+    local p="$1"
+    if [[ -f "$p/refs/main" && -d "$p/snapshots" ]]; then
+        local hash resolved
+        hash="$(cat "$p/refs/main")"
+        resolved="$p/snapshots/$hash"
+        if [[ -n "$hash" && -d "$resolved" ]]; then
+            echo "resolved snapshot: $p -> $resolved" >&2
+            echo "$resolved"
+            return 0
+        fi
+        echo "ERROR: refs/main=$hash but $resolved missing" >&2
+        return 1
+    fi
+    echo "$p"
+}
+MODEL_PATH="$(resolve_snapshot "$MODEL_PATH")" || exit 1
+
+# ---------------------------------------------------------------------------
+# Parse the recipe (runtime + bench + topology) into shell vars.
+# ---------------------------------------------------------------------------
+# Ensure PyYAML is available to the host python used for parsing.
+python3 -c 'import yaml' 2>/dev/null || pip install pyyaml -q 2>/dev/null \
+    || pip install --user pyyaml -q 2>/dev/null || true
+
+# Emit KEY=value lines and eval them (robust single-level command substitution;
+# avoids a nested read<<EOF/$(<<PY) heredoc that misparses on some shells).
+RECIPE_VARS="$(python3 - "$CONFIG_FILE" <<'PY'
+import sys, yaml
+r = yaml.safe_load(open(sys.argv[1]))
+rt = r["runtime"]; b = r["backend"]["sglang_config"]; bn = r["bench"]
+res = r.get("resources", {})
+def emit(k, v): print(f"{k}={v}")
+emit("IMAGE", rt["image"])
+# Attention backend: single (`attention_backend`) or split
+# (`prefill_attention_backend`/`decode_attention_backend`). Empty when absent so
+# the flag is dropped for a model that omits it.
+emit("ATTN", rt.get("attention_backend", ""))
+emit("PATTN", rt.get("prefill_attention_backend", ""))
+emit("DATTN", rt.get("decode_attention_backend", ""))
+emit("IB", rt["ib_devices"])
+emit("PPORT", rt["prefill_port"])
+emit("DPORT", rt["decode_port"])
+emit("PBOOT", rt["prefill_bootstrap_port"])
+emit("DBOOT", rt["decode_bootstrap_port"])
+emit("LBPORT", rt["lb_port"])
+emit("MEMFRAC", rt["mem_fraction_static"])
+emit("PAGE", rt["page_size"])
+emit("MAXREQ", rt["max_running_requests"])
+emit("MAXTOK", rt.get("max_total_tokens", ""))
+emit("CHUNK", rt["chunked_prefill_size"])
+# swa is DSV4-specific; emit empty when a model omits it so the flag is dropped.
+emit("SWA", rt.get("swa_full_tokens_ratio", ""))
+# 1 when the recipe carries a `model:` block (env + server_args written to
+# model_flags.sh); 0 for the DSV4 recipes, which keep the hardcoded DSV4 path.
+emit("HAS_MODEL", 1 if r.get("model") else 0)
+emit("PTP", b["prefill"]["tensor-parallel-size"])
+emit("DTP", b["decode"]["tensor-parallel-size"])
+emit("PEP", b["prefill"].get("expert-parallel-size", 1))
+emit("PDP", b["prefill"].get("data-parallel-size", 1))
+m = r.get("mtp", {}) or {}
+emit("MTP_ENABLED", 1 if m.get("enabled") else 0)
+emit("MTP_ALGO", m.get("algorithm", "EAGLE"))
+emit("MTP_STEPS", m.get("num_steps", 3))
+emit("MTP_TOPK", m.get("eagle_topk", 1))
+emit("MTP_DRAFT", m.get("num_draft_tokens", 4))
+# External draft checkpoint (EAGLE3 etc.); empty for DSV4's built-in EAGLE head.
+emit("MTP_DRAFT_PATH", m.get("draft_model_path", ""))
+# Worker counts double as node counts here: one server per node (TP == GPUs/node).
+# 1P1D today; bumping these reserves 2P2D / 1P3D / 3P1D. Multi-node-per-worker
+# (TP > GPUs/node, needs --dist-init-addr/--nnodes/--node-rank) is out of scope.
+emit("PW", res.get("prefill_workers", 1))
+emit("DW", res.get("decode_workers", 1))
+emit("CONCS", ",".join(str(c) for c in bn["concurrencies"]))
+emit("NPF", bn["num_prompts_factor"])
+emit("RRR", bn["random_range_ratio"])
+acc = bn.get("accuracy", {}) or {}
+emit("ACC_ENABLED", 1 if acc.get("enabled") else 0)
+emit("ACC_SHOTS", acc.get("num_shots", 8))
+emit("ACC_NQ", acc.get("num_questions", 1319))
+emit("ACC_THR", acc.get("threshold", 0.91))
+PY
+)"
+if [[ -z "$RECIPE_VARS" ]]; then
+    echo "ERROR: failed to parse recipe $CONFIG_FILE (empty output from python3/yaml)" >&2
+    exit 1
+fi
+eval "$RECIPE_VARS"
+# Optional image override from workflow_dispatch input.
+if [[ -n "${IMAGE_OVERRIDE:-}" ]]; then
+    IMAGE="$IMAGE_OVERRIDE"
+fi
+echo "recipe: image=$IMAGE attn=${ATTN:-$PATTN/$DATTN} ib=$IB ptp=$PTP dtp=$DTP concs=$CONCS isl=$ISL osl=$OSL"
+
+# ---------------------------------------------------------------------------
+# Shared NFS scratch (visible to login node + compute nodes). Raw bench output
+# lands here; the launcher normalizes it into GITHUB_WORKSPACE afterwards.
+# ---------------------------------------------------------------------------
+WORKDIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}"
+rm -rf "$WORKDIR"; mkdir -p "$WORKDIR"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Stage the workflow checkout on shared NFS so Slurm compute-node containers can
+# reinstall the same code SHA the workflow checked out. The container gets a
+# read-only mount and copies it to /tmp before mutating pyproject.toml.
+CHECKOUT_DOCKER_ARGS="-e SGLANG_USE_CHECKOUT_RUNTIME=$SGLANG_USE_CHECKOUT_RUNTIME"
+if [[ "$SGLANG_USE_CHECKOUT_RUNTIME" == "1" ]]; then
+    CHECKOUT_STAGE="$WORKDIR/checkout"
+    CHECKOUT_SHA="$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"
+    echo "Staging checkout runtime: sha=$CHECKOUT_SHA -> $CHECKOUT_STAGE"
+    rm -rf "$CHECKOUT_STAGE"
+    mkdir -p "$CHECKOUT_STAGE"
+    tar --exclude='__pycache__' --exclude='*.pyc' --exclude='.git/config' \
+        -C "$GITHUB_WORKSPACE" -cf - . | tar -C "$CHECKOUT_STAGE" -xf -
+    CHECKOUT_DOCKER_ARGS="$CHECKOUT_DOCKER_ARGS -e SGLANG_CHECKOUT_SHA=$CHECKOUT_SHA -v $CHECKOUT_STAGE:/sglang-checkout:ro"
+else
+    echo "SGLANG_USE_CHECKOUT_RUNTIME=0; using sglang package baked into image."
+fi
+
+# Accuracy-gate helpers (written when enabled). Pre-stage the GSM8K test set on
+# shared NFS from the login node (which has internet) so the in-container eval
+# doesn't depend on compute-node connectivity; fall back to in-container
+# download if the pre-fetch fails.
+if [[ "$ACC_ENABLED" == "1" ]]; then
+    GSM8K_URL="https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
+    curl -fsSL "$GSM8K_URL" -o "$WORKDIR/gsm8k_test.jsonl" 2>/dev/null \
+        && echo "gsm8k dataset staged at $WORKDIR/gsm8k_test.jsonl" \
+        || echo "WARN: gsm8k pre-stage failed; in-container download will be attempted"
+    cat > "$WORKDIR/check_acc.py" <<'PY'
+import sys
+acc, thr = float(sys.argv[1]), float(sys.argv[2])
+print(f"[gsm8k] accuracy={acc:.3f} threshold={thr}")
+sys.exit(0 if acc > thr else 1)
+PY
+fi
+
+# DSV4 load-bearing env (see test/registered/amd/test_deepseek_v4_flash_fp8.py).
+# SGLANG_DSV4_FP4_EXPERTS is precision-driven: true for fp4 weights, false for fp8.
+if [[ "$PRECISION" == "fp4" ]]; then
+    FP4_EXPERTS=true
+else
+    FP4_EXPERTS=false
+fi
+DSV4_ENV=(
+  -e SGLANG_DEFAULT_THINKING=1 -e SGLANG_DSV4_REASONING_EFFORT=max
+  -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -e SGLANG_USE_AITER=1
+  -e SGLANG_USE_ROCM700A=1 -e SGLANG_OPT_USE_FUSED_COMPRESS=true
+  -e SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true
+  -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
+  -e SGLANG_OPT_FP8_WO_A_GEMM=false -e SGLANG_OPT_USE_JIT_INDEXER_METADATA=false
+  -e SGLANG_OPT_USE_TOPK_V2=false -e SGLANG_OPT_USE_AITER_INDEXER=true
+  -e SGLANG_OPT_USE_TILELANG_INDEXER=false -e SGLANG_OPT_USE_TILELANG_MHC_PRE=false
+  -e SGLANG_OPT_USE_TILELANG_MHC_POST=false -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1
+  -e SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false -e SGLANG_ROCM_USE_MULTI_STREAM=false
+  -e AITER_BF16_FP8_MOE_BOUND=0 -e SGLANG_DSV4_FP4_EXPERTS=$FP4_EXPERTS
+)
+DSV4_ENV_STR="${DSV4_ENV[*]}"
+# A recipe carrying a `model:` block supplies its OWN docker env (below), so the
+# DSV4 env must not leak into it; the DSV4 recipes keep the string above.
+[[ "$HAS_MODEL" == "1" ]] && DSV4_ENV_STR=""
+MORI_ENV="-e MORI_DISABLE_AUTO_XGMI=1 -e NCCL_IB_HCA=ionic -e NCCL_IB_GID_INDEX=1 -e NCCL_CROSS_NIC=1"
+
+# Model-specific docker `-e` env + sglang server args from the recipe's optional
+# `model:` block, written as bash arrays to model_flags.sh (sourced by
+# prefill.sh/decode.sh). DSV4 recipes have no `model:` block -> empty arrays, so
+# their generated docker argv is unchanged. Each server arg + its value MUST be a
+# separate YAML list item so shlex.quote keeps "--foo" and "bar" as two tokens.
+python3 - "$CONFIG_FILE" "$WORKDIR/model_flags.sh" <<'PY'
+import shlex, sys, yaml
+r = yaml.safe_load(open(sys.argv[1]))
+model = r.get("model", {}) or {}
+env = model.get("env", {}) or {}
+server_args = model.get("server_args", []) or []
+# YAML true/false parse to Python bool; render lowercase so env values stay
+# byte-identical to shell (`=false`, not `=False`) -- SGLang parsing is
+# case-sensitive for some of these.
+def fmt(v):
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    return str(v)
+env_args = []
+for k, v in env.items():
+    env_args += ["-e", f"{k}={fmt(v)}"]
+def q(items):
+    return " ".join(shlex.quote(fmt(x)) for x in items)
+with open(sys.argv[2], "w") as f:
+    f.write(f"MODEL_ENV_ARGS=({q(env_args)})\n")
+    f.write(f"MODEL_SERVER_ARGS=({q(server_args)})\n")
+PY
+
+# Optional topology / speculative-decode flags driven by the recipe. Base recipes
+# (EP1/DP1, no mtp) leave EXTRA_FLAGS empty, preserving prior behavior exactly.
+EXTRA_FLAGS=""
+(( PDP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --enable-dp-attention --dp-size $PDP"
+(( PEP > 1 )) && EXTRA_FLAGS="$EXTRA_FLAGS --ep-size $PEP"
+[[ -n "$MAXTOK" ]] && EXTRA_FLAGS="$EXTRA_FLAGS --max-total-tokens $MAXTOK"
+if [[ "$MTP_ENABLED" == "1" ]]; then
+    EXTRA_FLAGS="$EXTRA_FLAGS --speculative-algorithm $MTP_ALGO \
+--speculative-num-steps $MTP_STEPS --speculative-eagle-topk $MTP_TOPK \
+--speculative-num-draft-tokens $MTP_DRAFT"
+    # EAGLE3 (and other draft-model algos) need an external draft checkpoint;
+    # built-in EAGLE (DSV4) omits draft_model_path and this stays unset.
+    if [[ -n "$MTP_DRAFT_PATH" ]]; then
+        DRAFT_RESOLVED="$(resolve_snapshot "$MTP_DRAFT_PATH")" || exit 1
+        EXTRA_FLAGS="$EXTRA_FLAGS --speculative-draft-model-path $DRAFT_RESOLVED"
+    fi
+fi
+echo "extra flags: ${EXTRA_FLAGS:-<none>} (pep=$PEP pdp=$PDP mtp=$MTP_ENABLED algo=$MTP_ALGO)"
+
+if [[ "$HAS_MODEL" == "1" ]]; then
+    # Generic path (e.g. Kimi): attention + swa from the recipe, model parsers /
+    # quirks ride MODEL_SERVER_ARGS. Single `--attention-backend` when the recipe
+    # sets `attention_backend`; split `--prefill-/--decode-attention-backend` when
+    # it sets the per-role keys. swa dropped when the recipe omits it.
+    ATTN_FLAGS=""
+    [[ -n "$ATTN" ]]  && ATTN_FLAGS="$ATTN_FLAGS --attention-backend $ATTN"
+    [[ -n "$PATTN" ]] && ATTN_FLAGS="$ATTN_FLAGS --prefill-attention-backend $PATTN"
+    [[ -n "$DATTN" ]] && ATTN_FLAGS="$ATTN_FLAGS --decode-attention-backend $DATTN"
+    SWA_FLAG=""
+    [[ -n "$SWA" ]] && SWA_FLAG=" --swa-full-tokens-ratio $SWA"
+    COMMON_FLAGS="--trust-remote-code --tp $PTP --disable-radix-cache \
+$ATTN_FLAGS --max-running-requests $MAXREQ --page-size $PAGE \
+--mem-fraction-static $MEMFRAC$SWA_FLAG \
+--chunked-prefill-size $CHUNK \
+--disaggregation-transfer-backend mori --disaggregation-ib-device $IB$EXTRA_FLAGS"
+else
+    # DSV4 path: byte-identical to the pre-Kimi launcher.
+    COMMON_FLAGS="--trust-remote-code --tp $PTP --disable-radix-cache \
+--attention-backend $ATTN --max-running-requests $MAXREQ --page-size $PAGE \
+--mem-fraction-static $MEMFRAC --swa-full-tokens-ratio $SWA \
+--chunked-prefill-size $CHUNK --disable-shared-experts-fusion \
+--tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 \
+--disaggregation-transfer-backend mori --disaggregation-ib-device $IB$EXTRA_FLAGS"
+fi
+
+DOCKER_COMMON="--rm --network host --ipc host --shm-size 32g --privileged \
+--security-opt seccomp=unconfined \
+--device /dev/kfd --device /dev/dri --device /dev/infiniband \
+-v /it-share:/it-share:ro -v $HOME:/host_home $CHECKOUT_DOCKER_ARGS"
+
+# ---------------------------------------------------------------------------
+# Write per-role scripts that srun dispatches to each compute node.
+# ---------------------------------------------------------------------------
+# These are UNQUOTED `<<EOF` heredocs, so $MORI_ENV/$DSV4_ENV_STR/$COMMON_FLAGS/
+# $IMAGE/$MODEL_PATH expand now (at generation). model_flags.sh is sourced at
+# runtime for the model's env/server arrays, so the `${MODEL_ENV_ARGS[@]}` /
+# `${MODEL_SERVER_ARGS[@]}` refs are backslash-escaped to survive into the script
+# and expand after `source`. For DSV4 those arrays are empty and $DSV4_ENV_STR is
+# set, so the resulting docker argv is byte-identical to the pre-Kimi launcher.
+cat > "$WORKDIR/install_checkout_sglang.sh" <<'EOF'
+#!/bin/bash
+set -euo pipefail
+
+case "${SGLANG_USE_CHECKOUT_RUNTIME:-1}" in
+  0|false|False|FALSE|no|No|NO|off|Off|OFF)
+    echo "[checkout-sglang] disabled; using image-baked sglang"
+    exit 0
+    ;;
+esac
+
+CHECKOUT_SRC="${CHECKOUT_SRC:-/sglang-checkout}"
+RUNTIME_CHECKOUT="${RUNTIME_CHECKOUT:-/tmp/sglang-checkout-runtime}"
+
+if [[ ! -f "$CHECKOUT_SRC/python/sglang/version.py" ]]; then
+  echo "[checkout-sglang] ERROR: invalid checkout mount: $CHECKOUT_SRC" >&2
+  exit 1
+fi
+
+echo "[checkout-sglang] reinstalling sglang from $CHECKOUT_SRC"
+rm -rf "$RUNTIME_CHECKOUT"
+mkdir -p "$RUNTIME_CHECKOUT"
+tar --exclude='__pycache__' --exclude='*.pyc' \
+  -C "$CHECKOUT_SRC" -cf - . | tar -C "$RUNTIME_CHECKOUT" -xf -
+
+git config --global --add safe.directory "$RUNTIME_CHECKOUT" || true
+
+# The ROCm pyproject variant is the one used by AMD CI. Mutate only the private
+# /tmp copy so prefill/decode/bench never race on the read-only checkout mount.
+rm -f "$RUNTIME_CHECKOUT/python/pyproject.toml"
+cp "$RUNTIME_CHECKOUT/python/pyproject_other.toml" "$RUNTIME_CHECKOUT/python/pyproject.toml"
+for f in README.md LICENSE; do
+  if [[ -f "$RUNTIME_CHECKOUT/$f" && ! -e "$RUNTIME_CHECKOUT/python/$f" ]]; then
+    cp "$RUNTIME_CHECKOUT/$f" "$RUNTIME_CHECKOUT/python/$f"
+  fi
+done
+
+python3 -m pip uninstall -y sglang || true
+python3 -m pip install --no-deps --no-build-isolation -e "$RUNTIME_CHECKOUT/python"
+
+export RUNTIME_CHECKOUT
+export PYTHONPATH="$RUNTIME_CHECKOUT/python:${PYTHONPATH:-}"
+python3 - <<'PY'
+import importlib.metadata
+import os
+import subprocess
+import sglang
+
+checkout = os.environ["RUNTIME_CHECKOUT"]
+expected = os.path.realpath(os.path.join(checkout, "python", "sglang")) + os.sep
+actual = os.path.realpath(os.path.dirname(sglang.__file__)) + os.sep
+try:
+    sha = subprocess.check_output(
+        ["git", "-C", checkout, "rev-parse", "HEAD"], text=True
+    ).strip()
+except Exception:
+    sha = os.environ.get("SGLANG_CHECKOUT_SHA", "unknown")
+
+print(f"[checkout-sglang] sha={sha}")
+print(f"[checkout-sglang] sglang_file={sglang.__file__}")
+print(f"[checkout-sglang] sglang_version={importlib.metadata.version('sglang')}")
+if not actual.startswith(expected):
+    raise SystemExit(f"sglang did not import from checkout: {sglang.__file__}")
+PY
+EOF
+
+cat > "$WORKDIR/install_checkout_router.sh" <<'EOF'
+#!/bin/bash
+set -euo pipefail
+
+case "${SGLANG_USE_CHECKOUT_RUNTIME:-1}" in
+  0|false|False|FALSE|no|No|NO|off|Off|OFF)
+    echo "[checkout-router] disabled; using image-baked sglang-router"
+    python3 - <<'PY' || true
+import importlib.metadata
+import sglang_router
+
+print(f"[checkout-router] sglang_router_file={sglang_router.__file__}")
+print(
+    "[checkout-router] sglang_router_version="
+    f"{importlib.metadata.version('sglang-router')}"
+)
+try:
+    import sglang_router.sglang_router_rs as rs
+
+    print(f"[checkout-router] sglang_router_rs_file={rs.__file__}")
+except Exception as exc:
+    print(f"[checkout-router] sglang_router_rs_import_error={exc}")
+PY
+    exit 0
+    ;;
+esac
+
+RUNTIME_CHECKOUT="${RUNTIME_CHECKOUT:-/tmp/sglang-checkout-runtime}"
+ROUTER_SRC="$RUNTIME_CHECKOUT/sgl-model-gateway/bindings/python"
+WHEEL_DIR="${SGLANG_ROUTER_WHEEL_DIR:-/tmp/sglang-router-wheels}"
+
+if [[ ! -f "$ROUTER_SRC/pyproject.toml" ]]; then
+  echo "[checkout-router] ERROR: invalid router checkout: $ROUTER_SRC" >&2
+  exit 1
+fi
+
+echo "[checkout-router] building sglang-router from $ROUTER_SRC"
+export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-4}"
+python3 -m maturin --version >/dev/null 2>&1 \
+  || python3 -m pip install --no-cache-dir "maturin<1.14"
+
+# Match the ROCm image build recipe when compiling from the checkout copy.
+if [[ -f "$RUNTIME_CHECKOUT/sgl-model-gateway/Cargo.toml" ]]; then
+  sed -i -E 's|^(smg-[a-zA-Z-]+)\s*=\s*"~1\.0\.0"|\1 = "=1.0.0"|' \
+    "$RUNTIME_CHECKOUT/sgl-model-gateway/Cargo.toml"
+fi
+
+rm -rf "$WHEEL_DIR"
+mkdir -p "$WHEEL_DIR"
+(
+  cd "$ROUTER_SRC"
+  ulimit -n 65536 || true
+  python3 -m maturin build --release --features vendored-openssl --out "$WHEEL_DIR"
+)
+
+python3 -m pip uninstall -y sglang-router || true
+python3 -m pip install --force-reinstall --no-deps "$WHEEL_DIR"/*.whl
+
+python3 - <<'PY'
+import importlib.metadata
+import sglang_router
+import sglang_router.sglang_router_rs as rs
+from sglang_router.sglang_router_rs import Router
+
+print(f"[checkout-router] sglang_router_file={sglang_router.__file__}")
+print(
+    "[checkout-router] sglang_router_version="
+    f"{importlib.metadata.version('sglang-router')}"
+)
+print(f"[checkout-router] sglang_router_rs_file={rs.__file__}")
+print(f"[checkout-router] Router={Router}")
+PY
+EOF
+
+cat > "$WORKDIR/prefill_entry.sh" <<EOF
+#!/bin/bash
+set -euo pipefail
+CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
+source "\$CIDIR/model_flags.sh"
+bash "\$CIDIR/install_checkout_sglang.sh"
+if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
+  export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
+fi
+exec python3 -m sglang.launch_server \
+  --model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \
+  $COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
+  --disaggregation-mode prefill --disaggregation-bootstrap-port $PBOOT
+EOF
+
+cat > "$WORKDIR/decode_entry.sh" <<EOF
+#!/bin/bash
+set -euo pipefail
+CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
+source "\$CIDIR/model_flags.sh"
+bash "\$CIDIR/install_checkout_sglang.sh"
+if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
+  export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
+fi
+exec python3 -m sglang.launch_server \
+  --model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \
+  $COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
+  --disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT
+EOF
+
+cat > "$WORKDIR/prefill.sh" <<EOF
+#!/bin/bash
+source "$WORKDIR/model_flags.sh"
+docker rm -f mi355x_prefill 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_prefill \
+  -e HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 $MORI_ENV $DSV4_ENV_STR "\${MODEL_ENV_ARGS[@]}" \
+  $IMAGE bash /host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}/prefill_entry.sh
+EOF
+
+cat > "$WORKDIR/decode.sh" <<EOF
+#!/bin/bash
+source "$WORKDIR/model_flags.sh"
+docker rm -f mi355x_decode 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_decode \
+  -e HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 $MORI_ENV $DSV4_ENV_STR "\${MODEL_ENV_ARGS[@]}" \
+  $IMAGE bash /host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}/decode_entry.sh
+EOF
+
+# Probe payload + validator (separate files to avoid quoting inside the
+# bench.sh `bash -lc '...'` block). One real request exercises the full
+# prefill->decode KV handoff before we commit to the whole sweep.
+cat > "$WORKDIR/probe.json" <<'JSON'
+{"text": "The capital of France is", "sampling_params": {"max_new_tokens": 16, "temperature": 0.0}}
+JSON
+cat > "$WORKDIR/assert_nonempty.py" <<'PY'
+import sys, json
+d = json.load(sys.stdin)
+t = d.get("text", "") if isinstance(d, dict) else ""
+if not (t and t.strip()):
+    print("[probe] empty/invalid output:", str(d)[:200])
+    sys.exit(1)
+print("[probe] ok:", t[:80].replace("\n", " "))
+PY
+
+# Bench script runs on the prefill node; \$PIP/\$DIP injected at srun time.
+cat > "$WORKDIR/bench.sh" <<EOF
+#!/bin/bash
+set -e
+PIP=\$1; DIP=\$2
+docker rm -f mi355x_bench 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_bench \
+  -e PIP=\$PIP -e DIP=\$DIP \
+  $IMAGE bash -lc '
+    CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
+    bash \$CIDIR/install_checkout_sglang.sh
+    if [ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]; then
+      export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
+    else
+      export PYTHONPATH=/sgl-workspace/sglang/python:\${PYTHONPATH:-}
+    fi
+    bash \$CIDIR/install_checkout_router.sh
+    echo "[wait] prefill"; for i in \$(seq 1 600); do curl -sf http://\$PIP:$PPORT/health >/dev/null && break; sleep 5; done
+    echo "[wait] decode";  for i in \$(seq 1 600); do curl -sf http://\$DIP:$DPORT/health >/dev/null && break; sleep 5; done
+    python3 -m sglang_router.launch_router \
+      --pd-disaggregation \
+      --prefill http://\$PIP:$PPORT $PBOOT \
+      --decode http://\$DIP:$DPORT \
+      --host 0.0.0.0 --port $LBPORT \
+      --disable-circuit-breaker &
+    for i in \$(seq 1 30); do curl -sf http://127.0.0.1:$LBPORT/health >/dev/null && break; sleep 2; done
+    echo "[probe] PD end-to-end check via LB"
+    curl -sf -X POST http://127.0.0.1:$LBPORT/generate \
+      -H "content-type: application/json" -d @\$CIDIR/probe.json > \$CIDIR/probe_out.json \
+      || { echo "[probe] request failed -- PD path not serving; aborting before sweep"; exit 1; }
+    python3 \$CIDIR/assert_nonempty.py < \$CIDIR/probe_out.json \
+      || { echo "[probe] empty/invalid generation; aborting before sweep"; exit 1; }
+    # Correctness gate runs BEFORE the perf sweep: if the model is wrong there
+    # is no point spending ~15min measuring how fast it is wrong, so a failure
+    # here exits immediately and the sweep never runs.
+    if [ "$ACC_ENABLED" = "1" ]; then
+      echo "=== GSM8K accuracy gate (num_questions=$ACC_NQ shots=$ACC_SHOTS) ==="
+      DP_ARG=""
+      [ -s \$CIDIR/gsm8k_test.jsonl ] && DP_ARG="--data-path \$CIDIR/gsm8k_test.jsonl"
+      python3 -m sglang.test.few_shot_gsm8k \
+        --num-shots $ACC_SHOTS --num-questions $ACC_NQ --parallel $MAXREQ \
+        --max-new-tokens 512 --host http://127.0.0.1 --port $LBPORT \
+        \$DP_ARG 2>&1 | tee \$CIDIR/gsm8k.log
+      ACC=\$(grep -oE "Accuracy: [0-9.]+" \$CIDIR/gsm8k.log | tail -1 | cut -d" " -f2)
+      [ -n "\$ACC" ] || { echo "[gsm8k] could not parse accuracy from harness output"; exit 1; }
+      python3 \$CIDIR/check_acc.py "\$ACC" "$ACC_THR" || { echo "[gsm8k] accuracy below threshold -- failing before sweep"; exit 1; }
+    fi
+    for C in ${CONCS//,/ }; do
+      echo "=== concurrency=\$C ==="
+      OUT=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}/raw_conc\${C}.json
+      rm -f \$OUT
+      python3 -m sglang.bench_serving --backend sglang \
+        --host 127.0.0.1 --port $LBPORT --model $MODEL_PATH \
+        --dataset-name random --random-input-len $ISL --random-output-len $OSL \
+        --random-range-ratio $RRR --max-concurrency \$C \
+        --num-prompts \$((C*$NPF)) --warmup-requests \$C \
+        --output-file \$OUT || true
+    done
+  '
+EOF
+chmod +x "$WORKDIR"/*.sh
+
+# ---------------------------------------------------------------------------
+# Orchestration drive (runs inside the salloc allocation on the login node).
+# ---------------------------------------------------------------------------
+# drive.sh splits the allocation into the first PW nodes (prefill) and the next
+# DW nodes (decode), launches one server per node, then benches. For 1P1D
+# (PW=DW=1) this is exactly prefill-on-node-A / decode-on-node-B. Larger PW/DW
+# reserve 2P2D / 1P3D / 3P1D: all servers come up, but the load balancer and
+# bench still target the first prefill + first decode (multi-P/D fan-out is the
+# remaining LB piece), so a >1 topology logs an explicit NOTE rather than
+# silently producing partial-coverage numbers.
+cat > "$WORKDIR/drive.sh" <<'DRIVE'
+#!/bin/bash
+set -x
+WORKDIR="$1"; PW="${2:-1}"; DW="${3:-1}"
+mapfile -t NODES < <(scontrol show hostnames "$SLURM_JOB_NODELIST")
+PNODES=("${NODES[@]:0:PW}")
+DNODES=("${NODES[@]:PW:DW}")
+PNODE="${PNODES[0]}"; DNODE="${DNODES[0]}"
+PIP=$(getent ahostsv4 "$PNODE" | head -1 | awk '{print $1}')
+DIP=$(getent ahostsv4 "$DNODE" | head -1 | awk '{print $1}')
+echo "[drive] prefill nodes: ${PNODES[*]} ; decode nodes: ${DNODES[*]}"
+echo "[drive] bench targets prefill=$PNODE($PIP) decode=$DNODE($DIP)"
+if (( PW > 1 || DW > 1 )); then
+  echo "[drive] NOTE: router + bench use the first prefill and first decode only;"
+  echo "[drive]       multi-prefill/multi-decode fan-out is not wired yet (LB work)."
+fi
+# Each server's srun runs here on the login node and returns exactly when its
+# compute-node container exits. Wrap it so the return code lands in a marker
+# file on shared NFS. The monitor then watches for markers instead of polling
+# PIDs -- unambiguous (no zombie/kill -0 guesswork) and it records which role
+# died and with what code. (A hung-but-alive server is NOT caught here; that is
+# bounded by bench.sh's health-wait timeout.)
+rm -f "$WORKDIR"/server_exit_* "$WORKDIR/bench_exit"
+for n in "${PNODES[@]}"; do
+  ( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/prefill.sh" > "$WORKDIR/prefill_$n.log" 2>&1
+    echo "prefill@$n rc=$?" > "$WORKDIR/server_exit_prefill_$n" ) &
+done
+for n in "${DNODES[@]}"; do
+  ( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/decode.sh" > "$WORKDIR/decode_$n.log" 2>&1
+    echo "decode@$n rc=$?" > "$WORKDIR/server_exit_decode_$n" ) &
+done
+sleep 5
+# Bench in the background with its own marker, so the wait loop is purely file
+# based: finish when bench writes its marker, abort if any server marker shows up
+# first (a server died before the sweep completed).
+( srun --overlap -N1 --nodelist="$PNODE" bash "$WORKDIR/bench.sh" "$PIP" "$DIP" > "$WORKDIR/bench.log" 2>&1
+  echo $? > "$WORKDIR/bench_exit" ) &
+BENCH_BG=$!
+# Stream bench output live and poll the markers with xtrace OFF, so the console
+# shows clean benchmark/accuracy output instead of a compgen/sleep trace every
+# 10s. (Mirrors NVIDIA's launch_gb200.sh, which set +x around its log stream.)
+touch "$WORKDIR/bench.log"
+tail -n +1 -F "$WORKDIR/bench.log" 2>/dev/null &
+TAIL_PID=$!
+set +x
+RC=0
+while [[ ! -f "$WORKDIR/bench_exit" ]]; do
+  if compgen -G "$WORKDIR/server_exit_*" > /dev/null; then
+    echo "[drive] ERROR: a server exited early before bench finished:"
+    cat "$WORKDIR"/server_exit_* || true
+    kill "$BENCH_BG" 2>/dev/null || true
+    RC=1
+    break
+  fi
+  sleep 10
+done
+set -x
+kill "$TAIL_PID" 2>/dev/null || true
+[[ "$RC" -eq 0 ]] && RC=$(cat "$WORKDIR/bench_exit" 2>/dev/null || echo 1)
+echo "[drive] bench finished (rc=$RC), tearing down"
+for n in "${PNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_prefill >/dev/null 2>&1 || true; done
+for n in "${DNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_decode  >/dev/null 2>&1 || true; done
+exit "$RC"
+DRIVE
+chmod +x "$WORKDIR/drive.sh"
+
+NODELIST_ARG=()
+[[ -n "${SLURM_NODELIST:-}" ]] && NODELIST_ARG=(--nodelist="$SLURM_NODELIST")
+
+# Request whole nodes so a co-scheduled job can't share a node and skew the
+# benchmark numbers. Toggle off with SLURM_EXCLUSIVE=0 on partitions that
+# disallow --exclusive.
+EXCLUSIVE_ARG=()
+[[ "${SLURM_EXCLUSIVE:-1}" == "1" ]] && EXCLUSIVE_ARG=(--exclusive)
+
+# Keep the scheduler off known-bad nodes (e.g. a host whose ionic RDMA driver
+# ABI mismatches the container, where MORI reports "no active RDMA device" and
+# the disagg server dies on init). Comma-separated node list.
+EXCLUDE_ARG=()
+[[ -n "${SLURM_EXCLUDE:-}" ]] && EXCLUDE_ARG=(--exclude="$SLURM_EXCLUDE")
+
+# One node per prefill/decode worker (TP == GPUs/node). 1P1D -> 2 nodes.
+TOTAL_NODES=$((PW + DW))
+
+# Name the allocation <RUNNER_NAME>-<GITHUB_RUN_ID>-<config> so the workflow's
+# cleanup steps can scancel precisely instead of a blanket `squeue --me` that
+# would kill a concurrent matrix leg. RUNNER_NAME alone is not assumed unique;
+# GITHUB_RUN_ID + config make the name unique per matrix leg regardless.
+JOB_NAME="mi355x-ci-${RUNNER_NAME:-norunner}-${GITHUB_RUN_ID:-0}-${MATRIX_CONFIG_NAME}"
+
+set +e
+salloc -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" \
+    --job-name "$JOB_NAME" -t "$TIME_LIMIT" \
+    bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW"
+SALLOC_RC=$?
+set -e
+
+# bench output already streamed live from drive.sh (tail -F). drive.sh exits
+# non-zero when a server died or bench failed; on failure dump bench.log + the
+# server logs (the actual root cause). We still fall through to normalize
+# whatever raw results the completed concurrencies produced -- partial perf data
+# is worth uploading -- and propagate the failure via the exit code at the end.
+if [[ "$SALLOC_RC" -ne 0 ]]; then
+    echo "ERROR: allocation/bench failed (rc=$SALLOC_RC); bench + server logs:" >&2
+    echo "--- bench.log (tail) ---"; tail -40 "$WORKDIR/bench.log" 2>/dev/null || true
+    for f in "$WORKDIR"/prefill_*.log "$WORKDIR"/decode_*.log; do
+        [[ -f "$f" ]] && { echo "--- $f (tail) ---"; tail -30 "$f"; }
+    done
+fi
+
+# Surface the GSM8K accuracy in the job summary -- it scrolls past in the live
+# log, and the perf table (collect-results/summarize.py) doesn't include it.
+if [[ "$ACC_ENABLED" == "1" && -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    ACC_LINE=$(grep -aoE "Accuracy: [0-9.]+" "$WORKDIR/bench.log" 2>/dev/null | tail -1 || true)
+    {
+        echo "### GSM8K accuracy gate — ${MATRIX_CONFIG_NAME}"
+        echo '```'
+        echo "${ACC_LINE:-Accuracy: <not found in bench.log>}   (threshold > ${ACC_THR})"
+        echo '```'
+    } >> "$GITHUB_STEP_SUMMARY"
+fi
+
+# ---------------------------------------------------------------------------
+# Normalize raw bench_serving output -> process_result.py schema.
+#
+# bench_serving and process_result.py disagree on field names, so we remap the
+# last JSON line of each raw file. If bench_serving ever renames an output
+# field, the KeyError raised here (rather than a silently wrong table) is the
+# signal to update this mapping. Field-by-field:
+#
+#   bench_serving key          ->  process_result.py key       (purpose)
+#   --------------------------     ------------------------     -------------------------
+#   max_concurrency            ->  max_concurrency             (sweep point; falls back to $C)
+#   total_throughput           ->  total_token_throughput      (in+out tok/s, tput_per_gpu)
+#   output_throughput          ->  output_throughput           (out tok/s, output_tput_per_gpu)
+#   median_ttft_ms             ->  median_ttft_ms              (TTFT; /1000 -> s)
+#   median_tpot_ms             ->  median_tpot_ms              (TPOT; -> interactivity)
+#   median_e2e_latency_ms      ->  median_e2el_ms              (E2E latency; /1000 -> s)
+#   (none; injected here)      ->  model_id                    (served model, from $MODEL_PATH)
+# ---------------------------------------------------------------------------
+TOTAL_GPUS=$((PTP + DTP))
+PROCESSED=0
+for C in ${CONCS//,/ }; do
+    RAW="$WORKDIR/raw_conc${C}.json"
+    [[ -f "$RAW" ]] || { echo "WARN: missing $RAW"; continue; }
+    DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${MATRIX_CONFIG_NAME}_conc${C}_gpus_${TOTAL_GPUS}_ctx_${PTP}_gen_${DTP}.json"
+    MODEL_ID="$MODEL_PATH" python3 - "$RAW" "$DEST" "$C" <<'PY'
+import json, os, sys
+raw_path, dest, conc = sys.argv[1], sys.argv[2], int(sys.argv[3])
+line = [l for l in open(raw_path).read().splitlines() if l.strip()][-1]
+r = json.loads(line)
+norm = {
+    "max_concurrency": r.get("max_concurrency") or conc,
+    "model_id": os.environ["MODEL_ID"],
+    "total_token_throughput": r["total_throughput"],
+    "output_throughput": r["output_throughput"],
+    "median_ttft_ms": r["median_ttft_ms"],
+    "median_tpot_ms": r["median_tpot_ms"],
+    "median_e2el_ms": r["median_e2e_latency_ms"],
+}
+json.dump(norm, open(dest, "w"), indent=2)
+print("normalized ->", dest)
+PY
+    PROCESSED=$((PROCESSED + 1))
+done
+
+# Propagate a benchmark/allocation failure even though we emitted partial
+# results above (the workflow uploads them with `always()`).
+if [[ "$SALLOC_RC" -ne 0 ]]; then
+    echo "ERROR: benchmark failed (rc=$SALLOC_RC); emitted $PROCESSED partial result file(s)." >&2
+    exit "$SALLOC_RC"
+fi
+if [[ "$PROCESSED" -eq 0 ]]; then
+    echo "ERROR: no result files produced" >&2
+    exit 1
+fi
+echo "Done. $PROCESSED result file(s) in $GITHUB_WORKSPACE."

scripts/ci/slurm/log_analysis_prompt.md

@@ -0,0 +1,230 @@
+# srtslurm Log Analysis
+
+You are an automated CI failure analyst. Your job is to analyze logs from a
+failed srtslurm job, determine the root cause, and **take action** by filing
+GitHub issues when the cause is clear.
+
+srtslurm is a Python-first orchestration framework for running distributed LLM
+inference benchmarks on SLURM clusters using SGLang and TRTLLM backends.
+
+## Architecture
+
+There are two repos involved:
+
+- **`NVIDIA/srt-slurm`**: The orchestration layer. It owns recipes (YAML configs)
+  that define which flags, environment variables, and topology to use when
+  launching SGLang workers. It controls `srtctl`, worker lifecycle, health
+  checks, and benchmark execution.
+- **`sgl-project/sglang`**: The inference engine. It owns the server, model
+  loading, CUDA kernels, MoE routing, attention backends, and all runtime code.
+
+When a recipe passes flags that SGLang doesn't support together, **that is a
+recipe bug in srt-slurm**, not an sglang bug — even though the error appears in
+SGLang code. The recipe is responsible for only requesting valid combinations.
+
+## Step 1: Read Logs
+
+List the directory contents, then read files in this priority order:
+
+### 1. `sweep_{job_id}.log`
+
+Read this first. It is the orchestration timeline.
+
+Look for:
+- stage transitions
+- worker readiness
+- benchmark start
+- exit codes
+- the last error before teardown
+
+### 2. `config.yaml`
+
+Read this to understand the flags being passed to workers. Pay close attention
+to flags on prefill vs decode workers — they often differ and mismatches are a
+common source of bugs.
+
+### 3. `benchmark.out`
+
+If present, this usually contains the benchmark-side exception or timeout.
+
+### 4. `artifacts/*/logs/aiperf_*.log`
+
+If present, these often contain framework-level initialization failures and
+HTTP/network issues.
+
+### 5. Worker logs
+
+Focus on errors that line up with the failure timestamp:
+- `{node}_prefill_w{N}.out`
+- `{node}_decode_w{N}.out`
+- `{node}_frontend_{N}.out`
+
+### 6. `infra.out`
+
+Use this to confirm infrastructure failures involving NATS, etcd, ports, or
+service health checks.
+
+## Step 2: Correlate Timestamps
+
+This is the most important analysis technique.
+
+Many warnings are harmless. The root cause is usually the error that occurs at
+the same time the orchestration log transitions into failure.
+
+1. Find the failure time in `sweep_{job_id}.log`.
+2. Search other logs for matching timestamps.
+3. Ignore earlier warnings if the job continued past them.
+4. Ignore cleanup/teardown errors — they are consequences, not causes.
+
+## Step 3: Classify the Failure
+
+Determine which category the failure falls into:
+
+### Category A: Recipe/Config Bug → file against `NVIDIA/srt-slurm`
+
+The recipe or config is passing invalid or incompatible flags to SGLang. Examples:
+- Incompatible flag combinations (e.g., `--moe-a2a-backend deepep` with
+  `--fp4-gemm-backend flashinfer_cutedsl` when no fused func exists for that pair)
+- Wrong environment variables for the topology
+- Incorrect worker counts, GPU assignments, or port configs
+- srtctl bugs, health check misconfigurations, orchestration logic errors
+
+**Key signal**: The error is in SGLang code but the `config.yaml` shows the
+recipe chose a flag combination that SGLang doesn't support. The fix belongs in
+the recipe, not in SGLang.
+
+### Category B: SGLang Bug → list suspect PRs (do NOT auto-file)
+
+A genuine bug in SGLang's runtime code. Examples:
+- CUDA OOM, NCCL timeout, or kernel crash with valid flags
+- Model loading failure for a supported model
+- Regression introduced by a recent commit
+
+For these, use `gh` to find recent commits:
+```
+gh api "repos/sgl-project/sglang/commits?since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)&per_page=50" --jq '.[] | "\(.sha[:8]) \(.commit.message | split("\n")[0])"'
+```
+Then check which files each suspect commit touched:
+```
+gh api repos/sgl-project/sglang/commits/<sha> --jq '.files[].filename'
+```
+List suspect PRs in the report. Do NOT auto-file issues against sglang.
+
+### Category C: Infra/Transient → do NOT file any issue
+
+Flaky infrastructure, transient network issues, SLURM scheduling problems.
+Just note it in the report.
+
+## Step 4: Write the Report
+
+Write the report to `/workspace/logs/ai_analysis.md`. This is mandatory.
+
+Use this structure:
+
+```markdown
+## Job Analysis: {job_id}
+
+### Root Cause
+One clear sentence. State the category (A/B/C) and which repo owns the fix.
+
+### Evidence
+- `file:line` — exact error text
+- `config.yaml` — the relevant flags that caused or contributed to the failure
+- Timestamps showing correlation
+
+### Timeline
+| Time | Event |
+|------|-------|
+| ... | ... |
+
+### Noise
+- Warnings that were NOT causal (and why)
+
+### Suspect PRs (sglang)
+(Only for Category B failures)
+- PR #NNNN: "title" — why this commit could be related based on files changed
+
+### Recommended Fix
+Concrete, actionable steps. Not generic advice. Reference specific files,
+flags, or config values that need to change.
+```
+
+## Step 5: File Issues
+
+This step is **mandatory** for Category A and Category B failures. You MUST
+take action — the whole point of this system is to create issues so humans
+can track and fix problems.
+
+### For Category A (recipe/config bugs) → file against `NVIDIA/srt-slurm`
+
+1. First, check for duplicates:
+   ```
+   gh issue list --repo NVIDIA/srt-slurm --search "<key error message>" --limit 5
+   ```
+2. If no duplicate exists, file the issue:
+   ```
+   gh issue create --repo NVIDIA/srt-slurm \
+     --title "<concise title>" \
+     --body "<body>"
+   ```
+
+The issue body MUST include:
+- **Summary**: One sentence describing the failure
+- **Error**: The exact error message and which log file/line it came from
+- **Config**: The relevant flags from `config.yaml` that caused the issue
+- **Job**: The job ID and model/precision/topology
+- **Suggested Fix**: What the recipe should change (e.g., "change
+  `moe-runner-backend` from `flashinfer_cutedsl` to `flashinfer_cutlass`
+  when `moe-a2a-backend` is `deepep`", or "add validation to reject this
+  combination")
+
+### For Category B (sglang bugs) → file against `sgl-project/sglang`
+
+1. First, check for duplicates:
+   ```
+   gh issue list --repo sgl-project/sglang --search "<key error message>" --limit 5
+   ```
+2. If no duplicate exists, file the issue:
+   ```
+   gh issue create --repo sgl-project/sglang \
+     --title "<concise title>" \
+     --body "<body>"
+   ```
+
+The issue body MUST include:
+- **Summary**: One sentence describing the failure
+- **Error**: The exact error message, traceback, and which log file it came from
+- **Repro context**: Model, precision, topology, relevant flags from `config.yaml`
+- **Suspect commits**: List any recent commits that may have caused this, with
+  links (e.g., `https://github.com/sgl-project/sglang/commit/<sha>`)
+- **Suggested Fix**: If you can identify the fix from reading the sglang source
+  in `/workspace/repos/sglang/`, include it. Otherwise, describe what needs to
+  change conceptually.
+
+### For Category C (infra/transient) → do NOT file any issue
+
+Just include the analysis in the report.
+
+## Common Signal Reference
+
+High-signal failures:
+- `NotImplementedError` with runner/backend combinations → Category A
+- `ReadTimeout` / `Connection refused` during benchmark → check if config-caused
+- `CUDA out of memory` → likely Category B (unless config requests too many GPUs)
+- `NCCL timeout` → could be B or C, check if topology is valid
+- `Model not found` → check if recipe has correct model path
+- Benchmark exit code failures → check benchmark.out for details
+
+Low-signal noise (ignore these):
+- dependency resolver warnings
+- cleanup warnings during teardown
+- keep-alive failures AFTER the main crash
+- import warnings unrelated to the active model
+- `pip`/`rustup`/`apt-get` warnings during setup
+
+## Safety
+
+- Do NOT include API keys, tokens, or secrets in issues or the report.
+- Do NOT file issues if you are uncertain about the root cause. Only file when
+  you have concrete evidence.
+- Do NOT file duplicate issues. Always search first.

scripts/ci/slurm/nightly-configs.yaml

@@ -0,0 +1,358 @@
+# Nightly benchmark configurations for srt-slurm powered runners.
+#
+# Structure mirrors InferenceX nvidia-master.yaml but only includes fields
+# actually needed by the runner — prefill/decode topology details are already
+# encoded in each srt-slurm recipe YAML and are not duplicated here.
+#
+# To add/remove concurrencies: edit conc-list for the relevant search-space entry.
+# To add a new runner:         add a new top-level block and create a corresponding
+#                              nightly-test-<runner>.yml workflow.
+# Never edit workflow YAML files directly for these changes.
+
+dsr1-fp8-gb200-dynamo-sglang:
+  model: deepseek-ai/DeepSeek-R1-0528
+  model-prefix: dsr1
+  runner: gb200
+  precision: fp8
+  framework: dynamo-sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1024, 2048, 4096, 6144]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp8/1k1k/max-tpt.yaml
+          config_file: recipes/gb200-fp8/1k1k/max-tpt.yaml
+
+        - conc-list: [4096]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp8/1k1k/ultra-tpt.yaml
+          config_file: recipes/gb200-fp8/1k1k/ultra-tpt.yaml
+
+dsr1-fp4-gb200-dynamo-sglang:
+  model: nvidia/DeepSeek-R1-0528-NVFP4-v2
+  model-prefix: dsr1
+  runner: gb200
+  precision: fp4
+  framework: dynamo-sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [512, 2048, 4096, 8192]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp4/1k1k/mid-curve.yaml
+          config_file: recipes/gb200-fp4/1k1k/mid-curve.yaml
+
+# AMD MI355X 2-node 1P1D disaggregation. Driven by
+# scripts/ci/slurm/launch_mi355x.sh, which reads each recipe's `runtime`,
+# `bench`, and `bench.accuracy` sections. Every nightly runs ALL four
+# DeepSeek-V4 model x precision combos below (full matrix; GitHub
+# strategy.matrix.config, fail-fast: false). Each runs a GSM8K accuracy
+# hard-gate before the perf sweep.
+#
+# model_path points at the shared NFS HuggingFace cache dir (models--org--name);
+# the launcher resolves the live snapshot via refs/main, so no hash is hardcoded.
+dsv4flash-fp8-mi355x-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml
+
+dsv4pro-fp8-mi355x-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d.yaml
+
+dsv4flash-fp4-mi355x-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml
+
+dsv4pro-fp4-mi355x-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d.yaml
+
+# AMD MI355X 2-node 1P1D disaggregation over MORI with extra topology / MTP
+# coverage on the SAME four DeepSeek-V4 model x precision combos as the base
+# MORI blocks above (which stay TP8, no MTP). Three variants per model:
+#   * -mtp        : TP8 + EAGLE MTP (recipe `mtp.enabled`)
+#   * -dp8ep8     : DP-attention 8 + narrow within-node EP8
+#   * -dp8ep8-mtp : DP8 + narrow EP8 + EAGLE MTP
+# launch_mi355x.sh reads expert-/data-parallel-size and the `mtp:` section to
+# append --ep-size / --enable-dp-attention --dp-size / --speculative-* flags.
+
+dsv4flash-fp8-mi355x-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml
+
+dsv4flash-fp8-mi355x-dp8ep8-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml
+
+dsv4flash-fp8-mi355x-dp8ep8-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml
+
+dsv4pro-fp8-mi355x-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        # conc256 excluded: disagg-decode SWA hybrid pool retract->get_cpu_copy
+        # is an upstream NotImplementedError (crashes decode). See recipe.
+        - conc-list: [1, 8, 16, 32, 64, 128]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-mtp.yaml
+
+dsv4pro-fp8-mi355x-dp8ep8-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8.yaml
+
+dsv4pro-fp8-mi355x-dp8ep8-mtp-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml
+
+dsv4flash-fp4-mi355x-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml
+
+dsv4flash-fp4-mi355x-dp8ep8-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml
+
+dsv4flash-fp4-mi355x-dp8ep8-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml
+
+dsv4pro-fp4-mi355x-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-mtp.yaml
+
+dsv4pro-fp4-mi355x-dp8ep8-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8.yaml
+
+dsv4pro-fp4-mi355x-dp8ep8-mtp-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml
+
+# Kimi-K2.6 (FP8) 2-node 1P1D. Demonstrates the launcher is model-agnostic: all
+# Kimi-specific config lives in the recipe's `model:` block + split attention
+# runtime, with nothing hardcoded in launch_mi355x.sh. Base + EAGLE3 MTP (the
+# MTP leg uses an external draft checkpoint via mtp.draft_model_path).
+kimik26-fp8-mi355x-sglang:
+  model: moonshotai/Kimi-K2.6
+  model-prefix: kimik26
+  model_path: /it-share/model_coverage/models--moonshotai--Kimi-K2.6
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d.yaml
+
+kimik26-fp8-mi355x-mtp-sglang:
+  model: moonshotai/Kimi-K2.6
+  model-prefix: kimik26
+  model_path: /it-share/model_coverage/models--moonshotai--Kimi-K2.6
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml

scripts/ci/slurm/process_result.py

@@ -0,0 +1,117 @@
+"""Process a raw srt-slurm benchmark result JSON into an aggregated format.
+
+Usage (called once per result file):
+    RESULT_FILENAME=<path_without_.json> PREFILL_GPUS=<n> DECODE_GPUS=<n> \\
+        RECIPE_FILE=<path_to_recipe.yaml> python3 process_result.py
+
+Required env vars:
+    RESULT_FILENAME   - path to the result file without the .json extension
+    FRAMEWORK         - e.g. dynamo-sglang
+    PRECISION         - e.g. fp8, fp4
+    MODEL_PREFIX      - short model label, e.g. dsr1
+    ISL               - input sequence length
+    OSL               - output sequence length
+    PREFILL_GPUS      - number of prefill GPUs (extracted from result filename)
+    DECODE_GPUS       - number of decode GPUs (extracted from result filename)
+
+Optional env vars:
+    RECIPE_FILE       - path to the srt-slurm recipe YAML; if set, topology
+                        fields (TP, EP, DP, workers) are parsed from it
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def require(var):
+    val = os.environ.get(var)
+    if val is None:
+        print(f"ERROR: Missing required env var: {var}", file=sys.stderr)
+        sys.exit(1)
+    return val
+
+
+result_filename = require("RESULT_FILENAME")
+framework = require("FRAMEWORK")
+precision = require("PRECISION")
+model_prefix = require("MODEL_PREFIX")
+isl = int(require("ISL"))
+osl = int(require("OSL"))
+prefill_gpus = int(require("PREFILL_GPUS"))
+decode_gpus = int(require("DECODE_GPUS"))
+
+with open(f"{result_filename}.json") as f:
+    raw = json.load(f)
+
+# ---------------------------------------------------------------------------
+# Topology — parse from recipe YAML if available, otherwise default to 0/"-"
+# ---------------------------------------------------------------------------
+prefill_tp = prefill_ep = prefill_dp_attn = 0
+prefill_num_workers = decode_tp = decode_ep = decode_dp_attn = decode_num_workers = 0
+
+recipe_file = os.environ.get("RECIPE_FILE")
+if recipe_file and Path(recipe_file).exists():
+    import yaml
+
+    with open(recipe_file) as f:
+        recipe = yaml.safe_load(f)
+
+    res = recipe.get("resources", {})
+    prefill_num_workers = res.get("prefill_workers", 0)
+    decode_num_workers = res.get("decode_workers", 0)
+
+    sgl = recipe.get("backend", {}).get("sglang_config", {})
+    p = sgl.get("prefill", {})
+    d = sgl.get("decode", {})
+
+    prefill_tp = p.get("tensor-parallel-size", 0)
+    prefill_ep = p.get("expert-parallel-size", 0)
+    prefill_dp_attn = p.get("data-parallel-size", "-")
+    decode_tp = d.get("tensor-parallel-size", 0)
+    decode_ep = d.get("expert-parallel-size", 0)
+    decode_dp_attn = d.get("data-parallel-size", "-")
+
+total_gpus = prefill_gpus + decode_gpus
+
+data = {
+    "hw": os.environ.get("HW", "gb200"),
+    "conc": int(raw["max_concurrency"]),
+    "model": raw["model_id"],
+    "infmax_model_prefix": model_prefix,
+    "framework": framework,
+    "precision": precision,
+    "isl": isl,
+    "osl": osl,
+    "is_multinode": True,
+    "disagg": True,
+    "num_prefill_gpu": prefill_gpus,
+    "num_decode_gpu": decode_gpus,
+    "prefill_num_workers": prefill_num_workers,
+    "prefill_tp": prefill_tp,
+    "prefill_ep": prefill_ep,
+    "prefill_dp_attention": prefill_dp_attn,
+    "decode_num_workers": decode_num_workers,
+    "decode_tp": decode_tp,
+    "decode_ep": decode_ep,
+    "decode_dp_attention": decode_dp_attn,
+    "tput_per_gpu": float(raw["total_token_throughput"]) / total_gpus,
+    "output_tput_per_gpu": float(raw["output_throughput"]) / decode_gpus,
+    "input_tput_per_gpu": (
+        float(raw["total_token_throughput"]) - float(raw["output_throughput"])
+    )
+    / prefill_gpus,
+}
+
+for key, value in raw.items():
+    if key.endswith("_ms"):
+        data[key.replace("_ms", "")] = float(value) / 1000.0
+    if "tpot" in key:
+        data[key.replace("_ms", "").replace("tpot", "intvty")] = 1000.0 / float(value)
+
+out_path = Path(result_filename).parent / f"agg_{Path(result_filename).name}.json"
+with open(out_path, "w") as f:
+    json.dump(data, f, indent=2)
+
+print(f"Written: {out_path}")

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,53 @@
+# MI355X DeepSeek-V4-Flash FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash FP4 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml

@@ -0,0 +1,55 @@
+# MI355X DeepSeek-V4-Flash (FP4) 2-node 1P1D disaggregation recipe.
+#
+# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,61 @@
+# MI355X DeepSeek-V4-Pro FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,52 @@
+# MI355X DeepSeek-V4-Pro FP4 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,61 @@
+# MI355X DeepSeek-V4-Pro FP4 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d.yaml

@@ -0,0 +1,56 @@
+# MI355X DeepSeek-V4-Pro (FP4) 2-node 1P1D disaggregation recipe.
+#
+# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
+# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
+# larger, so mem_fraction_static / max_running_requests may need tuning.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,53 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,62 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml

@@ -0,0 +1,54 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  max_total_tokens: 8551168
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path after the perf sweep. Mirrors the
+  # single-node registered test test/registered/amd/test_deepseek_v4_flash_fp8.py
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8-mtp.yaml

@@ -0,0 +1,61 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-dp8ep8.yaml

@@ -0,0 +1,52 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe — DP8 + narrow EP8.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 8
+      data-parallel-size: 8
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,68 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe — TP8 + MTP.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, and `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+# MTP / EAGLE speculative decoding (NextN head from the base model). Applied to
+# both prefill and decode. Flags mirror
+# test/registered/amd/test_deepseek_v4_pro_fp4_mtp.py.
+mtp:
+  enabled: true
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  # conc256 is excluded for this leg only: at conc256 the disagg-decode SWA
+  # hybrid KV pool fills, the scheduler retracts running requests, and the
+  # retract->offload_kv_cache path calls get_cpu_copy() which is unimplemented
+  # for the SWA hybrid pool (raises NotImplementedError, crashes the decode
+  # scheduler). Raising swa_full_tokens_ratio (tried up to 0.3) does not help
+  # -- usage climbs to fill the larger budget and still retracts. Until the
+  # upstream get_cpu_copy stub is implemented, cap this leg at 128.
+  concurrencies: [1, 8, 16, 32, 64, 128]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep (full GSM8K,
+  # 8-shot, accuracy > 0.91). A regression here fails the nightly even when
+  # throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d.yaml

@@ -0,0 +1,55 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe.
+#
+# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
+# larger, so mem_fraction_static / max_running_requests may need tuning.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml

@@ -0,0 +1,85 @@
+# MI355X Kimi-K2.6 (FP8) 2-node 1P1D disaggregation recipe (base + EAGLE3 MTP).
+#
+# Self-contained (no inheritance): same as 1p1d.yaml plus the `mtp:` block. Uses
+# EAGLE3 speculative decoding with an EXTERNAL draft checkpoint (unlike DSV4's
+# built-in EAGLE NextN head): the launcher resolves mtp.draft_model_path through
+# the HF-cache snapshot logic and appends --speculative-draft-model-path. The
+# draft dir must live under /it-share (the container's :ro mount). MTP is applied
+# to both prefill and decode. Mirrors the Kimi-K2.6 serving cookbook.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, `model`, `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+# Model-specific docker env + sglang server args (written verbatim via
+# model_flags.sh). Each server arg + value is a SEPARATE list item.
+model:
+  env:
+    SGLANG_USE_AITER: 1
+    SGLANG_ROCM_FUSED_DECODE_MLA: 0
+  server_args:
+    - --model-loader-extra-config
+    - '{"enable_multithread_load": true}'
+    - --watchdog-timeout
+    - 1200
+    - --reasoning-parser
+    - kimi_k2
+    - --tool-call-parser
+    - kimi_k2
+
+# EAGLE3 speculative decoding with an external draft checkpoint.
+mtp:
+  enabled: true
+  algorithm: EAGLE3
+  num_steps: 3
+  eagle_topk: 1
+  num_draft_tokens: 4
+  draft_model_path: /it-share/model_coverage/models--lightseekorg--kimi-k2.6-eagle3.1-mla
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  # Kimi uses split attention backends (aiter prefill / triton decode), not a
+  # single --attention-backend.
+  prefill_attention_backend: aiter
+  decode_attention_backend: triton
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep. Mirrors the
+  # registered single-node Kimi-K2.6 eval (full GSM8K, 8-shot, accuracy > 0.92).
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.92

scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d.yaml

@@ -0,0 +1,75 @@
+# MI355X Kimi-K2.6 (FP8) 2-node 1P1D disaggregation recipe (base).
+#
+# All Kimi-specific config lives in this recipe's `model:` block (docker env +
+# sglang server args) and `runtime` (split attention backends); nothing about
+# Kimi is hardcoded in launch_mi355x.sh. Mirrors the single-node registered test
+# test/registered/amd/accuracy/mi35x/test_kimi_k26_eval_mi35x.py (TP8, split
+# attention backends, multithread loader, GSM8K > 0.92).
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime`, `bench`, `model`, `mtp`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+# Model-specific docker env + sglang server args (written verbatim via
+# model_flags.sh). Each server arg + value is a SEPARATE list item.
+model:
+  env:
+    SGLANG_USE_AITER: 1
+    SGLANG_ROCM_FUSED_DECODE_MLA: 0
+  server_args:
+    - --model-loader-extra-config
+    - '{"enable_multithread_load": true}'
+    - --watchdog-timeout
+    - 1200
+    - --reasoning-parser
+    - kimi_k2
+    - --tool-call-parser
+    - kimi_k2
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  # Kimi uses split attention backends (aiter prefill / triton decode), not a
+  # single --attention-backend.
+  prefill_attention_backend: aiter
+  decode_attention_backend: triton
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep. Mirrors the
+  # registered single-node Kimi-K2.6 eval (full GSM8K, 8-shot, accuracy > 0.92).
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.92

scripts/ci/slurm/summarize.py

@@ -0,0 +1,128 @@
+"""Print a markdown summary table from processed benchmark results.
+
+Usage:
+    python3 summarize.py <results_dir>
+
+Reads all agg_*.json files recursively from <results_dir> and prints a
+markdown table to stdout (redirect to $GITHUB_STEP_SUMMARY to publish).
+"""
+
+import json
+import sys
+from pathlib import Path
+
+from tabulate import tabulate
+
+HEADERS = [
+    "Model",
+    "Served Model",
+    "Hardware",
+    "Framework",
+    "Precision",
+    "ISL",
+    "OSL",
+    "Prefill TP",
+    "Prefill EP",
+    "Prefill DP Attn",
+    "Prefill Workers",
+    "Prefill GPUs",
+    "Decode TP",
+    "Decode EP",
+    "Decode DP Attn",
+    "Decode Workers",
+    "Decode GPUs",
+    "Conc",
+    "TTFT (ms)",
+    "TPOT (ms)",
+    "Interactivity (tok/s/user)",
+    "E2EL (s)",
+    "TPUT per GPU",
+    "Output TPUT per GPU",
+    "Input TPUT per GPU",
+]
+
+
+def load_json(path):
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except Exception:
+        return None
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python3 summarize.py <results_dir>")
+        sys.exit(1)
+
+    results_dir = Path(sys.argv[1])
+    results = [
+        r
+        for path in results_dir.rglob("agg_*.json")
+        if (r := load_json(path)) and "is_multinode" in r
+    ]
+
+    if not results:
+        print("No processed result files found.")
+        return
+
+    results.sort(
+        key=lambda r: (
+            r["infmax_model_prefix"],
+            r["hw"],
+            r["framework"],
+            r["precision"],
+            r["isl"],
+            r["osl"],
+            r["prefill_tp"],
+            r["prefill_ep"],
+            r["decode_tp"],
+            r["decode_ep"],
+            r["conc"],
+        )
+    )
+
+    rows = [
+        [
+            r["infmax_model_prefix"],
+            r["model"],
+            r["hw"].upper(),
+            r["framework"].upper(),
+            r["precision"].upper(),
+            r["isl"],
+            r["osl"],
+            r["prefill_tp"],
+            r["prefill_ep"],
+            r["prefill_dp_attention"],
+            r["prefill_num_workers"],
+            r["num_prefill_gpu"],
+            r["decode_tp"],
+            r["decode_ep"],
+            r["decode_dp_attention"],
+            r["decode_num_workers"],
+            r["num_decode_gpu"],
+            r["conc"],
+            f"{r['median_ttft'] * 1000:.4f}",
+            f"{r['median_tpot'] * 1000:.4f}",
+            f"{r['median_intvty']:.4f}",
+            f"{r['median_e2el']:.4f}",
+            f"{r['tput_per_gpu']:.4f}",
+            f"{r['output_tput_per_gpu']:.4f}",
+            f"{r['input_tput_per_gpu']:.4f}",
+        ]
+        for r in results
+    ]
+
+    hw_label = "/".join(sorted({r["hw"].upper() for r in results}))
+    if hw_label == "GB200":
+        # NVIDIA GB200 nightly: keep the original hardcoded title untouched.
+        print("## GB200 Nightly Benchmark Results\n")
+    else:
+        # AMD (e.g. MI355X) nightly: derive the title from the result hardware.
+        print(f"## {hw_label} Nightly Benchmark Results\n")
+    print(tabulate(rows, headers=HEADERS, tablefmt="github"))
+    print()
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/stage_models_overrides.json

@@ -0,0 +1,31 @@
+{
+  "_comment": "Manual overrides for list_stage_models.py. by_file/by_suite ADD models the static scan cannot see (models built dynamically, read from configs, passed via CLI args). deny REMOVES false-positive ids the heuristic mistakes for models. Keys in by_file are repo-relative test paths (e.g. test/registered/foo/test_bar.py). suite_labels maps legacy suite= registrations (no runner_config) to the GH runner label(s) their dispatching workflow hardcodes in runs-on -- a list, because one suite can run on several labels (nightly-8-gpu-common). $b200_runner is the dynamic-b200 placeholder from runner_configs.yml. Deliberately absent: nightly-4-gpu-gb300-* (run as k8s pods, not GHA runners) and nightly-2-gpu (registered but dispatched by no workflow); both stay visible in unmapped_suites.",
+  "by_file": {},
+  "by_suite": {},
+  "suite_labels": {
+    "base-b-kernel-benchmark-1-gpu-large": ["1-gpu-h100"],
+    "base-b-kernel-unit-1-gpu-b200": ["$b200_runner"],
+    "base-b-kernel-unit-1-gpu-large": ["1-gpu-h100"],
+    "base-b-kernel-unit-8-gpu-h200": ["8-gpu-h200"],
+    "nightly-1-gpu": ["1-gpu-h100"],
+    "nightly-4-gpu": ["4-gpu-h100"],
+    "nightly-4-gpu-b200": ["$b200_runner"],
+    "nightly-8-gpu-b200": ["8-gpu-b200"],
+    "nightly-8-gpu-common": ["8-gpu-h200", "8-gpu-b200"],
+    "nightly-8-gpu-h200": ["8-gpu-h200"],
+    "nightly-eval-text-2-gpu": ["2-gpu-h100"],
+    "nightly-eval-vlm-2-gpu": ["2-gpu-h100"],
+    "nightly-kernel-1-gpu": ["1-gpu-h100"],
+    "nightly-kernel-8-gpu-h200": ["8-gpu-h200"],
+    "nightly-perf-text-2-gpu": ["2-gpu-h100"],
+    "nightly-perf-vlm-2-gpu": ["2-gpu-h100"],
+    "nightly-precision-8-gpu-h200": ["8-gpu-h200"],
+    "stress": ["8-gpu-h200"],
+    "weekly-8-gpu-h200": ["8-gpu-h200"]
+  },
+  "deny": [
+    "tok/req",
+    "qwen/qwen3",
+    "qwen/qwen3-vl"
+  ]
+}

scripts/ci/test_list_stage_models.py

@@ -0,0 +1,602 @@
+"""Unit tests for list_stage_models extraction logic.
+
+Pure-logic tests (stdlib only, no GPU, no sglang import) so they run in the
+ci-model-inventory workflow without installing dependencies:
+
+    python -m unittest discover -s scripts/ci -p 'test_list_stage_models.py'
+
+Guards the recall/precision contract of the static model extractor: model-id
+shape filtering, constant-table resolution (incl. tuple values), inline +
+name-reference extraction, override merge semantics, and the suite->files +
+inventory assembly that drives cache-warming.
+"""
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import list_stage_models as lsm  # noqa: E402
+
+# Repo root inferred from this file's location: <root>/scripts/ci/<this>.
+_REPO_ROOT = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+_REAL_CI_REGISTER = os.path.join(
+    _REPO_ROOT, "python", "sglang", "test", "ci", "ci_register.py"
+)
+
+
+def _write(root, rel, content):
+    path = os.path.join(root, rel)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(content)
+
+
+def _make_fake_repo(root, registered, helpers=None):
+    """Build a temp repo: copy the real ci_register.py, write test + helper files.
+
+    ``registered`` maps ``test/registered/...`` relpaths to file content;
+    ``helpers`` maps ``python/sglang/test/...`` relpaths to content.
+    """
+    dst = os.path.join(root, "python", "sglang", "test", "ci", "ci_register.py")
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+    shutil.copy(_REAL_CI_REGISTER, dst)
+    for rel, content in registered.items():
+        _write(root, os.path.join("test", "registered", rel), content)
+    for rel, content in (helpers or {}).items():
+        _write(root, os.path.join("python", "sglang", "test", rel), content)
+
+
+class LooksLikeModelId(unittest.TestCase):
+    def test_accepts_real_model_ids(self):
+        for value in (
+            "meta-llama/Llama-3.1-8B-Instruct",
+            "RedHatAI/Llama-3.2-3B-quantized.w8a8",  # dotted suffix is real
+            "cross-encoder/ms-marco-MiniLM-L6-v2",
+            "nvidia/DeepSeek-V3-0324-FP4",
+            "lmsys/sglang-ci-dsv3-test",
+            "Qwen/Qwen2-1.5B-Instruct-GGUF",  # -GGUF suffix, not .gguf extension
+        ):
+            self.assertTrue(lsm.looks_like_model_id(value), value)
+
+    def test_rejects_non_models(self):
+        for value in (
+            "text/plain",  # MIME
+            "application/json",  # MIME
+            "image/png",  # MIME
+            "Text/Plain",  # MIME, case-insensitive
+            "2/3",  # numeric ratio, no letters
+            "N/A",  # single-char sides
+            "configs/model.json",  # path with file extension
+            "org/weights.safetensors",  # weight-file extension
+            "org/model.bin",  # weight-file extension
+            "a/b/c",  # too many slashes
+            "/abs/path",  # leading slash
+            "./relative",  # leading dot
+            "just-a-string",  # no slash
+        ):
+            self.assertFalse(lsm.looks_like_model_id(value), value)
+
+    def test_deny_set_overrides(self):
+        value = "org/looks-like-a-model"
+        self.assertTrue(lsm.looks_like_model_id(value))
+        self.assertFalse(lsm.looks_like_model_id(value, deny={value}))
+
+
+class ConstantTable(unittest.TestCase):
+    def test_single_and_tuple_values(self):
+        source = (
+            'DEFAULT_MODEL = "meta-llama/Llama-3.1-8B-Instruct"\n'
+            'PAIR = ("OPEA/Qwen2.5-0.5B-int4", "Intel/Qwen2-0.5B-int4")\n'
+            'NOT_A_MODEL = "https://example.com/x"\n'
+            "PORT = 30000\n"
+        )
+        table = lsm.extract_constants_from_source(source)
+        self.assertEqual(table["DEFAULT_MODEL"], {"meta-llama/Llama-3.1-8B-Instruct"})
+        self.assertEqual(
+            table["PAIR"], {"OPEA/Qwen2.5-0.5B-int4", "Intel/Qwen2-0.5B-int4"}
+        )
+        self.assertNotIn("NOT_A_MODEL", table)
+        self.assertNotIn("PORT", table)
+
+    def test_annotated_assignment(self):
+        source = 'DEFAULT: str = "meta-llama/Llama-3.1-8B-Instruct"\n'
+        table = lsm.extract_constants_from_source(source)
+        self.assertEqual(table["DEFAULT"], {"meta-llama/Llama-3.1-8B-Instruct"})
+
+    def test_implicit_concatenation(self):
+        # Python folds adjacent string literals at parse time into one Constant.
+        source = 'M = (\n    "meta-llama/"\n    "Llama-3.1-8B-Instruct"\n)\n'
+        table = lsm.extract_constants_from_source(source)
+        self.assertEqual(table["M"], {"meta-llama/Llama-3.1-8B-Instruct"})
+
+    def test_deny_excludes_from_constant_table(self):
+        source = 'BAD = "weird/thing"\nGOOD = "meta-llama/Llama-3.1-8B-Instruct"\n'
+        table = lsm.extract_constants_from_source(source, deny={"weird/thing"})
+        self.assertNotIn("BAD", table)
+        self.assertIn("GOOD", table)
+
+
+class ExtractModels(unittest.TestCase):
+    def test_inline_and_name_reference(self):
+        const_table = {
+            "DEFAULT_MODEL_NAME_FOR_TEST": {"meta-llama/Llama-3.1-8B-Instruct"}
+        }
+        source = (
+            "from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST\n"
+            "class T:\n"
+            "    model = DEFAULT_MODEL_NAME_FOR_TEST\n"
+            '    draft = "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B"\n'
+        )
+        models = lsm.extract_models_from_source(source, const_table)
+        self.assertEqual(
+            models,
+            {
+                "meta-llama/Llama-3.1-8B-Instruct",
+                "lmsys/sglang-EAGLE3-LLaMA3.1-Instruct-8B",
+            },
+        )
+
+    def test_fstring_fragments_are_skipped(self):
+        # An f-string with a placeholder yields a truncated, non-existent id;
+        # it must NOT be picked up.
+        source = 'msg = f"meta-llama/Llama-3.1-8B-Instruct-{suffix}"\n'
+        self.assertEqual(lsm.extract_models_from_source(source, {}), set())
+
+    def test_fstring_name_placeholder_still_resolves(self):
+        # The literal fragment is skipped, but a constant referenced inside the
+        # f-string placeholder is still resolved.
+        const_table = {"DRAFT": {"lmsys/sglang-EAGLE-llama2-chat-7B"}}
+        source = 'path = f"prefix/{DRAFT}/topk"\n'
+        self.assertEqual(
+            lsm.extract_models_from_source(source, const_table),
+            {"lmsys/sglang-EAGLE-llama2-chat-7B"},
+        )
+
+    def test_model_less_file_yields_nothing(self):
+        source = (
+            "import torch\n"
+            "def test_kernel():\n"
+            "    assert torch.cuda.is_available() or True\n"
+        )
+        self.assertEqual(lsm.extract_models_from_source(source, {}), set())
+
+    def test_deny_removes_name_resolved_model(self):
+        # deny must apply to constant-resolved ids too, not just inline literals.
+        const_table = {"M": {"weird/thing"}}
+        source = "x = M\n"
+        self.assertIn(
+            "weird/thing", lsm.extract_models_from_source(source, const_table)
+        )
+        self.assertEqual(
+            lsm.extract_models_from_source(source, const_table, deny={"weird/thing"}),
+            set(),
+        )
+
+
+class Overrides(unittest.TestCase):
+    def test_load_defaults_when_missing(self):
+        ov = lsm.load_overrides(None)
+        self.assertEqual(
+            ov, {"by_file": {}, "by_suite": {}, "deny": [], "suite_labels": {}}
+        )
+
+    def test_null_values_fall_back_to_defaults(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "ov.json")
+            with open(path, "w", encoding="utf-8") as f:
+                f.write('{"deny": null, "by_file": null}')
+            ov = lsm.load_overrides(path)
+            self.assertEqual(
+                ov, {"by_file": {}, "by_suite": {}, "deny": [], "suite_labels": {}}
+            )
+
+
+class CollectSuiteFiles(unittest.TestCase):
+    REG = (
+        "import unittest\n"
+        "from sglang.test.ci.ci_register import register_cuda_ci, register_amd_ci\n"
+        "{calls}\n"
+        'MODEL = "{model}"\n'
+        'if __name__ == "__main__":\n    unittest.main()\n'
+    )
+
+    def _repo(self, tmp):
+        _make_fake_repo(
+            tmp,
+            registered={
+                # enabled CUDA, two suites in one file -> dedupe per suite
+                "a/test_a.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-x", '
+                        'runner_config="1-gpu")\n'
+                        'register_cuda_ci(est_time=1, suite="nightly-y", '
+                        "nightly=True)"
+                    ),
+                    model="meta-llama/Llama-3.1-8B-Instruct",
+                ),
+                # disabled CUDA -> excluded unless include_disabled
+                "b/test_b.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-z", '
+                        'runner_config="1-gpu", disabled="flaky")'
+                    ),
+                    model="Qwen/Qwen3-8B",
+                ),
+                # AMD only -> never in CUDA inventory
+                "c/test_c.py": self.REG.format(
+                    calls='register_amd_ci(est_time=1, suite="nightly-amd")',
+                    model="google/gemma-3-4b-it",
+                ),
+            },
+        )
+
+    def test_enabled_only_by_default(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            suites, nightly, errors, runner_configs = lsm.collect_suite_files(
+                tmp, "cuda"
+            )
+            self.assertEqual(set(suites), {"base-x-test-1-gpu", "nightly-y"})
+            self.assertEqual(errors, {})
+            self.assertTrue(nightly["nightly-y"])
+            self.assertFalse(nightly["base-x-test-1-gpu"])
+            # AMD suite never appears for the CUDA backend.
+            self.assertNotIn("nightly-amd", suites)
+            # Modern registrations carry their runner_config; legacy suite= has none.
+            self.assertEqual(runner_configs["base-x-test-1-gpu"], "1-gpu")
+            self.assertIsNone(runner_configs["nightly-y"])
+
+    def test_include_disabled(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            suites, _, _, _ = lsm.collect_suite_files(
+                tmp, "cuda", include_disabled=True
+            )
+            self.assertIn("base-z-test-1-gpu", suites)
+
+    def test_unparsable_registry_is_surfaced(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            _make_fake_repo(
+                tmp,
+                registered={
+                    # est_time missing -> RegistryVisitor raises ValueError
+                    "d/test_bad.py": (
+                        "from sglang.test.ci.ci_register import register_cuda_ci\n"
+                        'register_cuda_ci(suite="base-x")\n'
+                    ),
+                },
+            )
+            suites, _, errors, _ = lsm.collect_suite_files(tmp, "cuda")
+            self.assertEqual(suites, {})
+            self.assertIn("test/registered/d/test_bad.py", errors)
+
+
+class BuildInventory(unittest.TestCase):
+    def _repo(self, tmp):
+        _make_fake_repo(
+            tmp,
+            registered={
+                # resolves a model via an imported constant
+                "a/test_a.py": (
+                    "import unittest\n"
+                    "from sglang.test.ci.ci_register import register_cuda_ci\n"
+                    "from sglang.test.test_utils import DEFAULT_MODEL\n"
+                    'register_cuda_ci(est_time=1, stage="base-x", '
+                    'runner_config="1-gpu")\n'
+                    "MODEL = DEFAULT_MODEL\n"
+                    'if __name__ == "__main__":\n    unittest.main()\n'
+                ),
+                # model-less -> lands in unresolved_files (same suite as a)
+                "b/test_b.py": (
+                    "import unittest\n"
+                    "from sglang.test.ci.ci_register import register_cuda_ci\n"
+                    'register_cuda_ci(est_time=1, stage="base-x", '
+                    'runner_config="1-gpu")\n'
+                    'if __name__ == "__main__":\n    unittest.main()\n'
+                ),
+            },
+            helpers={"test_utils.py": 'DEFAULT_MODEL = "meta-llama/Llama-3.1-8B"\n'},
+        )
+
+    def test_resolution_and_unresolved(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            inv = lsm.build_inventory(tmp, "cuda", lsm.load_overrides(None), "sha123")
+            self.assertEqual(inv["generated_at_commit"], "sha123")
+            suite = inv["suites"]["base-x-test-1-gpu"]
+            self.assertEqual(suite["models"], ["meta-llama/Llama-3.1-8B"])
+            self.assertEqual(suite["test_file_count"], 2)
+            self.assertEqual(suite["unresolved_files"], ["test/registered/b/test_b.py"])
+            self.assertEqual(inv["model_count"], 1)
+            self.assertEqual(inv["parse_failures"], {})
+
+    def test_by_file_override_adds_and_clears_unresolved(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            overrides = {
+                "by_file": {"test/registered/b/test_b.py": ["org/extra-model"]},
+                "by_suite": {},
+                "deny": [],
+            }
+            inv = lsm.build_inventory(tmp, "cuda", overrides, "sha")
+            suite = inv["suites"]["base-x-test-1-gpu"]
+            self.assertIn("org/extra-model", suite["models"])
+            # by_file supplied a model for test_b -> no longer unresolved.
+            self.assertEqual(suite["unresolved_files"], [])
+
+    def test_by_suite_override_adds_model(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            overrides = {
+                "by_file": {},
+                "by_suite": {"base-x-test-1-gpu": ["org/suite-model"]},
+                "deny": [],
+            }
+            inv = lsm.build_inventory(tmp, "cuda", overrides, "sha")
+            self.assertIn(
+                "org/suite-model", inv["suites"]["base-x-test-1-gpu"]["models"]
+            )
+
+
+class RenderMarkdown(unittest.TestCase):
+    def test_table_and_glyphs(self):
+        inv = {
+            "backend": "cuda",
+            "generated_at_commit": "sha",
+            "suite_count": 2,
+            "model_count": 1,
+            "parse_failures": {},
+            "suites": {
+                "n-suite": {
+                    "nightly": True,
+                    "models": ["org/model"],
+                    "test_file_count": 1,
+                    "unresolved_files": [],
+                },
+                "empty": {
+                    "nightly": False,
+                    "models": [],
+                    "test_file_count": 2,
+                    "unresolved_files": ["x.py", "y.py"],
+                },
+            },
+        }
+        md = lsm.render_markdown(inv)
+        self.assertIn("| `n-suite` | ✓ | org/model | 0 |", md)
+        self.assertIn("| `empty` |  | _(none)_ | 2 |", md)
+
+    def test_parse_failures_line(self):
+        inv = {
+            "backend": "cuda",
+            "generated_at_commit": "sha",
+            "suite_count": 0,
+            "model_count": 0,
+            "parse_failures": {"x.py": "SyntaxError: bad"},
+            "suites": {},
+        }
+        self.assertIn("Unparsable files", lsm.render_markdown(inv))
+
+    def test_runner_label_table_and_unmapped_note(self):
+        inv = {
+            "backend": "cuda",
+            "generated_at_commit": "sha",
+            "suite_count": 1,
+            "model_count": 1,
+            "runner_label_count": 1,
+            "parse_failures": {},
+            "runner_labels": {
+                "1-gpu-h100": {"models": ["org/m"], "suites": ["s1", "s2"]}
+            },
+            "unmapped_suites": ["nightly-legacy"],
+            "suites": {
+                "s1": {"nightly": False, "models": ["org/m"], "unresolved_files": []}
+            },
+        }
+        md = lsm.render_markdown(inv)
+        self.assertIn("Per runner label", md)
+        self.assertIn("| `1-gpu-h100` | 2 | org/m |", md)
+        self.assertIn("no runner label: **1**", md)
+        self.assertIn("`nightly-legacy`", md)
+
+
+_FAKE_RUNNER_CONFIGS_YML = """\
+# comment
+_anchors:
+  default_install: &default scripts/ci/cuda/ci_install_dependency.sh
+
+runner_configs:
+  1-gpu:        { install: *default, artifact_version: v4, runs_on: 1-gpu-h100 }
+  deepep-1-gpu: { install: *default, artifact_version: v4, runs_on: 1-gpu-h100 }
+  4-gpu-b200:   { install: *default, artifact_version: v6, runs_on: $b200_runner }
+"""
+
+
+class LoadRunnerLabels(unittest.TestCase):
+    def _load(self, content):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "runner_configs.yml")
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(content)
+            return lsm.load_runner_labels(path)
+
+    def test_parses_flat_inline_maps(self):
+        labels = self._load(_FAKE_RUNNER_CONFIGS_YML)
+        self.assertEqual(
+            labels,
+            {
+                "1-gpu": "1-gpu-h100",
+                "deepep-1-gpu": "1-gpu-h100",
+                "4-gpu-b200": lsm.B200_SENTINEL,
+            },
+        )
+
+    def test_missing_runs_on_is_loud(self):
+        with self.assertRaises(ValueError):
+            self._load("runner_configs:\n  broken: { install: x }\n")
+
+    def test_empty_or_drifted_format_is_loud(self):
+        with self.assertRaises(ValueError):
+            self._load("something_else:\n  a: 1\n")
+
+    def test_real_repo_file(self):
+        """Anchor the stdlib parser to the actual runner_configs.yml: format
+        drift there must fail these tests (which the workflow runs before
+        generating the inventory), not silently empty the label aggregation."""
+        labels = lsm.load_runner_labels(
+            os.path.join(_REPO_ROOT, "scripts", "ci", "runner_configs.yml")
+        )
+        # Two configs sharing a label is the reason the aggregation exists.
+        self.assertEqual(labels["4-gpu-h100"], "4-gpu-h100")
+        self.assertEqual(labels["deepep-4-gpu-h100"], "4-gpu-h100")
+        self.assertEqual(labels["4-gpu-b200"], lsm.B200_SENTINEL)
+        self.assertGreaterEqual(len(labels), 10)
+
+
+class RunnerLabelAggregation(unittest.TestCase):
+    REG = (
+        "import unittest\n"
+        "from sglang.test.ci.ci_register import register_cuda_ci\n"
+        "{calls}\n"
+        'MODEL = "{model}"\n'
+        'if __name__ == "__main__":\n    unittest.main()\n'
+    )
+
+    def _repo(self, tmp):
+        _make_fake_repo(
+            tmp,
+            registered={
+                # Two suites on runner_configs that share the 1-gpu-h100 label.
+                "a/test_a.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-x", '
+                        'runner_config="1-gpu")'
+                    ),
+                    model="meta-llama/Llama-3.1-8B-Instruct",
+                ),
+                "b/test_b.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-y", '
+                        'runner_config="deepep-1-gpu")'
+                    ),
+                    model="Qwen/Qwen3-8B",
+                ),
+                # Sentinel-labeled config.
+                "c/test_c.py": self.REG.format(
+                    calls=(
+                        'register_cuda_ci(est_time=1, stage="base-z", '
+                        'runner_config="4-gpu-b200")'
+                    ),
+                    model="google/gemma-3-4b-it",
+                ),
+                # Legacy suite= with no runner_config -> unmapped.
+                "d/test_d.py": self.REG.format(
+                    calls='register_cuda_ci(est_time=1, suite="nightly-legacy")',
+                    model="openai/gpt-oss-20b",
+                ),
+            },
+        )
+        _write(
+            tmp,
+            os.path.join("scripts", "ci", "runner_configs.yml"),
+            _FAKE_RUNNER_CONFIGS_YML,
+        )
+
+    def test_union_per_label_and_unmapped(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            inv = lsm.build_inventory(tmp, "cuda", lsm.load_overrides(None), "sha")
+            self.assertEqual(inv["runner_label_count"], 2)
+            # Shared label carries the UNION of both suites' models.
+            shared = inv["runner_labels"]["1-gpu-h100"]
+            self.assertEqual(
+                shared["models"],
+                ["Qwen/Qwen3-8B", "meta-llama/Llama-3.1-8B-Instruct"],
+            )
+            self.assertEqual(
+                shared["suites"],
+                ["base-x-test-1-gpu", "base-y-test-deepep-1-gpu"],
+            )
+            # Sentinel stays literal without --b200-runner.
+            self.assertIn(lsm.B200_SENTINEL, inv["runner_labels"])
+            # Legacy suite is visible as unmapped, and still fully present
+            # (with its models) in the per-suite section.
+            self.assertEqual(inv["unmapped_suites"], ["nightly-legacy"])
+            self.assertEqual(
+                inv["suites"]["nightly-legacy"]["models"], ["openai/gpt-oss-20b"]
+            )
+            self.assertEqual(inv["suites"]["nightly-legacy"]["runner_config"], None)
+            self.assertEqual(
+                inv["suites"]["base-x-test-1-gpu"]["runner_config"], "1-gpu"
+            )
+
+    def test_b200_runner_substitution(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            inv = lsm.build_inventory(
+                tmp,
+                "cuda",
+                lsm.load_overrides(None),
+                "sha",
+                b200_runner="4-gpu-b200-dyn",
+            )
+            self.assertNotIn(lsm.B200_SENTINEL, inv["runner_labels"])
+            self.assertEqual(
+                inv["runner_labels"]["4-gpu-b200-dyn"]["models"],
+                ["google/gemma-3-4b-it"],
+            )
+
+    def test_suite_labels_override_maps_legacy_suite(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            overrides = lsm.load_overrides(None)
+            # Legacy suite dispatched by two workflows -> two labels; the
+            # sentinel in an override is substituted like a yml-derived one.
+            overrides["suite_labels"] = {
+                "nightly-legacy": ["1-gpu-h100", "$b200_runner"]
+            }
+            inv = lsm.build_inventory(
+                tmp, "cuda", overrides, "sha", b200_runner="b200-dyn"
+            )
+            self.assertEqual(inv["unmapped_suites"], [])
+            self.assertIn(
+                "openai/gpt-oss-20b", inv["runner_labels"]["1-gpu-h100"]["models"]
+            )
+            self.assertIn(
+                "openai/gpt-oss-20b", inv["runner_labels"]["b200-dyn"]["models"]
+            )
+            self.assertIn(
+                "nightly-legacy", inv["runner_labels"]["1-gpu-h100"]["suites"]
+            )
+
+    def test_missing_yml_leaves_all_suites_unmapped(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            self._repo(tmp)
+            os.remove(os.path.join(tmp, "scripts", "ci", "runner_configs.yml"))
+            inv = lsm.build_inventory(tmp, "cuda", lsm.load_overrides(None), "sha")
+            self.assertEqual(inv["runner_labels"], {})
+            self.assertEqual(len(inv["unmapped_suites"]), 4)
+
+
+class ResolveCommit(unittest.TestCase):
+    def test_explicit_arg_wins(self):
+        self.assertEqual(lsm.resolve_commit("abc", "/nonexistent"), "abc")
+
+    def test_env_fallback(self):
+        with mock.patch.dict(os.environ, {"GITHUB_SHA": "deadbeef"}, clear=True):
+            self.assertEqual(lsm.resolve_commit(None, "/nonexistent"), "deadbeef")
+
+    def test_unknown_when_no_git(self):
+        with tempfile.TemporaryDirectory() as tmp, mock.patch.dict(
+            os.environ, {}, clear=True
+        ):
+            self.assertEqual(lsm.resolve_commit(None, tmp), "unknown")
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/ci/update_est_time.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+"""Refresh est_time literals from sglang-ci-stats/model.json.
+
+Usage:
+    python scripts/ci/update_est_time.py [--dry-run] \\
+        [--model-url URL] [--summary-file PATH]
+"""
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+DEFAULT_MODEL_URL = (
+    "https://raw.githubusercontent.com/sgl-project/sglang-ci-stats/main/model.json"
+)
+
+# AMD / NPU live in separate workflows and are not scraped by sglang-ci-stats.
+BACKENDS = ("cuda", "cpu")
+
+# A change is "significant" if |delta| >= this many seconds AND the relative
+# change is at least SIGNIFICANT_REL_DELTA. Dual threshold filters out both
+# tiny absolute drifts on long tests and small-but-noisy relative swings on
+# short tests.
+SIGNIFICANT_ABS_DELTA = 30
+SIGNIFICANT_REL_DELTA = 0.3
+
+
+def fetch_model(url):
+    """Curl model.json. Fail loudly on network or parse errors -- the
+    weekly workflow will surface the failure rather than silently making
+    a no-op PR."""
+    out = subprocess.run(
+        ["curl", "--fail", "--silent", "--show-error", "--max-time", "30", url],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return json.loads(out.stdout)
+
+
+def make_patterns(suite):
+    """Yield regex objects that match `register_{backend}_ci(est_time=N, ...)`
+    for the given suite, covering both registration styles:
+
+      legacy: register_X_ci(est_time=N, suite="<full-suite>")
+      new:    register_X_ci(est_time=N, stage="<stage>", runner_config="<rc>")
+    """
+    stage_rc = None
+    if "-test-" in suite:
+        stage, _, rc = suite.partition("-test-")
+        stage_rc = (stage, rc)
+    for backend in BACKENDS:
+        yield re.compile(
+            rf"(register_{backend}_ci\(est_time=)(\d+)"
+            rf'(,\s*suite="{re.escape(suite)}")'
+        )
+        if stage_rc is not None:
+            stage, rc = stage_rc
+            yield re.compile(
+                rf"(register_{backend}_ci\(est_time=)(\d+)"
+                rf'(,\s*stage="{re.escape(stage)}",\s*runner_config="{re.escape(rc)}")'
+            )
+
+
+def update_files(model, dry_run=False):
+    """Walk `model.est`, apply each p90 to the matching register call.
+
+    Returns list of (relpath, suite, old, new) for every changed entry.
+    """
+    by_file = defaultdict(list)
+    for suite, files in model.get("est", {}).items():
+        for relpath, p90 in files.items():
+            by_file[relpath].append((suite, p90))
+
+    changes = []
+    for relpath, entries in sorted(by_file.items()):
+        filepath = REPO_ROOT / relpath
+        if not filepath.exists():
+            continue
+        content = filepath.read_text()
+        new_content = content
+
+        for suite, p90 in entries:
+            for pattern in make_patterns(suite):
+                match = pattern.search(new_content)
+                if match is None:
+                    continue
+                old_val = int(match.group(2))
+                if old_val != p90:
+                    new_content = pattern.sub(rf"\g<1>{p90}\3", new_content)
+                    changes.append((relpath, suite, old_val, p90))
+                    print(
+                        f"  {relpath}: suite={suite!r} " f"est_time {old_val} -> {p90}",
+                        file=sys.stderr,
+                    )
+                break  # one (file, suite) -> at most one register call
+
+        if new_content != content and not dry_run:
+            filepath.write_text(new_content)
+
+    return changes
+
+
+def is_significant(old, new):
+    delta = abs(new - old)
+    return (
+        delta >= SIGNIFICANT_ABS_DELTA and delta / max(old, 1) >= SIGNIFICANT_REL_DELTA
+    )
+
+
+def write_summary(changes, summary_file):
+    """Write a markdown summary of significant est_time changes."""
+    sig = [c for c in changes if is_significant(c[2], c[3])]
+    sig.sort(key=lambda c: abs(c[3] - c[2]), reverse=True)
+
+    lines = []
+    if sig:
+        lines.append(
+            f"### Significant est_time changes "
+            f"({len(sig)} of {len(changes)} updates)"
+        )
+        lines.append("")
+        lines.append("| File | Suite | Old (s) | New (s) | Δ |")
+        lines.append("| --- | --- | ---: | ---: | ---: |")
+        for relpath, suite, old, new in sig:
+            delta = new - old
+            sign = "+" if delta > 0 else ""
+            pct = round(delta / max(old, 1) * 100)
+            lines.append(
+                f"| `{Path(relpath).name}` | `{suite}` | "
+                f"{old} | {new} | {sign}{delta} ({sign}{pct}%) |"
+            )
+    else:
+        lines.append(
+            f"_{len(changes)} est_time update(s); none exceeded both "
+            f"±{SIGNIFICANT_ABS_DELTA}s and "
+            f"±{int(SIGNIFICANT_REL_DELTA * 100)}% thresholds._"
+        )
+
+    Path(summary_file).write_text("\n".join(lines) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--model-url",
+        default=DEFAULT_MODEL_URL,
+        help="URL of model.json from sglang-ci-stats (file:// is OK for testing)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print changes without modifying files",
+    )
+    parser.add_argument(
+        "--summary-file",
+        default=None,
+        help="Write a markdown summary of significant changes to this path",
+    )
+    args = parser.parse_args()
+
+    print(f"Fetching {args.model_url}", file=sys.stderr)
+    model = fetch_model(args.model_url)
+    print(
+        f"  model fit_window_start={model.get('fit_window_start')} "
+        f"fit_window_days={model.get('fit_window_days')} "
+        f"n_runs={model.get('n_runs')} "
+        f"n_suites={len(model.get('est', {}))}",
+        file=sys.stderr,
+    )
+
+    changes = update_files(model, dry_run=args.dry_run)
+
+    n_files = len({c[0] for c in changes})
+    action = "Would update" if args.dry_run else "Updated"
+    print(
+        f"\n{action} {len(changes)} est_time entries across {n_files} files",
+        file=sys.stderr,
+    )
+
+    if args.summary_file:
+        write_summary(changes, args.summary_file)
+        print(f"Wrote summary to {args.summary_file}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/ci_coverage_report.py

@@ -0,0 +1,649 @@
+#!/usr/bin/env python3
+"""
+CI Coverage Report Generator
+
+Collects all CI test registrations from test/registered/ and generates
+a coverage report organized by folder, backend, and suite.
+
+Usage:
+    python scripts/ci/utils/ci_coverage_report.py [--output-format markdown|json]
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+# Add the ci_register module path directly to avoid heavy sglang imports
+sys.path.insert(
+    0,
+    str(
+        Path(__file__).parent.parent.parent.parent / "python" / "sglang" / "test" / "ci"
+    ),
+)
+
+from ci_register import CIRegistry, HWBackend, ut_parse_one_file
+
+# Display order for backend tables / sections. The list is sourced from
+# HWBackend so a newly-added enum member can never be silently dropped from
+# the report -- if the assert below fires, add the new backend name here in
+# the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead
+# (highest test volume historically), then accelerators that have been
+# wired into the registry more recently (XPU, MUSA).
+BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA")
+assert set(BACKEND_DISPLAY_ORDER) == {
+    b.name for b in HWBackend
+}, "BACKEND_DISPLAY_ORDER is out of sync with HWBackend"
+
+# --------------------------------------------------------------------------- #
+# multimodal_gen test coverage
+#
+# multimodal_gen tests live under python/sglang/multimodal_gen/test/ and use
+# their own run_suite.py / partitioning framework, NOT the register_*_ci()
+# registry used under test/registered/. To surface them in the daily
+# overview we synthesize CIRegistry records from file paths + filename
+# tokens, using the rules below. These are heuristics derived from how the
+# multimodal_gen workflows (pr-test-{musa,npu,amd}.yml, pr-test-multimodal-
+# gen.yml) currently invoke each file -- they MAY drift if a workflow
+# stops/starts running a directory.
+# --------------------------------------------------------------------------- #
+MULTIMODAL_GEN_TEST_DIR = "python/sglang/multimodal_gen/test"
+
+# Subdirectory (relative to MULTIMODAL_GEN_TEST_DIR) -> backends those files
+# run on by default. Empty string is the top-level. Files whose tokenized
+# filename contains an explicit backend marker (see below) override this.
+_MM_GEN_SUBDIR_BACKENDS = {
+    # Top-level helper-style tests (e.g. test_consistency_metrics.py).
+    "": ("CUDA",),
+    # server/ top-level: pr-test-multimodal-gen.yml drives CUDA, pr-test-amd
+    # mirrors the same suite on AMD runners.
+    "server": ("CUDA", "AMD"),
+    "server/musa": ("MUSA",),
+    "server/ascend": ("NPU",),
+    "layers": ("CUDA",),
+    # unit/ are portable CPU-style unit tests. pr-test-amd now runs the `unit`
+    # suite on ROCm (multimodal-gen-unit-test-amd, both 7.0.0 and 7.2.0), so
+    # they are AMD-covered too, not CUDA-only.
+    "unit": ("CUDA", "AMD"),
+    "cli": ("CUDA",),
+    "manual": ("CUDA",),
+    # Standalone server/CLI single-file tests (restructured out of server/).
+    # Run on CUDA CI; AMD parity for these standalone files is TBD, so
+    # CUDA-only for now (previously matched no rule and were dropped entirely).
+    "single_test_file": ("CUDA",),
+    "single_test_file/component_accuracy": ("CUDA",),
+    # Nested unit suites run only on the CUDA lane today (they are not part of
+    # the AMD `unit` suite that multimodal-gen-unit-test-amd executes).
+    "unit/realtime": ("CUDA",),
+    "unit/sana_wm": ("CUDA",),
+    "unit/progressive_resolution": ("CUDA",),
+    # musa-named unit layer kernels.
+    "unit/musa/layers": ("MUSA",),
+}
+
+# Filenames that match `test_*.py` by convention but contain no real tests
+# (utility / fixture modules). Skipped before classification.
+_MM_GEN_HELPER_FILENAMES = frozenset({"test_utils.py"})
+
+# Filename token -> backend override. Tokenization is `stem.split("_")`, so a
+# file like `test_server_1_gpu_musa.py` yields tokens
+# {test, server, 1, gpu, musa} and matches `musa`. This correctly catches
+# both `test_musa_*.py` (musa-named kernels) and `test_*_musa*.py` (musa
+# variants of generic server tests). `nightly` is detected the same way and
+# flips the nightly flag without changing the backend.
+_MM_GEN_FILENAME_BACKEND_TOKENS = {
+    "musa": ("MUSA",),
+    "npu": ("NPU",),
+}
+
+
+def collect_all_tests(registered_dir: str) -> list[CIRegistry]:
+    """Collect all CI registrations from registered directory."""
+    files = glob.glob(f"{registered_dir}/**/*.py", recursive=True)
+    all_tests = []
+
+    for file in sorted(files):
+        try:
+            registries, _ = ut_parse_one_file(file)
+            all_tests.extend(registries)
+        except Exception as e:
+            print(f"Warning: Failed to parse {file}: {e}", file=sys.stderr)
+
+    return all_tests
+
+
+def collect_multimodal_gen_tests(
+    mm_gen_dir: str = MULTIMODAL_GEN_TEST_DIR,
+) -> list[CIRegistry]:
+    """Synthesize CIRegistry records for multimodal_gen tests.
+
+    multimodal_gen doesn't use register_*_ci(); see the module-level comment
+    above MULTIMODAL_GEN_TEST_DIR for the rules. Returns one record per
+    (file, backend) pair, matching the convention used for registered tests
+    that target multiple backends.
+    """
+    mm_gen_path = Path(mm_gen_dir)
+    if not mm_gen_path.is_dir():
+        return []
+
+    # Discover test files: anything matching test_*.py anywhere under the
+    # tree. The csrc/ and apps/ subtrees aren't part of the test/ directory
+    # so we don't need to exclude them.
+    test_files = sorted(mm_gen_path.glob("**/test_*.py"))
+    records: list[CIRegistry] = []
+
+    for file in test_files:
+        rel = file.relative_to(mm_gen_path)
+        subdir = "/".join(rel.parts[:-1])  # "" for top-level
+        filename_only = rel.parts[-1]
+
+        if filename_only in _MM_GEN_HELPER_FILENAMES:
+            continue  # test_utils.py and similar -- helper modules, not tests
+
+        # Tokenize stem to look for explicit backend / nightly markers.
+        stem_tokens = set(filename_only[:-3].split("_"))
+        nightly = "nightly" in stem_tokens
+
+        backends: tuple[str, ...] = ()
+        for token, override in _MM_GEN_FILENAME_BACKEND_TOKENS.items():
+            if token in stem_tokens:
+                backends = override
+                break
+        if not backends:
+            backends = _MM_GEN_SUBDIR_BACKENDS.get(subdir, ())
+
+        if not backends:
+            print(
+                f"Warning: multimodal_gen file {file} matches no backend "
+                f"rule (subdir={subdir!r}, tokens={sorted(stem_tokens)}); "
+                f"add a rule to _MM_GEN_SUBDIR_BACKENDS.",
+                file=sys.stderr,
+            )
+            continue
+
+        for backend_name in backends:
+            records.append(
+                CIRegistry(
+                    backend=HWBackend[backend_name],
+                    filename=str(file),
+                    # mm_gen does its own per-case partitioning -- file-level
+                    # estimates aren't available. Surfaced as 0 in the
+                    # by-suite section, which is correct (we don't know).
+                    est_time=0.0,
+                    suite=f"mm-gen-{backend_name.lower()}",
+                    nightly=nightly,
+                    disabled=None,
+                )
+            )
+
+    return records
+
+
+def get_folder_name(filename: str) -> str:
+    """Extract folder name from test filename.
+
+    Registered tests use test/registered/<folder>/test_*.py and map to
+    <folder>. multimodal_gen tests live outside that tree and get a virtual
+    `mm_gen/<subdir>` folder so they show up as their own rows in the
+    Folder Summary table.
+    """
+    parts = Path(filename).parts
+    if "multimodal_gen" in parts:
+        try:
+            mg_idx = parts.index("multimodal_gen")
+            test_idx = parts.index("test", mg_idx)
+            sub_parts = parts[test_idx + 1 : -1]  # strip 'test' anchor + file
+            if sub_parts:
+                return "mm_gen/" + "/".join(sub_parts)
+            return "mm_gen"
+        except ValueError:
+            pass  # fall through to default
+    if "registered" in parts:
+        idx = parts.index("registered")
+        if idx + 1 < len(parts) - 1:  # Has subfolder
+            return parts[idx + 1]
+    return "root"
+
+
+def get_test_basename(filename: str) -> str:
+    """Extract just the test file name from the path."""
+    return Path(filename).name
+
+
+def organize_test_data(tests: list[CIRegistry]) -> dict:
+    """Organize tests into various groupings."""
+    by_backend = defaultdict(list)
+    by_folder = defaultdict(list)
+    disabled_tests = []
+
+    for t in tests:
+        by_backend[t.backend.name].append(t)
+        by_folder[get_folder_name(t.filename)].append(t)
+        if t.disabled:
+            disabled_tests.append(t)
+
+    # Count unique test files (a file may be registered for multiple backends)
+    unique_files = set(t.filename for t in tests)
+    unique_enabled_files = set(t.filename for t in tests if not t.disabled)
+    unique_disabled_files = set(t.filename for t in tests if t.disabled)
+
+    return {
+        "total": len(tests),
+        "total_unique_files": len(unique_files),
+        "enabled": len(tests) - len(disabled_tests),
+        "enabled_unique_files": len(unique_enabled_files),
+        "disabled_count": len(disabled_tests),
+        "disabled_unique_files": len(unique_disabled_files),
+        "by_backend": by_backend,
+        "by_folder": by_folder,
+        "disabled_tests": disabled_tests,
+    }
+
+
+def generate_summary_section(data: dict) -> str:
+    """Generate the summary/overview section."""
+    lines = []
+    lines.append("# CI Coverage Overview\n")
+    lines.append(
+        f"**Unique Test Files:** {data['total_unique_files']} ({data['enabled_unique_files']} enabled, {data['disabled_unique_files']} disabled)\n"
+    )
+    lines.append(
+        f"**Total Registrations:** {data['total']} ({data['enabled']} enabled, {data['disabled_count']} disabled)\n"
+    )
+    lines.append(
+        "*Note: A test file may be registered for multiple backends (e.g., CUDA + AMD), so total registrations > unique files.*\n"
+    )
+
+    by_backend = data["by_backend"]
+    by_folder = data["by_folder"]
+    disabled_tests = data["disabled_tests"]
+
+    # Backend summary (collapsible)
+    lines.append("<details>")
+    lines.append("<summary><h2>Backend Summary</h2></summary>\n")
+    lines.append("| Backend | Total | Enabled | Disabled | Per-Commit | Nightly |")
+    lines.append("|---------|-------|---------|----------|------------|---------|")
+
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+        b_total = len(backend_tests)
+        b_disabled = sum(1 for t in backend_tests if t.disabled)
+        b_enabled = b_total - b_disabled
+        b_per_commit = sum(1 for t in backend_tests if not t.nightly and not t.disabled)
+        b_nightly = sum(1 for t in backend_tests if t.nightly and not t.disabled)
+        lines.append(
+            f"| {backend} | {b_total} | {b_enabled} | {b_disabled} | {b_per_commit} | {b_nightly} |"
+        )
+
+    lines.append("\n</details>\n")
+
+    # Folder summary (collapsible). Only show columns for backends that
+    # have at least one registered test across the whole report -- otherwise
+    # adding scaffolding for an unused backend would widen every row with a
+    # column of zeros.
+    active_backends = [b for b in BACKEND_DISPLAY_ORDER if by_backend.get(b)]
+    lines.append("<details>")
+    lines.append("<summary><h2>Folder Summary</h2></summary>\n")
+    header_cells = ["Folder", *active_backends, "Total"]
+    lines.append("| " + " | ".join(header_cells) + " |")
+    lines.append("|" + "|".join(["-" * max(len(c), 3) for c in header_cells]) + "|")
+
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        backend_counts = {b.name: 0 for b in HWBackend}
+        for t in folder_tests:
+            backend_counts[t.backend.name] += 1
+        row = [folder] + [str(backend_counts[b]) for b in active_backends]
+        row.append(str(len(folder_tests)))
+        lines.append("| " + " | ".join(row) + " |")
+
+    lines.append("\n</details>\n")
+
+    # Disabled tests section (collapsible)
+    if disabled_tests:
+        lines.append("<details>")
+        lines.append("<summary><h2>Disabled Tests</h2></summary>\n")
+        lines.append("| File | Backend | Suite | Reason |")
+        lines.append("|------|---------|-------|--------|")
+        for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+            test_name = get_test_basename(t.filename)
+            reason = t.disabled[:50] + "..." if len(t.disabled) > 50 else t.disabled
+            lines.append(
+                f"| `{test_name}` | {t.backend.name} | {t.effective_suite} | {reason} |"
+            )
+        lines.append("\n</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_by_folder_section(data: dict) -> str:
+    """Generate the 'All Tests by Folder' section."""
+    lines = []
+    by_folder = data["by_folder"]
+
+    lines.append("# All Tests by Folder\n")
+
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        lines.append("<details>")
+        lines.append(
+            f"<summary><h2>{folder}/ ({len(folder_tests)} tests)</h2></summary>\n"
+        )
+
+        # Group by backend within folder
+        folder_by_backend = defaultdict(list)
+        for t in folder_tests:
+            folder_by_backend[t.backend.name].append(t)
+
+        for backend in BACKEND_DISPLAY_ORDER:
+            backend_tests = folder_by_backend.get(backend, [])
+            if not backend_tests:
+                continue
+
+            lines.append(f"### {backend} ({len(backend_tests)} tests)\n")
+            lines.append("| Test File | Suite | Est. Time | Status |")
+            lines.append("|-----------|-------|-----------|--------|")
+
+            for t in sorted(backend_tests, key=lambda x: x.filename):
+                test_name = get_test_basename(t.filename)
+                status = (
+                    "Disabled"
+                    if t.disabled
+                    else ("Nightly" if t.nightly else "Per-Commit")
+                )
+                lines.append(
+                    f"| `{test_name}` | {t.effective_suite} | {t.est_time:.0f}s | {status} |"
+                )
+
+            lines.append("")
+
+        lines.append("</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_by_suite_section(data: dict) -> str:
+    """Generate the 'All Tests by Test Suite' section."""
+    lines = []
+    by_backend = data["by_backend"]
+
+    lines.append("# All Tests by Test Suite\n")
+
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+
+        b_total = len(backend_tests)
+        b_disabled = sum(1 for t in backend_tests if t.disabled)
+        b_enabled = b_total - b_disabled
+
+        lines.append("<details>")
+        lines.append(
+            f"<summary><h2>{backend} Backend ({b_enabled} enabled, {b_disabled} disabled)</h2></summary>\n"
+        )
+
+        # Group by suite within backend
+        backend_suites = defaultdict(list)
+        for t in backend_tests:
+            backend_suites[t.effective_suite].append(t)
+
+        for suite in sorted(backend_suites.keys()):
+            suite_tests = backend_suites[suite]
+            s_enabled = sum(1 for t in suite_tests if not t.disabled)
+            s_disabled = sum(1 for t in suite_tests if t.disabled)
+            s_est_time = sum(t.est_time for t in suite_tests if not t.disabled)
+            is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+            suite_type = "Nightly" if is_nightly else "Per-Commit"
+            lines.append("<details>")
+            lines.append(
+                f"<summary><h3>{suite} ({s_enabled} enabled, {s_disabled} disabled) - {suite_type}</h3></summary>\n"
+            )
+            lines.append(f"*Estimated total time: {s_est_time:.0f}s*\n")
+
+            lines.append("| Test File | Folder | Est. Time | Status |")
+            lines.append("|-----------|--------|-----------|--------|")
+
+            for t in sorted(suite_tests, key=lambda x: x.filename):
+                test_name = get_test_basename(t.filename)
+                folder = get_folder_name(t.filename)
+                if t.disabled:
+                    status = (
+                        f"Disabled: {t.disabled[:30]}..."
+                        if len(t.disabled) > 30
+                        else f"Disabled: {t.disabled}"
+                    )
+                else:
+                    status = "Nightly" if t.nightly else "Per-Commit"
+                lines.append(
+                    f"| `{test_name}` | {folder} | {t.est_time:.0f}s | {status} |"
+                )
+
+            lines.append("\n</details>\n")
+
+        lines.append("</details>\n")
+
+    return "\n".join(lines)
+
+
+def generate_markdown_report(tests: list[CIRegistry], section: str = "all") -> str:
+    """Generate markdown report for GitHub step summary."""
+    data = organize_test_data(tests)
+
+    if section == "summary":
+        return generate_summary_section(data)
+    elif section == "by-folder":
+        return generate_by_folder_section(data)
+    elif section == "by-suite":
+        return generate_by_suite_section(data)
+    else:  # "all"
+        parts = [
+            generate_summary_section(data),
+            "---",
+            generate_by_folder_section(data),
+            "---",
+            generate_by_suite_section(data),
+        ]
+        return "\n".join(parts)
+
+
+def generate_json_report(tests: list[CIRegistry]) -> str:
+    """Generate JSON report with detailed test listings."""
+    by_backend = defaultdict(list)
+    by_folder = defaultdict(list)
+
+    for t in tests:
+        by_backend[t.backend.name].append(t)
+        by_folder[get_folder_name(t.filename)].append(t)
+
+    disabled_tests = [t for t in tests if t.disabled]
+
+    # Build structured data
+    data = {
+        "summary": {
+            "total": len(tests),
+            "enabled": len(tests) - len(disabled_tests),
+            "disabled": len(disabled_tests),
+        },
+        "tests_by_folder": {},
+        "tests_by_suite": {},
+        "backend_summary": {},
+        "folder_summary": {},
+        "disabled_tests": [],
+    }
+
+    # Section 1: Tests by Folder
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        folder_by_backend = defaultdict(list)
+        for t in folder_tests:
+            folder_by_backend[t.backend.name].append(t)
+
+        data["tests_by_folder"][folder] = {
+            "total": len(folder_tests),
+            "backends": {},
+        }
+
+        for backend in BACKEND_DISPLAY_ORDER:
+            backend_tests = folder_by_backend.get(backend, [])
+            if backend_tests:
+                data["tests_by_folder"][folder]["backends"][backend] = [
+                    {
+                        "filename": get_test_basename(t.filename),
+                        "suite": t.effective_suite,
+                        "est_time": t.est_time,
+                        "status": (
+                            "disabled"
+                            if t.disabled
+                            else ("nightly" if t.nightly else "per-commit")
+                        ),
+                    }
+                    for t in sorted(backend_tests, key=lambda x: x.filename)
+                ]
+
+    # Section 2: Tests by Suite (Backend -> Suite)
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if not backend_tests:
+            continue
+
+        backend_suites = defaultdict(list)
+        for t in backend_tests:
+            backend_suites[t.effective_suite].append(t)
+
+        data["tests_by_suite"][backend] = {
+            "total": len(backend_tests),
+            "enabled": sum(1 for t in backend_tests if not t.disabled),
+            "disabled": sum(1 for t in backend_tests if t.disabled),
+            "suites": {},
+        }
+
+        for suite in sorted(backend_suites.keys()):
+            suite_tests = backend_suites[suite]
+            is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+            data["tests_by_suite"][backend]["suites"][suite] = {
+                "total": len(suite_tests),
+                "enabled": sum(1 for t in suite_tests if not t.disabled),
+                "disabled": sum(1 for t in suite_tests if t.disabled),
+                "est_time": sum(t.est_time for t in suite_tests if not t.disabled),
+                "type": "nightly" if is_nightly else "per-commit",
+                "tests": [
+                    {
+                        "filename": get_test_basename(t.filename),
+                        "folder": get_folder_name(t.filename),
+                        "est_time": t.est_time,
+                        "status": (
+                            "disabled"
+                            if t.disabled
+                            else ("nightly" if t.nightly else "per-commit")
+                        ),
+                        "disabled_reason": t.disabled if t.disabled else None,
+                    }
+                    for t in sorted(suite_tests, key=lambda x: x.filename)
+                ],
+            }
+
+    # Backend summary
+    for backend in BACKEND_DISPLAY_ORDER:
+        backend_tests = by_backend.get(backend, [])
+        if backend_tests:
+            data["backend_summary"][backend] = {
+                "total": len(backend_tests),
+                "enabled": sum(1 for t in backend_tests if not t.disabled),
+                "disabled": sum(1 for t in backend_tests if t.disabled),
+                "per_commit": sum(
+                    1 for t in backend_tests if not t.nightly and not t.disabled
+                ),
+                "nightly": sum(
+                    1 for t in backend_tests if t.nightly and not t.disabled
+                ),
+            }
+
+    # Folder summary -- one count per backend in HWBackend, in display
+    # order, so every registered backend shows up regardless of whether
+    # this folder has tests for it.
+    for folder in sorted(by_folder.keys()):
+        folder_tests = by_folder[folder]
+        backend_counts = {b: 0 for b in BACKEND_DISPLAY_ORDER}
+        for t in folder_tests:
+            backend_counts[t.backend.name] += 1
+        data["folder_summary"][folder] = {
+            **backend_counts,
+            "total": len(folder_tests),
+        }
+
+    # Disabled tests
+    for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+        data["disabled_tests"].append(
+            {
+                "filename": get_test_basename(t.filename),
+                "backend": t.backend.name,
+                "suite": t.effective_suite,
+                "reason": t.disabled,
+            }
+        )
+
+    return json.dumps(data, indent=2)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate CI coverage report")
+    parser.add_argument(
+        "--output-format",
+        choices=["markdown", "json"],
+        default="markdown",
+        help="Output format (default: markdown)",
+    )
+    parser.add_argument(
+        "--section",
+        choices=["all", "summary", "by-folder", "by-suite"],
+        default="all",
+        help="Which section to output (default: all). Only applies to markdown format.",
+    )
+    parser.add_argument(
+        "--registered-dir",
+        default="test/registered",
+        help="Path to registered test directory",
+    )
+    parser.add_argument(
+        "--multimodal-gen-dir",
+        default=MULTIMODAL_GEN_TEST_DIR,
+        help=(
+            "Path to multimodal_gen test directory. Pass empty string to "
+            "skip multimodal_gen tests entirely."
+        ),
+    )
+    args = parser.parse_args()
+
+    # Change to repo root if needed
+    script_dir = Path(__file__).parent.parent
+    repo_root = script_dir.parent.parent
+    os.chdir(repo_root)
+
+    tests = collect_all_tests(args.registered_dir)
+    if args.multimodal_gen_dir:
+        tests.extend(collect_multimodal_gen_tests(args.multimodal_gen_dir))
+
+    if args.output_format == "markdown":
+        report = generate_markdown_report(tests, section=args.section)
+    else:
+        report = generate_json_report(tests)
+
+    print(report)
+
+    # Write to GITHUB_STEP_SUMMARY if available
+    summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file and args.output_format == "markdown":
+        with open(summary_file, "a") as f:
+            f.write(report)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/cleanup_hf_cache.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+Clean up stale HuggingFace cache artifacts from previous failed downloads.
+
+This script removes incomplete marker files, temporary files, and lock files
+from the HuggingFace cache directory. These artifacts can accumulate from
+interrupted or failed downloads and may interfere with future downloads.
+"""
+
+import os
+import sys
+from pathlib import Path
+from typing import List
+
+try:
+    from huggingface_hub import constants
+
+    HF_HUB_AVAILABLE = True
+except ImportError:
+    print("Warning: huggingface_hub not available")
+    HF_HUB_AVAILABLE = False
+
+
+def get_hf_cache_dir() -> str:
+    """Get the HuggingFace cache directory."""
+    if HF_HUB_AVAILABLE:
+        return constants.HF_HUB_CACHE
+
+    # Fallback to environment variable or default
+    hf_home = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface"))
+    return os.path.join(hf_home, "hub")
+
+
+def find_stale_artifacts(cache_dir: str) -> List[Path]:
+    """
+    Find stale artifact files in the HuggingFace cache.
+
+    Args:
+        cache_dir: HuggingFace cache directory
+
+    Returns:
+        List of paths to stale artifact files
+    """
+    cache_path = Path(cache_dir)
+
+    if not cache_path.exists():
+        return []
+
+    # Patterns for stale files to clean up
+    patterns = [
+        "**/*.incomplete",  # Incomplete download markers
+        "**/*.tmp",  # Temporary files
+        "**/*.lock",  # Lock files from interrupted downloads
+    ]
+
+    stale_files = []
+    for pattern in patterns:
+        stale_files.extend(cache_path.glob(pattern))
+
+    return stale_files
+
+
+def cleanup_artifacts(artifacts: List[Path]) -> tuple[int, int]:
+    """
+    Remove stale artifact files.
+
+    Args:
+        artifacts: List of file paths to remove
+
+    Returns:
+        Tuple of (successful_removals, failed_removals)
+    """
+    successful = 0
+    failed = 0
+
+    for file_path in artifacts:
+        try:
+            file_path.unlink()
+            print(f"  Removed: {file_path}")
+            successful += 1
+        except Exception as e:
+            print(f"  Warning: Could not remove {file_path}: {e}")
+            failed += 1
+
+    return successful, failed
+
+
+def main() -> int:
+    """
+    Main cleanup logic.
+
+    Returns:
+        Always returns 0 (cleanup is best-effort and should not fail CI)
+    """
+    print("=" * 70)
+    print("HuggingFace Cache Cleanup")
+    print("=" * 70)
+
+    # Get cache directory
+    cache_dir = get_hf_cache_dir()
+    print(f"Cache directory: {cache_dir}")
+
+    if not os.path.exists(cache_dir):
+        print("Cache directory does not exist - nothing to clean")
+        return 0
+
+    print("-" * 70)
+
+    # Find stale artifacts
+    print("Scanning for stale artifacts...")
+    stale_artifacts = find_stale_artifacts(cache_dir)
+
+    if not stale_artifacts:
+        print("✓ No stale cache artifacts found")
+        return 0
+
+    # Clean up artifacts
+    print(f"Found {len(stale_artifacts)} stale artifact(s) to remove:")
+    successful, failed = cleanup_artifacts(stale_artifacts)
+
+    print("-" * 70)
+
+    # Summary
+    if failed > 0:
+        print(f"⚠ Cleaned up {successful} file(s), {failed} removal(s) failed")
+    else:
+        print(f"✓ Successfully cleaned up {successful} stale file(s)")
+
+    # Always return 0 - cleanup failures should not fail CI
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        exit_code = main()
+        sys.exit(exit_code)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(0)
+    except Exception as e:
+        print(f"ERROR: Unexpected error during cleanup: {e}")
+        import traceback
+
+        traceback.print_exc()
+        # Still return 0 - cleanup failures should not fail CI
+        sys.exit(0)

scripts/ci/utils/compute_partitions.py

@@ -0,0 +1,267 @@
+"""Sum est_time per per-commit suite and emit one $GITHUB_OUTPUT line
+keyed by suite name. Consumed by pr-test.yml stage jobs as
+`fromJson(needs.check-changes.outputs.partitions)['<suite>']`.
+
+    partitions={"base-b-test-1-gpu-small": {"size": 8, "arr": [0,...,7], "max_parallel": 2}, ...}
+"""
+
+import argparse
+import glob
+import importlib.util
+import json
+import math
+import os
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+import yaml  # PyYAML; preinstalled on ubuntu-latest GHA runners.
+
+REPO_ROOT = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+
+# Load ci_register.py directly: `import sglang.test...` pulls torch/orjson via
+# sglang.__init__ but check-changes runs on bare ubuntu-latest. ci_register
+# itself is stdlib-only (AST).
+_CI_REGISTER_PATH = os.path.join(
+    REPO_ROOT, "python", "sglang", "test", "ci", "ci_register.py"
+)
+_spec = importlib.util.spec_from_file_location("ci_register", _CI_REGISTER_PATH)
+_ci_register = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_ci_register)
+collect_tests = _ci_register.collect_tests
+HWBackend = _ci_register.HWBackend
+
+# pr-test-amd.yml / pr-test-npu.yml have their own dispatch.
+_TARGET_BACKENDS = {HWBackend.CUDA, HWBackend.CPU}
+
+# base-a is the critical-path entry gate; pin its fanout to sanity-coverage
+# defaults instead of est_time. max_parallel = size (no throttle).
+_BASE_A_OVERRIDES = {
+    "base-a-test-cpu": 8,
+    "base-a-test-1-gpu-small": 1,
+}
+
+_REUSABLE_STAGE_USES = "./.github/workflows/_pr-test-stage.yml"
+
+
+def load_run_timeouts(pr_test_yml_path: str) -> dict:
+    """Map `self_name -> run_timeout_minutes` from one pr-test*.yml. The input
+    is required in `_pr-test-stage.yml` -- KeyError surfaces missing.
+    Inline base-a-test-cpu is skipped (uses `_BASE_A_OVERRIDES`)."""
+    with open(pr_test_yml_path) as f:
+        wf = yaml.safe_load(f)
+    timeouts = {}
+    for job_id, job in (wf.get("jobs") or {}).items():
+        if not isinstance(job, dict) or job.get("uses") != _REUSABLE_STAGE_USES:
+            continue
+        with_ = job.get("with") or {}
+        suite = with_.get("self_name", job_id)
+        timeouts[suite] = int(with_["run_timeout_minutes"])
+    if not timeouts:
+        raise RuntimeError(
+            f"load_run_timeouts: no jobs matched uses={_REUSABLE_STAGE_USES!r} "
+            f"in {pr_test_yml_path}. The reusable workflow path likely "
+            "changed -- update _REUSABLE_STAGE_USES."
+        )
+    return timeouts
+
+
+def per_shard_target_seconds(suite: str, run_timeouts: dict) -> float:
+    """Per-shard wall budget = 0.75 * stage timeout. 0.75 is the inverse
+    of LPT's 4/3 worst-case approximation ratio, so the most imbalanced
+    LPT shard fills exactly the timeout."""
+    return 0.75 * run_timeouts[suite] * 60
+
+
+def discover_files(repo_root: str) -> list[str]:
+    test_dir = os.path.join(repo_root, "test")
+    files = [
+        f
+        for f in glob.glob(
+            os.path.join(test_dir, "registered", "**", "*.py"), recursive=True
+        )
+        if not f.endswith("/conftest.py") and not f.endswith("/__init__.py")
+    ]
+    jit_kernel_dir = os.path.join(repo_root, "python", "sglang", "jit_kernel")
+    files += glob.glob(
+        os.path.join(jit_kernel_dir, "tests", "**", "test_*.py"), recursive=True
+    )
+    files += glob.glob(
+        os.path.join(jit_kernel_dir, "benchmark", "**", "bench_*.py"), recursive=True
+    )
+    return files
+
+
+def load_partition_model(path):
+    """Read sglang-ci-stats' model.json; None on missing/unparsable.
+    Cross-repo schema -- guard against non-dict top-level."""
+    if not path or not os.path.exists(path):
+        return None
+    try:
+        with open(path) as f:
+            data = json.load(f)
+    except (OSError, json.JSONDecodeError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def compute_max_parallel(size: int) -> int:
+    return max(size // 3, 1)
+
+
+def compute_partitions(
+    tests, repo_root, run_timeouts, partition_model=None, full_parallel=False
+):
+    """Group per-commit tests by suite and emit partition metadata.
+
+    `run_timeouts`: `suite -> minutes` from `load_run_timeouts`.
+    `partition_model`: optional sglang-ci-stats `model.json`; per-file
+    `est` and per-suite `(coeff, bias)` each fall back independently to
+    in-source `est_time` / `(1.0, 0.0)`.
+    `full_parallel=True` lifts the matrix-fanout throttle.
+    """
+    # Allowlist: stages pr-test.yml dispatches. Stress / weekly /
+    # nightly-* live in test/registered/ but pr-test doesn't run them.
+    dispatched_suites = set(run_timeouts) | set(_BASE_A_OVERRIDES)
+    suite_tests = defaultdict(list)
+    for t in tests:
+        if t.backend not in _TARGET_BACKENDS:
+            continue
+        if t.nightly or t.disabled is not None:
+            continue
+        if t.effective_suite not in dispatched_suites:
+            continue
+        suite_tests[t.effective_suite].append(t)
+
+    est_table = (partition_model or {}).get("est", {})
+    fit_table = (partition_model or {}).get("fit", {})
+
+    result = {}
+    for suite, group in suite_tests.items():
+        live_est = est_table.get(suite, {})
+        total = 0.0
+        for t in group:
+            relpath = os.path.relpath(t.filename, repo_root)
+            total += live_est.get(relpath, t.est_time)
+
+        fit = fit_table.get(suite) or {}
+        coeff = fit.get("coeff", 1.0)
+        bias = fit.get("bias", 0.0)
+
+        # Each shard pays `bias` once, so size >= coeff*total / (target-bias).
+        if suite in _BASE_A_OVERRIDES:
+            size = _BASE_A_OVERRIDES[suite]
+            max_parallel = size
+        else:
+            target = per_shard_target_seconds(suite, run_timeouts)
+            budget = target - bias
+            if budget <= 0:
+                raise RuntimeError(
+                    f"Suite {suite!r}: fit bias={bias}s >= target={target}s. "
+                    "Investigate the fit or raise the stage's run_timeout_minutes."
+                )
+            ideal_size = math.ceil(coeff * total / budget)
+            # ideal_size > len(group) -> slowest single file alone exceeds
+            # the per-shard budget; surface via raise instead of empty shard.
+            if ideal_size > len(group):
+                raise RuntimeError(
+                    f"Suite {suite!r}: needs {ideal_size} shards but has only "
+                    f"{len(group)} test file(s). target={target:.0f}s, "
+                    f"coeff={coeff}, bias={bias}s, total_est={total:.0f}s."
+                )
+            size = max(1, ideal_size)
+            max_parallel = size if full_parallel else compute_max_parallel(size)
+        result[suite] = {
+            "size": size,
+            "arr": list(range(size)),
+            "max_parallel": max_parallel,
+        }
+    return result
+
+
+def format_fit_window(model: dict) -> str:
+    """Render the model's fit window as `[start, end)` for the step summary.
+
+    Surfacing the whole span keeps the lower-bound date from reading as a
+    staleness marker (it trails today by fit_window_days)."""
+    start = model.get("fit_window_start")
+    days = model.get("fit_window_days")
+    if not start or not isinstance(days, int):
+        return f"fit_window_start={start}"
+    end = (datetime.strptime(start[:10], "%Y-%m-%d") + timedelta(days=days)).strftime(
+        "%Y-%m-%d"
+    )
+    return f"fit over [{start[:10]}, {end}) ({days}d window)"
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo-root", default=REPO_ROOT)
+    parser.add_argument(
+        "--output-format",
+        choices=("gha", "json"),
+        default="gha",
+        help="`gha` emits `partitions=<json>` for $GITHUB_OUTPUT; `json` is raw",
+    )
+    parser.add_argument(
+        "--full-parallel",
+        choices=("true", "false"),
+        default="false",
+        help="Lift the max_parallel throttle (set by schedule / `high priority`)",
+    )
+    parser.add_argument(
+        "--partition-model-file",
+        default=None,
+        help="Path to sglang-ci-stats model.json (omit/missing -> static fallback)",
+    )
+    parser.add_argument(
+        "--pr-test-yml",
+        default=os.path.join(REPO_ROOT, ".github", "workflows", "pr-test.yml"),
+        help="Path to pr-test*.yml; per-stage `run_timeout_minutes` is read from here.",
+    )
+    args = parser.parse_args()
+
+    files = discover_files(args.repo_root)
+    # Warn-not-fail on unregistered files: run_suite.py catches this at
+    # test-execution time with sanity_check=True; dispatch should keep going.
+    all_tests = collect_tests(files, sanity_check=False)
+    partition_model = load_partition_model(args.partition_model_file)
+    run_timeouts = load_run_timeouts(args.pr_test_yml)
+
+    result = compute_partitions(
+        all_tests,
+        repo_root=args.repo_root,
+        run_timeouts=run_timeouts,
+        partition_model=partition_model,
+        full_parallel=(args.full_parallel == "true"),
+    )
+    payload = json.dumps(result, separators=(",", ":"), sort_keys=True)
+    if args.output_format == "gha":
+        print(f"partitions={payload}")
+    else:
+        print(payload)
+
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_path:
+        with open(summary_path, "a") as f:
+            f.write("## Partitions\n\n")
+            if partition_model is None:
+                src_note = "no live model -- static est_time + (coeff=1, bias=0)"
+            else:
+                src_note = (
+                    f"live model: {format_fit_window(partition_model)}, "
+                    f"`n_runs={partition_model.get('n_runs')}`"
+                )
+            f.write(
+                f"`full_parallel={args.full_parallel}` "
+                f"(`size//3` throttle is lifted when true); {src_note}\n\n"
+            )
+            f.write("| Suite | size | max_parallel |\n")
+            f.write("|---|---:|---:|\n")
+            for suite, info in sorted(result.items()):
+                f.write(f"| `{suite}` | {info['size']} | {info['max_parallel']} |\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/comparison_configs.json

@@ -0,0 +1,191 @@
+{
+    "_comment": "Per-model comparison config. Sampling params omitted where model defaults are correct — only override resolution, seed, and params that differ from defaults.",
+    "test_image_url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
+    "cases": [
+        {
+            "id": "flux1_dev_t2i_1024",
+            "model": "black-forest-labs/FLUX.1-dev",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --dit-layerwise-offload false --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "flux2_dev_t2i_1024",
+            "model": "black-forest-labs/FLUX.2-dev",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --dit-layerwise-offload false --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "qwen_image_2512_t2i_1024",
+            "model": "Qwen/Qwen-Image-2512",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "qwen_image_edit_2511",
+            "model": "Qwen/Qwen-Image-Edit-2511",
+            "task": "image-edit",
+            "prompt": "Make the cat wear a red hat",
+            "reference_image": true,
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "zimage_turbo_t2i_1024",
+            "model": "Tongyi-MAI/Z-Image-Turbo",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "wan22_t2v_a14b_720p",
+            "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 4,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "wan22_ti2v_5b_720p",
+            "model": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+            "task": "text-image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 1,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "ltx2.3_twostage_ti2v_2gpus",
+            "model": "Lightricks/LTX-2.3",
+            "task": "text-image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 768,
+            "height": 512,
+            "num_frames": 121,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --pipeline-class-name LTX2TwoStagePipeline --cfg-parallel-size 2",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "ideogram4_fp8_t2i_2gpu",
+            "model": "ideogram-ai/ideogram-4-fp8",
+            "task": "text-to-image",
+            "prompt": "A futuristic cyberpunk city at night, neon lights reflecting on wet streets",
+            "width": 1024,
+            "height": 1024,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2 --attention-backend fa",
+                    "extra_env": {}
+                }
+            }
+        },
+        {
+            "id": "cosmos3_super_t2v_2gpu",
+            "model": "nvidia/Cosmos3-Super",
+            "task": "text-to-video",
+            "prompt": "A cat and a dog baking a cake together in a kitchen.",
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 2,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --tp-size 2",
+                    "extra_env": {"SGLANG_DISABLE_COSMOS3_GUARDRAILS": "1"}
+                }
+            }
+        },
+        {
+            "id": "wan22_i2v_a14b_720p",
+            "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+            "task": "image-to-video",
+            "prompt": "The cat starts walking slowly towards the camera.",
+            "reference_image": true,
+            "width": 1280,
+            "height": 720,
+            "num_frames": 81,
+            "seed": 42,
+            "num_gpus": 4,
+            "frameworks": {
+                "sglang": {
+                    "serve_args": "--warmup --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory",
+                    "extra_env": {}
+                }
+            }
+        }
+    ]
+}

scripts/ci/utils/diffusion/compute_diffusion_partitions.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""
+Compute dynamic partitions for diffusion CI tests.
+
+This script runs on lightweight CI runners without sglang dependencies and uses
+AST parsing to extract parametrized cases plus standalone files from source.
+"""
+
+import argparse
+import importlib.util
+import json
+import math
+import os
+import sys
+from pathlib import Path
+
+from diffusion_case_parser import (
+    BASELINE_REL_PATH,
+    RUN_SUITE_REL_PATH,
+    DiffusionSuiteInfo,
+    collect_diffusion_suites,
+    resolve_case_config_path,
+)
+
+
+def _load_partitioning_helpers():
+    repo_root = Path(__file__).resolve().parents[4]
+    helper_path = repo_root / "python/sglang/multimodal_gen/test/partitioning.py"
+    spec = importlib.util.spec_from_file_location(
+        "diffusion_test_partitioning", helper_path
+    )
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module.PartitionItem, module.partition_items_by_lpt
+
+
+PartitionItem, partition_items_by_lpt = _load_partitioning_helpers()
+
+SUITE_OUTPUT_NAMES = {"1-gpu": "1gpu", "2-gpu": "2gpu", "1-gpu-b200": "b200"}
+
+USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true")
+
+if USE_NPU_CONFIGS:
+    SUITE_OUTPUT_NAMES = {"1-npu": "1npu", "2-npu": "2npu"}
+
+DEFAULT_STANDALONE_EST_TIME_SECONDS = 300.0
+
+
+def validate_suite_case_coverage(suites: dict[str, DiffusionSuiteInfo]) -> None:
+    """
+    Guardrail: dynamic diffusion suites must contain parametrized cases.
+    """
+    suites_with_no_cases = []
+    for suite_name in SUITE_OUTPUT_NAMES:
+        suite_info = suites.get(suite_name)
+        if suite_info is None:
+            print(f"Error: Required suite '{suite_name}' not found in parsed suites.")
+            sys.exit(1)
+        if len(suite_info.cases) == 0:
+            suites_with_no_cases.append(suite_name)
+
+    if suites_with_no_cases:
+        joined = ", ".join(suites_with_no_cases)
+        print(
+            "Error: Parsed zero parametrized cases for diffusion suites: "
+            f"{joined}. This usually means run_suite case imports changed but "
+            "diffusion parser logic was not updated."
+        )
+        sys.exit(1)
+
+
+def compute_partition_count(
+    total_time_seconds: float,
+    min_time_seconds: float,
+    target_time_seconds: float,
+    max_time_seconds: float,
+    max_partitions: int,
+) -> int:
+    if total_time_seconds <= 0:
+        return 0
+
+    min_partition_count = max(1, math.ceil(total_time_seconds / max_time_seconds))
+    max_partition_count = max(1, math.floor(total_time_seconds / min_time_seconds))
+
+    min_partition_count = min(min_partition_count, max_partitions)
+    max_partition_count = min(max_partition_count, max_partitions)
+
+    if max_partition_count < min_partition_count:
+        fallback_count = math.ceil(total_time_seconds / target_time_seconds)
+        return max(1, min(fallback_count, max_partitions))
+
+    preferred_count = math.ceil(total_time_seconds / target_time_seconds)
+    preferred_count = max(1, min(preferred_count, max_partitions))
+    return max(min_partition_count, min(preferred_count, max_partition_count))
+
+
+def build_partition_items(
+    suite_info: DiffusionSuiteInfo, include_standalone: bool = True
+) -> list[PartitionItem]:
+    items = [
+        PartitionItem(kind="case", item_id=case.case_id, est_time=case.est_time)
+        for case in suite_info.cases
+    ]
+    if not include_standalone:
+        return items
+
+    items.extend(
+        PartitionItem(
+            kind="standalone",
+            item_id=standalone_file,
+            est_time=suite_info.standalone_est_times.get(
+                standalone_file, DEFAULT_STANDALONE_EST_TIME_SECONDS
+            ),
+            used_fallback_estimate=(
+                standalone_file in suite_info.missing_standalone_estimates
+            ),
+        )
+        for standalone_file in suite_info.standalone_files
+    )
+    return items
+
+
+def build_matrix(partition_count: int) -> dict:
+    if partition_count <= 0:
+        return {"include": []}
+    return {"include": [{"part": i} for i in range(partition_count)]}
+
+
+def build_partition_plan(
+    suite_name: str,
+    partitions: list[list[PartitionItem]],
+) -> dict:
+    return {
+        "suite": suite_name,
+        "partition_count": len(partitions),
+        "partitions": [
+            {
+                "part": idx,
+                "case_ids": [item.item_id for item in partition if item.kind == "case"],
+                "standalone_files": [
+                    item.item_id for item in partition if item.kind == "standalone"
+                ],
+                "missing_standalone_estimates": [
+                    item.item_id
+                    for item in partition
+                    if item.kind == "standalone" and item.used_fallback_estimate
+                ],
+                "estimated_time": round(sum(item.est_time for item in partition), 1),
+            }
+            for idx, partition in enumerate(partitions)
+        ],
+    }
+
+
+def output_github_value(name: str, value: dict) -> None:
+    value_json = json.dumps(value, separators=(",", ":"))
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as f:
+            f.write(f"{name}={value_json}\n")
+    print(f"{name}={value_json}")
+
+
+def output_github_scalar(name: str, value: str) -> None:
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as f:
+            f.write(f"{name}={value}\n")
+    print(f"{name}={value}")
+
+
+def print_suite_summary(
+    suite_name: str,
+    suite_info: DiffusionSuiteInfo,
+    partitions: list[list[PartitionItem]],
+    include_standalone: bool = True,
+) -> None:
+    total_time = sum(
+        item.est_time
+        for item in build_partition_items(
+            suite_info, include_standalone=include_standalone
+        )
+    )
+    print(f"{suite_name.upper()} suite:")
+    print(f"  Cases: {len(suite_info.cases)}")
+    standalone_label = "Standalone files"
+    if not include_standalone:
+        standalone_label = "Standalone files ignored"
+    print(f"  {standalone_label}: {len(suite_info.standalone_files)}")
+    print(
+        f"  Missing standalone estimates: {len(suite_info.missing_standalone_estimates)}"
+    )
+    if suite_info.missing_standalone_estimates:
+        print(
+            f"  Fallback standalone estimate: "
+            f"{DEFAULT_STANDALONE_EST_TIME_SECONDS:.1f}s"
+        )
+        for standalone_file in suite_info.missing_standalone_estimates:
+            print(f"    - {standalone_file}")
+    print(f"  Total estimated time: {total_time:.1f}s ({total_time/60:.1f} min)")
+    print(f"  Selected partitions: {len(partitions)}")
+    print()
+
+    print("  Partition assignments:")
+    for idx, partition in enumerate(partitions):
+        partition_time = sum(item.est_time for item in partition)
+        print(f"    Partition {idx}:")
+        print(
+            f"      Estimated time: {partition_time:.1f}s ({partition_time/60:.1f} min)"
+        )
+        for item in partition:
+            fallback_suffix = (
+                ", fallback estimate"
+                if item.kind == "standalone" and item.used_fallback_estimate
+                else ""
+            )
+            print(
+                f"      - {item.kind}: {item.item_id} "
+                f"({item.est_time:.1f}s{fallback_suffix})"
+            )
+    print()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Compute diffusion test partitions for CI"
+    )
+    parser.add_argument(
+        "--min-time",
+        type=float,
+        default=1200.0,
+        help="Minimum desired partition time in seconds (default: 1200 = 20 minutes)",
+    )
+    parser.add_argument(
+        "--target-time",
+        type=float,
+        default=1800.0,
+        help="Preferred partition time in seconds (default: 1800 = 30 minutes)",
+    )
+    parser.add_argument(
+        "--max-time",
+        type=float,
+        default=2400.0,
+        help="Maximum desired partition time in seconds (default: 2400 = 40 minutes)",
+    )
+    parser.add_argument(
+        "--max-partitions",
+        type=int,
+        default=10,
+        help="Maximum number of partitions (default: 10)",
+    )
+    parser.add_argument(
+        "--parametrized-only",
+        action="store_true",
+        help="Only partition DiffusionTestCase parametrized cases.",
+    )
+    args = parser.parse_args()
+
+    script_dir = Path(__file__).resolve().parent
+    repo_root = script_dir.parent.parent.parent.parent
+
+    baseline_path = repo_root / BASELINE_REL_PATH
+    run_suite_path = repo_root / RUN_SUITE_REL_PATH
+
+    if not run_suite_path.exists():
+        print(f"Error: Run suite not found: {run_suite_path}")
+        sys.exit(1)
+    try:
+        if USE_NPU_CONFIGS:
+            case_config_path = (
+                repo_root
+                / "python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py"
+            )
+        else:
+            case_config_path = resolve_case_config_path(repo_root, run_suite_path)
+    except (RuntimeError, FileNotFoundError) as exc:
+        print(f"Error: {exc}")
+        sys.exit(1)
+
+    suites = collect_diffusion_suites(
+        case_config_path,
+        run_suite_path,
+        baseline_path,
+    )
+    validate_suite_case_coverage(suites)
+
+    print("=== Diffusion Partition Computation ===")
+    print(f"Min partition time: {args.min_time}s ({args.min_time/60:.1f} min)")
+    print(f"Target partition time: {args.target_time}s ({args.target_time/60:.1f} min)")
+    print(f"Max partition time: {args.max_time}s ({args.max_time/60:.1f} min)")
+    print()
+
+    for suite_name, suite_info in suites.items():
+        if suite_name not in SUITE_OUTPUT_NAMES:
+            continue
+
+        items = build_partition_items(
+            suite_info, include_standalone=not args.parametrized_only
+        )
+        total_time = sum(item.est_time for item in items)
+        partition_count = compute_partition_count(
+            total_time_seconds=total_time,
+            min_time_seconds=args.min_time,
+            target_time_seconds=args.target_time,
+            max_time_seconds=args.max_time,
+            max_partitions=args.max_partitions,
+        )
+        partitions = partition_items_by_lpt(items, partition_count)
+
+        print_suite_summary(
+            suite_name,
+            suite_info,
+            partitions,
+            include_standalone=not args.parametrized_only,
+        )
+
+        output_name = SUITE_OUTPUT_NAMES[suite_name]
+        output_github_value(f"matrix-{output_name}", build_matrix(partition_count))
+        output_github_scalar(f"partition-count-{output_name}", str(partition_count))
+        output_github_value(
+            f"plan-{output_name}", build_partition_plan(suite_name, partitions)
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/diffusion_case_parser.py

@@ -0,0 +1,517 @@
+#!/usr/bin/env python3
+"""
+AST-based parser for diffusion test cases.
+
+This module parses the diffusion case source and run_suite.py using AST to
+extract test case information without requiring sglang dependencies. The case
+source file is discovered from ONE_GPU_CASES/TWO_GPU_CASES imports in
+run_suite.py so CI keeps a single source of truth.
+
+Usage:
+    # From sibling scripts in this directory:
+    from diffusion_case_parser import collect_diffusion_suites, resolve_case_config_path
+    case_config_path = resolve_case_config_path(repo_root, run_suite_path)
+    suites = collect_diffusion_suites(case_config_path, run_suite_path, baseline_path)
+"""
+
+import ast
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional
+
+# Mapping from list variable names to suite names
+CASE_LIST_TO_SUITE = {
+    "ONE_GPU_CASES": "1-gpu",
+    "ONE_GPU_CASES_A": "1-gpu",
+    "ONE_GPU_CASES_B": "1-gpu",
+    "ONE_GPU_CASES_C": "1-gpu-b200",
+    "ONE_GPU_MODELOPT_FP8_CASES": "1-gpu",
+    "ONE_GPU_MODELOPT_CASES": "1-gpu-b200",
+    "ONE_GPU_B200_CASES": "1-gpu-b200",
+    "TWO_GPU_CASES": "2-gpu",
+    "TWO_GPU_CASES_A": "2-gpu",
+    "TWO_GPU_CASES_B": "2-gpu",
+}
+
+# Default estimated time for cases without baseline (5 minutes)
+DEFAULT_EST_TIME_SECONDS = 300.0
+
+# Fixed overhead for server startup when estimated_full_test_time_s is not set
+STARTUP_OVERHEAD_SECONDS = 120.0
+
+# Paths relative to repository root
+BASELINE_REL_PATH = "python/sglang/multimodal_gen/test/server/perf_baselines"
+BASELINE_PLATFORM_ORDER = ("h100", "b200", "5090")
+RUN_SUITE_REL_PATH = "python/sglang/multimodal_gen/test/run_suite.py"
+
+USE_NPU_CONFIGS = os.getenv("USE_NPU_CONFIGS", "0").lower() in ("1", "true")
+
+if USE_NPU_CONFIGS:
+    BASELINE_REL_PATH = (
+        "python/sglang/multimodal_gen/test/server/perf_baselines_npu.json"
+    )
+    CASE_LIST_TO_SUITE = {
+        "ONE_NPU_CASES": "1-npu",
+        "TWO_NPU_CASES": "2-npu",
+    }
+
+
+@dataclass
+class DiffusionCaseInfo:
+    """Information about a single diffusion test case."""
+
+    case_id: str  # e.g., "qwen_image_t2i"
+    suite: str  # "1-gpu" or "2-gpu"
+    est_time: float  # estimated time in seconds
+
+
+@dataclass
+class DiffusionSuiteInfo:
+    """Complete information for a test suite."""
+
+    suite: str  # "1-gpu" or "2-gpu"
+    cases: List[DiffusionCaseInfo]  # parametrized test cases
+    standalone_files: List[str]  # standalone test files
+    standalone_est_times: Dict[str, float]  # standalone file -> estimated seconds
+    missing_standalone_estimates: List[
+        str
+    ]  # standalone files without configured estimate
+
+
+class DiffusionTestCaseVisitor(ast.NodeVisitor):
+    """
+    AST visitor to extract DiffusionTestCase definitions from the case config.
+
+    Parses assignments like:
+        ONE_GPU_CASES_A: list[DiffusionTestCase] = [
+            DiffusionTestCase("case_id", ...),
+            ...
+        ]
+    """
+
+    def __init__(self):
+        self.cases: Dict[str, List[str]] = {}  # list_name -> [case_id, ...]
+        self.factory_case_ids: Dict[str, str] = {}
+
+    def visit_Module(self, node: ast.Module):
+        for stmt in node.body:
+            if not isinstance(stmt, ast.FunctionDef):
+                continue
+            case_id = self._extract_factory_case_id(stmt)
+            if case_id:
+                self.factory_case_ids[stmt.name] = case_id
+
+        self.generic_visit(node)
+
+    def visit_Expr(self, node: ast.Expr):
+        """Handle ``LIST.append(...)`` mutations at any nesting level.
+
+        Previously only module-top-level ``ast.Expr`` statements were scanned for
+        ``.append()`` calls, so cases registered under a platform guard such as
+        ``if not current_platform.is_hip(): ONE_GPU_CASES.append(...)`` (used by
+        ``hunyuan3d_shape_gen`` and ``turbo_wan2_1_t2v_1.3b``) were invisible to
+        the partition planner and therefore never scheduled in CI. Visiting every
+        ``Expr`` lets ``generic_visit`` reach appends inside ``if``/``else`` blocks.
+        """
+        self._process_expr(node.value)
+        self.generic_visit(node)
+
+    def visit_Assign(self, node: ast.Assign):
+        self._process_assignment(node.targets, node.value)
+        self.generic_visit(node)
+
+    def visit_AnnAssign(self, node: ast.AnnAssign):
+        if node.target and node.value:
+            self._process_assignment([node.target], node.value)
+        self.generic_visit(node)
+
+    def visit_AugAssign(self, node: ast.AugAssign):
+        self._process_aug_assignment(node.target, node.op, node.value)
+        self.generic_visit(node)
+
+    def _process_assignment(self, targets: List[ast.AST], value: ast.AST):
+        """Process an assignment to extract case IDs."""
+        for target in targets:
+            if isinstance(target, ast.Name):
+                list_name = target.id
+                case_ids = self._extract_case_ids(value)
+                if case_ids is not None:
+                    self.cases[list_name] = case_ids
+
+    def _process_aug_assignment(self, target: ast.AST, op: ast.AST, value: ast.AST):
+        """Process `+=` style assignment to merge case lists."""
+        if not isinstance(target, ast.Name) or not isinstance(op, ast.Add):
+            return
+
+        if isinstance(value, ast.Name):
+            target_suite = CASE_LIST_TO_SUITE.get(target.id)
+            value_suite = CASE_LIST_TO_SUITE.get(value.id)
+            if target_suite and value_suite and target_suite != value_suite:
+                return
+
+        rhs_case_ids = self._extract_case_ids(value)
+        if rhs_case_ids is None:
+            return
+
+        lhs_case_ids = self.cases.get(target.id, [])
+        self.cases[target.id] = [*lhs_case_ids, *rhs_case_ids]
+
+    def _process_expr(self, node: ast.AST):
+        """Process list mutation calls such as `ONE_GPU_CASES.append(...)`."""
+        if not isinstance(node, ast.Call):
+            return
+        if not isinstance(node.func, ast.Attribute):
+            return
+        if node.func.attr != "append":
+            return
+        if not isinstance(node.func.value, ast.Name):
+            return
+        list_name = node.func.value.id
+        if list_name not in CASE_LIST_TO_SUITE:
+            return
+        if len(node.args) != 1:
+            return
+
+        case_id = self._extract_case_id_from_call(node.args[0])
+        if case_id:
+            self.cases.setdefault(list_name, []).append(case_id)
+
+    def _extract_case_ids(self, node: ast.AST) -> Optional[List[str]]:
+        """Extract case IDs from a supported expression."""
+        if isinstance(node, ast.List):
+            return self._extract_case_ids_from_list(node)
+
+        if isinstance(node, ast.Name):
+            # Reference to a previously parsed list variable.
+            if node.id not in self.cases:
+                return None
+            return list(self.cases[node.id])
+
+        if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+            left_ids = self._extract_case_ids(node.left)
+            right_ids = self._extract_case_ids(node.right)
+            if left_ids is None or right_ids is None:
+                return None
+            return [*left_ids, *right_ids]
+
+        return None
+
+    def _extract_case_ids_from_list(self, node: ast.List) -> List[str]:
+        """Extract case IDs from a literal list of DiffusionTestCase calls."""
+        case_ids = []
+        for elt in node.elts:
+            if isinstance(elt, ast.Starred):
+                starred_case_ids = self._extract_case_ids(elt.value)
+                if starred_case_ids:
+                    case_ids.extend(starred_case_ids)
+                continue
+            case_id = self._extract_case_id_from_call(elt)
+            if case_id:
+                case_ids.append(case_id)
+        return case_ids
+
+    def _extract_case_id_from_call(self, node: ast.AST) -> Optional[str]:
+        """Extract case_id from DiffusionTestCase(...) call."""
+        if not isinstance(node, ast.Call):
+            return None
+
+        # First positional argument is the case_id.
+        if isinstance(node.func, ast.Name) and node.func.id in {
+            "DiffusionTestCase",
+            "_make_modelopt_ci_case",
+        }:
+            if node.args and isinstance(node.args[0], ast.Constant):
+                return node.args[0].value
+        if isinstance(node.func, ast.Name) and not node.args:
+            return self.factory_case_ids.get(node.func.id)
+
+        return None
+
+    def _extract_factory_case_id(self, node: ast.FunctionDef) -> Optional[str]:
+        for child in ast.walk(node):
+            if not isinstance(child, ast.Return) or child.value is None:
+                continue
+            case_id = self._extract_case_id_from_call(child.value)
+            if case_id:
+                return case_id
+        return None
+
+
+def resolve_case_config_path(repo_root: Path, run_suite_path: Path) -> Path:
+    """
+    Resolve the diffusion case config path from run_suite imports.
+
+    run_suite.py must import BOTH ONE_GPU_CASES and TWO_GPU_CASES from the same
+    module. That imported module is treated as the single source of truth.
+    """
+    with open(run_suite_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    tree = ast.parse(content, filename=str(run_suite_path))
+    one_gpu_module: Optional[str] = None
+    two_gpu_module: Optional[str] = None
+
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.ImportFrom) or not node.module:
+            continue
+        imported_names = {alias.name for alias in node.names}
+        if "ONE_GPU_CASES" in imported_names:
+            one_gpu_module = node.module
+        if "TWO_GPU_CASES" in imported_names:
+            two_gpu_module = node.module
+
+    if one_gpu_module is None or two_gpu_module is None:
+        raise RuntimeError(
+            "run_suite.py must import BOTH ONE_GPU_CASES and TWO_GPU_CASES."
+        )
+    if one_gpu_module != two_gpu_module:
+        raise RuntimeError(
+            "run_suite.py imports ONE_GPU_CASES and TWO_GPU_CASES from different "
+            f"modules: {one_gpu_module} vs {two_gpu_module}"
+        )
+
+    rel_path = Path(*one_gpu_module.split(".")).with_suffix(".py")
+    candidates = [repo_root / rel_path, repo_root / "python" / rel_path]
+    case_config_path = next((path for path in candidates if path.exists()), None)
+    if case_config_path is None:
+        raise FileNotFoundError(
+            "Resolved case config from run_suite does not exist. Checked: "
+            + ", ".join(str(path) for path in candidates)
+        )
+    return case_config_path
+
+
+class RunSuiteVisitor(ast.NodeVisitor):
+    """
+    AST visitor to extract standalone metadata from run_suite.py.
+
+    Parses:
+        STANDALONE_FILES = {
+            "1-gpu": ["test_lora_format_adapter.py"],
+            "2-gpu": [],
+        }
+    """
+
+    def __init__(self):
+        self.standalone_files: Dict[str, List[str]] = {}
+        self.standalone_est_times: Dict[str, Dict[str, float]] = {}
+
+    def visit_Assign(self, node: ast.Assign):
+        for target in node.targets:
+            if isinstance(target, ast.Name) and target.id == "STANDALONE_FILES":
+                self.standalone_files = self._extract_file_dict(node.value)
+            if (
+                isinstance(target, ast.Name)
+                and target.id == "STANDALONE_FILE_EST_TIMES"
+            ):
+                self.standalone_est_times = self._extract_est_time_dict(node.value)
+        self.generic_visit(node)
+
+    def _extract_file_dict(self, node: ast.AST) -> Dict[str, List[str]]:
+        """Extract dictionary of suite -> file list."""
+        result = {}
+        if isinstance(node, ast.Dict):
+            for key, value in zip(node.keys, node.values):
+                if isinstance(key, ast.Constant) and isinstance(value, ast.List):
+                    suite = key.value
+                    files = [
+                        elt.value for elt in value.elts if isinstance(elt, ast.Constant)
+                    ]
+                    result[suite] = files
+        return result
+
+    def _extract_est_time_dict(self, node: ast.AST) -> Dict[str, Dict[str, float]]:
+        """Extract dictionary of suite -> standalone file -> estimated seconds."""
+        result = {}
+        if not isinstance(node, ast.Dict):
+            return result
+
+        for key, value in zip(node.keys, node.values):
+            if not isinstance(key, ast.Constant) or not isinstance(value, ast.Dict):
+                continue
+
+            suite = key.value
+            suite_est_times = {}
+            for inner_key, inner_value in zip(value.keys, value.values):
+                if not (
+                    isinstance(inner_key, ast.Constant)
+                    and isinstance(inner_value, ast.Constant)
+                ):
+                    continue
+                suite_est_times[inner_key.value] = float(inner_value.value)
+            result[suite] = suite_est_times
+
+        return result
+
+
+def _iter_baseline_paths(baseline_path: Path) -> List[Path]:
+    if baseline_path.is_file():
+        return [baseline_path]
+    if not baseline_path.is_dir():
+        return []
+
+    ordered_paths = [
+        baseline_path / f"{platform}.json" for platform in BASELINE_PLATFORM_ORDER
+    ]
+    ordered_paths.extend(
+        path
+        for path in sorted(baseline_path.glob("*.json"))
+        if path not in ordered_paths
+    )
+    return [path for path in ordered_paths if path.exists()]
+
+
+def load_baselines(baseline_path: Path) -> Dict[str, float]:
+    """
+    Load performance baselines from a JSON file or platform baseline directory.
+
+    Returns:
+        Dictionary mapping case_id to estimated time in seconds.
+    """
+    baselines = {}
+    for path in _iter_baseline_paths(baseline_path):
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        scenarios = data.get("scenarios", {})
+        for case_id, scenario in scenarios.items():
+            if scenario.get("estimated_full_test_time_s") is not None:
+                est_time = scenario["estimated_full_test_time_s"]
+            else:
+                expected_e2e_ms = scenario.get("expected_e2e_ms", 0)
+                est_time = expected_e2e_ms / 1000.0 + STARTUP_OVERHEAD_SECONDS
+            baselines.setdefault(case_id, est_time)
+
+    return baselines
+
+
+def get_case_est_time(case_id: str, baselines: Dict[str, float]) -> float:
+    """Get estimated time for a case, with fallback to default."""
+    return baselines.get(case_id, DEFAULT_EST_TIME_SECONDS)
+
+
+def parse_testcase_configs(config_path: Path) -> Dict[str, List[str]]:
+    """
+    Parse a diffusion case config file to extract case IDs.
+
+    Returns:
+        Dictionary mapping list name to case IDs.
+        e.g., {"ONE_GPU_CASES_A": ["qwen_image_t2i", ...], ...}
+    """
+    with open(config_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    tree = ast.parse(content, filename=str(config_path))
+    visitor = DiffusionTestCaseVisitor()
+    visitor.visit(tree)
+
+    return visitor.cases
+
+
+def parse_run_suite_standalone_data(
+    run_suite_path: Path,
+) -> tuple[Dict[str, List[str]], Dict[str, Dict[str, float]]]:
+    """
+    Parse run_suite.py to extract standalone file metadata.
+
+    Returns:
+        Tuple of:
+          - suite -> standalone file list
+          - suite -> standalone file -> estimated seconds
+    """
+    with open(run_suite_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    tree = ast.parse(content, filename=str(run_suite_path))
+    visitor = RunSuiteVisitor()
+    visitor.visit(tree)
+
+    return visitor.standalone_files, visitor.standalone_est_times
+
+
+def validate_standalone_est_times(
+    standalone_files: Dict[str, List[str]],
+    standalone_est_times: Dict[str, Dict[str, float]],
+) -> Dict[str, List[str]]:
+    missing_by_suite = {}
+    for suite, files in standalone_files.items():
+        suite_est_times = standalone_est_times.get(suite, {})
+        missing = [
+            standalone_file
+            for standalone_file in files
+            if standalone_file not in suite_est_times
+        ]
+        if missing:
+            missing_by_suite[suite] = missing
+    return missing_by_suite
+
+
+def collect_diffusion_suites(
+    case_config_path: Path,
+    run_suite_path: Path,
+    baseline_path: Path,
+) -> Dict[str, DiffusionSuiteInfo]:
+    """
+    Collect all diffusion test suite information using AST parsing.
+
+    Args:
+        case_config_path: Path to case config (resolved from run_suite.py)
+        run_suite_path: Path to run_suite.py
+        baseline_path: Path to perf_baselines/ or a single baseline JSON file
+
+    Returns:
+        Dictionary mapping suite name to DiffusionSuiteInfo.
+    """
+    # Parse case IDs from the single source case config.
+    case_lists = parse_testcase_configs(case_config_path)
+
+    # Parse standalone files from run_suite.py
+    standalone_files, standalone_est_times = parse_run_suite_standalone_data(
+        run_suite_path
+    )
+    missing_standalone_estimates = validate_standalone_est_times(
+        standalone_files, standalone_est_times
+    )
+
+    # Load baselines for time estimation
+    baselines = load_baselines(baseline_path)
+
+    # Build suite info
+    suites = {}
+    for list_name, suite in CASE_LIST_TO_SUITE.items():
+        case_ids = case_lists.get(list_name, [])
+        cases = [
+            DiffusionCaseInfo(
+                case_id=cid,
+                suite=suite,
+                est_time=get_case_est_time(cid, baselines),
+            )
+            for cid in case_ids
+        ]
+
+        if suite not in suites:
+            suites[suite] = DiffusionSuiteInfo(
+                suite=suite,
+                cases=[],
+                standalone_files=standalone_files.get(suite, []),
+                standalone_est_times=dict(standalone_est_times.get(suite, {})),
+                missing_standalone_estimates=list(
+                    missing_standalone_estimates.get(suite, [])
+                ),
+            )
+        suites[suite].cases.extend(cases)
+
+    # Dedupe duplicated case IDs while preserving first-seen order.
+    for suite_info in suites.values():
+        seen_case_ids = set()
+        deduped_cases = []
+        for case in suite_info.cases:
+            if case.case_id in seen_case_ids:
+                continue
+            seen_case_ids.add(case.case_id)
+            deduped_cases.append(case)
+        suite_info.cases = deduped_cases
+
+    return suites

scripts/ci/utils/diffusion/generate_diffusion_dashboard.py

@@ -0,0 +1,835 @@
+"""Generate a Markdown dashboard for SGLang-Diffusion nightly benchmarks.
+
+Reads current comparison results + historical data from sgl-project/ci-data repo
+and produces a Markdown report with tables and trend charts saved as PNG files.
+
+Usage:
+    python3 scripts/ci/utils/diffusion/generate_diffusion_dashboard.py \
+        --results comparison-results.json \
+        --output dashboard.md \
+        --charts-dir comparison-charts/ \
+        --history-dir history/           # optional, local history JSONs
+        --fetch-history                  # fetch from GitHub API instead
+"""
+
+import argparse
+import json
+import os
+from datetime import datetime, timezone
+
+# ---------------------------------------------------------------------------
+# History fetching (from sgl-project/ci-data repo via GitHub API)
+# ---------------------------------------------------------------------------
+
+CI_DATA_REPO_OWNER = "sgl-project"
+CI_DATA_REPO_NAME = "ci-data"
+CI_DATA_BRANCH = "main"
+HISTORY_PREFIX = "diffusion-comparisons"
+MAX_HISTORY_RUNS = 29
+
+# Base URL for chart images pushed to sgl-project/ci-data
+CHARTS_RAW_BASE_URL = (
+    f"https://raw.githubusercontent.com/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
+    f"/{CI_DATA_BRANCH}/{HISTORY_PREFIX}/charts"
+)
+
+
+def _github_get(url: str, token: str) -> dict | list | None:
+    """Simple GET to GitHub API."""
+    from urllib.error import HTTPError
+    from urllib.request import Request, urlopen
+
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    req = Request(url, headers=headers)
+    try:
+        with urlopen(req) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except HTTPError as e:
+        print(f"  Warning: GitHub API request failed ({e.code}): {url}")
+        return None
+    except Exception as e:
+        print(f"  Warning: GitHub API request error: {e}")
+        return None
+
+
+def fetch_history_from_github(token: str) -> list[dict]:
+    """Fetch recent comparison result JSONs from sgl-project/ci-data repo."""
+    print("Fetching historical comparison data from GitHub...")
+    url = (
+        f"https://api.github.com/repos/{CI_DATA_REPO_OWNER}/{CI_DATA_REPO_NAME}"
+        f"/contents/{HISTORY_PREFIX}?ref={CI_DATA_BRANCH}"
+    )
+    listing = _github_get(url, token)
+    if not listing or not isinstance(listing, list):
+        print("  No historical data found.")
+        return []
+
+    # Filter JSON files and sort by name (date prefix) descending
+    json_files = sorted(
+        [f for f in listing if f["name"].endswith(".json")],
+        key=lambda f: f["name"],
+        reverse=True,
+    )[:MAX_HISTORY_RUNS]
+
+    history = []
+    for entry in json_files:
+        raw_url = entry.get("download_url")
+        if not raw_url:
+            continue
+        data = _github_get(raw_url, token)
+        if data and isinstance(data, dict):
+            history.append(data)
+    print(f"  Loaded {len(history)} historical run(s).")
+    return history
+
+
+def load_history_from_dir(history_dir: str) -> list[dict]:
+    """Load historical JSONs from a local directory."""
+    if not os.path.isdir(history_dir):
+        return []
+    files = sorted(
+        [f for f in os.listdir(history_dir) if f.endswith(".json")],
+        reverse=True,
+    )[:MAX_HISTORY_RUNS]
+    history = []
+    for fname in files:
+        try:
+            with open(os.path.join(history_dir, fname)) as f:
+                history.append(json.load(f))
+        except Exception:
+            pass
+    return history
+
+
+# ---------------------------------------------------------------------------
+# Dashboard generation
+# ---------------------------------------------------------------------------
+
+
+def _fmt_latency(val: float | None) -> str:
+    if val is None:
+        return "N/A"
+    return f"{val:.2f}"
+
+
+def _fmt_speedup(sglang_lat: float | None, other_lat: float | None) -> str:
+    if sglang_lat is None or other_lat is None or sglang_lat <= 0:
+        return "N/A"
+    ratio = other_lat / sglang_lat
+    return f"{ratio:.2f}x"
+
+
+def _short_date(ts: str) -> str:
+    """Extract short date from ISO timestamp."""
+    try:
+        dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+        return dt.strftime("%b %d")
+    except Exception:
+        return ts[:10]
+
+
+def _short_sha(sha: str) -> str:
+    return sha[:7] if sha and sha != "unknown" else "?"
+
+
+def _assess_risk(
+    cid: str,
+    current_cases: dict[str, dict[str, float | None]],
+    history: list[dict],
+    other_frameworks: list[str],
+) -> tuple[str, str]:
+    """Assess risk for a given case, returning (emoji, reason).
+
+    Rules (checked in order):
+    - N/A latency → ❌ broken
+    - History exists: SGLang latency >5% vs avg of last 3 runs → ⚠️ regression
+    - Competitor exists & SGLang slower → 🔴 competitive risk
+    - SGLang faster than all competitors by >20% → 🟢 strong advantage
+    - SGLang faster than all competitors by ≤20% → 🟡 moderate advantage
+    - Default → ✅ stable
+    """
+    sg_lat = current_cases.get(cid, {}).get("sglang")
+
+    # Broken: sglang latency is N/A
+    if sg_lat is None:
+        return "❌", f"{cid}: SGLang latency is N/A (broken)"
+
+    # Check regression against 3-run historical average
+    if history:
+        hist_lats: list[float] = []
+        for run in history[:3]:
+            run_cases = _extract_case_results(run)
+            h_lat = run_cases.get(cid, {}).get("sglang")
+            if h_lat is not None:
+                hist_lats.append(h_lat)
+        if hist_lats:
+            avg_3 = sum(hist_lats) / len(hist_lats)
+            if avg_3 > 0 and (sg_lat - avg_3) / avg_3 > 0.05:
+                pct = (sg_lat - avg_3) / avg_3 * 100
+                return (
+                    "⚠️",
+                    f"{cid}: SGLang regression +{pct:.1f}% vs 3-run avg "
+                    f"({sg_lat:.2f}s vs {avg_3:.2f}s)",
+                )
+
+    # Check competitive risk
+    if other_frameworks:
+        competitor_lats: dict[str, float] = {}
+        for ofw in other_frameworks:
+            olat = current_cases.get(cid, {}).get(ofw)
+            if olat is not None:
+                competitor_lats[ofw] = olat
+
+        if competitor_lats:
+            # SGLang slower than any competitor?
+            for ofw, olat in competitor_lats.items():
+                if sg_lat > olat:
+                    return (
+                        "🔴",
+                        f"{cid}: SGLang slower than {ofw} "
+                        f"({sg_lat:.2f}s vs {olat:.2f}s)",
+                    )
+
+            # SGLang faster — check margin
+            min_competitor = min(competitor_lats.values())
+            advantage = (min_competitor - sg_lat) / min_competitor
+            if advantage > 0.20:
+                return "🟢", ""
+            else:
+                return "🟡", ""
+
+    # Default: stable
+    return "✅", ""
+
+
+def _trend_emoji(current: float | None, previous: float | None) -> str:
+    if current is None or previous is None:
+        return ""
+    diff_pct = (current - previous) / previous * 100
+    if diff_pct < -2:
+        return " :arrow_down:"  # faster (good)
+    elif diff_pct > 2:
+        return " :arrow_up:"  # slower (bad)
+    return " :left_right_arrow:"
+
+
+def _extract_case_results(run_data: dict) -> dict[str, dict[str, float | None]]:
+    """Extract {case_id: {framework: latency}} from a run."""
+    mapping: dict[str, dict[str, float | None]] = {}
+    for r in run_data.get("results", []):
+        cid = r["case_id"]
+        fw = r["framework"]
+        if cid not in mapping:
+            mapping[cid] = {}
+        mapping[cid][fw] = r.get("latency_s")
+    return mapping
+
+
+def _sanitize_filename(name: str) -> str:
+    """Sanitize a case ID to be a safe filename."""
+    return name.replace("/", "_").replace(" ", "_").replace(":", "_")
+
+
+def generate_dashboard(
+    current: dict,
+    history: list[dict],
+    charts_dir: str | None = None,
+) -> tuple[str, list[str]]:
+    """Generate full markdown dashboard.
+
+    Returns (markdown_string, alert_reasons) where alert_reasons is a list of
+    human-readable strings for cases that need attention (empty if all is well).
+
+    If charts_dir is provided, saves chart PNGs as files to that directory
+    and references them via raw.githubusercontent URLs. Otherwise, charts
+    are omitted.
+
+    Returns the markdown string.
+    """
+    lines: list[str] = []
+    lines.append("# SGLang-Diffusion Nightly Performance Dashboard\n")
+    ts = current.get("timestamp", datetime.now(timezone.utc).isoformat())
+    sha = current.get("commit_sha", "unknown")
+    lines.append(f"*Generated: {_short_date(ts)} | Commit: `{_short_sha(sha)}`*\n")
+
+    current_cases = _extract_case_results(current)
+    case_ids = list(current_cases.keys())
+
+    # ---- Regression detection ----
+    REGRESSION_THRESHOLD = 0.05  # 5%
+    regressions: list[str] = []
+    if history:
+        prev_cases = _extract_case_results(history[0])
+        for cid in case_ids:
+            for fw in ("sglang", "vllm-omni"):
+                cur = current_cases.get(cid, {}).get(fw)
+                prev = prev_cases.get(cid, {}).get(fw)
+                if cur and prev and prev > 0:
+                    pct = (cur - prev) / prev
+                    if pct > REGRESSION_THRESHOLD:
+                        regressions.append(
+                            f"**{cid}** ({fw}): {prev:.2f}s -> {cur:.2f}s "
+                            f"(+{pct*100:.1f}%)"
+                        )
+
+    if regressions:
+        lines.append("> [!WARNING]\n> **Performance Regression Detected**\n>")
+        for reg in regressions:
+            lines.append(f"> - {reg}")
+        lines.append("\n")
+
+    # Discover all frameworks present in results
+    all_frameworks = []
+    seen_fw = set()
+    for r in current.get("results", []):
+        fw = r["framework"]
+        if fw not in seen_fw:
+            all_frameworks.append(fw)
+            seen_fw.add(fw)
+    # Ensure sglang is first
+    if "sglang" in all_frameworks:
+        all_frameworks.remove("sglang")
+        all_frameworks.insert(0, "sglang")
+    other_frameworks = [fw for fw in all_frameworks if fw != "sglang"]
+
+    # ---- Section 1: SGLang-Diffusion performance (current run) ----
+    lines.append("## SGLang-Diffusion Performance\n")
+
+    # Compute risk assessments for all cases
+    risk_map: dict[str, tuple[str, str]] = {}
+    for cid in case_ids:
+        risk_map[cid] = _assess_risk(cid, current_cases, history, other_frameworks)
+
+    # Dynamic header
+    header = "| Model | Risk |"
+    sep = "|-------|------|"
+    for fw in all_frameworks:
+        header += f" {fw} (s) |"
+        sep += "---------|"
+    for ofw in other_frameworks:
+        header += f" vs {ofw} |"
+        sep += "---------|"
+    lines.append(header)
+    lines.append(sep)
+
+    # One row per case (deduplicated by case_id)
+    seen_cases = set()
+    for r in current.get("results", []):
+        cid = r["case_id"]
+        if cid in seen_cases:
+            continue
+        seen_cases.add(cid)
+
+        case_fws = current_cases.get(cid, {})
+        sg_lat = case_fws.get("sglang")
+
+        risk_emoji, _ = risk_map.get(cid, ("✅", ""))
+        row = f"| {r['model'].split('/')[-1]} | {risk_emoji} |"
+        # Latency columns -- bold the fastest
+        lats = {fw: case_fws.get(fw) for fw in all_frameworks}
+        valid_lats = [v for v in lats.values() if v is not None]
+        min_lat = min(valid_lats) if valid_lats else None
+        for fw in all_frameworks:
+            lat = lats[fw]
+            if lat is not None and min_lat is not None and lat == min_lat:
+                row += f" **{_fmt_latency(lat)}** |"
+            else:
+                row += f" {_fmt_latency(lat)} |"
+        # Speedup columns
+        for ofw in other_frameworks:
+            row += f" {_fmt_speedup(sg_lat, case_fws.get(ofw))} |"
+        lines.append(row)
+
+    # ---- Section 2: Speedup-over-time vs. other frameworks (rendered only when present) ----
+    if history and other_frameworks:
+        lines.append("\n## SGLang vs vLLM-Omni Speedup Over Time\n")
+
+        header = "| Date |"
+        sep = "|------|"
+        for cid in case_ids:
+            header += f" {cid} |"
+            sep += "---------|"
+        lines.append(header)
+        lines.append(sep)
+
+        all_runs = [current] + history
+        for run in all_runs:
+            run_cases = _extract_case_results(run)
+            date = _short_date(run.get("timestamp", ""))
+            row = f"| {date} |"
+            for cid in case_ids:
+                sg = run_cases.get(cid, {}).get("sglang")
+                vl = run_cases.get(cid, {}).get("vllm-omni")
+                row += f" {_fmt_speedup(sg, vl)} |"
+            lines.append(row)
+
+    # ---- Section 4: Matplotlib Trend Charts (saved as PNG files) ----
+    if history and charts_dir:
+        all_runs = list(reversed([current] + history))  # chronological order
+
+        def _chart_label(run: dict) -> str:
+            d = _short_date(run.get("timestamp", ""))
+            s = _short_sha(run.get("commit_sha", ""))
+            return f"{d}\n({s})"
+
+        try:
+            import matplotlib
+
+            matplotlib.use("Agg")
+            import matplotlib.pyplot as plt
+
+            os.makedirs(charts_dir, exist_ok=True)
+
+            # Per-case latency trend charts
+            for cid in case_ids:
+                labels = []
+                sg_vals = []
+                vl_vals = []
+                for run in all_runs:
+                    run_cases = _extract_case_results(run)
+                    sg = run_cases.get(cid, {}).get("sglang")
+                    vl = run_cases.get(cid, {}).get("vllm-omni")
+                    if sg is None:
+                        continue
+                    labels.append(_chart_label(run))
+                    sg_vals.append(sg)
+                    vl_vals.append(vl)
+
+                if not sg_vals:
+                    continue
+
+                has_vl = any(v is not None for v in vl_vals)
+                fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.2), 4))
+
+                # SGLang line
+                ax.plot(
+                    range(len(sg_vals)),
+                    sg_vals,
+                    "o-",
+                    color="#2563eb",
+                    linewidth=2,
+                    markersize=6,
+                    label="SGLang",
+                )
+                for i, v in enumerate(sg_vals):
+                    ax.annotate(
+                        f"{v:.2f}s",
+                        (i, v),
+                        textcoords="offset points",
+                        xytext=(0, 10),
+                        ha="center",
+                        fontsize=8,
+                        fontweight="bold",
+                        color="#2563eb",
+                    )
+
+                # vLLM-Omni line (if data exists)
+                if has_vl:
+                    vl_clean = [v if v is not None else float("nan") for v in vl_vals]
+                    ax.plot(
+                        range(len(vl_clean)),
+                        vl_clean,
+                        "s--",
+                        color="#dc2626",
+                        linewidth=2,
+                        markersize=5,
+                        label="vLLM-Omni",
+                    )
+                    for i, v in enumerate(vl_vals):
+                        if v is not None:
+                            ax.annotate(
+                                f"{v:.2f}s",
+                                (i, v),
+                                textcoords="offset points",
+                                xytext=(0, -14),
+                                ha="center",
+                                fontsize=8,
+                                color="#dc2626",
+                            )
+
+                ax.set_xticks(range(len(labels)))
+                ax.set_xticklabels(labels, fontsize=7)
+                ax.set_ylabel("Latency (s)")
+                ax.set_title(f"Latency Trend -- {cid}", fontsize=11, fontweight="bold")
+                ax.legend(loc="lower right", fontsize=8, framealpha=0.8)
+                ax.grid(True, alpha=0.3)
+                all_vals = sg_vals + [v for v in vl_vals if v is not None]
+                y_min = min(all_vals)
+                y_max = max(all_vals)
+                y_range = y_max - y_min if y_max > y_min else max(y_max * 0.1, 0.1)
+                ax.set_ylim(
+                    bottom=max(0, y_min - y_range * 0.3),
+                    top=y_max + y_range * 0.3,
+                )
+
+                filename = f"latency_{_sanitize_filename(cid)}.png"
+                chart_path = os.path.join(charts_dir, filename)
+                fig.savefig(chart_path, format="png", dpi=120, bbox_inches="tight")
+                plt.close(fig)
+                print(f"  Saved chart: {chart_path}")
+
+                chart_url = f"{CHARTS_RAW_BASE_URL}/{filename}"
+                lines.append(f"\n### Latency Trend: {cid}\n")
+                lines.append(f"![Latency Trend {cid}]({chart_url})\n")
+
+            # Speedup trend chart (only if multiple frameworks)
+            if other_frameworks:
+                fig, ax = plt.subplots(figsize=(max(6, len(all_runs) * 1.2), 4))
+                colors = ["#2563eb", "#dc2626", "#16a34a", "#ea580c"]
+                for ci_idx, cid in enumerate(case_ids):
+                    speedups = []
+                    run_labels = []
+                    for run in all_runs:
+                        run_cases = _extract_case_results(run)
+                        sg = run_cases.get(cid, {}).get("sglang")
+                        vl = run_cases.get(cid, {}).get("vllm-omni")
+                        if sg and vl and sg > 0:
+                            speedups.append(vl / sg)
+                        else:
+                            speedups.append(None)
+                        run_labels.append(_chart_label(run))
+                    clean = [v if v is not None else float("nan") for v in speedups]
+                    ax.plot(
+                        range(len(clean)),
+                        clean,
+                        "o-",
+                        color=colors[ci_idx % len(colors)],
+                        linewidth=2,
+                        markersize=5,
+                        label=cid,
+                    )
+
+                ax.set_xticks(range(len(run_labels)))
+                ax.set_xticklabels(run_labels, fontsize=7)
+                ax.set_ylabel("Speedup (x)")
+                ax.set_title(
+                    "SGLang Speedup Over vLLM-Omni", fontsize=11, fontweight="bold"
+                )
+                ax.axhline(y=1.0, color="gray", linestyle=":", alpha=0.5)
+                ax.legend(loc="upper left", fontsize=7)
+                ax.grid(True, alpha=0.3)
+
+                filename = "speedup_trend.png"
+                chart_path = os.path.join(charts_dir, filename)
+                fig.savefig(chart_path, format="png", dpi=120, bbox_inches="tight")
+                plt.close(fig)
+                print(f"  Saved chart: {chart_path}")
+
+                chart_url = f"{CHARTS_RAW_BASE_URL}/{filename}"
+                lines.append("\n### Speedup Trend (SGLang vs vLLM-Omni)\n")
+                lines.append(f"![Speedup Trend]({chart_url})\n")
+
+        except ImportError:
+            lines.append("\n*Charts unavailable (matplotlib not installed)*\n")
+
+    # ---- SGLang Performance Trend (raw data table, at the end) ----
+    if history:
+        lines.append(f"\n## SGLang Performance Trend (Last {len(history) + 1} Runs)\n")
+
+        header = "| Date | Commit |"
+        sep = "|------|--------|"
+        for cid in case_ids:
+            header += f" {cid} (s) |"
+            sep += "---------|"
+        header += " Trend |"
+        sep += "-------|"
+        lines.append(header)
+        lines.append(sep)
+
+        all_runs = [current] + history
+        for i, run in enumerate(all_runs):
+            run_cases = _extract_case_results(run)
+            date = _short_date(run.get("timestamp", ""))
+            sha_s = _short_sha(run.get("commit_sha", ""))
+            row = f"| {date} | `{sha_s}` |"
+            for cid in case_ids:
+                lat = run_cases.get(cid, {}).get("sglang")
+                row += f" {_fmt_latency(lat)} |"
+            if i + 1 < len(all_runs):
+                prev_cases = _extract_case_results(all_runs[i + 1])
+                emojis = []
+                for cid in case_ids:
+                    cur = run_cases.get(cid, {}).get("sglang")
+                    prev = prev_cases.get(cid, {}).get("sglang")
+                    emojis.append(_trend_emoji(cur, prev))
+                row += " ".join(emojis) + " |"
+            else:
+                row += " -- |"
+            lines.append(row)
+
+    # ---- Risk Notification ----
+    alert_cases = [
+        (cid, emoji, reason)
+        for cid, (emoji, reason) in risk_map.items()
+        if emoji in ("⚠️", "🔴", "❌")
+    ]
+    if alert_cases:
+        lines.append("\n> [!CAUTION]")
+        lines.append("> **Action Required — Performance Alert**")
+        lines.append(">")
+        lines.append("> The following cases need attention:")
+        for _cid, _emoji, reason in alert_cases:
+            lines.append(f"> - {reason}")
+        lines.append("")
+
+    # Footer
+    lines.append("\n---")
+    lines.append(
+        "*Generated by `generate_diffusion_dashboard.py` in SGLang nightly CI.*"
+    )
+
+    alert_reasons = [reason for _, _, reason in alert_cases]
+    return "\n".join(lines) + "\n", alert_reasons
+
+
+ALERT_ASSIGNEES = ["mickqian", "bbuf", "yhyang201"]
+ALERT_LABEL = "perf-regression"
+
+
+ALERT_ISSUE_TITLE = "[Diffusion CI] Performance regression tracker"
+
+
+def _find_alert_issue(repo: str) -> tuple[str | None, bool]:
+    """Find the perf-regression tracker issue (open OR closed).
+
+    Returns (issue_number, is_open).  Prefers an open issue; if none,
+    returns the most recent closed one so it can be reopened.
+    """
+    import subprocess
+
+    for state in ("open", "closed"):
+        result = subprocess.run(
+            [
+                "gh",
+                "issue",
+                "list",
+                "--repo",
+                repo,
+                "--label",
+                ALERT_LABEL,
+                "--state",
+                state,
+                "--json",
+                "number",
+                "--limit",
+                "1",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if result.returncode != 0 or not result.stdout.strip():
+            continue
+        issues = json.loads(result.stdout)
+        if issues:
+            return str(issues[0]["number"]), state == "open"
+    return None, False
+
+
+def _create_alert_issue(alert_reasons: list[str]) -> None:
+    """Create or update the single perf-regression tracker issue.
+
+    Logic:
+    - If an open issue exists  → add a comment with the new alert.
+    - If a closed issue exists → reopen it, then add a comment.
+    - If no issue exists       → create one.
+
+    This guarantees at most one tracker issue ever exists.
+
+    Uses `gh` (GitHub CLI) which is available in all GitHub Actions runners.
+    Falls back silently outside CI.
+    """
+    import subprocess
+
+    run_url = ""
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    repo = os.environ.get("GITHUB_REPOSITORY", "sgl-project/sglang")
+    server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+    if run_id:
+        run_url = f"{server_url}/{repo}/actions/runs/{run_id}"
+
+    date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    body_lines = [
+        f"## Performance Alert — {date}",
+        "",
+        "The nightly diffusion benchmark detected the following issue(s):",
+        "",
+    ]
+    for reason in alert_reasons:
+        body_lines.append(f"- {reason}")
+    if run_url:
+        body_lines += ["", f"**CI Run:** {run_url}"]
+    body = "\n".join(body_lines)
+
+    try:
+        existing, is_open = _find_alert_issue(repo)
+
+        if existing:
+            # Reopen if closed
+            if not is_open:
+                subprocess.run(
+                    [
+                        "gh",
+                        "issue",
+                        "reopen",
+                        existing,
+                        "--repo",
+                        repo,
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=30,
+                )
+                print(f"Reopened alert issue #{existing}")
+
+            # Add comment
+            result = subprocess.run(
+                [
+                    "gh",
+                    "issue",
+                    "comment",
+                    existing,
+                    "--repo",
+                    repo,
+                    "--body",
+                    body,
+                ],
+                capture_output=True,
+                text=True,
+                timeout=30,
+            )
+            if result.returncode == 0:
+                print(f"Commented on alert issue #{existing}")
+            else:
+                print(
+                    f"Warning: failed to comment on issue #{existing} "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+        else:
+            # Create a new issue
+            cmd = [
+                "gh",
+                "issue",
+                "create",
+                "--repo",
+                repo,
+                "--title",
+                ALERT_ISSUE_TITLE,
+                "--body",
+                body,
+                "--label",
+                ALERT_LABEL,
+            ]
+            for user in ALERT_ASSIGNEES:
+                cmd += ["--assignee", user]
+
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+            if result.returncode == 0:
+                print(f"Created alert issue: {result.stdout.strip()}")
+            else:
+                print(
+                    f"Warning: failed to create alert issue "
+                    f"(rc={result.returncode}): {result.stderr.strip()}"
+                )
+    except FileNotFoundError:
+        print("Warning: `gh` CLI not found — skipping alert issue creation")
+    except Exception as e:
+        print(f"Warning: failed to create/update alert issue: {e}")
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate SGLang-Diffusion nightly benchmark dashboard"
+    )
+    parser.add_argument(
+        "--results",
+        required=True,
+        help="Path to comparison-results.json from current run",
+    )
+    parser.add_argument(
+        "--output",
+        default="dashboard.md",
+        help="Output markdown file path",
+    )
+    parser.add_argument(
+        "--charts-dir",
+        default="comparison-charts",
+        help="Directory to save chart PNG files (default: comparison-charts/)",
+    )
+    parser.add_argument(
+        "--history-dir",
+        default=None,
+        help="Local directory containing historical comparison JSONs",
+    )
+    parser.add_argument(
+        "--fetch-history",
+        action="store_true",
+        help="Fetch history from ci-data GitHub repo",
+    )
+    parser.add_argument(
+        "--step-summary",
+        action="store_true",
+        help="Also write to $GITHUB_STEP_SUMMARY",
+    )
+
+    args = parser.parse_args()
+
+    # Load current results
+    with open(args.results) as f:
+        current = json.load(f)
+    print(f"Loaded current results: {len(current.get('results', []))} entries")
+
+    # Load history
+    history: list[dict] = []
+    if args.fetch_history:
+        token = os.environ.get("GH_PAT_FOR_NIGHTLY_CI_DATA") or os.environ.get(
+            "GITHUB_TOKEN"
+        )
+        if token:
+            history = fetch_history_from_github(token)
+        else:
+            print("Warning: No GitHub token available, skipping history fetch")
+    elif args.history_dir:
+        history = load_history_from_dir(args.history_dir)
+        print(f"Loaded {len(history)} historical run(s) from {args.history_dir}")
+
+    # Generate dashboard
+    markdown, alert_reasons = generate_dashboard(
+        current, history, charts_dir=args.charts_dir
+    )
+
+    # Write output
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    with open(args.output, "w") as f:
+        f.write(markdown)
+    print(f"Dashboard written to {args.output}")
+
+    # Write to GitHub Step Summary
+    if args.step_summary:
+        summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+        if summary_file:
+            with open(summary_file, "a") as f:
+                f.write(markdown)
+            print("Dashboard appended to $GITHUB_STEP_SUMMARY")
+        else:
+            print("Warning: $GITHUB_STEP_SUMMARY not set, skipping")
+
+    # Create GitHub Issue for performance alerts (so assignees get notified)
+    if alert_reasons:
+        _create_alert_issue(alert_reasons)
+    else:
+        print("No performance alerts — skipping issue creation.")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/publish_comparison_results.py

@@ -0,0 +1,231 @@
+"""Publish SGLang-Diffusion nightly benchmark results to sgl-project/ci-data repo.
+
+Pushes comparison-results.json, dashboard.md, and chart PNG files to the
+ci-data repository for historical tracking. Chart PNGs are stored under
+diffusion-comparisons/charts/ so they can be referenced via
+raw.githubusercontent URLs in the dashboard markdown (GitHub Step Summary
+blocks data: URIs).
+
+Usage:
+    python3 scripts/ci/utils/diffusion/publish_comparison_results.py \
+        --results comparison-results.json \
+        --dashboard dashboard.md \
+        --charts-dir comparison-charts/
+"""
+
+import argparse
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Reuse GitHub API helpers from publish_traces.
+# Support both direct script execution and package-style imports.
+if __package__:
+    from ..publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+else:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    from publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+
+# Repository configuration
+REPO_OWNER = "sgl-project"
+REPO_NAME = "ci-data"
+BRANCH = "main"
+STORAGE_PREFIX = "diffusion-comparisons"
+
+
+def _collect_chart_files(charts_dir: str) -> list[tuple[str, bytes]]:
+    """Collect PNG chart files from directory for upload."""
+    files: list[tuple[str, bytes]] = []
+    if not charts_dir or not os.path.isdir(charts_dir):
+        return files
+
+    for entry in sorted(os.listdir(charts_dir)):
+        if not entry.lower().endswith(".png"):
+            continue
+        full_path = os.path.join(charts_dir, entry)
+        if not os.path.isfile(full_path):
+            continue
+        with open(full_path, "rb") as f:
+            content = f.read()
+        # Store charts under diffusion-comparisons/charts/
+        repo_path = f"{STORAGE_PREFIX}/charts/{entry}"
+        files.append((repo_path, content))
+
+    return files
+
+
+def publish_comparison(
+    results_path: str,
+    dashboard_path: str | None = None,
+    charts_dir: str | None = None,
+) -> None:
+    """Publish comparison results, dashboard, and charts to ci-data repo."""
+    token = os.environ.get("GH_PAT_FOR_NIGHTLY_CI_DATA") or os.environ.get(
+        "GITHUB_TOKEN"
+    )
+    if not token:
+        print("Error: GH_PAT_FOR_NIGHTLY_CI_DATA or GITHUB_TOKEN not set")
+        sys.exit(1)
+
+    run_id = os.environ.get("GITHUB_RUN_ID", "local")
+    run_number = os.environ.get("GITHUB_RUN_NUMBER", "0")
+
+    # Verify permissions
+    perm = verify_token_permissions(REPO_OWNER, REPO_NAME, token)
+    if perm == "rate_limited":
+        print("Warning: Rate limited, skipping publish")
+        return
+    elif not perm:
+        print("Error: Token permission verification failed")
+        sys.exit(1)
+
+    # Prepare files to upload
+    files_to_upload: list[tuple[str, bytes]] = []
+
+    # Results JSON: stored with date prefix for chronological ordering
+    date_prefix = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    results_target = f"{STORAGE_PREFIX}/{date_prefix}_{run_id}.json"
+    with open(results_path, "rb") as f:
+        files_to_upload.append((results_target, f.read()))
+
+    # Dashboard markdown: always overwrite latest
+    if dashboard_path and os.path.exists(dashboard_path):
+        dashboard_target = f"{STORAGE_PREFIX}/dashboard.md"
+        with open(dashboard_path, "rb") as f:
+            files_to_upload.append((dashboard_target, f.read()))
+
+    # Chart PNG files
+    chart_files = _collect_chart_files(charts_dir)
+    if chart_files:
+        print(f"Found {len(chart_files)} chart PNG(s) to upload")
+        files_to_upload.extend(chart_files)
+
+    print(f"Publishing {len(files_to_upload)} file(s) to {REPO_OWNER}/{REPO_NAME}")
+
+    # Create blobs
+    try:
+        tree_items = create_blobs(REPO_OWNER, REPO_NAME, files_to_upload, token)
+    except Exception as e:
+        if is_rate_limit_error(e):
+            print("Warning: Rate limited during blob creation, skipping")
+            return
+        if is_permission_error(e):
+            print(f"Error: No write permission to {REPO_OWNER}/{REPO_NAME}")
+            sys.exit(1)
+        raise
+
+    # Commit with retry (handle concurrent writes)
+    max_retries = 5
+    retry_delay = 5
+
+    for attempt in range(max_retries):
+        try:
+            branch_sha = get_branch_sha(REPO_OWNER, REPO_NAME, BRANCH, token)
+            tree_sha = get_tree_sha(REPO_OWNER, REPO_NAME, branch_sha, token)
+
+            new_tree_sha = create_tree(
+                REPO_OWNER, REPO_NAME, tree_sha, tree_items, token
+            )
+
+            commit_msg = (
+                f"Diffusion comparison results for run {run_id} (#{run_number})"
+            )
+            commit_sha = create_commit(
+                REPO_OWNER, REPO_NAME, new_tree_sha, branch_sha, commit_msg, token
+            )
+
+            update_branch_ref(REPO_OWNER, REPO_NAME, BRANCH, commit_sha, token)
+            print(
+                f"Successfully published comparison results (commit {commit_sha[:7]})"
+            )
+            return
+
+        except Exception as e:
+            is_retryable = False
+            if hasattr(e, "error_body"):
+                body = getattr(e, "error_body", "")
+                if "Update is not a fast forward" in body:
+                    is_retryable = True
+                elif "Object does not exist" in body:
+                    is_retryable = True
+
+            from urllib.error import HTTPError
+
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                is_retryable = True
+
+            if is_rate_limit_error(e):
+                print("Warning: Rate limited, skipping publish")
+                return
+
+            if is_permission_error(e):
+                print(f"Error: No write permission to {REPO_OWNER}/{REPO_NAME}")
+                sys.exit(1)
+
+            if is_retryable and attempt < max_retries - 1:
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed, retrying in {retry_delay}s..."
+                )
+                time.sleep(retry_delay)
+            else:
+                print(f"Failed to publish after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish SGLang-Diffusion nightly benchmark results to ci-data"
+    )
+    parser.add_argument(
+        "--results",
+        required=True,
+        help="Path to comparison-results.json",
+    )
+    parser.add_argument(
+        "--dashboard",
+        default=None,
+        help="Path to dashboard.md (optional)",
+    )
+    parser.add_argument(
+        "--charts-dir",
+        default=None,
+        help="Directory containing chart PNG files to upload (optional)",
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.results):
+        print(f"Error: Results file not found: {args.results}")
+        sys.exit(1)
+
+    publish_comparison(
+        results_path=args.results,
+        dashboard_path=args.dashboard,
+        charts_dir=args.charts_dir,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/publish_diffusion_gt.py

@@ -0,0 +1,507 @@
+"""
+Publish diffusion CI ground-truth images to sgl-project/ci-data
+via the GitHub API (same pattern as publish_traces.py).
+"""
+
+import argparse
+import base64
+import hashlib
+import io
+import json
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.error import HTTPError
+
+import numpy as np
+from PIL import Image, ImageFilter
+
+# Reuse GitHub API helpers from publish_traces.
+# Support both direct script execution and package-style imports.
+if __package__:
+    from ..publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        make_github_request,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+else:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    from publish_traces import (
+        create_blobs,
+        create_commit,
+        create_tree,
+        get_branch_sha,
+        get_tree_sha,
+        is_permission_error,
+        is_rate_limit_error,
+        make_github_request,
+        update_branch_ref,
+        verify_token_permissions,
+    )
+
+REPO_OWNER = "sgl-project"
+REPO_NAME = "ci-data"
+BRANCH = "main"
+DEFAULT_TARGET_DIR = "diffusion-ci/consistency_gt/sglang_generated"
+
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
+QUALITY_MAX_SIDE = 256
+LOW_DETAIL_STD_THRESHOLD = 0.075
+LOW_DETAIL_ENTROPY_THRESHOLD = 0.55
+LOW_DETAIL_BLUR_RESIDUAL_THRESHOLD = 0.035
+LOW_DETAIL_GRADIENT_P95_THRESHOLD = 0.045
+RANDOM_NOISE_CORRELATION_THRESHOLD = 0.55
+RANDOM_NOISE_LOW_FREQUENCY_THRESHOLD = 0.20
+RANDOM_NOISE_BLUR_RESIDUAL_THRESHOLD = 0.045
+OLD_NEW_MIN_SSIM = 0.20
+OLD_NEW_MAX_MEAN_ABS_DIFF = 45.0
+
+
+@dataclass(frozen=True)
+class ImageQualityMetrics:
+    luminance_std: float
+    entropy: float
+    blur_residual: float
+    gradient_p95: float
+    neighbor_correlation: float
+    low_frequency_ratio: float
+
+
+@dataclass(frozen=True)
+class OldNewMetrics:
+    ssim: float
+    mean_abs_diff: float
+
+
+def collect_images(source_dir, target_dir):
+    """Collect image files from source_dir and return list of (repo_path, content) tuples."""
+    files = []
+    for entry in sorted(os.listdir(source_dir)):
+        ext = os.path.splitext(entry)[1].lower()
+        if ext not in IMAGE_EXTENSIONS:
+            continue
+        full_path = os.path.join(source_dir, entry)
+        if not os.path.isfile(full_path):
+            continue
+        with open(full_path, "rb") as f:
+            content = f.read()
+        repo_path = f"{target_dir}/{entry}"
+        files.append((repo_path, content))
+    return files
+
+
+def git_blob_sha(content):
+    header = f"blob {len(content)}\0".encode()
+    return hashlib.sha1(header + content).hexdigest()
+
+
+def get_remote_blob_shas(repo_owner, repo_name, target_dir, token):
+    return {
+        path: item["sha"]
+        for path, item in get_remote_image_entries(
+            repo_owner, repo_name, target_dir, token
+        ).items()
+    }
+
+
+def get_remote_image_entries(repo_owner, repo_name, target_dir, token):
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents/"
+        f"{target_dir}?ref={BRANCH}"
+    )
+    try:
+        response = make_github_request(url, token)
+    except HTTPError as e:
+        if e.code == 404:
+            return {}
+        raise
+    entries = json.loads(response)
+    return {
+        item["path"]: item
+        for item in entries
+        if item.get("type") == "file"
+        and "sha" in item
+        and os.path.splitext(item["path"])[1].lower() in IMAGE_EXTENSIONS
+    }
+
+
+def filter_changed_files(files, remote_blob_shas):
+    return [
+        (path, content)
+        for path, content in files
+        if remote_blob_shas.get(path) != git_blob_sha(content)
+    ]
+
+
+def get_remote_blob_content(repo_owner, repo_name, blob_sha, token):
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs/{blob_sha}"
+    response = make_github_request(url, token)
+    blob = json.loads(response)
+    if blob.get("encoding") != "base64":
+        raise ValueError(
+            f"Unexpected blob encoding for {blob_sha}: {blob.get('encoding')}"
+        )
+    return base64.b64decode(blob["content"])
+
+
+def _load_quality_image(content):
+    with Image.open(io.BytesIO(content)) as image:
+        image = image.convert("RGB")
+        image.thumbnail((QUALITY_MAX_SIDE, QUALITY_MAX_SIDE), Image.Resampling.BICUBIC)
+        return image.copy()
+
+
+def _image_to_rgb_array(image):
+    return np.asarray(image, dtype=np.float32)
+
+
+def _luminance(rgb):
+    return 0.299 * rgb[..., 0] + 0.587 * rgb[..., 1] + 0.114 * rgb[..., 2]
+
+
+def _neighbor_correlation(luma):
+    def corr(a, b):
+        a = a.ravel()
+        b = b.ravel()
+        if a.std() < 1e-6 or b.std() < 1e-6:
+            return 1.0
+        return float(np.corrcoef(a, b)[0, 1])
+
+    return (corr(luma[:, 1:], luma[:, :-1]) + corr(luma[1:, :], luma[:-1, :])) / 2
+
+
+def _low_frequency_ratio(luma):
+    centered = luma - luma.mean()
+    power = np.abs(np.fft.fftshift(np.fft.fft2(centered))) ** 2
+    total_power = power.sum()
+    if total_power < 1e-12:
+        return 0.0
+
+    height, width = luma.shape
+    y, x = np.ogrid[:height, :width]
+    center_y = height // 2
+    center_x = width // 2
+    radius = np.sqrt((y - center_y) ** 2 + (x - center_x) ** 2)
+    low_frequency_radius = min(height, width) * 0.08
+    return float(power[radius <= low_frequency_radius].sum() / total_power)
+
+
+def compute_image_quality_metrics(content):
+    image = _load_quality_image(content)
+    rgb = _image_to_rgb_array(image)
+    luma = _luminance(rgb) / 255.0
+
+    gradients = np.concatenate(
+        [
+            np.abs(np.diff(luma, axis=1)).ravel(),
+            np.abs(np.diff(luma, axis=0)).ravel(),
+        ]
+    )
+    histogram, _ = np.histogram(luma, bins=32, range=(0, 1))
+    probabilities = histogram / histogram.sum()
+    nonzero_probabilities = probabilities[probabilities > 0]
+    entropy = float(
+        -(nonzero_probabilities * np.log2(nonzero_probabilities)).sum() / 5.0
+    )
+    blurred = _image_to_rgb_array(image.filter(ImageFilter.GaussianBlur(radius=3)))
+
+    return ImageQualityMetrics(
+        luminance_std=float(luma.std()),
+        entropy=entropy,
+        blur_residual=float(np.mean(np.abs(rgb - blurred)) / 255.0),
+        gradient_p95=float(np.percentile(gradients, 95)),
+        neighbor_correlation=_neighbor_correlation(luma),
+        low_frequency_ratio=_low_frequency_ratio(luma),
+    )
+
+
+def get_quality_failure_reasons(metrics):
+    reasons = []
+    low_detail_static = (
+        metrics.luminance_std < LOW_DETAIL_STD_THRESHOLD
+        and metrics.entropy < LOW_DETAIL_ENTROPY_THRESHOLD
+        and (
+            metrics.blur_residual < LOW_DETAIL_BLUR_RESIDUAL_THRESHOLD
+            or metrics.gradient_p95 < LOW_DETAIL_GRADIENT_P95_THRESHOLD
+        )
+    )
+    high_frequency_noise = (
+        metrics.neighbor_correlation < RANDOM_NOISE_CORRELATION_THRESHOLD
+        and metrics.low_frequency_ratio < RANDOM_NOISE_LOW_FREQUENCY_THRESHOLD
+        and metrics.blur_residual > RANDOM_NOISE_BLUR_RESIDUAL_THRESHOLD
+    )
+    if low_detail_static:
+        reasons.append("low-contrast low-detail output")
+    if high_frequency_noise:
+        reasons.append("high-frequency random noise")
+    return reasons
+
+
+def _resize_for_old_new_compare(content, size=None):
+    with Image.open(io.BytesIO(content)) as image:
+        image = image.convert("RGB")
+        if size is None:
+            image.thumbnail(
+                (QUALITY_MAX_SIDE, QUALITY_MAX_SIDE), Image.Resampling.BICUBIC
+            )
+        else:
+            image = image.resize(size, Image.Resampling.BICUBIC)
+        return _image_to_rgb_array(image)
+
+
+def compute_old_new_metrics(old_content, new_content):
+    old_rgb = _resize_for_old_new_compare(old_content)
+    new_rgb = _resize_for_old_new_compare(
+        new_content, size=(old_rgb.shape[1], old_rgb.shape[0])
+    )
+    old_luma = _luminance(old_rgb) / 255.0
+    new_luma = _luminance(new_rgb) / 255.0
+
+    old_mean = old_luma.mean()
+    new_mean = new_luma.mean()
+    old_variance = old_luma.var()
+    new_variance = new_luma.var()
+    covariance = ((old_luma - old_mean) * (new_luma - new_mean)).mean()
+    c1 = 0.01**2
+    c2 = 0.03**2
+    ssim = (
+        (2 * old_mean * new_mean + c1)
+        * (2 * covariance + c2)
+        / ((old_mean**2 + new_mean**2 + c1) * (old_variance + new_variance + c2))
+    )
+
+    return OldNewMetrics(
+        ssim=float(ssim),
+        mean_abs_diff=float(np.abs(old_rgb - new_rgb).mean()),
+    )
+
+
+def _format_quality_metrics(metrics):
+    return (
+        f"std={metrics.luminance_std:.4f}, entropy={metrics.entropy:.4f}, "
+        f"blur_residual={metrics.blur_residual:.4f}, "
+        f"gradient_p95={metrics.gradient_p95:.4f}, "
+        f"neighbor_corr={metrics.neighbor_correlation:.4f}, "
+        f"low_freq={metrics.low_frequency_ratio:.4f}"
+    )
+
+
+def _format_old_new_metrics(metrics):
+    return f"ssim={metrics.ssim:.4f}, mean_abs_diff={metrics.mean_abs_diff:.2f}"
+
+
+def validate_gt_files(files_to_upload, changed_files, remote_image_entries, token):
+    failures = []
+    for path, content in files_to_upload:
+        quality_metrics = compute_image_quality_metrics(content)
+        quality_reasons = get_quality_failure_reasons(quality_metrics)
+        if quality_reasons:
+            failures.append(
+                f"{path}: {', '.join(quality_reasons)} "
+                f"({_format_quality_metrics(quality_metrics)})"
+            )
+
+    for path, content in changed_files:
+        remote_entry = remote_image_entries.get(path)
+        if not remote_entry:
+            continue
+
+        old_content = get_remote_blob_content(
+            REPO_OWNER, REPO_NAME, remote_entry["sha"], token
+        )
+        old_quality_metrics = compute_image_quality_metrics(old_content)
+        old_quality_reasons = get_quality_failure_reasons(old_quality_metrics)
+        if old_quality_reasons:
+            print(
+                f"Skipping old/new drift check for {path} because existing GT is "
+                f"already suspicious: {', '.join(old_quality_reasons)} "
+                f"({_format_quality_metrics(old_quality_metrics)})"
+            )
+            continue
+
+        old_new_metrics = compute_old_new_metrics(old_content, content)
+        if (
+            old_new_metrics.ssim < OLD_NEW_MIN_SSIM
+            and old_new_metrics.mean_abs_diff > OLD_NEW_MAX_MEAN_ABS_DIFF
+        ):
+            failures.append(
+                f"{path}: changed too far from existing GT "
+                f"({_format_old_new_metrics(old_new_metrics)})"
+            )
+
+    if not failures:
+        print(
+            f"GT quality gate passed for {len(files_to_upload)} generated image(s) "
+            f"and {len(changed_files)} changed image(s)."
+        )
+        return
+
+    print("GT quality gate failed; refusing to publish suspicious image updates:")
+    for failure in failures:
+        print(f"  - {failure}")
+    sys.exit(1)
+
+
+def check_quality(source_dir, target_dir=None):
+    target_dir = target_dir or DEFAULT_TARGET_DIR
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    files_to_upload = collect_images(source_dir, target_dir)
+    if not files_to_upload:
+        print(f"No image files found in {source_dir}")
+        return
+
+    remote_image_entries = get_remote_image_entries(
+        REPO_OWNER, REPO_NAME, target_dir, token
+    )
+    remote_blob_shas = {
+        path: item["sha"] for path, item in remote_image_entries.items()
+    }
+    changed_files = filter_changed_files(files_to_upload, remote_blob_shas)
+    validate_gt_files(files_to_upload, changed_files, remote_image_entries, token)
+
+
+def publish(source_dir, target_dir=None):
+    target_dir = target_dir or DEFAULT_TARGET_DIR
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    files_to_upload = collect_images(source_dir, target_dir)
+    if not files_to_upload:
+        print(f"No image files found in {source_dir}")
+        return
+
+    print(
+        f"Found {len(files_to_upload)} image(s) to upload to {REPO_OWNER}/{REPO_NAME}/{target_dir}"
+    )
+
+    # Verify token
+    perm = verify_token_permissions(REPO_OWNER, REPO_NAME, token)
+    if perm == "rate_limited":
+        print("GitHub API rate-limited, skipping upload.")
+        return
+    if not perm:
+        print("Token permission verification failed.")
+        sys.exit(1)
+
+    # Commit with retry (handle concurrent pushes)
+    max_retries = 5
+    for attempt in range(max_retries):
+        try:
+            branch_sha = get_branch_sha(REPO_OWNER, REPO_NAME, BRANCH, token)
+            tree_sha = get_tree_sha(REPO_OWNER, REPO_NAME, branch_sha, token)
+            remote_image_entries = get_remote_image_entries(
+                REPO_OWNER, REPO_NAME, target_dir, token
+            )
+            remote_blob_shas = {
+                path: item["sha"] for path, item in remote_image_entries.items()
+            }
+            changed_files = filter_changed_files(files_to_upload, remote_blob_shas)
+            validate_gt_files(
+                files_to_upload, changed_files, remote_image_entries, token
+            )
+            if not changed_files:
+                print("No image changes to publish.")
+                return
+
+            try:
+                tree_items = create_blobs(REPO_OWNER, REPO_NAME, changed_files, token)
+            except Exception as e:
+                if is_rate_limit_error(e):
+                    print("Rate-limited during blob creation, skipping.")
+                    return
+                if is_permission_error(e):
+                    print(
+                        f"ERROR: Token lacks write permission to {REPO_OWNER}/{REPO_NAME}. "
+                        "Update GH_PAT_FOR_NIGHTLY_CI_DATA with a token that has contents:write."
+                    )
+                    sys.exit(1)
+                raise
+
+            new_tree_sha = create_tree(
+                REPO_OWNER, REPO_NAME, tree_sha, tree_items, token
+            )
+            if new_tree_sha == tree_sha:
+                print("No tree changes to publish.")
+                return
+
+            commit_msg = f"diffusion-ci: update images in {target_dir} ({len(changed_files)} files) [automated]"
+            commit_sha = create_commit(
+                REPO_OWNER, REPO_NAME, new_tree_sha, branch_sha, commit_msg, token
+            )
+            update_branch_ref(REPO_OWNER, REPO_NAME, BRANCH, commit_sha, token)
+            print(
+                f"Successfully pushed {len(changed_files)} changed images (commit {commit_sha[:10]})"
+            )
+            return
+        except Exception as e:
+            if is_rate_limit_error(e):
+                print("Rate-limited, skipping.")
+                return
+            if is_permission_error(e):
+                print(f"ERROR: permission denied to {REPO_OWNER}/{REPO_NAME}")
+                sys.exit(1)
+
+            retryable = False
+            if hasattr(e, "error_body"):
+                if "Update is not a fast forward" in e.error_body:
+                    retryable = True
+                elif "Object does not exist" in e.error_body:
+                    retryable = True
+
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                retryable = True
+
+            if retryable and attempt < max_retries - 1:
+                import time
+
+                wait = 2**attempt
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed, retrying in {wait}s..."
+                )
+                time.sleep(wait)
+            else:
+                print(f"Failed after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish diffusion GT images to GitHub"
+    )
+    parser.add_argument(
+        "--source-dir", required=True, help="Directory containing GT images"
+    )
+    parser.add_argument(
+        "--target-dir",
+        required=False,
+        default=None,
+        help=f"Target directory in the remote repo (default: {DEFAULT_TARGET_DIR})",
+    )
+    parser.add_argument(
+        "--check-only",
+        action="store_true",
+        help="Validate generated GT images without publishing them",
+    )
+    args = parser.parse_args()
+    if args.check_only:
+        check_quality(args.source_dir, args.target_dir)
+    else:
+        publish(args.source_dir, args.target_dir)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/save_diffusion_metrics.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""Collect and save diffusion performance metrics for artifact collection in CI.
+
+This script reads diffusion test results from the pytest stash and saves them
+with metadata for the performance dashboard.
+
+Usage:
+    python3 scripts/ci/utils/diffusion/save_diffusion_metrics.py \
+        --gpu-config 1-gpu-h100 \
+        --run-id 12345678 \
+        --output test/diffusion-metrics-1gpu.json \
+        --results-json test/diffusion-results.json
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def load_diffusion_results(results_file: str) -> list[dict]:
+    """Load diffusion performance results from JSON file."""
+    if not os.path.exists(results_file):
+        print(f"Warning: Results file not found: {results_file}")
+        return []
+
+    try:
+        with open(results_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return data if isinstance(data, list) else [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {results_file}: {e}")
+        return []
+
+
+def transform_diffusion_result(result: dict, gpu_config: str) -> dict:
+    """Transform a diffusion result to match dashboard expectations.
+
+    Dashboard expects:
+    - Separate test_name, class_name
+    - Numeric metrics in consistent units
+    - Optional modality field
+    """
+    return {
+        "test_name": result.get("test_name"),
+        "class_name": result.get("class_name"),
+        "modality": result.get("modality", "image"),
+        "e2e_ms": result.get("e2e_ms"),
+        "avg_denoise_ms": result.get("avg_denoise_ms"),
+        "median_denoise_ms": result.get("median_denoise_ms"),
+        "stage_metrics": result.get("stage_metrics", {}),
+        "sampled_steps": result.get("sampled_steps", {}),
+        # Video-specific metrics (if present)
+        "frames_per_second": result.get("frames_per_second"),
+        "total_frames": result.get("total_frames"),
+        "avg_frame_time_ms": result.get("avg_frame_time_ms"),
+    }
+
+
+def group_results_by_class(results: list[dict], gpu_config: str) -> list[dict]:
+    """Group diffusion results by test class (suite).
+
+    Returns list with one entry per test class, containing all tests in that class.
+    """
+    groups = {}
+
+    for result in results:
+        class_name = result.get("class_name", "unknown")
+
+        if class_name not in groups:
+            groups[class_name] = {
+                "gpu_config": gpu_config,
+                "test_suite": class_name,
+                "tests": [],
+            }
+
+        transformed = transform_diffusion_result(result, gpu_config)
+        groups[class_name]["tests"].append(transformed)
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    run_id: str,
+    output_file: str,
+    results_file: str,
+) -> bool:
+    """Collect diffusion metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Load diffusion results
+    raw_results = load_diffusion_results(results_file)
+    print(f"Loaded {len(raw_results)} diffusion test result(s)")
+
+    # Group by test class
+    grouped = group_results_by_class(raw_results, gpu_config)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "test_type": "diffusion",
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not raw_results:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved diffusion metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect diffusion performance metrics from test results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 1-gpu-h100, 2-gpu-h100)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--results-json",
+        required=True,
+        help="Path to diffusion results JSON file",
+    )
+
+    args = parser.parse_args()
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        run_id=args.run_id,
+        output_file=args.output,
+        results_file=args.results_json,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/diffusion/verify_diffusion_coverage.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python3
+"""
+Verify 100% coverage of diffusion test cases.
+
+This script checks that all expected test cases were executed across all partitions.
+Designed to run in the CI summary job after all partition jobs complete.
+
+Usage:
+    python scripts/ci/utils/diffusion/verify_diffusion_coverage.py --reports-dir <path>
+
+Exit codes:
+    0 - All cases executed (100% coverage)
+    1 - Missing cases detected (coverage < 100%)
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from diffusion_case_parser import (
+    BASELINE_REL_PATH,
+    RUN_SUITE_REL_PATH,
+    collect_diffusion_suites,
+    resolve_case_config_path,
+)
+
+DYNAMIC_SUITES = {"1-gpu", "2-gpu"}
+
+
+def load_execution_reports(reports_dir: Path) -> list[dict]:
+    """Load all execution report JSON files from the given directory."""
+    reports = []
+    for json_file in reports_dir.glob("**/execution_report_*.json"):
+        with open(json_file, "r", encoding="utf-8") as f:
+            reports.append(json.load(f))
+    return reports
+
+
+def get_expected_cases(repo_root: Path) -> dict[str, set[str]]:
+    """
+    Get all expected cases from case config and run_suite.py.
+
+    Returns:
+        Dictionary mapping suite name to set of expected case IDs.
+        Standalone files are represented as "standalone:<filename>".
+    """
+    baseline_path = repo_root / BASELINE_REL_PATH
+    run_suite_path = repo_root / RUN_SUITE_REL_PATH
+    case_config_path = resolve_case_config_path(repo_root, run_suite_path)
+
+    suites = collect_diffusion_suites(
+        case_config_path,
+        run_suite_path,
+        baseline_path,
+    )
+
+    expected = {}
+    for suite_name, suite_info in suites.items():
+        if suite_name not in DYNAMIC_SUITES:
+            continue
+        case_ids = set(case.case_id for case in suite_info.cases)
+        # Add standalone files as special case IDs
+        for standalone_file in suite_info.standalone_files:
+            case_ids.add(f"standalone:{standalone_file}")
+        expected[suite_name] = case_ids
+
+    empty_dynamic_suites = [
+        suite_name
+        for suite_name in DYNAMIC_SUITES
+        if suite_name in expected
+        and not any(
+            not case_id.startswith("standalone:") for case_id in expected[suite_name]
+        )
+    ]
+    if empty_dynamic_suites:
+        raise RuntimeError(
+            "Parsed zero parametrized cases for diffusion suites: "
+            + ", ".join(sorted(empty_dynamic_suites))
+            + ". Refuse to pass coverage verification."
+        )
+
+    return expected
+
+
+def collect_executed_cases(reports: list[dict]) -> dict[str, set[str]]:
+    """
+    Collect all executed cases from execution reports.
+
+    Returns:
+        Dictionary mapping suite name to set of executed case IDs.
+    """
+    executed = {}
+    for report in reports:
+        suite = report["suite"]
+        if suite not in executed:
+            executed[suite] = set()
+
+        executed_cases = report.get("executed_cases", [])
+        if executed_cases:
+            executed[suite].update(executed_cases)
+        elif report["is_standalone"]:
+            standalone_file = report["standalone_file"]
+            executed[suite].add(f"standalone:{standalone_file}")
+
+    return executed
+
+
+def collect_case_results(reports: list[dict]) -> dict[str, dict[str, str]]:
+    """
+    Collect case results (pass/fail/error status) from execution reports.
+
+    Returns:
+        Dictionary mapping suite name to {case_id: status} dictionary.
+    """
+    results = {}
+    for report in reports:
+        suite = report["suite"]
+        if suite not in results:
+            results[suite] = {}
+
+        # Get case_results from report (empty dict for legacy reports)
+        case_results = report.get("case_results", {})
+        results[suite].update(case_results)
+
+    return results
+
+
+def collect_missing_standalone_estimates(reports: list[dict]) -> dict[str, set[str]]:
+    missing_by_suite: dict[str, set[str]] = {}
+    for report in reports:
+        suite = report["suite"]
+        missing = report.get("missing_standalone_estimates", [])
+        if not missing:
+            continue
+        missing_by_suite.setdefault(suite, set()).update(missing)
+    return missing_by_suite
+
+
+def collect_standalone_measurements(reports: list[dict]) -> dict[tuple[str, str], dict]:
+    measurements: dict[tuple[str, str], dict] = {}
+    for report in reports:
+        for measurement in report.get("standalone_measurements", []):
+            key = (measurement["suite"], measurement["standalone_file"])
+            measurements[key] = measurement
+    return measurements
+
+
+def print_missing_standalone_estimates_summary(
+    missing_by_suite: dict[str, set[str]],
+    measurements: dict[tuple[str, str], dict],
+) -> None:
+    if not missing_by_suite:
+        return
+
+    print("\n" + "=" * 60)
+    print(
+        "Add standalone estimate(s) to "
+        "python/sglang/multimodal_gen/test/run_suite.py"
+    )
+    print("=" * 60)
+    print("The following standalone file(s) used fallback estimate 300.0s.")
+    print("Update STANDALONE_FILE_EST_TIMES with the measured runtime below:\n")
+
+    for suite in sorted(missing_by_suite):
+        print(f'"{suite}": {{')
+        for standalone_file in sorted(missing_by_suite[suite]):
+            measurement = measurements.get((suite, standalone_file))
+            measured_time = (
+                measurement["measured_full_test_time_s"] if measurement else 300.0
+            )
+            print(f'    "{standalone_file}": {measured_time:.1f},')
+        print("}\n")
+
+
+def verify_coverage(
+    expected: dict[str, set[str]],
+    executed: dict[str, set[str]],
+) -> tuple[bool, dict[str, set[str]]]:
+    """
+    Verify that all expected cases were executed.
+
+    Returns:
+        Tuple of (is_complete, missing_cases_by_suite)
+    """
+    missing = {}
+    for suite, expected_cases in expected.items():
+        executed_cases = executed.get(suite, set())
+        suite_missing = expected_cases - executed_cases
+        if suite_missing:
+            missing[suite] = suite_missing
+
+    return len(missing) == 0, missing
+
+
+def print_results_summary(
+    case_results: dict[str, dict[str, str]],
+) -> tuple[int, int, int]:
+    """
+    Print test results summary and return counts.
+
+    Returns:
+        Tuple of (passed_count, failed_count, error_count)
+    """
+    # Check if we have any results data
+    total_results = sum(len(results) for results in case_results.values())
+    if total_results == 0:
+        print("\nTest Results: No results data available (legacy reports)")
+        return (0, 0, 0)
+
+    # Count by status
+    passed_count = 0
+    failed_count = 0
+    error_count = 0
+    failed_cases: dict[str, list[str]] = {}
+
+    for suite, results in case_results.items():
+        for case_id, status in results.items():
+            if status == "pass":
+                passed_count += 1
+            elif status == "fail":
+                failed_count += 1
+                if suite not in failed_cases:
+                    failed_cases[suite] = []
+                failed_cases[suite].append(case_id)
+            elif status == "error":
+                error_count += 1
+                if suite not in failed_cases:
+                    failed_cases[suite] = []
+                failed_cases[suite].append(f"{case_id} (error)")
+
+    # Print summary
+    total = passed_count + failed_count + error_count
+    print("\n" + "=" * 60)
+    print("Test Results Summary")
+    print("=" * 60)
+    print(f"  Total executed: {total}")
+    print(f"  ✅ Passed: {passed_count}")
+    print(f"  ❌ Failed: {failed_count}")
+    if error_count > 0:
+        print(f"  ⚠️  Errors: {error_count}")
+
+    # Print failed cases if any
+    if failed_cases:
+        print("\nFailed cases:")
+        for suite, cases in sorted(failed_cases.items()):
+            print(f"  {suite}:")
+            for case_id in sorted(cases):
+                print(f"    - {case_id}")
+
+    return (passed_count, failed_count, error_count)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Verify 100% coverage of diffusion test cases"
+    )
+    parser.add_argument(
+        "--reports-dir",
+        type=str,
+        required=True,
+        help="Directory containing execution report JSON files",
+    )
+    args = parser.parse_args()
+
+    # Determine repository root
+    script_dir = Path(__file__).resolve().parent
+    repo_root = script_dir.parent.parent.parent.parent
+
+    reports_dir = Path(args.reports_dir)
+
+    print("=" * 60)
+    print("Diffusion CI Coverage Verification")
+    print("=" * 60)
+
+    # Load execution reports
+    reports = load_execution_reports(reports_dir)
+    print(f"\nLoaded {len(reports)} execution reports")
+
+    if not reports:
+        print("\nERROR: No execution reports found!")
+        print(f"Expected reports in: {reports_dir}")
+        sys.exit(1)
+
+    # Get expected cases
+    try:
+        expected = get_expected_cases(repo_root)
+    except (RuntimeError, FileNotFoundError) as exc:
+        print(f"\nERROR: {exc}")
+        sys.exit(1)
+    print("\nExpected cases by suite:")
+    for suite, cases in expected.items():
+        print(f"  {suite}: {len(cases)} cases")
+
+    # Collect executed cases
+    executed = collect_executed_cases(reports)
+    print("\nExecuted cases by suite:")
+    for suite, cases in executed.items():
+        print(f"  {suite}: {len(cases)} cases")
+
+    # Collect case results
+    case_results = collect_case_results(reports)
+    missing_standalone_estimates = collect_missing_standalone_estimates(reports)
+    standalone_measurements = collect_standalone_measurements(reports)
+
+    # Verify coverage
+    is_complete, missing = verify_coverage(expected, executed)
+
+    if is_complete:
+        print("\n" + "=" * 60)
+        print("✅ COVERAGE: 100% - All test cases executed")
+        print("=" * 60)
+    else:
+        print("\n" + "=" * 60)
+        print("❌ COVERAGE FAILURE: Missing test cases detected")
+        print("=" * 60)
+        for suite, cases in missing.items():
+            print(f"\n{suite.upper()} suite - Missing {len(cases)} case(s):")
+            for case_id in sorted(cases):
+                print(f"  - {case_id}")
+
+    # Print test results summary
+    passed_count, failed_count, error_count = print_results_summary(case_results)
+    print_missing_standalone_estimates_summary(
+        missing_standalone_estimates, standalone_measurements
+    )
+
+    # Exit with appropriate code
+    if not is_complete:
+        sys.exit(1)
+    elif missing_standalone_estimates:
+        sys.exit(1)
+    elif failed_count > 0 or error_count > 0:
+        print("\n" + "=" * 60)
+        print("⚠️  WARNING: Some tests failed but coverage is complete")
+        print("=" * 60)
+        sys.exit(0)  # Coverage is complete, failures are visible in results
+    else:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/docker_build_metadata_args.py

@@ -0,0 +1,119 @@
+import argparse
+import datetime
+import json
+import sys
+
+MOVING_TAGS = {"dev", "dev-cu12", "dev-cu13", "latest"}
+
+
+def render_tag_template(tag: str, version: str, date: str, short_sha: str) -> str:
+    return (
+        tag.replace("{version}", version)
+        .replace("{date}", date)
+        .replace("{short_sha}", short_sha)
+    )
+
+
+def is_moving_tag(tag: str) -> bool:
+    return tag in MOVING_TAGS or tag.startswith("latest-")
+
+
+def select_tag(
+    tag_config: str, cuda: str, version: str, date: str, short_sha: str
+) -> str:
+    entries = json.loads(tag_config)
+    for entry in entries:
+        if entry.get("cuda") != cuda:
+            continue
+
+        tags = [
+            render_tag_template(tag, version, date, short_sha)
+            for tag in entry.get("tags", [])
+        ]
+        if not tags:
+            raise ValueError(f"No tags configured for CUDA variant {cuda}")
+
+        for tag in tags:
+            if not is_moving_tag(tag):
+                return tag
+        return tags[0]
+
+    raise ValueError(f"CUDA variant {cuda} not found in tag_config")
+
+
+def build_arg_tokens(
+    *,
+    cuda: str,
+    tag_config: str,
+    image_repo: str,
+    version: str,
+    build_commit: str,
+    build_url: str,
+    date: str,
+    short_sha: str,
+) -> list[str]:
+    image_tag = select_tag(tag_config, cuda, version, date, short_sha)
+    build_args = {
+        "SGLANG_BUILD_COMMIT": build_commit,
+        "SGLANG_BUILD_URL": build_url,
+        "SGLANG_IMAGE_TAG": f"{image_repo}:{image_tag}",
+    }
+
+    tokens = []
+    for key, value in build_args.items():
+        tokens.extend(["--build-arg", f"{key}={value}"])
+    return tokens
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Emit docker build arguments for SGLang image metadata."
+    )
+    parser.add_argument("--cuda", required=True, help="CUDA variant from tag_config.")
+    parser.add_argument("--tag-config", required=True, help="Docker tag JSON config.")
+    parser.add_argument("--image-repo", required=True, help="Docker image repository.")
+    parser.add_argument("--sgl-version", default="", help="SGLang release version.")
+    parser.add_argument(
+        "--build-commit",
+        required=True,
+        help="Commit checked out for the Docker build.",
+    )
+    parser.add_argument("--build-url", default="", help="CI run URL.")
+    parser.add_argument(
+        "--date",
+        default=datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d"),
+        help="Date used for {date} tag templates.",
+    )
+    parser.add_argument(
+        "--short-sha",
+        default="",
+        help="Short SHA used for {short_sha}; defaults to build commit prefix.",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    short_sha = args.short_sha or args.build_commit[:8]
+
+    try:
+        tokens = build_arg_tokens(
+            cuda=args.cuda,
+            tag_config=args.tag_config,
+            image_repo=args.image_repo,
+            version=args.sgl_version,
+            build_commit=args.build_commit,
+            build_url=args.build_url,
+            date=args.date,
+            short_sha=short_sha,
+        )
+    except (json.JSONDecodeError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+    print("\n".join(tokens))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/utils/install_protoc.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Ensure protoc is installed for router build (gRPC protobuf compilation).
+set -euxo pipefail
+
+if command -v protoc >/dev/null 2>&1 && protoc --version >/dev/null 2>&1; then
+    echo "protoc already installed: $(protoc --version)"
+    exit 0
+fi
+
+if command -v protoc >/dev/null 2>&1; then
+    echo "protoc found but not runnable, reinstalling..."
+else
+    echo "protoc not found, installing..."
+fi
+
+ARCH=$(uname -m)
+
+if command -v apt-get &> /dev/null; then
+    # Ubuntu/Debian
+    apt-get update || true # May fail due to unrelated broken packages
+    PROTOC_APT_PACKAGES=(wget unzip)
+    apt-get install -y --no-install-recommends "${PROTOC_APT_PACKAGES[@]}" || {
+        echo "Warning: apt-get install failed, checking if required packages are available..."
+        for pkg in "${PROTOC_APT_PACKAGES[@]}"; do
+            if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
+                echo "ERROR: Required package $pkg is not installed and apt-get failed"
+                exit 1
+            fi
+        done
+        echo "All required packages are already installed, continuing..."
+    }
+elif command -v yum &> /dev/null; then
+    # RHEL/CentOS
+    yum update -y
+    yum install -y wget unzip
+else
+    echo "ERROR: Neither apt-get nor yum found; cannot install protoc"
+    exit 1
+fi
+
+if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
+    PROTOC_ARCH="aarch_64"
+else
+    PROTOC_ARCH="x86_64"
+fi
+PROTOC_ZIP="protoc-32.0-linux-${PROTOC_ARCH}.zip"
+(
+    cd /tmp
+    wget "https://github.com/protocolbuffers/protobuf/releases/download/v32.0/${PROTOC_ZIP}"
+    unzip -o "${PROTOC_ZIP}" -d /usr/local
+    rm -f "${PROTOC_ZIP}"
+)
+protoc --version

scripts/ci/utils/install_rust_protoc.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Install protoc and a Rust toolchain (rustup/cargo). Required by setuptools-rust
+# to build the bundled native gRPC extension (rust/sglang-grpc) when installing
+# the main `sglang` wheel from source. Idempotent — both helpers no-op if
+# already installed.
+#
+# protoc installs system-wide (/usr/local) and apt deps, so it needs root.
+# rustup installs per-user under $HOME/.cargo, so it must run as the calling
+# user (running it under sudo would put cargo in /root/.cargo and the rest of
+# the job wouldn't find it).
+set -euxo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [ "$(id -u)" = "0" ]; then
+    SUDO=""
+elif command -v sudo >/dev/null 2>&1; then
+    SUDO="sudo"
+else
+    SUDO=""
+fi
+
+${SUDO} bash "${SCRIPT_DIR}/install_protoc.sh"
+bash "${SCRIPT_DIR}/install_rustup.sh"

scripts/ci/utils/install_rustup.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Ensure a Rust toolchain (rustc/cargo) is installed for crates built from
+# source, e.g. the native gRPC extension bundled into the sglang wheel via
+# setuptools-rust. Minimum supported version is 1.85 (edition 2024).
+set -euxo pipefail
+
+# Make cargo/rustc visible to the rest of this shell and to subsequent
+# GitHub Actions steps in the same job.
+export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}"
+if [ -n "${GITHUB_PATH:-}" ]; then
+    # Self-heal if _runner_file_commands/ disappears mid-job on some self-hosted
+    # runners; the runner reads this file by its registered UUID at step end, so
+    # recreating the path keeps PATH propagation working for subsequent steps.
+    mkdir -p "$(dirname "${GITHUB_PATH}")" 2>/dev/null || true
+    echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> "${GITHUB_PATH}" || true
+fi
+
+if command -v cargo >/dev/null 2>&1 && command -v rustc >/dev/null 2>&1; then
+    echo "rust already installed: $(rustc --version), $(cargo --version)"
+    exit 0
+fi
+
+echo "rust not found, installing via rustup..."
+
+# rustup.rs requires curl — make sure it's present.
+if ! command -v curl >/dev/null 2>&1; then
+    if command -v apt-get &> /dev/null; then
+        apt-get update || true
+        apt-get install -y --no-install-recommends curl ca-certificates
+    elif command -v yum &> /dev/null; then
+        yum install -y curl ca-certificates
+    else
+        echo "ERROR: curl is required to install rustup, but no supported package manager was found"
+        exit 1
+    fi
+fi
+
+if [ -n "${RUSTUP_CACHE_URL:-}" ]; then
+    # An in-cluster HTTP mirror is available (e.g. on NPU runners).
+    export RUSTUP_DIST_SERVER="${RUSTUP_CACHE_URL}/rustup"
+    export RUSTUP_UPDATE_ROOT="${RUSTUP_CACHE_URL}/rustup/rustup"
+    case "$(uname -m)" in
+        x86_64)  RUSTUP_ARCH="x86_64-unknown-linux-gnu" ;;
+        aarch64) RUSTUP_ARCH="aarch64-unknown-linux-gnu" ;;
+        *) echo "ERROR: unsupported arch $(uname -m)"; exit 1 ;;
+    esac
+    RUSTUP_TMP="$(mktemp -d)"
+    trap 'rm -rf "${RUSTUP_TMP}"' EXIT
+    curl --retry 3 --retry-delay 2 -sSfL \
+        "${RUSTUP_UPDATE_ROOT}/dist/${RUSTUP_ARCH}/rustup-init" \
+        -o "${RUSTUP_TMP}/rustup-init"
+    chmod +x "${RUSTUP_TMP}/rustup-init"
+    "${RUSTUP_TMP}/rustup-init" -y --no-modify-path
+else
+    curl --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 -sSf https://sh.rustup.rs \
+        | sh -s -- -y --no-modify-path
+fi
+
+rustc --version
+cargo --version

scripts/ci/utils/merge_metrics.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Merge per-partition metrics into a consolidated metrics file.
+
+This script reads all per-partition metric JSON files and consolidates them
+into a single JSON file with run-level metadata.
+
+Usage:
+    python3 scripts/ci/utils/merge_metrics.py \
+        --input-dir metrics/ \
+        --output consolidated-metrics-12345678.json \
+        --run-id 12345678 \
+        --commit-sha abc123def456
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_partition_files(input_dir: str) -> list[str]:
+    """Find all partition metric files in the input directory."""
+    patterns = [
+        os.path.join(input_dir, "**/metrics-*.json"),
+        os.path.join(input_dir, "**/diffusion-metrics-*.json"),
+        os.path.join(input_dir, "**/comparison-metrics-*.json"),
+    ]
+    files = set()
+    for pattern in patterns:
+        files.update(glob.glob(pattern, recursive=True))
+    return list(files)
+
+
+def load_partition_metrics(filepath: str) -> dict | None:
+    """Load a partition metrics file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to load {filepath}: {e}")
+        return None
+
+
+def merge_metrics(
+    input_dir: str,
+    output_file: str,
+    run_id: str,
+    commit_sha: str,
+    branch: str | None = None,
+) -> bool:
+    """Merge all partition metrics into a consolidated file."""
+    run_date = datetime.now(timezone.utc).isoformat()
+
+    # Find all partition files
+    partition_files = find_partition_files(input_dir)
+    print(f"Found {len(partition_files)} partition file(s)")
+
+    all_results = []
+    if not partition_files:
+        print("No partition metrics files found")
+    else:
+        # Load all partition files
+        for filepath in sorted(partition_files):
+            print(f"  Reading: {filepath}")
+            metrics = load_partition_metrics(filepath)
+            if metrics and "results" in metrics:
+                all_results.extend(metrics["results"])
+        print(f"Total results collected: {len(all_results)}")
+
+    # Create consolidated structure
+    consolidated = {
+        "run_id": run_id,
+        "run_date": run_date,
+        "commit_sha": commit_sha,
+        "branch": branch,
+        "results": all_results,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(consolidated, f, indent=2)
+
+        if not partition_files:
+            print(f"Created empty consolidated file: {output_file}")
+        else:
+            print(f"Saved consolidated metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing consolidated file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Merge per-partition metrics into consolidated file"
+    )
+    parser.add_argument(
+        "--input-dir",
+        required=True,
+        help="Directory containing partition metric files",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for consolidated metrics JSON",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--commit-sha",
+        required=True,
+        help="Git commit SHA",
+    )
+    parser.add_argument(
+        "--branch",
+        default=None,
+        help="Git branch name (optional)",
+    )
+
+    args = parser.parse_args()
+
+    success = merge_metrics(
+        input_dir=args.input_dir,
+        output_file=args.output,
+        run_id=args.run_id,
+        commit_sha=args.commit_sha,
+        branch=args.branch,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/prevalidate_cached_models.py

@@ -0,0 +1,407 @@
+#!/usr/bin/env python3
+"""
+Pre-validate all cached HuggingFace models to provide detailed feedback.
+
+This script runs once during CI initialization (in prepare_runner.sh) to:
+1. Scan snapshots in ~/.cache/huggingface/hub/ (with time/quantity limits)
+2. Validate completeness (config/tokenizer/weights)
+3. Output detailed failure reasons for debugging
+
+NOTE: This script no longer writes shared validation markers. Each test run
+independently validates its cache using per-run markers to avoid cross-runner
+cache state pollution.
+"""
+
+import glob
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Add python directory to path to import sglang modules
+REPO_ROOT = Path(__file__).parent.parent.parent.parent
+sys.path.insert(0, str(REPO_ROOT / "python"))
+
+from sglang.srt.model_loader.ci_weight_validation import (  # noqa: E402
+    _validate_diffusion_model,
+    validate_cache_with_detailed_reason,
+)
+
+# Limits to avoid spending too much time on validation
+MAX_VALIDATION_TIME_SECONDS = 300  # Max 5 minutes total
+
+
+def find_all_hf_snapshots():
+    """
+    Find all HuggingFace snapshots in cache.
+
+    Returns:
+        List of (model_name, snapshot_dir) tuples, sorted by mtime (newest first)
+    """
+    hf_home = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface"))
+    hub_dir = os.path.join(hf_home, "hub")
+
+    if not os.path.isdir(hub_dir):
+        print(f"HF hub directory not found: {hub_dir}")
+        return []
+
+    snapshots = []
+
+    # Pattern: models--org--model/snapshots/hash
+    for model_dir in glob.glob(os.path.join(hub_dir, "models--*")):
+        # Extract model name from directory (models--org--model -> org/model)
+        dir_name = os.path.basename(model_dir)
+        if not dir_name.startswith("models--"):
+            continue
+
+        # models--meta-llama--Llama-2-7b-hf -> meta-llama/Llama-2-7b-hf
+        # Handle multi-part names: models--a--b--c -> a/b-c (join parts 1+ with /)
+        parts = dir_name.split("--")
+        if len(parts) < 3 or parts[0] != "models":
+            # Invalid format, skip
+            continue
+        # Standard format: models--org--repo -> org/repo
+        # Extended format: models--org--repo--extra -> org/repo-extra (join with -)
+        model_name = parts[1] + "/" + "-".join(parts[2:])
+
+        snapshots_dir = os.path.join(model_dir, "snapshots")
+        if not os.path.isdir(snapshots_dir):
+            continue
+
+        # Find all snapshot hashes
+        for snapshot_hash_dir in os.listdir(snapshots_dir):
+            snapshot_path = os.path.join(snapshots_dir, snapshot_hash_dir)
+            if os.path.isdir(snapshot_path):
+                try:
+                    mtime = os.path.getmtime(snapshot_path)
+                    snapshots.append((model_name, snapshot_path, mtime))
+                except OSError:
+                    continue
+
+    # Sort by mtime (newest first) - prioritize recently used models
+    snapshots.sort(key=lambda x: x[2], reverse=True)
+
+    # Return without mtime
+    return [(name, path) for name, path, _ in snapshots]
+
+
+def is_transformers_text_model(snapshot_dir):
+    """
+    Check if a snapshot is a transformers text model.
+
+    Only excludes (returns False) for models with STRONG evidence of being
+    diffusers/generation pipelines. Uses conservative heuristics to avoid
+    false negatives on multimodal LLMs with tokenizers.
+
+    Args:
+        snapshot_dir: Path to snapshot directory
+
+    Returns:
+        True if this looks like a transformers text model, False otherwise (N/A)
+    """
+    # Check for diffusers pipeline markers (strong evidence)
+    diffusers_markers = [
+        "model_index.json",  # Diffusers pipeline config
+        "scheduler",  # Scheduler directory (diffusers)
+    ]
+    if any(
+        os.path.exists(os.path.join(snapshot_dir, marker))
+        for marker in diffusers_markers
+    ):
+        return False
+
+    config_path = os.path.join(snapshot_dir, "config.json")
+    if not os.path.exists(config_path):
+        # No config.json - likely not a transformers model
+        return False
+
+    try:
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = json.load(f)
+
+        # Check for explicit diffusers/generation model types (conservative keywords)
+        model_type = config.get("_class_name") or config.get("model_type")
+        if model_type:
+            model_type_lower = str(model_type).lower()
+            # Only exclude clear diffusion/generation models
+            if any(
+                keyword in model_type_lower
+                for keyword in [
+                    "diffusion",
+                    "unet",
+                    "vae",
+                    "controlnet",
+                    "stable-diffusion",
+                    "latent-diffusion",
+                ]
+            ):
+                return False
+
+        # Check architectures for explicit generation/diffusion classes
+        architectures = config.get("architectures", [])
+        if architectures:
+            arch_str = " ".join(architectures).lower()
+            # Conservative: only exclude obvious diffusion/generation architectures
+            # Use word boundaries to avoid false positives (e.g., "dit" in "conditional")
+            for keyword in [
+                "diffusion",
+                "unet2d",
+                "unet3d",
+                "vaedecoder",  # More specific than "vae"
+                "vaeencoder",
+                "controlnet",
+                "autoencoder",
+                "ditmodel",  # Diffusion Transformer - use more specific pattern
+                "pixart",  # PixArt diffusion model
+            ]:
+                if keyword in arch_str:
+                    return False
+
+        # Check for standalone vision encoder/image processor (no text component)
+        # Only if model name explicitly indicates non-text usage
+        model_name = config.get("_name_or_path", "").lower()
+
+        if any(
+            keyword in model_name
+            for keyword in [
+                "image-edit-",  # Pure image editing (e.g., Qwen-Image-Edit)
+                "-image-editing",
+                "dit-",  # DiT generation models
+                "pixart-",  # PixArt generation models
+            ]
+        ):
+            # Additional check: does it have tokenizer? If yes, might be multimodal LLM
+            has_tokenizer = any(
+                os.path.exists(os.path.join(snapshot_dir, fname))
+                for fname in ["tokenizer.json", "tokenizer.model", "tiktoken.model"]
+            )
+            if not has_tokenizer:
+                # Image-edit model without tokenizer -> likely pure vision pipeline
+                return False
+
+        # Default: assume it's a transformers text/multimodal model
+        # Even if it lacks tokenizer, let validation report the actual error
+        # (better false positive than false negative for text models)
+        return True
+
+    except (json.JSONDecodeError, OSError, KeyError):
+        # Can't parse config - assume it's transformers and let validation report failure
+        return True
+
+
+def scan_weight_files(snapshot_dir):
+    """
+    Scan for weight files in a snapshot.
+
+    Returns:
+        List of weight file paths, or empty list if scan fails
+    """
+    weight_files = []
+
+    # First, look for index files
+    index_patterns = ["*.safetensors.index.json", "pytorch_model.bin.index.json"]
+    index_files = []
+    for pattern in index_patterns:
+        index_files.extend(glob.glob(os.path.join(snapshot_dir, pattern)))
+
+    # If we have safetensors index, collect shards from it
+    for index_file in index_files:
+        if index_file.endswith(".safetensors.index.json"):
+            try:
+                with open(index_file, "r", encoding="utf-8") as f:
+                    index_data = json.load(f)
+                weight_map = index_data.get("weight_map", {})
+                for weight_file in set(weight_map.values()):
+                    weight_path = os.path.join(snapshot_dir, weight_file)
+                    if os.path.exists(weight_path):
+                        weight_files.append(weight_path)
+            except Exception as e:
+                print(
+                    f"  Warning: Failed to parse index {os.path.basename(index_file)}: {e}"
+                )
+
+    # If no index found or no shards from index, do recursive glob
+    if not weight_files:
+        matched = glob.glob(
+            os.path.join(snapshot_dir, "**/*.safetensors"), recursive=True
+        )
+        MAX_WEIGHT_FILES = 1000
+        if len(matched) > MAX_WEIGHT_FILES:
+            print(
+                f"  Warning: Too many safetensors files ({len(matched)} > {MAX_WEIGHT_FILES})"
+            )
+            return []
+
+        for f in matched:
+            if os.path.exists(f):  # Filter out broken symlinks
+                weight_files.append(f)
+
+    return weight_files
+
+
+def validate_snapshot(model_name, snapshot_dir, weight_files, validated_cache):
+    """
+    Validate a snapshot and return detailed status.
+
+    Uses in-process cache to avoid duplicate validation within the same run.
+
+    Args:
+        model_name: Model identifier
+        snapshot_dir: Path to snapshot directory
+        weight_files: List of weight files to validate
+        validated_cache: Dict to track already-validated snapshots in this run
+
+    Returns:
+        Tuple of (result, reason):
+        - (True, None) if validation passed
+        - (False, reason_str) if validation failed
+        - (None, None) if skipped (already validated in this run)
+    """
+    # Fast path: check in-process cache first
+    if snapshot_dir in validated_cache:
+        return None, None  # Already validated in this run, skip
+
+    try:
+        # Perform validation with detailed reason
+        is_complete, reason = validate_cache_with_detailed_reason(
+            snapshot_dir=snapshot_dir,
+            weight_files=weight_files,
+            model_name_or_path=model_name,
+        )
+
+        # Cache result to avoid re-validation in this run
+        validated_cache[snapshot_dir] = (is_complete, reason)
+
+        return is_complete, reason
+
+    except Exception as e:
+        error_msg = f"Validation raised exception: {e}"
+        return False, error_msg
+
+
+def main():
+    start_time = time.time()
+
+    print("=" * 70)
+    print("CI_OFFLINE: Pre-validating cached HuggingFace models")
+    print("=" * 70)
+    print(f"Max time: {MAX_VALIDATION_TIME_SECONDS}s")
+    print()
+
+    print("Scanning HuggingFace cache for models...")
+    snapshots = find_all_hf_snapshots()
+
+    if not snapshots:
+        print("No cached models found, skipping validation")
+        print("=" * 70)
+        return
+
+    print(f"Found {len(snapshots)} snapshot(s) in cache")
+    print()
+
+    validated_count = 0
+    failed_count = 0
+    skipped_count = 0
+    processed_count = 0
+
+    # In-process cache to avoid re-validating same snapshot in this run
+    validated_cache = {}
+
+    for model_name, snapshot_dir in snapshots:
+        # Check time limit
+        elapsed = time.time() - start_time
+        if elapsed > MAX_VALIDATION_TIME_SECONDS:
+            print()
+            print(
+                f"Time limit reached ({elapsed:.1f}s > {MAX_VALIDATION_TIME_SECONDS}s)"
+            )
+            print(
+                f"Stopping validation, {len(snapshots) - processed_count} snapshots remaining"
+            )
+            break
+
+        snapshot_hash = os.path.basename(snapshot_dir)
+        print(
+            f"[{processed_count + 1}/{len(snapshots)}] {model_name} ({snapshot_hash[:8]}...)"
+        )
+        processed_count += 1
+
+        # Determine model type by checking for model_index.json (diffusers pipeline marker)
+        model_index_path = os.path.join(snapshot_dir, "model_index.json")
+        is_diffusion_model = os.path.exists(model_index_path)
+
+        if is_diffusion_model:
+            # This is a diffusers pipeline - use diffusion validation
+            try:
+                is_valid, reason = _validate_diffusion_model(snapshot_dir)
+
+                if is_valid:
+                    print("  PASS (diffusion) - Cache complete & valid")
+                    validated_count += 1
+                else:
+                    print(f"  FAIL (diffusion) - {reason}")
+                    failed_count += 1
+
+            except Exception as e:
+                print(f"  FAIL (diffusion) - Validation raised exception: {e}")
+                failed_count += 1
+
+            continue
+
+        # Transformers model - use standard validation
+        # First check if this looks like a transformers text model
+        if not is_transformers_text_model(snapshot_dir):
+            # Not a recognized model type, skip
+            print(
+                "  SKIP (unknown type) - Not a diffusers pipeline or transformers model"
+            )
+            skipped_count += 1
+            continue
+
+        # Scan weight files
+        weight_files = scan_weight_files(snapshot_dir)
+
+        if not weight_files:
+            print("  SKIP (no weights) - empty or incomplete download")
+            skipped_count += 1
+            continue
+
+        # Validate
+        try:
+            result, reason = validate_snapshot(
+                model_name, snapshot_dir, weight_files, validated_cache
+            )
+
+            if result is True:
+                print("  PASS - Cache complete & valid")
+                validated_count += 1
+            elif result is False:
+                # Print detailed failure reason
+                if reason:
+                    print(f"  FAIL (incomplete) - {reason}")
+                else:
+                    print("  FAIL (incomplete) - cache validation failed")
+                failed_count += 1
+            else:  # None (skipped)
+                print("  SKIP (already validated in this run)")
+                skipped_count += 1
+
+        except Exception as e:
+            print(f"  FAIL (error) - Validation raised exception: {e}")
+            failed_count += 1
+
+    elapsed_total = time.time() - start_time
+
+    print()
+    print("=" * 70)
+    print(f"Validation summary (completed in {elapsed_total:.1f}s):")
+    print(f"  PASS (complete & valid):      {validated_count}")
+    print(f"  FAIL (incomplete/corrupted):  {failed_count}")
+    print(f"  SKIP (no weights/duplicate):  {skipped_count}")
+    print(f"  Total processed:              {processed_count}/{len(snapshots)}")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/publish_traces.py

@@ -0,0 +1,517 @@
+"""
+Publish performance traces to GitHub repository
+"""
+
+import argparse
+import base64
+import json
+import os
+import sys
+import time
+import warnings
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+
+def is_rate_limit_error(e):
+    """Check if an exception is a GitHub rate limit error (not permission error)"""
+    if not isinstance(e, HTTPError):
+        return False
+    if e.code == 429:
+        return True
+    if e.code == 403:
+        # 403 can be rate limit OR permission error - check the message
+        error_body = getattr(e, "error_body", "")
+        if isinstance(error_body, str):
+            # Rate limit errors contain specific phrases
+            rate_limit_phrases = [
+                "rate limit",
+                "abuse detection",
+                "secondary rate limit",
+            ]
+            return any(phrase in error_body.lower() for phrase in rate_limit_phrases)
+    return False
+
+
+def is_permission_error(e):
+    """Check if an exception is a GitHub permission error"""
+    if not isinstance(e, HTTPError) or e.code != 403:
+        return False
+    error_body = getattr(e, "error_body", "")
+    if isinstance(error_body, str):
+        permission_phrases = [
+            "resource not accessible",
+            "must have push access",
+            "permission",
+            "denied",
+        ]
+        return any(phrase in error_body.lower() for phrase in permission_phrases)
+    return False
+
+
+def make_github_request(url, token, method="GET", data=None):
+    """Make authenticated request to GitHub API"""
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        # "User-Agent": "sglang-ci",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+
+    if data:
+        headers["Content-Type"] = "application/json"
+        data = json.dumps(data).encode("utf-8")
+
+    req = Request(url, data=data, headers=headers, method=method)
+
+    try:
+        with urlopen(req) as response:
+            return response.read().decode("utf-8")
+    except HTTPError as e:
+        print(f"GitHub API request failed: {e}")
+        try:
+            error_body = e.read().decode("utf-8")
+            print(f"Error response body: {error_body}")
+            e.error_body = error_body  # Attach for later inspection
+        except Exception:
+            e.error_body = ""
+        raise
+    except Exception as e:
+        print(f"GitHub API request failed with a non-HTTP error: {e}")
+        raise
+
+
+def verify_token_permissions(repo_owner, repo_name, token):
+    """Verify that the token has necessary permissions for the repository"""
+    print("Verifying token permissions...")
+
+    checks = [
+        (
+            f"https://api.github.com/repos/{repo_owner}/{repo_name}",  # Check if we can access the repository
+            "Repository access verified",
+        ),
+        (
+            f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents",  # Check if we can read the repository contents
+            "Repository contents access verified",
+        ),
+    ]
+
+    for url, success_message in checks:
+        try:
+            response = make_github_request(url, token)
+            if success_message == "Repository access verified":
+                repo_data = json.loads(response)
+                print(f"{success_message}: {repo_data['full_name']}")
+            else:
+                print(success_message)
+        except Exception as e:
+            if is_rate_limit_error(e):
+                warnings.warn(
+                    "GitHub API rate limit exceeded during token verification."
+                )
+                return "rate_limited"
+            print(f"Failed to verify permissions for {url}: {e}")
+            return False
+
+    return True
+
+
+def get_branch_sha(repo_owner, repo_name, branch, token):
+    """Get SHA of the branch head"""
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}"
+    )
+    response = make_github_request(url, token)
+    data = json.loads(response)
+    return data["object"]["sha"]
+
+
+def get_tree_sha(repo_owner, repo_name, commit_sha, token):
+    """Get tree SHA from commit"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}"
+    response = make_github_request(url, token)
+    data = json.loads(response)
+    return data["tree"]["sha"]
+
+
+def create_blob(repo_owner, repo_name, content, token, max_retries=3):
+    """Create a blob with file content"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs"
+
+    # Encode content as base64 for GitHub API
+    content_b64 = base64.b64encode(content).decode("utf-8")
+
+    data = {"content": content_b64, "encoding": "base64"}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            return json.loads(response)["sha"]
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt  # Exponential backoff: 1s, 2s, 4s
+                print(
+                    f"Blob creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def create_blobs(repo_owner, repo_name, files, token):
+    """Create blobs for all files and return tree items with blob SHAs"""
+    tree_items = []
+    for i, (file_path, content) in enumerate(files):
+        # Create blob first to get SHA
+        blob_sha = create_blob(repo_owner, repo_name, content, token)
+        tree_items.append(
+            {
+                "path": file_path,
+                "mode": "100644",
+                "type": "blob",
+                "sha": blob_sha,
+            }
+        )
+        # Progress indicator for large uploads
+        if (i + 1) % 10 == 0 or (i + 1) == len(files):
+            print(f"Created {i + 1}/{len(files)} blobs...")
+    return tree_items
+
+
+def create_tree(repo_owner, repo_name, base_tree_sha, tree_items, token, max_retries=3):
+    """Create a new tree from pre-created blob SHAs"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/trees"
+
+    data = {"base_tree": base_tree_sha, "tree": tree_items}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            return json.loads(response)["sha"]
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Tree creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def create_commit(
+    repo_owner, repo_name, tree_sha, parent_sha, message, token, max_retries=3
+):
+    """Create a new commit"""
+    url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits"
+
+    data = {"tree": tree_sha, "parents": [parent_sha], "message": message}
+
+    for attempt in range(max_retries):
+        try:
+            response = make_github_request(url, token, method="POST", data=data)
+            commit_sha = json.loads(response)["sha"]
+
+            # Verify the commit was actually created
+            verify_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}"
+            verify_response = make_github_request(verify_url, token)
+            verify_data = json.loads(verify_response)
+            if verify_data["sha"] != commit_sha:
+                raise Exception(
+                    f"Commit verification failed: expected {commit_sha}, got {verify_data['sha']}"
+                )
+
+            return commit_sha
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Commit creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retries=3):
+    """Update branch reference to point to new commit"""
+    url = (
+        f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}"
+    )
+
+    data = {"sha": commit_sha}
+
+    for attempt in range(max_retries):
+        try:
+            make_github_request(url, token, method="PATCH", data=data)
+            return
+        except HTTPError as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            # Check if this is an "Object does not exist" error
+            is_object_not_exist = False
+            if hasattr(e, "error_body"):
+                try:
+                    error_data = json.loads(e.error_body)
+                    if "Object does not exist" in error_data.get("message", ""):
+                        is_object_not_exist = True
+                except Exception:
+                    pass
+
+            if is_object_not_exist and attempt < max_retries - 1:
+                # This might be a transient consistency issue - wait and retry
+                wait_time = 2**attempt
+                print(
+                    f"Branch update failed with 'Object does not exist' (attempt {attempt + 1}/{max_retries}), waiting {wait_time}s for consistency..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+        except Exception as e:
+            # Don't retry on rate limit errors - fail fast
+            if is_rate_limit_error(e):
+                raise
+
+            if attempt < max_retries - 1:
+                wait_time = 2**attempt
+                print(
+                    f"Branch update failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
+                )
+                time.sleep(wait_time)
+            else:
+                raise
+
+
+def copy_trace_files(source_dir, target_base_path):
+    """Copy trace files and return list of files to upload.
+
+    Only uploads traces from TP rank 0 to avoid duplicated data across tensor parallel ranks.
+    """
+    files_to_upload = []
+
+    if not os.path.exists(source_dir):
+        print(f"Warning: Traces directory {source_dir} does not exist")
+        return files_to_upload
+
+    # Walk through source directory and find .json.gz files
+    for root, dirs, files in os.walk(source_dir):
+        for file in files:
+            if file.endswith(".json.gz"):
+
+                # Only upload TP rank 0 traces to avoid duplicates across tensor parallel ranks
+                if "TP-" in file and "TP-0" not in file:
+                    continue
+
+                source_file = os.path.join(root, file)
+                # Calculate relative path from source_dir
+                rel_path = os.path.relpath(source_file, source_dir)
+                target_path = f"{target_base_path}/{rel_path}"
+
+                # Read file content
+                with open(source_file, "rb") as f:
+                    content = f.read()
+
+                files_to_upload.append((target_path, content))
+
+    return files_to_upload
+
+
+def publish_traces(traces_dir, run_id, run_number):
+    """Publish traces from a single directory to GitHub repository in a single commit"""
+    target_base_path = f"traces/{run_id}"
+    files_to_upload = copy_trace_files(traces_dir, target_base_path)
+
+    if not files_to_upload:
+        print("No trace files found to upload")
+        return
+
+    print(f"Found {len(files_to_upload)} files to upload")
+    publish_traces_from_files(files_to_upload, run_id, run_number)
+
+
+def publish_traces_from_files(files_to_upload, run_id, run_number):
+    """Publish pre-collected trace files to GitHub repository in a single commit"""
+    # Get environment variables
+    token = os.getenv("GITHUB_TOKEN")
+    if not token:
+        print("Error: GITHUB_TOKEN environment variable not set")
+        sys.exit(1)
+
+    # Repository configuration
+    repo_owner = "sgl-project"
+    repo_name = "ci-data"
+    branch = "main"
+
+    # Verify token permissions before proceeding
+    permission_check = verify_token_permissions(repo_owner, repo_name, token)
+    if permission_check == "rate_limited":
+        warnings.warn(
+            "Skipping trace upload due to GitHub API rate limit. "
+            "This is expected during high CI activity and does not indicate a test failure."
+        )
+        return
+    elif not permission_check:
+        print(
+            "Token permission verification failed. Please check the token permissions."
+        )
+        sys.exit(1)
+
+    max_retries = 5
+    retry_delay = 5  # seconds
+
+    # Create blobs once before retry loop to avoid re-uploading on failures
+    try:
+        tree_items = create_blobs(repo_owner, repo_name, files_to_upload, token)
+    except Exception as e:
+        # Check for rate limit errors during blob creation
+        if is_rate_limit_error(e):
+            warnings.warn(
+                "GitHub API rate limit exceeded during blob creation. Skipping trace upload."
+            )
+            return
+        # Check for permission errors - these should fail loudly
+        if is_permission_error(e):
+            print(
+                f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                "'contents: write' permission for the repository."
+            )
+            sys.exit(1)
+        print(f"Failed to create blobs: {e}")
+        raise
+
+    for attempt in range(max_retries):
+        try:
+            # Get current branch head
+            branch_sha = get_branch_sha(repo_owner, repo_name, branch, token)
+            print(f"Current branch head: {branch_sha}")
+
+            # Get current tree
+            tree_sha = get_tree_sha(repo_owner, repo_name, branch_sha, token)
+            print(f"Current tree SHA: {tree_sha}")
+
+            # Create new tree with pre-created blobs
+            new_tree_sha = create_tree(
+                repo_owner, repo_name, tree_sha, tree_items, token
+            )
+            print(f"Created new tree: {new_tree_sha}")
+
+            # Create commit
+            commit_message = f"Nightly traces for run {run_id} at {run_number} ({len(files_to_upload)} files)"
+            commit_sha = create_commit(
+                repo_owner,
+                repo_name,
+                new_tree_sha,
+                branch_sha,
+                commit_message,
+                token,
+            )
+            print(f"Created commit: {commit_sha}")
+
+            # Update branch reference
+            update_branch_ref(repo_owner, repo_name, branch, commit_sha, token)
+            print("Updated branch reference")
+
+            print("Successfully published all traces in a single commit")
+            return
+
+        except Exception as e:
+            # Check for retryable errors
+            is_retryable = False
+            error_type = "unknown"
+
+            if hasattr(e, "error_body"):
+                if "Update is not a fast forward" in e.error_body:
+                    is_retryable = True
+                    error_type = "fast-forward conflict"
+                elif "Object does not exist" in e.error_body:
+                    is_retryable = True
+                    error_type = "object consistency"
+
+            # Also retry on HTTP errors that might be transient
+            if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
+                is_retryable = True
+                error_type = f"HTTP {e.code}"
+
+            # Check for rate limit errors (non-fatal - just warn and skip)
+            if is_rate_limit_error(e):
+                warnings.warn("GitHub API rate limit exceeded. Skipping trace upload.")
+                return
+
+            # Check for permission errors - these should fail loudly
+            if is_permission_error(e):
+                print(
+                    f"ERROR: Token does not have write permission to {repo_owner}/{repo_name}. "
+                    "Please update the GH_PAT_FOR_NIGHTLY_CI_DATA secret with a token that has "
+                    "'contents: write' permission for the repository."
+                )
+                sys.exit(1)
+
+            if is_retryable and attempt < max_retries - 1:
+                print(
+                    f"Attempt {attempt + 1}/{max_retries} failed ({error_type}). Retrying in {retry_delay} seconds..."
+                )
+                time.sleep(retry_delay)
+            else:
+                print(f"Failed to publish traces after {attempt + 1} attempts: {e}")
+                raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Publish performance traces to GitHub repository"
+    )
+    parser.add_argument(
+        "--traces-dir",
+        type=str,
+        action="append",
+        dest="traces_dirs",
+        required=True,
+        help="Traces directory to publish (can be specified multiple times)",
+    )
+    args = parser.parse_args()
+
+    # Get environment variables
+    run_id = os.getenv("GITHUB_RUN_ID", "test")
+    run_number = os.getenv("GITHUB_RUN_NUMBER", "12345")
+
+    if not run_id or not run_number:
+        print(
+            "Error: GITHUB_RUN_ID and GITHUB_RUN_NUMBER environment variables must be set"
+        )
+        sys.exit(1)
+
+    # Collect trace files from all directories
+    target_base_path = f"traces/{run_id}"
+    all_files = []
+    for traces_dir in args.traces_dirs:
+        print(f"Processing traces from directory: {traces_dir}")
+        files = copy_trace_files(traces_dir, target_base_path)
+        all_files.extend(files)
+
+    if not all_files:
+        print("No trace files found to upload across all directories")
+        return
+
+    print(f"Found {len(all_files)} total files to upload")
+
+    # Publish all collected traces in a single commit
+    publish_traces_from_files(all_files, run_id, run_number)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/runner_utilization_report.py

@@ -0,0 +1,763 @@
+#!/usr/bin/env python3
+"""
+Runner Utilization Report
+
+Analyzes GitHub Actions job data to calculate runner utilization metrics.
+Reports idle time, active time, and utilization percentage per runner label.
+"""
+
+import argparse
+import json
+import os
+import random
+import subprocess
+import time
+from collections import Counter, defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timedelta, timezone
+
+# Labels to skip when grouping runners (GitHub default labels)
+DEFAULT_LABELS_TO_IGNORE = {"self-hosted", "Linux", "X64", "ARM64"}
+GITHUB_HOSTED_LABELS = {"ubuntu-latest", "ubuntu-22.04", "ubuntu-24.04"}
+
+# Human-facing job outcome buckets, in display order, with emoji.
+STATUS_ORDER = ("pass", "fail", "cancel", "running", "queued")
+STATUS_EMOJI = {
+    "pass": "✅",
+    "fail": "❌",
+    "cancel": "🚫",
+    "running": "🔄",
+    "queued": "⏳",
+}
+
+
+def format_status_counts(counts: dict) -> str:
+    """Compact per-label outcome summary, e.g. '✅120 ❌3 🔄2 ⏳4'."""
+    parts = [f"{STATUS_EMOJI[s]}{counts[s]}" for s in STATUS_ORDER if counts.get(s)]
+    return " ".join(parts) if parts else "—"
+
+
+def run_gh_command(args: list[str], max_retries: int = 10) -> dict:
+    """Run gh CLI command and return JSON result.
+
+    Retries on transient failures (5xx, secondary rate limits, network
+    blips) with exponential backoff. The previous fail-fast behavior
+    combined with `except Exception: return None` in the threadpool
+    callers caused entire workflow runs to be silently dropped from
+    the utilization numerator whenever GH API hiccuped, severely
+    undercounting busy time on busy days.
+    """
+    last_err = ""
+    for attempt in range(max_retries):
+        result = subprocess.run(
+            ["gh", "api"] + args,
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode == 0:
+            return json.loads(result.stdout)
+        last_err = result.stderr or "(no stderr)"
+        # Detect retryable conditions: HTTP 5xx, secondary rate limit, abuse
+        # detection, network resets. 4xx other than 429 are non-retryable.
+        retryable = any(
+            s in last_err
+            for s in (
+                "rate limit",
+                "abuse",
+                "Internal Server Error",
+                "502",
+                "503",
+                "504",
+                "Bad Gateway",
+                "Gateway Time-out",
+                "connection reset",
+                "Connection reset",
+                "EOF",
+                "timeout",
+            )
+        )
+        if not retryable:
+            break
+        # Exponential backoff with jitter, capped at 60s.
+        delay = min(60, (2**attempt) + random.uniform(0, 1))
+        time.sleep(delay)
+    raise Exception(f"gh api failed after {max_retries} attempts: {last_err[:300]}")
+
+
+def get_workflow_runs(repo: str, hours: int = 24) -> list[dict]:
+    """Get workflow runs from the last N hours."""
+    since = datetime.now(timezone.utc) - timedelta(hours=hours)
+
+    runs = []
+    page = 1
+    while True:
+        data = run_gh_command(
+            [
+                f"repos/{repo}/actions/runs?per_page=100&page={page}",
+            ]
+        )
+        page_runs = data.get("workflow_runs", [])
+
+        # Filter by time
+        for run in page_runs:
+            created_at = parse_time(run.get("created_at"))
+            if created_at and created_at >= since:
+                runs.append(run)
+            elif created_at and created_at < since:
+                # Runs are ordered by created_at desc, so we can stop
+                return runs
+
+        if len(page_runs) < 100:
+            break
+        page += 1
+        if page > 50:  # Safety limit (5000 runs)
+            break
+    return runs
+
+
+def get_jobs_for_run(repo: str, run_id: int) -> list[dict]:
+    """Get all jobs for a workflow run, including all retry attempts.
+
+    `filter=all` is required so that re-run attempts of the same job
+    appear separately. Each attempt consumed host time on the runner
+    pool, so for utilization we want them all summed in. The default
+    (`filter=latest`) only returns the most recent attempt and silently
+    hides time spent on prior retries.
+    """
+    jobs = []
+    page = 1
+    while True:
+        data = run_gh_command(
+            [
+                f"repos/{repo}/actions/runs/{run_id}/jobs"
+                f"?per_page=100&page={page}&filter=all",
+            ]
+        )
+        jobs.extend(data.get("jobs", []))
+        if len(data.get("jobs", [])) < 100:
+            break
+        page += 1
+        if page > 20:  # Safety limit (2000 jobs per run)
+            break
+    return jobs
+
+
+def get_runners(repo: str, online_only: bool = True) -> list[dict]:
+    """Get all self-hosted runners with pagination. Returns empty if no permission."""
+    try:
+        all_runners = []
+        page = 1
+        while True:
+            data = run_gh_command(
+                [f"repos/{repo}/actions/runners?per_page=100&page={page}"]
+            )
+            runners = data.get("runners", [])
+            all_runners.extend(runners)
+            if len(runners) < 100:
+                break
+            page += 1
+            if page > 10:  # Safety limit
+                break
+        if online_only:
+            all_runners = [r for r in all_runners if r.get("status") == "online"]
+        return all_runners
+    except Exception as e:
+        print(f"Warning: Cannot access runners API (need admin): {e}")
+        return []
+
+
+def parse_time(time_str: str) -> datetime:
+    """Parse ISO timestamp to datetime."""
+    if not time_str:
+        return None
+    return datetime.fromisoformat(time_str.replace("Z", "+00:00"))
+
+
+def classify_job(job: dict, now: datetime):
+    """Derive the queue-wait and busy interval for a single job.
+
+    Returns a job_info dict, or None when the job neither waited for nor
+    occupied a runner (skipped / cancelled-before-start / missing data).
+
+    The queue wait runs from when the job entered the runner queue
+    (`created_at`) until a runner picked it up (`started_at`) — or until
+    `now` if it is still waiting.
+
+    GitHub API gotcha this exists to handle: a still-queued job reports
+    status="queued", runner_name="" and `started_at` set to a PLACEHOLDER
+    equal to `created_at` (not null). The previous code required both a
+    runner_name and a `completed_at`, so every in-flight wait — the
+    multi-hour 8-gpu jobs still sitting in the queue, i.e. the worst cases —
+    was dropped, undercounting max/avg queue time. We therefore measure a
+    queued job's wait against `now` rather than its bogus `started_at`, and
+    don't require completion.
+    """
+    status = job.get("status")
+    runner_name = job.get("runner_name") or ""
+    created_at = parse_time(job.get("created_at"))
+    started_at = parse_time(job.get("started_at"))
+    completed_at = parse_time(job.get("completed_at"))
+
+    if status == "queued":
+        # Still waiting for a runner; ignore the placeholder started_at.
+        queue_end, start, end = now, None, None
+    elif status == "in_progress" and started_at is not None:
+        # Running now: the wait is final and it still occupies the runner.
+        queue_end, start, end = started_at, started_at, now
+    elif (
+        status == "completed"
+        and started_at is not None
+        and completed_at is not None
+        and runner_name
+    ):
+        queue_end, start, end = started_at, started_at, completed_at
+    else:
+        # Skipped, cancelled before start, or missing timestamps: never
+        # waited for or occupied a runner.
+        return None
+
+    if created_at is None:
+        return None
+
+    queue_time = max(0.0, (queue_end - created_at).total_seconds())
+    duration = (end - start).total_seconds() if start is not None else 0.0
+    labels = [
+        label
+        for label in job.get("labels", [])
+        if label not in DEFAULT_LABELS_TO_IGNORE | GITHUB_HOSTED_LABELS
+    ]
+
+    # Human-facing outcome bucket used by the report's status breakdown.
+    if status == "queued":
+        outcome = "queued"
+    elif status == "in_progress":
+        outcome = "running"
+    else:  # completed and actually ran
+        outcome = {"success": "pass", "cancelled": "cancel"}.get(
+            job.get("conclusion"), "fail"
+        )
+
+    return {
+        "start": start,
+        "end": end,
+        "created_at": created_at,
+        "queue_end": queue_end,
+        "duration": duration,
+        "queue_time": queue_time,
+        "job_name": job.get("name", ""),
+        "runner_name": runner_name,
+        "labels": labels,
+        "status": outcome,
+        "html_url": job.get("html_url", ""),
+    }
+
+
+def calculate_concurrency_metrics(
+    jobs: list,
+    window_start: datetime,
+    window_end: datetime,
+    num_runners: int,
+) -> dict:
+    """Sweep-line algorithm: peak/avg concurrent, saturation time, peak queue."""
+    if not jobs:
+        return {
+            "peak_concurrent": 0,
+            "avg_concurrent": 0.0,
+            "saturation_seconds": 0,
+            "saturation_pct": 0.0,
+            "peak_queue": 0,
+        }
+    window_seconds = (window_end - window_start).total_seconds()
+    if window_seconds <= 0:
+        return {
+            "peak_concurrent": 0,
+            "avg_concurrent": 0.0,
+            "saturation_seconds": 0,
+            "saturation_pct": 0.0,
+            "peak_queue": 0,
+        }
+    running_events = []
+    for job in jobs:
+        start, end = job["start"], job["end"]
+        # Still-queued jobs have no running interval yet (start/end are None).
+        if start is None or end is None:
+            continue
+        if end < window_start or start > window_end:
+            continue
+        running_events.append((max(start, window_start), 1))
+        running_events.append((min(end, window_end), -1))
+    queue_events = []
+    for job in jobs:
+        created_at = job.get("created_at")
+        # The wait ends when a runner picks the job up, or `now` if it is
+        # still queued (queue_end was set to now upstream). Counting the
+        # still-open waits is what makes peak_queue reflect the real backlog.
+        queue_end = job.get("queue_end") or job["start"]
+        if created_at and queue_end and created_at < queue_end:
+            if queue_end < window_start or created_at > window_end:
+                continue
+            queue_events.append((max(created_at, window_start), 1))
+            queue_events.append((min(queue_end, window_end), -1))
+    running_events.sort(key=lambda e: (e[0], e[1] == 1))
+    current_running = 0
+    peak_running = 0
+    prev_time = window_start
+    total_running_seconds = 0.0
+    saturation_seconds = 0.0
+    for event_time, delta in running_events:
+        td = (event_time - prev_time).total_seconds()
+        if td > 0:
+            total_running_seconds += current_running * td
+            if current_running >= num_runners:
+                saturation_seconds += td
+        current_running += delta
+        peak_running = max(peak_running, current_running)
+        prev_time = event_time
+    if prev_time < window_end:
+        td = (window_end - prev_time).total_seconds()
+        total_running_seconds += current_running * td
+        if current_running >= num_runners:
+            saturation_seconds += td
+    queue_events.sort(key=lambda e: (e[0], e[1] == 1))
+    current_queued = 0
+    peak_queue = 0
+    for _, delta in queue_events:
+        current_queued += delta
+        peak_queue = max(peak_queue, current_queued)
+    avg_concurrent = total_running_seconds / window_seconds if window_seconds > 0 else 0
+    return {
+        "peak_concurrent": peak_running,
+        "avg_concurrent": avg_concurrent,
+        "saturation_seconds": saturation_seconds,
+        "saturation_pct": (
+            (saturation_seconds / window_seconds * 100) if window_seconds > 0 else 0
+        ),
+        "peak_queue": peak_queue,
+    }
+
+
+_NON_GPU_WORKFLOW_HINTS = (
+    "lint",
+    "deploy",
+    "release",
+    "publish",
+    "docs",
+    "doc",
+    "mintlify",
+    "runner utilization",  # this very script
+    "tag-and-rerun",
+    "auto",  # auto-merge etc.
+    "label",
+    "stale",
+    "dependabot",
+    "codeql",
+)
+
+
+def _likely_no_gpu_jobs(workflow_name: str) -> bool:
+    """Heuristic: skip per-run job-fetch for workflows that don't dispatch
+    to self-hosted GPU runners. The GH API rate limit (~5000 req/hr per
+    token) is the bottleneck on busy 24h windows where ~4000 workflow
+    runs fire — but only a fraction of those (pr-test, nightly-test,
+    pr-test-*kernel, etc.) actually run on GPU runners. Skipping the
+    docs/lint/release runs cuts the API call budget by 2-4x.
+    """
+    if not workflow_name:
+        return False
+    n = workflow_name.lower()
+    return any(h in n for h in _NON_GPU_WORKFLOW_HINTS)
+
+
+def calculate_utilization(repo: str, hours: int = 24, runner_filter: str = None):
+    """Calculate runner utilization metrics."""
+
+    print(f"Fetching workflow runs from last {hours} hours...")
+    all_runs = get_workflow_runs(repo, hours)
+    runs = [r for r in all_runs if not _likely_no_gpu_jobs(r.get("name", ""))]
+    skipped = len(all_runs) - len(runs)
+    print(
+        f"Found {len(all_runs)} workflow runs "
+        f"({skipped} skipped as non-GPU: docs/lint/release/etc.)"
+    )
+
+    # Try to get online runners from API
+    print("Fetching online runners...")
+    runners = get_runners(repo, online_only=True)
+
+    # Build label -> set of online runner names from API
+    api_label_runners = defaultdict(set)
+    if runners:
+        for runner in runners:
+            for label in runner.get("labels", []):
+                label_name = label.get("name", "")
+                if label_name not in DEFAULT_LABELS_TO_IGNORE:
+                    api_label_runners[label_name].add(runner["name"])
+        print(f"Got {len(runners)} online runners from API")
+    else:
+        print("No runner API access, will use observed runners from job data")
+
+    # Track runners seen in jobs (for labels not in API or when API unavailable)
+    job_label_runners = defaultdict(set)
+    label_jobs = defaultdict(list)  # label -> list of job_info
+    # Per-host accumulation: each physical machine appears once regardless of
+    # how many overlapping labels it advertises. This is what we use for the
+    # "Per Host Utilization" section (the source-of-truth view).
+    host_jobs = defaultdict(list)  # runner_name -> list of job_info
+    host_labels = defaultdict(set)  # runner_name -> set of labels it ran jobs under
+
+    # Fetch jobs for all runs in parallel. Cap concurrency lower than the
+    # GH API secondary rate-limit threshold to avoid bursts that silently
+    # drop runs even with retries.
+    total_runs = len(runs)
+    print(f"Fetching jobs for {total_runs} runs in parallel...")
+
+    def fetch_jobs_for_run(run):
+        """Fetch jobs for a single run.
+
+        Returns (run_id, jobs, error_msg). `error_msg` is None on success.
+        We surface failures rather than silently dropping the run so the
+        caller can report how many runs' jobs are missing — silently
+        dropping previously caused 4-gpu-b200 (and every other label) to
+        report wildly different numbers depending on transient API hiccups.
+        """
+        try:
+            return (run["id"], get_jobs_for_run(repo, run["id"]), None)
+        except Exception as e:
+            return (run["id"], None, str(e)[:200])
+
+    all_jobs = []
+    failed_runs = []
+    # Concurrency=4 with longer retry budget keeps us well below the GH
+    # API secondary rate-limit threshold (~10 req/s). On a 24h window
+    # with ~1500 GPU-relevant runs (post-filter), this completes in ~5
+    # min and almost never hits the rate limit.
+    with ThreadPoolExecutor(max_workers=4) as executor:
+        futures = [executor.submit(fetch_jobs_for_run, run) for run in runs]
+        completed = 0
+        for future in as_completed(futures):
+            completed += 1
+            if completed % 100 == 0:
+                print(
+                    f"Fetched jobs for {completed}/{total_runs} runs "
+                    f"({len(failed_runs)} failed so far)..."
+                )
+            run_id, jobs, err = future.result()
+            if err:
+                failed_runs.append((run_id, err))
+            elif jobs:
+                all_jobs.extend(jobs)
+
+    print(f"Processing {len(all_jobs)} jobs...")
+    if failed_runs:
+        print(
+            f"WARNING: {len(failed_runs)}/{total_runs} runs failed to fetch "
+            f"after retries. Utilization will be undercounted. "
+            f"First few errors:"
+        )
+        for rid, err in failed_runs[:5]:
+            print(f"  run {rid}: {err}")
+    fetch_failure_pct = len(failed_runs) / total_runs * 100 if total_runs > 0 else 0
+
+    # `now` anchors the wait of jobs that are still queued or running. It is
+    # captured once so every in-flight job is measured against a single
+    # reference (matches window_end below to within processing time).
+    now = datetime.now(timezone.utc)
+    all_job_infos = []  # one entry per job (deduped across labels) for detail views
+    for job in all_jobs:
+        job_info = classify_job(job, now)
+        if job_info is None:
+            continue
+        all_job_infos.append(job_info)
+        runner_name = job_info["runner_name"]
+
+        # Per-host busy time only applies to jobs that actually occupied a
+        # runner (ran or still running); a still-queued job has no host yet.
+        if job_info["start"] is not None and runner_name:
+            host_jobs[runner_name].append(job_info)
+
+        for label in job_info["labels"]:
+            if runner_name:
+                job_label_runners[label].add(runner_name)
+                host_labels[runner_name].add(label)
+            label_jobs[label].append(job_info)
+
+    # Merge API runners and job-observed runners
+    # Prefer API count (online runners) when available
+    # Include labels seen only on still-queued jobs (no online runner, no
+    # completed job under them yet) so a fully-backed-up pool still reports.
+    all_labels = (
+        set(api_label_runners.keys())
+        | set(job_label_runners.keys())
+        | set(label_jobs.keys())
+    )
+
+    # Filter labels if specified
+    if runner_filter:
+        all_labels = {lbl for lbl in all_labels if runner_filter in lbl}
+
+    print(f"Tracking {len(all_labels)} runner labels: {sorted(all_labels)}")
+
+    window_seconds = hours * 3600
+    window_end = datetime.now(timezone.utc)
+    window_start = window_end - timedelta(hours=hours)
+
+    # Per-host window-clamped busy time (each physical machine counted once).
+    # This is the source of truth for how loaded each host actually is.
+    host_busy_seconds = {}
+    for host, jobs in host_jobs.items():
+        busy = 0.0
+        for j in jobs:
+            cs = max(j["start"], window_start)
+            ce = min(j["end"], window_end)
+            if ce > cs:
+                busy += (ce - cs).total_seconds()
+        host_busy_seconds[host] = busy
+
+    results = []
+    for label in sorted(all_labels):
+        # Hosts to attribute to this label = union of currently-online
+        # runners advertising the label PLUS hosts that actually ran a
+        # job under it during the window. The union catches hosts that
+        # went offline mid-window (their busy time is still real
+        # capacity consumed) and hosts that came online late.
+        hosts = api_label_runners.get(label, set()) | job_label_runners.get(
+            label, set()
+        )
+        num_runners = len(hosts) if hosts else 1
+
+        # Pool busy time: sum of busy time across the hosts that could
+        # serve this label, regardless of which sibling label actually
+        # dispatched the job. This is the right denominator/numerator for
+        # asking "how saturated is the underlying hardware that this
+        # label depends on?" — sibling labels (e.g. `4-gpu-b200` and
+        # `4-gpu-b200-low-disk`) compete for the same physical machines,
+        # so their busy time should not be double-counted into separate
+        # capacity buckets.
+        active_seconds = sum(host_busy_seconds.get(h, 0.0) for h in hosts)
+        capacity_seconds = num_runners * window_seconds
+        utilization = (
+            (active_seconds / capacity_seconds * 100) if capacity_seconds > 0 else 0
+        )
+
+        # Job count + queue stats stay label-specific (only jobs that
+        # were dispatched under THIS label).
+        jobs = label_jobs.get(label, [])
+        queue_times = [j["queue_time"] for j in jobs if j["queue_time"] > 0]
+        avg_queue = sum(queue_times) / len(queue_times) if queue_times else 0
+        max_queue = max(queue_times) if queue_times else 0
+        # Outcome breakdown for this label (pass/fail/cancel/running/queued).
+        status_counts = dict(Counter(j["status"] for j in jobs))
+
+        # Concurrency / saturation / queue-depth metrics. Use observed
+        # peak as effective capacity if it's lower than the API count
+        # (e.g. for autoscaling pools where most listeners sit idle).
+        conc_initial = calculate_concurrency_metrics(
+            jobs, window_start, window_end, num_runners
+        )
+        effective_runners = (
+            min(num_runners, conc_initial["peak_concurrent"]) or num_runners
+        )
+        if effective_runners < num_runners and effective_runners > 0:
+            conc = calculate_concurrency_metrics(
+                jobs, window_start, window_end, effective_runners
+            )
+        else:
+            conc = conc_initial
+
+        results.append(
+            {
+                "label": label,
+                "num_runners": num_runners,
+                "effective_runners": effective_runners,
+                "num_jobs": len(jobs),
+                "total_active_hours": active_seconds / 3600,
+                "utilization_pct": utilization,
+                "avg_queue_min": avg_queue / 60,
+                "max_queue_min": max_queue / 60,
+                "peak_concurrent": conc_initial["peak_concurrent"],
+                "avg_concurrent": conc["avg_concurrent"],
+                "saturation_hours": conc["saturation_seconds"] / 3600,
+                "saturation_pct": conc["saturation_pct"],
+                "peak_queue": conc["peak_queue"],
+                "status_counts": status_counts,
+            }
+        )
+
+    # Per-job detail (deduped across labels), longest waits first, for the
+    # links + status section of the report.
+    longest_waits = sorted(all_job_infos, key=lambda j: j["queue_time"], reverse=True)
+    return results, fetch_failure_pct, longest_waits
+
+
+def format_report(
+    results: list[dict],
+    hours: int,
+    fetch_failure_pct: float = 0.0,
+    longest_waits: list = None,
+    top_n: int = 20,
+) -> str:
+    """One compact summary table — original schema, fixed columns.
+
+    Active (hrs) and Utilization now reflect the actual host pool's
+    busy time (sum across all jobs on the hosts that advertise this
+    label, regardless of which sibling label dispatched them). This
+    makes the column meaningful for shared host pools — e.g.
+    `4-gpu-b200` and `4-gpu-b200-low-disk` both consume the same
+    physical hosts, so their utilization now reflects real hardware
+    saturation instead of being divided across labels.
+    """
+    lines = [
+        "# Runner Utilization Report",
+        "",
+        f"**Time window:** Last {hours} hours · "
+        f"**Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
+        "",
+    ]
+    if fetch_failure_pct > 1.0:
+        lines.append(
+            f"⚠️ **Data completeness warning**: {fetch_failure_pct:.0f}% of "
+            f"GPU-relevant workflow runs failed to fetch jobs after retries "
+            f"(GH API rate limit). Active hours and utilization below are "
+            f"under-counted by approximately this fraction."
+        )
+        lines.append("")
+    lines.extend(
+        [
+            "| Label | Runners | Jobs | Active (hrs) | Utilization | Avg Queue | Max Queue | Status |",
+            "|-------|---------|------|--------------|-------------|-----------|-----------|--------|",
+        ]
+    )
+    for r in results:
+        bar = "█" * int(r["utilization_pct"] / 10) + "░" * (
+            10 - int(r["utilization_pct"] / 10)
+        )
+        lines.append(
+            f"| {r['label']} | {r['num_runners']} | {r['num_jobs']} | "
+            f"{r['total_active_hours']:.1f} | "
+            f"{r['utilization_pct']:.1f}% {bar} | "
+            f"{r['avg_queue_min']:.1f}m | {r['max_queue_min']:.1f}m | "
+            f"{format_status_counts(r.get('status_counts', {}))} |"
+        )
+
+    # Longest queue waits — links to the actual jobs, with live status, so the
+    # worst waits (including jobs still queued/running right now) are one click
+    # away. This is the detail behind the Max Queue column.
+    waits = [j for j in (longest_waits or []) if j.get("queue_time", 0) > 0][:top_n]
+    if waits:
+        lines.extend(
+            [
+                "",
+                f"## Longest Queue Waits (top {len(waits)})",
+                "",
+                "| Wait | Status | Label | Job |",
+                "|------|--------|-------|-----|",
+            ]
+        )
+        for j in waits:
+            status = j.get("status", "")
+            emoji = STATUS_EMOJI.get(status, "")
+            label = ", ".join(j.get("labels", [])) or "—"
+            name = j.get("job_name", "job")
+            url = j.get("html_url", "")
+            job_cell = f"[{name}]({url})" if url else name
+            lines.append(
+                f"| {j['queue_time'] / 60:.0f}m | {emoji} {status} | "
+                f"{label} | {job_cell} |"
+            )
+
+    # Concurrency Analysis section
+    lines.extend(
+        [
+            "",
+            "## Concurrency Analysis",
+            "",
+            "| Label | Runners (API/Effective) | Peak Concurrent | Avg Concurrent | Saturation Time | Peak Queue |",
+            "|-------|-------------------------|-----------------|----------------|-----------------|------------|",
+        ]
+    )
+    for r in results:
+        effective = r["effective_runners"]
+        avg_pct = (r["avg_concurrent"] / effective * 100) if effective > 0 else 0
+        runner_str = (
+            f"{r['num_runners']}/{effective}"
+            if effective != r["num_runners"]
+            else str(r["num_runners"])
+        )
+        lines.append(
+            f"| {r['label']} | {runner_str} | "
+            f"{r['peak_concurrent']} | "
+            f"{r['avg_concurrent']:.1f} ({avg_pct:.0f}%) | "
+            f"{r['saturation_hours']:.1f}h ({r['saturation_pct']:.0f}%) | "
+            f"{r['peak_queue']} jobs |"
+        )
+
+    # Recommendations
+    lines.extend(["", "## Recommendations", ""])
+    has_recs = False
+    for r in results:
+        label = r["label"]
+        sat_pct = r["saturation_pct"]
+        peak_q = r["peak_queue"]
+        effective = r["effective_runners"]
+        avg_pct = (r["avg_concurrent"] / effective * 100) if effective > 0 else 0
+        if sat_pct > 50 or peak_q > 5:
+            lines.append(
+                f"⚠️ **{label}**: High saturation ({sat_pct:.0f}%) "
+                f"with queue buildup ({peak_q} jobs). Consider adding runners."
+            )
+            has_recs = True
+        elif sat_pct > 20 or peak_q > 0:
+            lines.append(
+                f"📊 **{label}**: Moderate saturation ({sat_pct:.0f}%), "
+                f"peak queue {peak_q} jobs. Monitor for trends."
+            )
+            has_recs = True
+        elif avg_pct < 30 and r["num_jobs"] > 0:
+            lines.append(
+                f"💡 **{label}**: Low average utilization ({avg_pct:.0f}%). "
+                f"Runner pool may be oversized."
+            )
+            has_recs = True
+        else:
+            lines.append(f"✓ **{label}**: Healthy utilization with minimal queueing.")
+    if not has_recs and results:
+        lines.append("All runner pools have healthy utilization.")
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate runner utilization report")
+    parser.add_argument("--repo", default="sgl-project/sglang", help="GitHub repo")
+    parser.add_argument(
+        "--hours", type=float, default=24, help="Time window in hours (fractional ok)"
+    )
+    parser.add_argument(
+        "--filter", type=str, help="Filter runner labels (e.g., '5090', 'h200')"
+    )
+    parser.add_argument("--output", type=str, help="Output file (default: stdout)")
+    args = parser.parse_args()
+
+    results, fetch_failure_pct, longest_waits = calculate_utilization(
+        args.repo, args.hours, args.filter
+    )
+    report = format_report(
+        results, args.hours, fetch_failure_pct, longest_waits=longest_waits
+    )
+
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(report)
+        print(f"Report written to {args.output}")
+    else:
+        print(report)
+
+    # Also write to GITHUB_STEP_SUMMARY if available
+    summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_file:
+        with open(summary_file, "a") as f:
+            f.write(report)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/save_metrics.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""Collect and save performance metrics from nightly benchmark results.
+
+This script reads benchmark result JSON files from performance profile directories
+and saves them with metadata for artifact collection in CI.
+
+Usage:
+    python3 scripts/ci/utils/save_metrics.py \
+        --gpu-config 8-gpu-h200 \
+        --partition 0 \
+        --run-id 12345678 \
+        --output test/metrics-8gpu-h200-partition-0.json
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_result_files(search_dirs: list[str]) -> list[str]:
+    """Find all results_*.json files in the given directories."""
+    result_files = set()
+    for search_dir in search_dirs:
+        if os.path.exists(search_dir):
+            pattern = os.path.join(search_dir, "**/results_*.json")
+            result_files.update(glob.glob(pattern, recursive=True))
+    return list(result_files)
+
+
+def parse_result_file(filepath: str) -> list[dict]:
+    """Parse a benchmark result JSON file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, list):
+            return data
+        return [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {filepath}: {e}")
+        return []
+
+
+def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict:
+    """Transform a benchmark result to the metrics schema.
+
+    Note: input_len and output_len are preserved here for the flat benchmarks list,
+    but are also used as grouping keys in benchmarks_by_io_len.
+    """
+    # Handle None values safely for numeric conversions
+    latency = result.get("latency")
+    last_ttft = result.get("last_ttft")
+
+    return {
+        "batch_size": result.get("batch_size"),
+        "input_len": result.get("input_len"),
+        "output_len": result.get("output_len"),
+        "latency_ms": latency * 1000 if latency is not None else None,
+        "input_throughput": result.get("input_throughput"),
+        "output_throughput": result.get("output_throughput"),
+        "overall_throughput": result.get("overall_throughput"),
+        "ttft_ms": last_ttft * 1000 if last_ttft is not None else None,
+        "acc_length": result.get("acc_length"),
+    }
+
+
+def get_io_len_key(input_len: int, output_len: int) -> str:
+    """Generate a key for input/output length combination."""
+    return f"{input_len}_{output_len}"
+
+
+def group_results_by_model(
+    results: list[dict], gpu_config: str, partition: int
+) -> list[dict]:
+    """Group benchmark results by model, variant, and server_args.
+
+    Results are organized with two benchmark structures:
+    - benchmarks: flat list of all benchmarks (for backward compatibility)
+    - benchmarks_by_io_len: nested structure grouped by input/output length combinations
+    """
+    groups = {}
+
+    for result in results:
+        model_path = result.get("model_path", "unknown")
+        run_name = result.get("run_name", "default")
+        variant = run_name if run_name != "default" else None
+        server_args = result.get("server_args")
+        # Convert server_args list to tuple for use as dict key (lists are not hashable)
+        server_args_key = tuple(server_args) if server_args else None
+
+        key = (model_path, variant, server_args_key)
+        if key not in groups:
+            groups[key] = {
+                "gpu_config": gpu_config,
+                "partition": partition,
+                "model": model_path,
+                "variant": variant,
+                "server_args": server_args,
+                "benchmarks": [],
+                "benchmarks_by_io_len": {},
+            }
+
+        transformed = transform_benchmark_result(result, gpu_config, partition)
+
+        # Add to flat benchmarks list (backward compatibility)
+        groups[key]["benchmarks"].append(transformed)
+
+        # Add to nested benchmarks_by_io_len structure
+        input_len = result.get("input_len")
+        output_len = result.get("output_len")
+        if input_len is not None and output_len is not None:
+            io_key = get_io_len_key(input_len, output_len)
+            if io_key not in groups[key]["benchmarks_by_io_len"]:
+                groups[key]["benchmarks_by_io_len"][io_key] = {
+                    "input_len": input_len,
+                    "output_len": output_len,
+                    "benchmarks": [],
+                }
+            # For the nested structure, exclude input_len and output_len from individual benchmarks
+            # since they're already in the parent
+            nested_benchmark = {
+                k: v
+                for k, v in transformed.items()
+                if k not in ("input_len", "output_len")
+            }
+            groups[key]["benchmarks_by_io_len"][io_key]["benchmarks"].append(
+                nested_benchmark
+            )
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    partition: int,
+    run_id: str,
+    output_file: str,
+    search_dirs: list[str],
+) -> bool:
+    """Collect metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Find all result files
+    result_files = find_result_files(search_dirs)
+    print(f"Found {len(result_files)} result file(s)")
+
+    grouped = []
+    if not result_files:
+        print("No benchmark result files found")
+    else:
+        # Parse all result files
+        all_results = []
+        for filepath in sorted(result_files):
+            print(f"  Reading: {filepath}")
+            results = parse_result_file(filepath)
+            all_results.extend(results)
+        print(f"Total benchmark results: {len(all_results)}")
+
+        # Group by model/variant
+        grouped = group_results_by_model(all_results, gpu_config, partition)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "partition": partition,
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not result_files:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect performance metrics from benchmark results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 8-gpu-h200, 8-gpu-b200)",
+    )
+    parser.add_argument(
+        "--partition",
+        type=int,
+        required=True,
+        help="Partition number (0, 1, 2, etc.)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--search-dir",
+        action="append",
+        default=[],
+        dest="search_dirs",
+        help="Directory to search for result files (can be specified multiple times)",
+    )
+
+    args = parser.parse_args()
+
+    # Default search directories if none specified
+    search_dirs = args.search_dirs or [
+        "test/performance_profiles_8_gpu",
+        "test/performance_profiles_text_models",
+        "test/performance_profiles_vlms",
+        "test",
+        ".",
+    ]
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        partition=args.partition,
+        run_id=args.run_id,
+        output_file=args.output,
+        search_dirs=search_dirs,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/utils/test_runner_utilization_report.py

@@ -0,0 +1,195 @@
+"""Unit tests for runner_utilization_report.classify_job.
+
+Pure-logic tests (no GitHub API, stdlib only) so they run in the
+runner-utilization workflow without installing dependencies:
+
+    python -m unittest discover -s scripts/ci/utils -p 'test_runner_utilization_report.py'
+
+Regression guard for the queue-time underestimation bug: jobs still
+waiting in the runner queue (or still running) used to be dropped because
+the old code required a runner_name and a completed_at, so multi-hour
+8-gpu waits never showed up in max/avg queue time.
+"""
+
+import os
+import sys
+import unittest
+from datetime import datetime, timedelta, timezone
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import runner_utilization_report as rur  # noqa: E402
+
+NOW = datetime(2026, 5, 27, 21, 50, 56, tzinfo=timezone.utc)
+CREATED = NOW - timedelta(hours=4)  # entered the queue 4h ago
+
+
+def _job(**kw):
+    base = {
+        "name": "base-c-test-8-gpu-h200 / base-c-test-8-gpu-h200 (3)",
+        "status": "completed",
+        "conclusion": "success",
+        "runner_name": "h200-wk03",
+        "labels": ["self-hosted", "X64", "8-gpu-h200"],
+        "created_at": CREATED.isoformat().replace("+00:00", "Z"),
+        "started_at": None,
+        "completed_at": None,
+        "html_url": "https://github.com/o/r/actions/runs/1/job/2",
+    }
+    base.update(kw)
+    return base
+
+
+def _iso(dt):
+    return dt.isoformat().replace("+00:00", "Z")
+
+
+class TestClassifyJob(unittest.TestCase):
+    def test_queued_job_counts_ongoing_wait(self):
+        """The core bug: a still-queued job reports started_at == created_at
+        (placeholder) and no completed_at. Its wait must be now - created_at,
+        not 0, and it must not be dropped."""
+        job = _job(
+            status="queued",
+            runner_name="",
+            started_at=_iso(CREATED),  # GitHub placeholder == created_at
+            completed_at=None,
+        )
+        info = rur.classify_job(job, NOW)
+        self.assertIsNotNone(info)
+        self.assertAlmostEqual(info["queue_time"], 4 * 3600, delta=1)
+        self.assertIsNone(info["start"])  # no runner occupied yet
+        self.assertEqual(info["labels"], ["8-gpu-h200"])  # generic labels dropped
+
+    def test_in_progress_job_counts_final_wait(self):
+        """A running job's wait is final (started - created); old code dropped
+        it for lacking completed_at."""
+        started = CREATED + timedelta(hours=3)
+        job = _job(status="in_progress", started_at=_iso(started), completed_at=None)
+        info = rur.classify_job(job, NOW)
+        self.assertIsNotNone(info)
+        self.assertAlmostEqual(info["queue_time"], 3 * 3600, delta=1)
+        self.assertEqual(info["start"], started)
+        self.assertEqual(info["end"], NOW)  # still occupying the runner
+
+    def test_completed_job_unchanged(self):
+        started = CREATED + timedelta(minutes=30)
+        completed = CREATED + timedelta(minutes=90)
+        job = _job(started_at=_iso(started), completed_at=_iso(completed))
+        info = rur.classify_job(job, NOW)
+        self.assertAlmostEqual(info["queue_time"], 30 * 60, delta=1)
+        self.assertAlmostEqual(info["duration"], 60 * 60, delta=1)
+        self.assertEqual(info["end"], completed)
+
+    def test_skipped_job_dropped(self):
+        """Skipped / cancelled-before-start jobs never waited for a runner."""
+        job = _job(status="completed", runner_name="", started_at=None)
+        self.assertIsNone(rur.classify_job(job, NOW))
+
+    def test_queued_without_created_dropped(self):
+        job = _job(status="queued", runner_name="", created_at=None, started_at=None)
+        self.assertIsNone(rur.classify_job(job, NOW))
+
+
+class TestConcurrencyHandlesQueuedJobs(unittest.TestCase):
+    def test_queued_job_does_not_crash_and_counts_in_peak_queue(self):
+        window_start = NOW - timedelta(hours=24)
+        queued = rur.classify_job(
+            _job(status="queued", runner_name="", started_at=_iso(CREATED)), NOW
+        )
+        ran = rur.classify_job(
+            _job(
+                started_at=_iso(CREATED + timedelta(hours=1)),
+                completed_at=_iso(CREATED + timedelta(hours=2)),
+            ),
+            NOW,
+        )
+        conc = rur.calculate_concurrency_metrics(
+            [queued, ran], window_start, NOW, num_runners=2
+        )
+        # Both jobs were waiting at CREATED before either started -> peak 2.
+        self.assertEqual(conc["peak_queue"], 2)
+
+
+class TestStatusAndFormatting(unittest.TestCase):
+    def test_status_mapping_and_url(self):
+        for conclusion, expected in (
+            ("success", "pass"),
+            ("failure", "fail"),
+            ("timed_out", "fail"),
+            ("cancelled", "cancel"),
+        ):
+            info = rur.classify_job(
+                _job(
+                    conclusion=conclusion,
+                    started_at=_iso(CREATED + timedelta(minutes=5)),
+                    completed_at=_iso(CREATED + timedelta(minutes=10)),
+                ),
+                NOW,
+            )
+            self.assertEqual(info["status"], expected)
+            self.assertEqual(
+                info["html_url"], "https://github.com/o/r/actions/runs/1/job/2"
+            )
+
+        self.assertEqual(
+            rur.classify_job(
+                _job(status="queued", runner_name="", started_at=_iso(CREATED)), NOW
+            )["status"],
+            "queued",
+        )
+        self.assertEqual(
+            rur.classify_job(
+                _job(
+                    status="in_progress", started_at=_iso(CREATED + timedelta(hours=1))
+                ),
+                NOW,
+            )["status"],
+            "running",
+        )
+
+    def test_format_status_counts(self):
+        self.assertEqual(rur.format_status_counts({}), "—")
+        cell = rur.format_status_counts({"pass": 5, "queued": 2, "fail": 0})
+        self.assertIn("✅5", cell)
+        self.assertIn("⏳2", cell)
+        self.assertNotIn("❌", cell)  # zero counts omitted
+
+    def test_format_report_has_links_and_status(self):
+        results = [
+            {
+                "label": "8-gpu-h200",
+                "num_runners": 4,
+                "effective_runners": 4,
+                "num_jobs": 2,
+                "total_active_hours": 1.0,
+                "utilization_pct": 50.0,
+                "avg_queue_min": 100.0,
+                "max_queue_min": 264.0,
+                "peak_concurrent": 1,
+                "avg_concurrent": 0.5,
+                "saturation_hours": 0.0,
+                "saturation_pct": 0.0,
+                "peak_queue": 2,
+                "status_counts": {"pass": 1, "queued": 1},
+            }
+        ]
+        url = "https://github.com/sgl-project/sglang/actions/runs/1/job/2"
+        waits = [
+            {
+                "queue_time": 264 * 60,
+                "status": "queued",
+                "labels": ["8-gpu-h200"],
+                "job_name": "base-c-test-8-gpu-h200 (3)",
+                "html_url": url,
+            }
+        ]
+        report = rur.format_report(results, 24, 0.0, longest_waits=waits)
+        self.assertIn("| Status |", report)  # new main-table column
+        self.assertIn("Longest Queue Waits", report)
+        self.assertIn(f"]({url})", report)  # clickable job link
+        self.assertIn("264m", report)
+        self.assertIn("⏳", report)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/ci/utils/update_rerun_test_status.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""
+Update the per-batch status icon in a /rerun-test reply comment.
+
+State machine for one batch line (anchored by a unique HTML-comment marker
+written by the slash-command handler):
+
+  dispatched   ⏳ ... <!--rrt:i-->          (handler, on dispatch)
+  running      🔄 ... <!--rrt:i-->          (start-beacon, on the test runner)
+  done         ✅/❌ ... <!--rrt:i:done-->  (finalizer, after the test job)
+
+The leading 🚀 on the line is the visual anchor that this came from a
+slash-command trigger and is preserved across all states.
+
+Idempotency:
+- :done marker present  -> no-op (covers reruns and start-after-finalizer race)
+- running and line already has 🔄  -> no-op
+- marker not found after retries  -> warn and exit 0 so a single placeholder
+  glitch does not amplify into N noisy job failures.
+
+Concurrent updates against the same comment from different batches can race
+on the body (read-modify-write of the same field). The finalizer
+serializes itself via job-level concurrency; the beacon does not, because
+splitting it into its own job would re-queue the GPU runner. Worst case
+for the beacon race is one missed 🔄 flicker - comment stays consistent.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+RETRY_DELAYS_SEC = [0, 5, 15]
+
+STATUS_ICONS = {
+    "running": "🔄",
+    "success": "✅",
+    "failure": "❌",
+}
+
+TERMINAL_STATUSES = {"success", "failure"}
+
+
+def gh_request(method, url, token, body=None):
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, method=method, headers=headers)
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            return resp.status, resp.read().decode()
+    except urllib.error.HTTPError as e:
+        return e.code, e.read().decode()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--comment-id", required=True, type=int)
+    ap.add_argument(
+        "--marker", required=True, help="Per-batch marker, e.g. <!--rrt:0-->"
+    )
+    ap.add_argument(
+        "--status",
+        required=True,
+        choices=list(STATUS_ICONS.keys()),
+    )
+    ap.add_argument("--repo", required=True, help="owner/repo")
+    args = ap.parse_args()
+
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        print("ERROR: GITHUB_TOKEN not set")
+        return 1
+
+    icon = STATUS_ICONS[args.status]
+    is_terminal = args.status in TERMINAL_STATUSES
+    done_marker = args.marker.replace("-->", ":done-->")
+
+    url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}"
+
+    body = None
+    for attempt, delay in enumerate(RETRY_DELAYS_SEC):
+        if delay:
+            time.sleep(delay)
+        status, text = gh_request("GET", url, token)
+        if status != 200:
+            print(f"GET failed: {status} {text}")
+            return 1
+        body = json.loads(text).get("body") or ""
+        if done_marker in body:
+            print(f"Marker {done_marker} already present; nothing to do.")
+            return 0
+        if args.marker in body:
+            break
+        print(
+            f"Marker {args.marker} not found "
+            f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry."
+        )
+    else:
+        print(
+            f"WARNING: marker {args.marker} not found after "
+            f"{len(RETRY_DELAYS_SEC)} attempts; skipping. "
+            f"The handler may have failed to edit the placeholder comment."
+        )
+        return 0
+
+    new_lines = []
+    for line in body.splitlines(keepends=True):
+        if args.marker in line:
+            if not is_terminal and icon in line:
+                print(f"Line already has {icon}; nothing to do.")
+                return 0
+            for prior in ("⏳", "🔄"):
+                if prior in line:
+                    line = line.replace(prior, icon, 1)
+                    break
+            if is_terminal:
+                line = line.replace(args.marker, done_marker)
+        new_lines.append(line)
+
+    new_body = "".join(new_lines)
+    status, text = gh_request("PATCH", url, token, body={"body": new_body})
+    if status != 200:
+        print(f"PATCH failed: {status} {text}")
+        return 1
+    print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/xpu/xpu_ci_start_container.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+set -euo pipefail
+
+# Start the Intel XPU CI container (ci_sglang_xpu) using the intel/sglang-dev:latest
+# image published by .github/workflows/release-docker-intel-xpu-nightly.yml.
+#
+# Pulls the :latest tag and starts a long-running container that subsequent
+# steps `docker exec` into.
+
+CONTAINER_NAME="ci_sglang_xpu"
+IMAGE_REPO="intel/sglang-dev"
+IMAGE_TAG="latest"
+CUSTOM_IMAGE=""
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --custom-image) CUSTOM_IMAGE="$2"; shift 2;;
+    --container-name) CONTAINER_NAME="$2"; shift 2;;
+    --image-tag) IMAGE_TAG="$2"; shift 2;;
+    -h|--help)
+      echo "Usage: $0 [OPTIONS]"
+      echo "Options:"
+      echo "  --custom-image IMAGE     Use a specific Docker image directly"
+      echo "  --container-name NAME    Override container name (default: ${CONTAINER_NAME})"
+      echo "  --image-tag TAG          Tag of ${IMAGE_REPO} to pull (default: ${IMAGE_TAG})"
+      exit 0
+      ;;
+    *) echo "Unknown option $1"; exit 1;;
+  esac
+done
+
+# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
+retry_with_backoff() {
+  local max_attempts=$1; shift
+  local attempt=1
+  local wait_secs=30
+  local jitter=$(( RANDOM % 30 ))
+  while true; do
+    if "$@"; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "Error: '$*' failed after ${max_attempts} attempts" >&2
+      return 1
+    fi
+    local sleep_time=$(( wait_secs + jitter ))
+    echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s..." >&2
+    sleep "${sleep_time}"
+    (( attempt++ ))
+    (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
+    jitter=$(( RANDOM % 30 ))
+  done
+}
+
+# Authenticate to Docker Hub when credentials are present (avoids anonymous pull
+# rate limits). Both vars are optional; falls back to unauthenticated pulls.
+if [[ -n "${DOCKERHUB_INTEL_USERNAME:-}" && -n "${DOCKERHUB_INTEL_TOKEN:-}" ]]; then
+  echo "Logging in to Docker Hub..."
+  if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_INTEL_TOKEN}" | docker login -u "${DOCKERHUB_INTEL_USERNAME}" --password-stdin >/dev/null 2>&1'; then
+    echo "Docker Hub login successful"
+  else
+    echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
+  fi
+fi
+
+if [[ -n "${CUSTOM_IMAGE}" ]]; then
+  IMAGE="${CUSTOM_IMAGE}"
+  echo "Using custom image: ${IMAGE}"
+else
+  IMAGE="${IMAGE_REPO}:${IMAGE_TAG}"
+  echo "Using image: ${IMAGE}"
+fi
+# Always pull so each stage runs the registry's current image; the cleanup
+# step removes the image after the stage so the runner doesn't accumulate
+# stale layers across runs.
+retry_with_backoff 6 docker pull "${IMAGE}"
+
+# Export the resolved image so the cleanup step can rmi the exact tag used.
+if [[ -n "${GITHUB_ENV:-}" ]]; then
+  echo "CI_SGLANG_XPU_IMAGE=${IMAGE}" >> "${GITHUB_ENV}"
+fi
+
+# Remove any stale container of the same name so re-runs are idempotent.
+if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then
+  echo "Removing existing container: ${CONTAINER_NAME}"
+  docker rm -f "${CONTAINER_NAME}" >/dev/null
+fi
+
+VIDEO_GID=$(getent group video | cut -d: -f3)
+RENDER_GID=$(getent group render | cut -d: -f3)
+
+HF_TOKEN_FILE="${HOME}/huggingface_token.txt"
+HF_TOKEN_VALUE=""
+if [[ -n "${HF_TOKEN:-}" ]]; then
+  HF_TOKEN_VALUE="${HF_TOKEN}"
+elif [[ -r "${HF_TOKEN_FILE}" ]]; then
+  HF_TOKEN_VALUE=$(cat "${HF_TOKEN_FILE}")
+fi
+
+echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}"
+docker run -dt \
+  --shm-size 8g \
+  --group-add 992 \
+  ${VIDEO_GID:+--group-add "${VIDEO_GID}"} \
+  ${RENDER_GID:+--group-add "${RENDER_GID}"} \
+  --device /dev/dri \
+  -v /dev/dri/by-path:/dev/dri/by-path \
+  -v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
+  -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
+  -e HF_TOKEN="${HF_TOKEN_VALUE}" \
+  --name "${CONTAINER_NAME}" \
+  "${IMAGE}"
+
+# Mark the workspace mount as a safe directory so git operations as root
+# inside the container don't trip the cross-user repo guard.
+docker exec "${CONTAINER_NAME}" git config --global --add safe.directory /sglang-checkout || true

scripts/ci_monitor/README.md

@@ -0,0 +1,42 @@
+# SGLang CI failure monitoring
+
+Scripts used by [.github/workflows/ci-failure-monitor.yml](../../.github/workflows/ci-failure-monitor.yml): scheduled failure analysis.
+
+## Tools
+
+1. **Failures Analyzer** (`ci_failures_analysis.py`): Tracks consecutive failures, identifies flaky jobs, and monitors runner health across PR Test / Nightly workflows (Nvidia, AMD, Intel, XPU, NPU).
+
+## Installation
+
+```bash
+pip install requests
+```
+
+## Usage
+
+### Failures Analyzer
+
+```bash
+export GITHUB_TOKEN="your_token_here"
+
+python ci_failures_analysis.py --token $GITHUB_TOKEN --limit 50 --threshold 2
+python ci_failures_analysis.py --token $GITHUB_TOKEN --limit 300 --threshold 2
+python ci_failures_analysis.py --token $GITHUB_TOKEN --limit 500 --threshold 3
+```
+
+## Token permissions
+
+The GitHub token needs `repo` and `workflow` scopes to read CI run data; otherwise API calls may return 404.
+
+### Failures Analyzer parameters
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `--token` | Required | GitHub Personal Access Token |
+| `--limit` | 500 | Number of workflow runs to analyze |
+| `--threshold` | 3 | Alert threshold for consecutive failures |
+| `--output` | None | Output JSON file (optional) |
+
+## Historical note
+
+The former **CI Monitor** workflow (`ci-monitor.yml`) and its analyzers (`ci_analyzer.py`, `ci_analyzer_perf.py`, `ci_analyzer_balance.py`) were removed as redundant; use this failure monitor workflow and scripts for ongoing CI health alerts.

scripts/code_sync/check_commits.py

@@ -0,0 +1,483 @@
+"""
+List commits in the private repo that need to be synced to the OSS repo.
+
+NOTE:
+1. This script resolves the git root automatically and can be run anywhere
+   inside the repo.
+
+This script will:
+1. Find the most recent sync commit (message starts with
+   "[Automated PR] Copy OSS code from commit").
+2. Scan commits after that point and keep those that touch the configured paths.
+3. Compare added diff lines in relevant files against OSS main.
+4. Print a markdown summary with commit links and write it to GitHub Step Summary.
+
+Usage:
+python3 scripts/code_sync/check_commits.py
+"""
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set, Tuple
+
+# Allow sibling imports regardless of the working directory.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from utils import (  # noqa: E402
+    FOLDER_NAMES,
+    get_last_sync_commit,
+    write_github_step_summary,
+)
+
+# --- Configuration Begin ---
+private_repo = "your-org/sglang-private-repo"
+oss_repo_url = "https://github.com/sgl-project/sglang.git"
+oss_repo_branch = "main"
+default_oss_repo_dir = ".oss_repo"
+# --- Configuration End ---
+
+
+@dataclass
+class CommitInfo:
+    commit_hash: str
+    subject: str
+    commit_date: str
+    relevant_files: List[str]
+    synced_lines: int
+    total_added_lines: int
+
+
+def check_dependencies() -> None:
+    """Check for required command-line tools."""
+    if not shutil.which("git"):
+        raise EnvironmentError("git is not installed or not in PATH.")
+
+
+def get_repo_root() -> str:
+    try:
+        output = subprocess.run(
+            ["git", "rev-parse", "--show-toplevel"],
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout.strip()
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Unable to determine git repo root: {e.stderr or e}") from e
+
+    if not output:
+        raise RuntimeError("Unable to determine git repo root.")
+    return os.path.abspath(output)
+
+
+def get_repo_from_origin(repo_root: str) -> str:
+    """Try to infer the repo slug (owner/name) from git remote.origin.url."""
+    try:
+        url = subprocess.run(
+            ["git", "config", "--get", "remote.origin.url"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=repo_root,
+        ).stdout.strip()
+    except subprocess.CalledProcessError:
+        return private_repo
+
+    if url.startswith("git@github.com:"):
+        repo = url.split("git@github.com:", 1)[1]
+    elif url.startswith("https://github.com/"):
+        repo = url.split("https://github.com/", 1)[1]
+    else:
+        return private_repo
+
+    if repo.endswith(".git"):
+        repo = repo[: -len(".git")]
+    return repo or private_repo
+
+
+def get_default_oss_repo_path(repo_root: str) -> str:
+    env_path = os.environ.get("OSS_REPO_PATH")
+    if env_path:
+        return os.path.abspath(env_path)
+    return os.path.abspath(os.path.join(repo_root, default_oss_repo_dir))
+
+
+def ensure_oss_repo(oss_repo_path: str, repo_url: str, branch: str) -> str:
+    oss_repo_path = os.path.abspath(oss_repo_path)
+    if os.path.exists(oss_repo_path) and not os.path.isdir(oss_repo_path):
+        raise RuntimeError(f"OSS repo path is not a directory: {oss_repo_path}")
+
+    if os.path.isdir(os.path.join(oss_repo_path, ".git")):
+        try:
+            subprocess.run(
+                ["git", "-C", oss_repo_path, "rev-parse", "--is-inside-work-tree"],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(
+                f"OSS repo path exists but is not a git repo: {oss_repo_path}"
+            ) from e
+
+        subprocess.run(
+            ["git", "-C", oss_repo_path, "fetch", "origin", branch, "--depth", "1"],
+            check=True,
+        )
+        return oss_repo_path
+
+    parent_dir = os.path.dirname(oss_repo_path)
+    if parent_dir and not os.path.isdir(parent_dir):
+        os.makedirs(parent_dir, exist_ok=True)
+    subprocess.run(
+        ["git", "clone", "--depth", "1", "--branch", branch, repo_url, oss_repo_path],
+        check=True,
+    )
+    return oss_repo_path
+
+
+def get_commits_since(repo_root: str, last_sync_hash: Optional[str]) -> List[str]:
+    """Get commit hashes from last sync commit (exclusive) to HEAD."""
+    try:
+        if last_sync_hash:
+            command = ["git", "rev-list", f"{last_sync_hash}..HEAD"]
+        else:
+            command = ["git", "rev-list", "HEAD"]
+        result = subprocess.run(
+            command, capture_output=True, text=True, check=True, cwd=repo_root
+        ).stdout.strip()
+        return [line for line in result.split("\n") if line]
+    except subprocess.CalledProcessError as e:
+        print(f"Error getting commit list: {e.stderr}")
+        return []
+
+
+def get_changed_files(repo_root: str, commit_hash: str) -> List[str]:
+    try:
+        output = subprocess.run(
+            ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=repo_root,
+        ).stdout.strip()
+        return [line for line in output.split("\n") if line]
+    except subprocess.CalledProcessError as e:
+        print(f"Error getting changed files for {commit_hash}: {e.stderr}")
+        return []
+
+
+def is_relevant_path(changed_file: str, path_prefix: str) -> bool:
+    if changed_file == path_prefix:
+        return True
+    return changed_file.startswith(f"{path_prefix}/")
+
+
+def get_relevant_files(changed_files: List[str]) -> List[str]:
+    return [
+        changed_file
+        for changed_file in changed_files
+        if any(is_relevant_path(changed_file, path) for path in FOLDER_NAMES)
+    ]
+
+
+def get_added_lines_by_file(
+    repo_root: str, commit_hash: str, relevant_files: List[str]
+) -> Dict[str, List[str]]:
+    if not relevant_files:
+        return {}
+
+    command = [
+        "git",
+        "show",
+        "--no-color",
+        "--unified=0",
+        "--format=",
+        commit_hash,
+        "--",
+    ] + relevant_files
+    try:
+        output = subprocess.run(
+            command, capture_output=True, text=True, check=True, cwd=repo_root
+        ).stdout
+    except subprocess.CalledProcessError as e:
+        print(f"Error getting diff for {commit_hash}: {e.stderr}")
+        return {}
+
+    added_lines: Dict[str, List[str]] = {path: [] for path in relevant_files}
+    relevant_set = set(relevant_files)
+    current_file: Optional[str] = None
+    for line in output.splitlines():
+        if line.startswith("diff --git "):
+            current_file = None
+            continue
+        if line.startswith("+++ "):
+            file_path = None
+            if line.startswith("+++ b/"):
+                file_path = line[6:]
+            else:
+                candidate = line[4:]
+                if candidate == "/dev/null":
+                    file_path = None
+                elif candidate.startswith("b/") or candidate.startswith("a/"):
+                    file_path = candidate[2:]
+                else:
+                    file_path = candidate
+
+            if file_path in relevant_set:
+                current_file = file_path
+            else:
+                current_file = None
+            continue
+
+        if current_file and line.startswith("+") and not line.startswith("+++ "):
+            added_lines[current_file].append(line[1:])
+
+    return added_lines
+
+
+def get_oss_file_lines(
+    oss_repo_path: str,
+    oss_ref: str,
+    file_path: str,
+    cache: Dict[str, Optional[Set[str]]],
+) -> Optional[Set[str]]:
+    if file_path in cache:
+        return cache[file_path]
+    try:
+        output = subprocess.run(
+            ["git", "-C", oss_repo_path, "show", f"{oss_ref}:{file_path}"],
+            capture_output=True,
+            text=True,
+            errors="replace",
+            check=True,
+        ).stdout
+    except subprocess.CalledProcessError:
+        cache[file_path] = None
+        return None
+
+    lines = output.splitlines()
+    line_set = set(lines)
+    cache[file_path] = line_set
+    return line_set
+
+
+def count_synced_lines(
+    added_lines_by_file: Dict[str, List[str]],
+    oss_repo_path: str,
+    oss_ref: str,
+    oss_file_cache: Dict[str, Optional[Set[str]]],
+) -> Tuple[int, int]:
+    total_added_lines = 0
+    synced_lines = 0
+    for file_path, lines in added_lines_by_file.items():
+        total_added_lines += len(lines)
+        if not lines:
+            continue
+        oss_lines = get_oss_file_lines(
+            oss_repo_path, oss_ref, file_path, oss_file_cache
+        )
+        if not oss_lines:
+            continue
+        for line in lines:
+            if line in oss_lines:
+                synced_lines += 1
+    return synced_lines, total_added_lines
+
+
+def get_commit_summary(repo_root: str, commit_hash: str) -> Tuple[str, str]:
+    """Return (subject, date) for a commit."""
+    try:
+        output = subprocess.run(
+            ["git", "show", "-s", "--format=%s%x00%ad", "--date=short", commit_hash],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=repo_root,
+        ).stdout.strip()
+        subject, commit_date = output.split("\x00", 1)
+    except subprocess.CalledProcessError as e:
+        print(f"Error getting commit subject for {commit_hash}: {e.stderr}")
+        subject = "(unknown subject)"
+        commit_date = "(unknown date)"
+    return subject, commit_date
+
+
+def format_files_list(relevant_files: List[str]) -> str:
+    return "\n".join([f"- {file_path}" for file_path in relevant_files])
+
+
+def format_last_sync_block(
+    repo: str, subject: str, commit_hash: str, commit_date: str
+) -> str:
+    short_hash = commit_hash[:9]
+    commit_url = f"https://github.com/{repo}/commit/{commit_hash}"
+    return "\n".join(
+        [
+            "## Last sync",
+            "",
+            f"#### {subject}",
+            f"date: {commit_date}",
+            f"commit: [{short_hash}]({commit_url})",
+            "",
+        ]
+    )
+
+
+def format_commit_block(
+    repo: str,
+    subject: str,
+    commit_hash: str,
+    commit_date: str,
+    relevant_files: List[str],
+    synced_lines: int,
+    total_added_lines: int,
+) -> str:
+    short_hash = commit_hash[:9]
+    commit_url = f"https://github.com/{repo}/commit/{commit_hash}"
+    files_str = format_files_list(relevant_files) if relevant_files else "- None"
+    status_icon = "✅" if synced_lines == total_added_lines else "❌"
+    status_line = (
+        f"status: {status_icon} {synced_lines}/{total_added_lines} lines synced"
+    )
+    return "\n".join(
+        [
+            f"#### {subject}",
+            status_line,
+            f"date: {commit_date}",
+            "files to sync:",
+            files_str,
+            "",
+            f"commit: [{short_hash}]({commit_url})",
+            "",
+        ]
+    )
+
+
+def format_output(
+    repo: str,
+    last_sync: Optional[Tuple[str, str, str]],
+    commits: List[CommitInfo],
+) -> str:
+    lines: List[str] = []
+    if last_sync:
+        subject, commit_hash, commit_date = last_sync
+        lines.append(format_last_sync_block(repo, subject, commit_hash, commit_date))
+    else:
+        lines.extend(["## Last sync", "", "No sync commit found.", ""])
+
+    lines.extend(["## Commits to sync", ""])
+    if not commits:
+        lines.append("No commits need to be synced.")
+        return "\n".join(lines) + "\n"
+
+    for commit in commits:
+        lines.append(
+            format_commit_block(
+                repo,
+                commit.subject,
+                commit.commit_hash,
+                commit.commit_date,
+                commit.relevant_files,
+                commit.synced_lines,
+                commit.total_added_lines,
+            )
+        )
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="List commits in the private repo that need to be synced to OSS."
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=0,
+        help="Limit number of commits printed (0 means no limit).",
+    )
+    parser.add_argument(
+        "--oss-repo-path",
+        default=None,
+        help="Path to OSS repo clone (default: $OSS_REPO_PATH or .oss_repo).",
+    )
+    parser.add_argument(
+        "--oss-repo-url",
+        default=oss_repo_url,
+        help="OSS repo URL (default: https://github.com/sgl-project/sglang.git).",
+    )
+    parser.add_argument(
+        "--oss-branch",
+        default=oss_repo_branch,
+        help="OSS repo branch to check (default: main).",
+    )
+    args = parser.parse_args()
+
+    check_dependencies()
+    repo_root = get_repo_root()
+    oss_repo_path = (
+        os.path.abspath(args.oss_repo_path)
+        if args.oss_repo_path
+        else get_default_oss_repo_path(repo_root)
+    )
+
+    repo = get_repo_from_origin(repo_root)
+    last_sync_hash = get_last_sync_commit(repo_root)
+    last_sync_block = None
+    if last_sync_hash:
+        last_sync_subject, last_sync_date = get_commit_summary(
+            repo_root, last_sync_hash
+        )
+        last_sync_block = (last_sync_subject, last_sync_hash, last_sync_date)
+
+    commits = get_commits_since(repo_root, last_sync_hash)
+    if args.limit > 0:
+        commits = commits[: args.limit]
+
+    relevant_commit_inputs: List[Tuple[str, List[str]]] = []
+    for commit_hash in commits:
+        changed_files = get_changed_files(repo_root, commit_hash)
+        if not changed_files:
+            continue
+        relevant_files = get_relevant_files(changed_files)
+        if relevant_files:
+            relevant_commit_inputs.append((commit_hash, relevant_files))
+
+    relevant_commits: List[CommitInfo] = []
+    if relevant_commit_inputs:
+        oss_repo_path = ensure_oss_repo(
+            oss_repo_path, args.oss_repo_url, args.oss_branch
+        )
+        oss_ref = f"origin/{args.oss_branch}"
+        oss_file_cache: Dict[str, Optional[Set[str]]] = {}
+        for commit_hash, relevant_files in relevant_commit_inputs:
+            subject, commit_date = get_commit_summary(repo_root, commit_hash)
+            added_lines_by_file = get_added_lines_by_file(
+                repo_root, commit_hash, relevant_files
+            )
+            synced_lines, total_added_lines = count_synced_lines(
+                added_lines_by_file, oss_repo_path, oss_ref, oss_file_cache
+            )
+            relevant_commits.append(
+                CommitInfo(
+                    commit_hash=commit_hash,
+                    subject=subject,
+                    commit_date=commit_date,
+                    relevant_files=relevant_files,
+                    synced_lines=synced_lines,
+                    total_added_lines=total_added_lines,
+                )
+            )
+
+    output = format_output(repo, last_sync_block, relevant_commits)
+    print(output)
+    if os.environ.get("GITHUB_STEP_SUMMARY"):
+        write_github_step_summary(output)
+
+
+if __name__ == "__main__":
+    main()

scripts/code_sync/copy_from_oss.py

@@ -0,0 +1,273 @@
+"""
+Sync code from OSS repo to the local repo and open a PR if changes exist.
+
+NOTE:
+1. You need to execute this script in the git root folder.
+2. A GH_TOKEN environment variable is required to create the pull request.
+  - see also https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
+
+This script will:
+1. Clone the sgl-project/sglang repository (or use a local copy).
+2. Sync specified files and directories using rsync.
+3. Check if the sync operation resulted in any changes.
+4. If there are changes:
+   a. Create a new branch.
+   b. Commit and push the changes.
+   c. Open a pull request using the GitHub CLI (gh).
+
+Usage:
+# Run the full sync and PR creation process
+python3 scripts/copy_from_oss.py
+
+# Perform a dry run without making any actual changes
+python3 scripts/copy_from_oss.py --dry-run
+
+# Use a local directory as the source instead of cloning
+python3 scripts/copy_from_oss.py --local-dir ~/projects/sglang
+"""
+
+import argparse
+import datetime
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+# Allow sibling imports regardless of the working directory.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from utils import FOLDER_NAMES, write_github_step_summary  # noqa: E402
+
+# --- Configuration Begin ---
+private_repo = "your-org/sglang-private-repo"
+# --- Configuration End ---
+
+
+def check_dependencies():
+    """Check for required command-line tools."""
+    if not shutil.which("git"):
+        raise EnvironmentError("git is not installed or not in PATH.")
+    if not shutil.which("gh"):
+        raise EnvironmentError("GitHub CLI (gh) is not installed or not in PATH.")
+    print("✅ All dependencies (git, gh) are available.")
+
+
+def checkout_main(dry_run):
+    """Checkout to the main branch."""
+    commands = [
+        "git checkout main",
+        "git reset --hard",
+    ]
+    for cmd in commands:
+        print(f"Run: {cmd}")
+        if not dry_run:
+            try:
+                subprocess.run(cmd, shell=True, check=True, capture_output=True)
+            except subprocess.CalledProcessError as e:
+                print(f"Git command failed: {e.stderr.decode()}")
+                raise
+    print("✅ Checkout the main branch.")
+
+
+def get_source_folder(args):
+    """
+    Prepare the source repository, either by cloning from GitHub or using a local directory.
+    Returns the path to the source repo root, a temporary directory path (if created),
+    and the short commit hash.
+    """
+    temp_dir = None
+    if args.local_dir:
+        oss_root = os.path.expanduser(args.local_dir)
+        if not os.path.exists(oss_root):
+            raise FileNotFoundError(
+                f"Specified local directory {oss_root} does not exist."
+            )
+        print(f"Using local directory as the source: {oss_root}")
+    else:
+        temp_dir = tempfile.mkdtemp()
+        oss_root = temp_dir
+        print(f"Created temporary directory: {oss_root}")
+
+        repo_url = "https://github.com/sgl-project/sglang.git"
+        try:
+            subprocess.run(
+                [
+                    "git",
+                    "clone",
+                    "--single-branch",
+                    "--branch",
+                    "main",
+                    repo_url,
+                    temp_dir,
+                ],
+                check=True,
+                capture_output=True,
+            )
+            print(f"Successfully cloned repository to {temp_dir}")
+        except subprocess.CalledProcessError as e:
+            print(f"Error cloning repository: {e.stderr.decode()}")
+            raise
+
+    commit_hash = subprocess.run(
+        ["git", "-C", oss_root, "rev-parse", "HEAD"],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()[:8]
+    print(f"✅ Get source OSS code at commit: {commit_hash}")
+    return oss_root, temp_dir, commit_hash
+
+
+def sync_directories(oss_root, sync_paths, dry_run):
+    """Sync specified directories from oss_root to current working directory."""
+    rsync_commands = []
+    for folder_name in sync_paths:
+        target_name = f"{oss_root}/{folder_name}"
+        src_name = "./" + "/".join(folder_name.split("/")[:-1])
+        cmd = f"rsync -r --delete {target_name} {src_name}"
+        rsync_commands.append(cmd)
+
+    for cmd in rsync_commands:
+        try:
+            print(f"Run: {cmd}")
+            if not dry_run:
+                subprocess.run(cmd, shell=True, check=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Error executing command '{cmd}': {e}")
+            raise
+    print(f"✅ Sync all folders.")
+
+
+def check_for_changes():
+    """Check if there are any uncommitted git changes."""
+    # This command exits with 1 if there are changes, 0 otherwise.
+    result = subprocess.run(["git", "diff", "--quiet"])
+    return result.returncode != 0
+
+
+def create_and_push_branch(branch_name, commit_message, dry_run):
+    """Create a new branch, commit all changes, and push to origin."""
+    commands = [
+        f"git checkout -b {branch_name}",
+        "git config user.name 'github-actions[bot]'",
+        "git config user.email 'github-actions[bot]@users.noreply.github.com'",
+        "git add .",
+        f"git commit -m '{commit_message}'",
+        f"git push origin {branch_name} --force",
+    ]
+    print("\nCreating and pushing git branch...")
+    for cmd in commands:
+        print(f"Run: {cmd}")
+        if not dry_run:
+            try:
+                subprocess.run(cmd, shell=True, check=True, capture_output=True)
+            except subprocess.CalledProcessError as e:
+                print(f"Git command failed: {e.stderr.decode()}")
+                raise
+
+
+def create_pull_request(branch_name, title, body, dry_run):
+    """Create a pull request using the GitHub CLI."""
+    gh_token = os.getenv("GH_TOKEN")
+    if not gh_token:
+        print(
+            "\n⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation."
+        )
+        if not dry_run:
+            return
+
+    print("\nCreating pull request...")
+    command = [
+        "gh",
+        "pr",
+        "create",
+        "--base",
+        "main",
+        "--head",
+        branch_name,
+        "--repo",
+        private_repo,
+        "--title",
+        title,
+        "--body",
+        body,
+    ]
+    print(f"Run: {' '.join(command)}")
+    if not dry_run:
+        env = os.environ.copy()
+        env["GH_TOKEN"] = gh_token
+        try:
+            result = subprocess.run(
+                command, check=True, capture_output=True, text=True, env=env
+            )
+            pr_url = result.stdout.strip()
+            msg = f"✅ Successfully created pull request: {pr_url}"
+            print(msg)
+            write_github_step_summary(msg)
+        except subprocess.CalledProcessError as e:
+            print(f"Error creating pull request: {e.stderr}")
+            raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Copy code from OSS and open a PR if changes are detected."
+    )
+    parser.add_argument(
+        "--local-dir",
+        type=str,
+        help="Path to local SGLang directory to use instead of cloning from GitHub.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Dry run the script without executing git, rsync, or gh commands.",
+    )
+    args = parser.parse_args()
+
+    check_dependencies()
+    checkout_main(args.dry_run)
+
+    oss_root, temp_dir, oss_commit = get_source_folder(args)
+
+    try:
+        # Sync directories
+        sync_directories(oss_root, FOLDER_NAMES, args.dry_run)
+
+        # Check for changes and create PR if necessary
+        if not check_for_changes():
+            msg = "😴 No changes detected. The code is already in sync."
+            print(msg)
+            write_github_step_summary(msg)
+            return
+
+        print("✅ Changes detected. Proceeding to create a PR.")
+
+        current_date = datetime.datetime.now().strftime("%Y%m%d")
+        branch_name = f"copy-from-oss-{oss_commit}-{current_date}"
+        commit_message = f"Copy OSS code from {oss_commit} on {current_date}"
+        pr_title = (
+            f"[Automated PR] Copy OSS code from commit {oss_commit} on {current_date}"
+        )
+        pr_body = (
+            f"Copy OSS code from https://github.com/sgl-project/sglang/commit/{oss_commit} on {current_date}."
+            "\n\n---\n\n"
+            "*This is an automated PR created by scripts/copy_from_oss.py.*"
+        )
+
+        create_and_push_branch(branch_name, commit_message, args.dry_run)
+        create_pull_request(branch_name, pr_title, pr_body, args.dry_run)
+
+    finally:
+        # Remove temporary directory if it was created
+        if temp_dir:
+            try:
+                shutil.rmtree(temp_dir)
+                print(f"\nRemoved temporary directory: {temp_dir}")
+            except OSError as e:
+                print(f"Error removing temporary directory {temp_dir}: {e}")
+
+
+if __name__ == "__main__":
+    main()

scripts/code_sync/copy_to_oss.py

@@ -0,0 +1,591 @@
+"""
+Sync a specific commit from the local private repo to the OSS upstream and open a PR.
+
+NOTE:
+1. You need to execute this script in the git root folder.
+2. A GH_TOKEN environment variable is required to create the pull request.
+  - see also https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
+
+This script will:
+1. Take a commit hash as an argument (or use the latest commit by default).
+2. Create a patch for that commit.
+3. Filter the patch to only include changes in specified directories.
+4. Clone the sgl-project/sglang repository.
+5. Create a new branch in the OSS repo.
+6. Apply the filtered patch, commit, and force push.
+7. Open a pull request to the OSS repo using the GitHub CLI (gh).
+
+Usage:
+# Sync the latest commit from the current branch
+python3 scripts/copy_to_oss.py
+
+# Run the full sync and PR creation process for a given commit
+python3 scripts/copy_to_oss.py --commit <commit_hash>
+
+# Perform a dry run without making any actual changes
+python3 scripts/copy_to_oss.py --commit <commit_hash> --dry-run
+"""
+
+import argparse
+import datetime
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+# Allow sibling imports regardless of the working directory.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from utils import (  # noqa: E402
+    FOLDER_NAMES,
+    find_latest_oss_sync_commit,
+    write_github_step_summary,
+)
+
+
+def get_commit_info(commit_ref):
+    """
+    Retrieves the hash and message of a specific commit.
+
+    Args:
+        commit_ref (str): The commit hash, tag, or branch to inspect (e.g., 'HEAD').
+
+    Returns:
+        A tuple containing the (commit_hash, commit_message),
+        or (None, None) if an error occurs.
+    """
+    try:
+        # Use a custom format to get the hash (%H) and the full message (%B)
+        # separated by a null character for safe parsing.
+        command = ["git", "log", "-1", f"--pretty=%H%x00%B", commit_ref]
+        result = subprocess.run(
+            command, capture_output=True, text=True, check=True, encoding="utf-8"
+        )
+
+        # Split the output by the null character separator
+        commit_hash, commit_message = result.stdout.strip().split("\x00", 1)
+        return commit_hash, commit_message
+
+    except FileNotFoundError:
+        print("❌ Error: 'git' command not found. Is Git installed and in your PATH?")
+    except subprocess.CalledProcessError as e:
+        print(f"❌ Error getting commit info for '{commit_ref}': {e.stderr.strip()}")
+        print(
+            "Hint: Make sure you are running this from within a Git repository and the commit exists."
+        )
+
+    return None, None
+
+
+def check_dependencies():
+    """Check for required command-line tools."""
+    if not shutil.which("git"):
+        raise EnvironmentError("git is not installed or not in PATH.")
+    if not shutil.which("gh"):
+        raise EnvironmentError("GitHub CLI (gh) is not installed or not in PATH.")
+    print("✅ All dependencies (git, gh) are available.")
+
+
+def create_filtered_patch(commit_hash, dry_run):
+    """
+    Create a patch file for the given commit, containing only changes
+    to files and directories specified in `folder_names`.
+    """
+    print(f"Creating a filtered patch for commit {commit_hash}")
+
+    try:
+        # Get the list of all files changed in the commit
+        changed_files_raw = subprocess.run(
+            ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash],
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout
+        changed_files = changed_files_raw.strip().split("\n")
+
+        # Filter the list of files
+        relevant_files = [
+            f for f in changed_files if any(f.startswith(path) for path in FOLDER_NAMES)
+        ]
+
+        if not relevant_files:
+            msg = "\n😴 No relevant file changes found in this commit. Exiting."
+            print(msg)
+            write_github_step_summary(msg)
+            return None, None
+
+        print("Found relevant changes in the following files:")
+        for f in relevant_files:
+            print(f"  - {f}")
+
+        # Create a patch containing only the changes for the relevant files
+        patch_command = [
+            "git",
+            "format-patch",
+            "--stdout",
+            f"{commit_hash}^..{commit_hash}",
+            "--",
+        ] + relevant_files
+
+        print(f"Run: {' '.join(patch_command)}")
+
+        patch_content = subprocess.run(
+            patch_command, capture_output=True, text=True, check=True
+        ).stdout
+
+        # Save the patch to a temporary file
+        patch_file = tempfile.NamedTemporaryFile(
+            mode="w", delete=False, suffix=".patch", encoding="utf-8"
+        )
+        patch_file.write(patch_content)
+        patch_file.close()
+
+        print(f"✅ Filtered patch created successfully at: {patch_file.name}")
+        return patch_file.name, relevant_files
+
+    except subprocess.CalledProcessError as e:
+        print(f"Error creating patch: {e.stderr}")
+        raise
+
+
+def get_oss_repo(dry_run):
+    """
+    Clones the OSS repository into a temporary directory.
+    Returns the path to the repo root and the temp directory itself.
+    """
+    gh_token = os.getenv("GH_TOKEN")
+    if not gh_token:
+        print(
+            "⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation."
+        )
+        if not dry_run:
+            return
+
+    temp_dir = tempfile.mkdtemp()
+    oss_root = os.path.join(temp_dir, "sglang")
+    print(f"\nCreated temporary directory for OSS repo: {temp_dir}")
+
+    repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git"
+    command = ["git", "clone", repo_url, oss_root]
+
+    print(f"Run: {' '.join(command)}")
+    if not dry_run:
+        try:
+            subprocess.run(command, check=True, capture_output=True)
+            print(f"✅ Successfully cloned repository to {oss_root}")
+        except subprocess.CalledProcessError as e:
+            print(f"Error cloning repository: {e.stderr.decode()}")
+            shutil.rmtree(temp_dir)
+            raise
+
+    return oss_root, temp_dir
+
+
+def _apply_patch(patch_file, dry_run):
+    """
+    Try to apply a patch, falling back to --3way merge if a clean apply fails.
+
+    Returns True if the patch was applied cleanly.
+    Returns False if conflicts were encountered (changes are still staged
+    with conflict markers so a PR can be created for manual resolution).
+    """
+    # --- Attempt 1: clean git apply ---
+    apply_cmd = ["git", "apply", patch_file]
+    print(f"Run: {' '.join(apply_cmd)}")
+    if dry_run:
+        return True
+
+    result = subprocess.run(apply_cmd, capture_output=True, text=True)
+    if result.returncode == 0:
+        print("✅ Patch applied cleanly.")
+        return True
+
+    print(f"⚠️  Clean apply failed:\n{result.stderr.strip()}")
+    print("Falling back to git apply --3way ...\n")
+
+    # --- Attempt 2: three-way merge ---
+    threeway_cmd = ["git", "apply", "--3way", patch_file]
+    print(f"Run: {' '.join(threeway_cmd)}")
+    result_3way = subprocess.run(threeway_cmd, capture_output=True, text=True)
+
+    if result_3way.returncode == 0:
+        print("✅ Patch applied via --3way merge (no conflicts).")
+        return True
+
+    # --- --3way left conflict markers in the working tree ---
+    print(f"⚠️  --3way merge had conflicts:\n{result_3way.stderr.strip()}\n")
+
+    # Show which hunks conflict
+    check_cmd = ["git", "apply", "--check", "--verbose", patch_file]
+    print(f"Run: {' '.join(check_cmd)}")
+    check_result = subprocess.run(check_cmd, capture_output=True, text=True)
+    conflict_details = (check_result.stdout + check_result.stderr).strip()
+    print(
+        f"\n--- Conflict details ---\n{conflict_details}\n--- End conflict details ---\n"
+    )
+
+    # Show git diff if --3way left conflict markers
+    diff_result = subprocess.run(["git", "diff"], capture_output=True, text=True)
+    if diff_result.stdout.strip():
+        print(
+            f"\n--- git diff (conflict markers) ---\n"
+            f"{diff_result.stdout.strip()}\n"
+            f"--- End git diff ---\n"
+        )
+
+    # Read the patch content for the summary
+    with open(patch_file, "r", encoding="utf-8") as pf:
+        patch_content = pf.read()
+
+    # Print the patch to stdout so it's visible in the CI logs
+    separator = "=" * 72
+    print(
+        f"\n{separator}\n"
+        f"PATCH CONTENT (apply this manually):\n"
+        f"{separator}\n"
+        f"{patch_content}\n"
+        f"{separator}\n"
+    )
+
+    # Write a rich summary to the GitHub Actions step summary
+    summary_lines = [
+        "\n## ⚠️ Patch had conflicts — PR created for manual resolution\n",
+        "### Conflict details\n",
+        f"```\n{conflict_details}\n```\n",
+    ]
+    if diff_result.stdout.strip():
+        summary_lines.append("### git diff (conflict markers)\n")
+        summary_lines.append(f"```diff\n{diff_result.stdout.strip()}\n```\n")
+    summary_lines.append("### Patch to apply manually\n")
+    summary_lines.append(
+        "<details><summary>Click to expand full patch</summary>\n\n"
+        f"```diff\n{patch_content}\n```\n"
+        "</details>\n"
+    )
+    write_github_step_summary("".join(summary_lines))
+
+    return False
+
+
+def apply_patch_and_push(
+    oss_root, patch_file, branch_name, commit_message, base_oss_commit, dry_run
+):
+    """
+    In the OSS repo, create a branch from base_oss_commit, apply the patch,
+    commit, and push.
+
+    Args:
+        base_oss_commit: The OSS commit hash to branch from (the last sync
+            point). If None, the current HEAD (main) is used.
+
+    Returns True if the patch applied cleanly, False if there were conflicts
+    (the conflicted state is still committed and pushed so a PR can be opened).
+    """
+    print("\nApplying patch and pushing to OSS repo...")
+
+    original_cwd = os.getcwd()
+    if not dry_run:
+        os.chdir(oss_root)
+
+    applied_cleanly = True
+    try:
+        # Check out a new branch from the base OSS commit
+        if base_oss_commit:
+            checkout_cmd = ["git", "checkout", "-b", branch_name, base_oss_commit]
+        else:
+            checkout_cmd = ["git", "checkout", "-b", branch_name]
+        print(f"Run: {' '.join(checkout_cmd)}")
+        if not dry_run:
+            subprocess.run(checkout_cmd, check=True, capture_output=True, text=True)
+
+        # Apply the patch (with --3way fallback and diagnostics)
+        applied_cleanly = _apply_patch(patch_file, dry_run)
+
+        # Configure git user and stage changes
+        post_apply_commands = [
+            ["git", "config", "user.name", "github-actions[bot]"],
+            [
+                "git",
+                "config",
+                "user.email",
+                "github-actions[bot]@users.noreply.github.com",
+            ],
+            ["git", "add", "."],
+        ]
+
+        for cmd_list in post_apply_commands:
+            print(f"Run: {' '.join(cmd_list)}")
+            if not dry_run:
+                subprocess.run(cmd_list, check=True, capture_output=True, text=True)
+
+        # Handle commit separately to pass multi-line message safely via stdin
+        commit_cmd = ["git", "commit", "-F", "-"]
+        print(f"Run: {' '.join(commit_cmd)}")
+        if not dry_run:
+            print(f"Commit Message:\n---\n{commit_message}\n---")
+            subprocess.run(
+                commit_cmd,
+                input=commit_message,
+                text=True,
+                check=True,
+                capture_output=True,
+            )
+
+        # Push the changes
+        push_cmd = ["git", "push", "origin", branch_name, "--force"]
+        print(f"Run: {' '.join(push_cmd)}")
+        if not dry_run:
+            subprocess.run(push_cmd, check=True, capture_output=True, text=True)
+
+    except subprocess.CalledProcessError as e:
+        print(f"Git command failed: {e.stderr}")
+        raise
+    finally:
+        if not dry_run:
+            os.chdir(original_cwd)
+
+    if applied_cleanly:
+        print("✅ Branch created, patch applied cleanly, and pushed successfully.")
+    else:
+        print(
+            "⚠️  Branch created and pushed with conflict markers. "
+            "A PR will be opened for manual resolution."
+        )
+
+    return applied_cleanly
+
+
+def create_pull_request(oss_root, branch_name, title, body, dry_run):
+    """Create a pull request in the OSS repo using the GitHub CLI."""
+    gh_token = os.getenv("GH_TOKEN")
+    if not gh_token:
+        print(
+            "⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation."
+        )
+        if not dry_run:
+            return
+
+    print("\nCreating pull request...")
+    command = [
+        "gh",
+        "pr",
+        "create",
+        "--base",
+        "main",
+        "--head",
+        branch_name,
+        "--repo",
+        "sgl-project/sglang",
+        "--title",
+        title,
+        "--body",
+        body,
+    ]
+
+    print(f"Run: {' '.join(command)}")
+    if not dry_run:
+        env = os.environ.copy()
+        env["GH_TOKEN"] = gh_token
+        try:
+            result = subprocess.run(
+                command,
+                check=True,
+                capture_output=True,
+                text=True,
+                env=env,
+                cwd=oss_root,
+            )
+            msg = f"✅ Successfully created pull request: {result.stdout.strip()}"
+            print(msg)
+            write_github_step_summary(msg)
+        except subprocess.CalledProcessError as e:
+            print(f"Error creating pull request: {e.stderr}")
+            # Check if a PR already exists
+            if "A pull request for" in e.stderr and "already exists" in e.stderr:
+                print("ℹ️ A PR for this branch likely already exists.")
+            else:
+                raise
+
+
+def get_commit_author(commit_hash):
+    """Get the author name and email of a commit."""
+    try:
+        author_name = subprocess.run(
+            ["git", "show", "-s", "--format=%an", commit_hash],
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout.strip()
+        author_email = subprocess.run(
+            ["git", "show", "-s", "--format=%ae", commit_hash],
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout.strip()
+        return author_name, author_email
+    except subprocess.CalledProcessError as e:
+        print(f"Error getting commit author for {commit_hash}: {e.stderr}")
+        raise
+
+
+def get_all_co_author_lines(commit_hash, commit_message):
+    """
+    Build a deduplicated list of Co-authored-by lines that includes both
+    the primary commit author and any Co-authored-by trailers already
+    present in the commit message.
+
+    Returns a list of unique "Co-authored-by: Name <email>" strings.
+    """
+    seen = set()
+    co_author_lines = []
+
+    def _add(name, email):
+        key = (name.strip(), email.strip().lower())
+        if key not in seen:
+            seen.add(key)
+            co_author_lines.append(f"Co-authored-by: {name.strip()} <{email.strip()}>")
+
+    # 1. Primary author of the commit
+    author_name, author_email = get_commit_author(commit_hash)
+    _add(author_name, author_email)
+
+    # 2. Existing Co-authored-by trailers in the commit message
+    for line in commit_message.splitlines():
+        m = re.match(r"^\s*Co-authored-by:\s*(.+?)\s*<([^>]+)>", line, re.IGNORECASE)
+        if m:
+            _add(m.group(1), m.group(2))
+
+    return co_author_lines
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Copy a commit from the private repo to OSS and open a PR."
+    )
+    parser.add_argument(
+        "--commit",
+        type=str,
+        default="LAST",
+        help="The commit hash to sync. Defaults to 'LAST' to use the latest commit.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Dry run the script without executing git, rsync, or gh commands.",
+    )
+    args = parser.parse_args()
+
+    check_dependencies()
+
+    commit_ref = "HEAD" if args.commit == "LAST" else args.commit
+    commit_hash, original_commit_message = get_commit_info(commit_ref)
+
+    if not commit_hash:
+        return  # Exit if we couldn't get commit info
+
+    # Display the details of the commit being processed
+    if args.commit == "LAST":
+        summary = (
+            f"\nℹ️ No commit specified. Using the last commit:\n"
+            f"  - **Hash:** `{commit_hash}`\n"
+            f"  - **Message:** {original_commit_message}\n\n"
+        )
+    else:
+        summary = (
+            f"\nℹ️ Using specified commit:\n"
+            f"  - **Hash:** `{commit_hash}`\n"
+            f"  - **Message:** {original_commit_message}\n\n"
+        )
+    print(summary)
+    write_github_step_summary(summary)
+
+    short_hash = commit_hash[:8]
+
+    patch_file = None
+    temp_dir = None
+    try:
+        # 1. Create a filtered patch from the local repo
+        patch_file, relevant_files = create_filtered_patch(commit_hash, args.dry_run)
+        if not patch_file:
+            return
+
+        # 2. Get the OSS repo
+        oss_root, temp_dir = get_oss_repo(args.dry_run)
+
+        # 3. Find the latest OSS commit that was synced into sglang-private.
+        #    This is the correct base for our patch, since the private repo's
+        #    code is based on this sync point.
+        base_oss_commit = find_latest_oss_sync_commit()
+        if base_oss_commit:
+            print(f"ℹ️  Will branch from OSS commit {base_oss_commit}")
+        else:
+            print(
+                "⚠️  Could not determine latest OSS sync commit. "
+                "Falling back to OSS main HEAD."
+            )
+
+        # 4. Get all co-author lines (primary author + trailers from commit message)
+        co_author_lines = get_all_co_author_lines(commit_hash, original_commit_message)
+        authors_display = "\n".join(co_author_lines)
+
+        # 5. Prepare content for the commit and PR based on changed files
+        file_list_str = "\n".join([f"- {f}" for f in relevant_files])
+        filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files])
+        if len(filename_list_str) > 40:
+            filename_list_str = filename_list_str[:40] + "..."
+        current_date = datetime.datetime.now().strftime("%Y%m%d")
+        pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})"
+
+        # 6. Create branch from the last synced OSS commit, apply patch, and push
+        branch_name = f"sync-{short_hash}-{current_date}"
+        co_authors_block = "\n".join(co_author_lines)
+        commit_message = f"{pr_title}\n\n{co_authors_block}"
+        applied_cleanly = apply_patch_and_push(
+            oss_root,
+            patch_file,
+            branch_name,
+            commit_message,
+            base_oss_commit,
+            args.dry_run,
+        )
+
+        # 7. Adjust PR title and body when there are conflicts
+        if not applied_cleanly:
+            pr_title = (
+                f"[Auto Sync][⚠️ Conflicts] Update {filename_list_str} ({current_date})"
+            )
+
+        pr_body_parts = [
+            f"Sync changes from commit `{short_hash}`.\n",
+            f"**Files Changed:**\n{file_list_str}\n",
+            f"**Authors:**\n{authors_display}",
+        ]
+        if not applied_cleanly:
+            pr_body_parts.append(
+                "\n\n⚠️ **This patch had merge conflicts.** "
+                "The branch contains conflict markers that must be resolved manually. "
+                "Please check the CI logs for the full patch and conflict details."
+            )
+        pr_body_parts.append(
+            f"\n\n---\n\n"
+            f"*This is an automated PR created by scripts/copy_to_oss.py.*"
+        )
+        pr_body = "\n".join(pr_body_parts)
+
+        # 8. Create Pull Request
+        create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run)
+
+    finally:
+        # Cleanup temporary files
+        if patch_file and os.path.exists(patch_file):
+            os.remove(patch_file)
+            print(f"\nRemoved temporary patch file: {patch_file}")
+        if temp_dir and os.path.exists(temp_dir):
+            shutil.rmtree(temp_dir)
+            print(f"Removed temporary directory: {temp_dir}")
+
+
+if __name__ == "__main__":
+    main()

scripts/code_sync/guideline.md

@@ -0,0 +1,51 @@
+### Sync Code Between OSS and Private Fork
+
+You can use the following principles and tools to sync the code between a private fork and the OSS repo [sgl-project/sglang](https://github.com/sgl-project/sglang/tree/main).
+It learns from [Copybara](https://github.com/google/copybara), a tool used at Google for maintaining open-source code synchronization.
+
+## Principals
+
+- The core folders (e.g., `python/sglang/srt`) are 100% mirrored between the private fork and OSS repo.
+- The OSS repo is the single source of truth. If one commit changes `python/sglang/srt` in the private repo, the change should be synced to the OSS repo as soon as possible with the action B below.
+- The common code (e.g., base classes, well-known techniques in the industry without private secrets) goes to `python/sglang/srt`. The private-specific code (e.g., with private-specific features, confidential info) goes to `python/sglang/private` .
+- Anytime you want to make private changes to a file or class under `python/sglang/srt`, duplicate the file and move it under `python/sglang/private`. You can achieve code reuse by importing and inheriting.
+
+## How to sync the code bidirectionally
+### Action A: Copy code from OSS to private
+
+- We can run this action: [Open A PR to Copy Code From OSS](https://github.com/sgl-project/sglang/tree/main/.github/workflows/open-pr-copy-from-oss.yml)
+    - It opens a PR to copy all files under certain folders (e.g., `python/sglang/srt` , `test/srt` , `sgl-kernel` ) from the OSS main branch to the private fork.
+    - Since the OSS repo is the single source of truth, this action copies files and overwrites any changes in the private fork. To prevent the private changes from being overwritten, you need to ensure all private changes are merged into the OSS repo before running this action.
+- This action will be run automatically every day and can also be triggered manually.
+
+### Action B: Copy diff from private to OSS
+
+- We can run this action: [Open A PR to Copy Code To OSS](https://github.com/sgl-project/sglang/tree/main/.github/workflows/open-pr-copy-to-oss.yml)
+    - It opens a PR to apply the diff of one specific commit of the private fork to the OSS main branch. It will only pick the changes under certain folders (e.g., `python/sglang/srt` , `test/srt` , `sgl-kernel` ) and ignore changes under private folders (e.g., `python/sglang/private` )
+    - For example, you can have a PR that changes both `python/sglang/srt` and `python/sglang/private/srt`. Once you merge the PR into the private repo, `python/sglang/srt` becomes desynced between the two repos. You need to run this action on your merge commit immediately to open a PR to send your diff to the OSS repo. Then, we need to merge the OSS PR as soon as possible. Once your OSS PR is merged, we can run action A again.
+    - Action A copies files directly, but Action B applies diff. This is because OSS is the source of truth; action A can just copy files. Action B cannot copy, so it uses diff instead.
+- This action currently needs a manual trigger in order to prevent incidental code leaks. One can also consider making it automatic.
+
+## Examples
+- If you want to have some private server arguments, you can create a new file `python/sglang/private/server_args.py`. It defines a class that inherits the oss ServerArgs.
+    ```python
+    from sglang.srt.server_args import ServerArgs as ServerArgsOSS
+
+    @dataclasses.dataclass
+    class ServerArgs(ServerArgsOSS):
+        private_flag: str = "foo"
+
+        @staticmethod
+        def add_cli_args(parser: argparse.ArgumentParser):
+            # Get all public args
+            ServerArgsOSS.add_cli_args(parser)
+
+            # Add your private flags
+            parser.add_argument(
+                "--private-flag",
+                type=str,
+                default=ServerArgs.private_flag,
+            )
+    ```
+- Similarly, you can inherit `Engine` and override its fields. You can override `server_args_class` to use your own ServerArgs,
+  override `init_tokenizer_manager_func` to use your own TokenizerManager, override `run_scheduler_process_func` to use your own scheduler.

scripts/code_sync/install_github_cli.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Check if gh is installed before attempting to install it
+if ! command -v gh &> /dev/null
+then
+echo "GitHub CLI not found. Installing now..."
+(type -p wget >/dev/null || ( apt update &&  apt install wget -y)) \
+&&  mkdir -p -m 755 /etc/apt/keyrings \
+&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
+&& cat $out |  tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
+&&  chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+&&  mkdir -p -m 755 /etc/apt/sources.list.d \
+&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" |  tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+&&  apt update \
+&&  apt install gh -y
+else
+echo "GitHub CLI is already installed. Skipping installation."
+fi

scripts/code_sync/utils.py

@@ -0,0 +1,135 @@
+"""
+Shared constants and helpers for code-sync scripts.
+"""
+
+import os
+import re
+import subprocess
+from typing import Optional
+
+# --- Configuration Begin ---
+# List of folders and files to copy to / from the OSS repo.
+# Changes outside these paths will be ignored.
+FOLDER_NAMES = [
+    "3rdparty",
+    "assets",
+    "benchmark",
+    "docker",
+    "docs",
+    "examples",
+    "python/sglang/lang",
+    "python/sglang/jit_kernel",
+    "python/sglang/srt",
+    "python/sglang/test",
+    "python/sglang/utils.py",
+    "python/sglang/README.md",
+    "sgl-kernel",
+    "test/manual",
+    "test/registered",
+    "test/README.md",
+    "test/run_suite.py",
+    "README.md",
+]
+
+SYNC_COMMIT_PREFIX = r"\[Automated PR\] Copy OSS code from commit"
+# --- Configuration End ---
+
+
+def write_github_step_summary(content: str) -> None:
+    """Append *content* to the GitHub Actions step summary (no-op outside CI)."""
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if not summary_path:
+        return
+    with open(summary_path, "a") as f:
+        f.write(content)
+
+
+def get_last_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
+    """
+    Find the most recent sync commit that copied from OSS.
+
+    Returns the full private-repo commit hash, or None if not found.
+    The match is restricted to commits whose **subject** starts with the
+    sync prefix so that unrelated commits mentioning the phrase in their
+    body are ignored.
+    """
+    subject_pattern = re.compile("^" + SYNC_COMMIT_PREFIX)
+
+    try:
+        cmd = [
+            "git",
+            "log",
+            "--all",
+            "--grep",
+            SYNC_COMMIT_PREFIX,
+            "--format=%H %s",
+        ]
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=repo_root,
+        ).stdout.strip()
+
+        for line in result.splitlines():
+            # Format: "<full_hash> <subject>"
+            parts = line.split(" ", 1)
+            if len(parts) != 2:
+                continue
+            commit_hash, subject = parts
+            if subject_pattern.search(subject):
+                return commit_hash
+
+        return None
+    except subprocess.CalledProcessError as e:
+        print(f"Error finding last sync commit: {e.stderr}")
+        return None
+
+
+def find_latest_oss_sync_commit(repo_root: Optional[str] = None) -> Optional[str]:
+    """
+    Search the private repo history for the latest commit whose **subject**
+    matches "[Automated PR] Copy OSS code from commit {commit_id} on {date}"
+    and return the embedded **OSS** commit hash.
+
+    Returns the short OSS commit hash string, or None if not found.
+    """
+    oss_hash_pattern = re.compile("^" + SYNC_COMMIT_PREFIX + r" ([0-9a-f]+)")
+
+    try:
+        # --grep filters on the full message body, so we request subject-only
+        # output and validate the pattern against the subject ourselves.
+        result = subprocess.run(
+            [
+                "git",
+                "log",
+                "--all",
+                "--grep",
+                SYNC_COMMIT_PREFIX,
+                "--pretty=%s",
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=repo_root,
+        )
+
+        for subject in result.stdout.strip().splitlines():
+            m = oss_hash_pattern.search(subject)
+            if m:
+                oss_commit = m.group(1)
+                print(
+                    f"✅ Latest OSS sync commit found: {oss_commit} "
+                    f"(from: {subject})"
+                )
+                return oss_commit
+
+        print(
+            "⚠️  No '[Automated PR] Copy OSS code from commit ...' " "found in history."
+        )
+        return None
+
+    except subprocess.CalledProcessError as e:
+        print(f"Error searching for OSS sync commits: {e.stderr.strip()}")
+        return None

scripts/convert_otel_2_perfetto.py

@@ -0,0 +1,468 @@
+import argparse
+import bisect
+import json
+import time
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Tuple
+
+parser = argparse.ArgumentParser(
+    description="Convert SGLang OTEL trace files to Perfetto format.",
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+)
+parser.add_argument(
+    "-i",
+    "--input",
+    dest="input_file",
+    required=True,
+    type=str,
+    help="Path to the input OTEL trace file (JSON or JSONL format).",
+)
+parser.add_argument(
+    "-o",
+    "--output",
+    dest="output_file",
+    type=str,
+    default="sglang_trace_perfetto.json",
+    help="Path to the output Perfetto JSON file.",
+)
+parser.add_argument(
+    "-f", "--torch-file", dest="torch_file", help="specify torch profile file"
+)
+
+args = parser.parse_args()
+
+perfetto_data = None
+if args.torch_file:
+    with open(args.torch_file, "r", encoding="utf-8") as file:
+        perfetto_data = json.load(file)
+        baseline = perfetto_data["baseTimeNanoseconds"]
+else:
+    baseline = 0
+
+
+def id_generator():
+    i = 0
+    while True:
+        yield i
+        i += 1
+
+
+relation_id_gen = id_generator()
+
+
+class SpanLayoutContainer:
+    def __init__(self):
+        self.intervals = []
+
+    def check_overlap(self, start, end):
+        idx = bisect.bisect_left(self.intervals, (start, float("-inf")))
+
+        if idx > 0:
+            prev_start, prev_end = self.intervals[idx - 1]
+            if prev_end > start:
+                return True
+
+        if idx < len(self.intervals):
+            next_start, next_end = self.intervals[idx]
+            if next_start < end:
+                return True
+        return False
+
+    def insert_span(self, start, end):
+        bisect.insort_left(self.intervals, (start, end))
+
+
+def new_metadata_level1(name: str, pid):
+    return {
+        "name": "process_name",
+        "ph": "M",
+        "pid": pid,
+        "args": {"name": name},
+    }
+
+
+def new_metadata_level2(name: str, pid, slot_seq):
+    return {
+        "name": "thread_name",
+        "ph": "M",
+        "pid": pid,
+        "tid": slot_seq,
+        "args": {"name": name},
+    }
+
+
+def __find_line(graph, trans_graph_status, slot_meta_data, pid, start, end):
+    if pid in trans_graph_status:
+        line = trans_graph_status[pid]
+        if start == end:
+            return line
+        # check conflict
+        if not graph[pid][line].check_overlap(start, end):
+            return line
+
+    if pid not in graph:
+        line = 1
+        graph[pid] = {line: SpanLayoutContainer()}
+        trans_graph_status[pid] = line
+        slot_meta_data.append(new_metadata_level2("slot", pid, line))
+        return line
+
+    for line in graph[pid]:
+        if not graph[pid][line].check_overlap(start, end):
+            trans_graph_status[pid] = line
+            return line
+
+    new_line = len(graph[pid]) + 1
+    graph[pid][new_line] = SpanLayoutContainer()
+    trans_graph_status[pid] = new_line
+    slot_meta_data.append(new_metadata_level2("slot", pid, new_line))
+    return new_line
+
+
+OtelSpan = Dict[str, Any]
+
+
+def load_otel_data(path: str | Path):
+    p = Path(path)
+    with p.open("rt", encoding="utf-8") as f:
+        first = f.read(1)
+        f.seek(0)
+        if first == "[":
+            data = json.load(f)  # JSON array
+        else:
+            data = [json.loads(line) for line in f if line.strip()]  # JSONL
+    return data
+
+
+def extract_all_otel_spans(otel_data):
+    engine_otel_spans = []
+    smg_otel_spans = []
+    for line_data in otel_data:
+        for resource_spans in line_data["resourceSpans"]:
+            # filter: only keep spans which service.name is 'sglang' or 'smg'
+            service_name = ""
+            for attr in resource_spans["resource"]["attributes"]:
+                if attr["key"] == "service.name":
+                    service_name = attr["value"]["stringValue"]
+
+            if service_name == "sglang":
+                spans_ref = engine_otel_spans
+            elif service_name == "smg":
+                spans_ref = smg_otel_spans
+            else:
+                continue
+
+            for scope_spans in resource_spans["scopeSpans"]:
+                for span in scope_spans["spans"]:
+                    if "attributes" in span:
+                        attributes_dict = {
+                            attr.get("key"): next(
+                                iter(attr.get("value", {}).values()), None
+                            )
+                            for attr in span["attributes"]
+                        }
+                        span["attributes"] = attributes_dict
+                    else:
+                        span["attributes"] = {}
+                    spans_ref.append(span)
+    return engine_otel_spans, smg_otel_spans
+
+
+def build_otel_span_tree(otel_spans):
+    span_id_map = {span["spanId"]: span for span in otel_spans}
+    for span in otel_spans:
+        span["child"] = []
+
+    root_spans = []
+
+    for span in otel_spans:
+        parent_span_id = span.get("parentSpanId", "")
+        module_name = span.get("attributes", {}).get("module", "")
+        if module_name == "sglang::request" or module_name == "sglang::mooncake":
+            root_spans.append(span)
+        elif parent_span_id in span_id_map:
+            parent_span = span_id_map[parent_span_id]
+            parent_span["child"].append(span)
+
+        link_spans = []
+        if "links" in span:
+            for link in span["links"]:
+                link_span = span_id_map.get(link["spanId"])
+                if link_span:
+                    link_spans.append(link_span)
+            span["links"] = link_spans
+
+    return root_spans
+
+
+def __convert_to_perfetto_span(span, rid, bootstrap_room, pid, host_id):
+    if bootstrap_room:
+        span["attributes"]["bootstrap_room"] = bootstrap_room
+    if rid:
+        span["attributes"]["rid"] = rid
+    if host_id:
+        span["host_id"] = host_id
+    span["pid"] = pid
+
+    span["startTimeUnixNano"] = int(span["startTimeUnixNano"])
+    span["endTimeUnixNano"] = int(span["endTimeUnixNano"]) - 1000
+    ts = span["startTimeUnixNano"]
+    dur = span["endTimeUnixNano"] - ts
+
+    perfetto_span = {
+        "ph": "X",
+        "name": span.get("name", "unknown"),
+        "cat": "sglang",
+        "ts": (ts - baseline) / 1000.0,
+        "dur": dur / 1000.0,
+        "pid": pid,
+        "tid": 0,
+        "args": span["attributes"],
+    }
+
+    span["perfetto_span"] = perfetto_span
+
+    for child_span in span["child"]:
+        __convert_to_perfetto_span(child_span, rid, bootstrap_room, pid, host_id)
+
+
+def generate_perfetto_span(engine_root_spans, smg_otel_spans, thread_meta_data):
+    for root_span in engine_root_spans:
+        root_span["spans"] = []
+
+        rid = root_span["attributes"]["rid"]
+        bootstrap_room = root_span["attributes"].get("bootstrap_room", "")
+
+        for thread_span in root_span["child"]:
+            pid = int(thread_span["attributes"]["pid"])
+            host_id = thread_span["attributes"]["host_id"]
+            thread_name = f'{thread_span["attributes"]["host_id"][:8]}:{thread_span["attributes"]["thread_label"]}'
+            if "pp_rank" in thread_span["attributes"]:
+                thread_name += f"-PP{thread_span['attributes']['pp_rank']}"
+            if "dp_rank" in thread_span["attributes"]:
+                thread_name += f"-DP{thread_span['attributes']['dp_rank']}"
+            if "tp_rank" in thread_span["attributes"]:
+                thread_name += f"-TP{thread_span['attributes']['tp_rank']}"
+
+            if pid not in thread_meta_data:
+                thread_meta_data[pid] = new_metadata_level1(thread_name, pid)
+
+            for span in thread_span["child"]:
+                __convert_to_perfetto_span(span, rid, bootstrap_room, pid, host_id)
+                root_span["spans"].append(span)
+
+    smg_pid = "smg"
+    thread_meta_data[smg_pid] = new_metadata_level1("smg", smg_pid)
+    for span in smg_otel_spans:
+        span["pid"] = smg_pid
+        __convert_to_perfetto_span(span, None, None, smg_pid, None)
+
+
+def __set_span_tid(span, line):
+    span["perfetto_span"]["tid"] = line
+
+    for child_span in span["child"]:
+        __set_span_tid(child_span, line)
+
+
+def generate_perfetto_span_layout(engine_root_spans, smg_otel_spans, slot_meta_data):
+    for root_span in engine_root_spans:
+        root_span["spans"] = sorted(
+            root_span["spans"], key=lambda x: int(x["startTimeUnixNano"])
+        )
+
+    engine_root_spans = sorted(
+        engine_root_spans, key=lambda x: int(x["spans"][0]["startTimeUnixNano"])
+    )
+    graph = {}
+    for root_span in engine_root_spans:
+        req_thread_status = {}
+        for span in root_span["spans"]:
+            line = __find_line(
+                graph,
+                req_thread_status,
+                slot_meta_data,
+                span["perfetto_span"]["pid"],
+                span["startTimeUnixNano"],
+                span["endTimeUnixNano"],
+            )
+            graph[span["perfetto_span"]["pid"]][line].insert_span(
+                span["startTimeUnixNano"], span["endTimeUnixNano"]
+            )
+            __set_span_tid(span, line)
+
+    smg_otel_spans = sorted(smg_otel_spans, key=lambda x: int(x["startTimeUnixNano"]))
+    req_thread_status = {}
+    for span in smg_otel_spans:
+        line = __find_line(
+            graph,
+            req_thread_status,
+            slot_meta_data,
+            span["perfetto_span"]["pid"],
+            span["startTimeUnixNano"],
+            span["endTimeUnixNano"],
+        )
+        graph[span["perfetto_span"]["pid"]][line].insert_span(
+            span["startTimeUnixNano"], span["endTimeUnixNano"]
+        )
+        span["perfetto_span"]["tid"] = line
+
+
+def __convert_to_perfetto_events(span):
+    span["perfetto_events"] = []
+    if "events" in span:
+        for event in span["events"]:
+            attributes_dict = {
+                attr.get("key"): next(iter(attr.get("value", {}).values()), None)
+                for attr in event["attributes"]
+            }
+            perfetto_event = {
+                "ph": "i",
+                "cat": "sglang",
+                "ts": (int(event["timeUnixNano"]) - baseline) / 1000.0,
+                "pid": span["perfetto_span"]["pid"],
+                "tid": span["perfetto_span"]["tid"],
+                "name": event.get("name", "unknown"),
+                "args": attributes_dict,
+            }
+
+            span["perfetto_events"].append(perfetto_event)
+
+    for child_span in span["child"]:
+        __convert_to_perfetto_events(child_span)
+
+
+def generate_perfetto_events(engine_root_spans, smg_otel_spans):
+    spans = [span for root_span in engine_root_spans for span in root_span["spans"]]
+
+    for span in spans:
+        __convert_to_perfetto_events(span)
+
+    for span in smg_otel_spans:
+        __convert_to_perfetto_events(span)
+
+
+def generate_perfetto_links(engine_root_spans, smg_otel_spans):
+    # build link between engine span and smg span
+    span_id_map = {span["spanId"]: span for span in smg_otel_spans}
+
+    for root_span in engine_root_spans:
+        if "parentSpanId" in root_span and root_span["parentSpanId"] in span_id_map:
+            parent_span = span_id_map[root_span["parentSpanId"]]
+            root_span["spans"][0]["links"] = [parent_span]
+
+        for span in root_span["spans"]:
+            span["perfetto_links"] = []
+
+            if "links" in span:
+                for link_span in span["links"]:
+                    try:
+                        link_perfetto_span = link_span["perfetto_span"]
+                    except (KeyError, AttributeError):
+                        continue
+
+                    if "correlation" in link_perfetto_span["args"]:
+                        id = link_perfetto_span["args"]["correlation"]
+                    else:
+                        id = next(relation_id_gen)
+                        link_perfetto_span["args"]["correlation"] = id
+
+                    perfetto_start_node = {
+                        "ph": "s",
+                        "id": id,
+                        "pid": link_perfetto_span["pid"],
+                        "tid": link_perfetto_span["tid"],
+                        "ts": link_perfetto_span["ts"],
+                        "cat": "ac2g",
+                        "name": "ac2g",
+                    }
+
+                    perfetto_end_node = {
+                        "ph": "f",
+                        "id": id,
+                        "pid": span["perfetto_span"]["pid"],
+                        "tid": span["perfetto_span"]["tid"],
+                        "ts": span["perfetto_span"]["ts"],
+                        "cat": "ac2g",
+                        "name": "ac2g",
+                        "bp": "e",
+                    }
+
+                    span["perfetto_links"].append(perfetto_start_node)
+                    span["perfetto_links"].append(perfetto_end_node)
+
+
+def __gather_one_span(span):
+    elems = []
+    elems.append(span["perfetto_span"])
+    if "perfetto_events" in span:
+        elems.extend(span["perfetto_events"])
+    if "perfetto_links" in span:
+        elems.extend(span["perfetto_links"])
+
+    for child_span in span["child"]:
+        elems.extend(__gather_one_span(child_span))
+
+    return elems
+
+
+def gather_all_perfetto_elems(
+    engine_root_spans, smg_otel_spans, thread_meta_data, slot_meta_data
+):
+    elems = []
+    elems.extend(thread_meta_data.values())
+    elems.extend(slot_meta_data)
+    for root_span in engine_root_spans:
+        for span in root_span["spans"]:
+            elems.extend(__gather_one_span(span))
+
+    for span in smg_otel_spans:
+        elems.append(span["perfetto_span"])
+        elems.extend(span["perfetto_events"])
+
+    return elems
+
+
+def write_json(perfetto_elems):
+    global perfetto_data
+
+    if args.torch_file:
+        perfetto_data["traceEvents"].extend(perfetto_elems)
+        filered_data = [
+            item
+            for item in perfetto_data["traceEvents"]
+            if item.get("cat") != "gpu_user_annotation"
+        ]
+        perfetto_data["traceEvents"] = filered_data
+    else:
+        perfetto_data = perfetto_elems
+
+    with open(args.output_file, "w", encoding="utf-8") as file:
+        json.dump(perfetto_data, file, ensure_ascii=False, indent=4)
+
+
+def main():
+    start_time = time.time()
+    otel_data = load_otel_data(args.input_file)
+    engine_otel_spans, smg_otel_spans = extract_all_otel_spans(otel_data)
+    engine_root_spans = build_otel_span_tree(engine_otel_spans)
+    thread_meta_data = {}
+    generate_perfetto_span(engine_root_spans, smg_otel_spans, thread_meta_data)
+    slot_meta_data = []
+    generate_perfetto_span_layout(engine_root_spans, smg_otel_spans, slot_meta_data)
+    generate_perfetto_events(engine_root_spans, smg_otel_spans)
+    generate_perfetto_links(engine_root_spans, smg_otel_spans)
+    perfetto_elems = gather_all_perfetto_elems(
+        engine_root_spans, smg_otel_spans, thread_meta_data, slot_meta_data
+    )
+    write_json(perfetto_elems)
+    end_time = time.time()
+    execution_time = end_time - start_time
+    print(f"\nConversion finished successfully!")
+    print(f"Output written to: {args.output_file}")
+    print(f"Execution time: {execution_time * 1000:.4f} ms")
+
+
+if __name__ == "__main__":
+    main()

scripts/export_deepseek_nextn.py

@@ -0,0 +1,115 @@
+"""
+Export NextN layer for DeepSeek-V3/R1 model. The exported model can be used for speculative decoding.
+
+Usage:
+python3 export_deepseek_nextn.py --input-dir /path/to/DeepSeek-V3 --output-dir /path/to/DeepSeek-V3-NextN
+"""
+
+import argparse
+import json
+import os
+import shutil
+
+from safetensors import safe_open
+from safetensors.torch import save_file
+from transformers import AutoConfig
+
+
+def get_nextn_layer_id(config):
+    if not hasattr(config, "num_hidden_layers"):
+        raise ValueError("'num_hidden_layers' not found in model config.")
+    return config.num_hidden_layers
+
+
+def update_and_save_config(config, output_dir):
+    new_config = config.to_dict()
+    new_config.update(
+        {
+            "num_hidden_layers": 1,
+            "architectures": ["DeepseekV3ForCausalLMNextN"],
+        }
+    )
+    with open(os.path.join(output_dir, "config.json"), "w") as f:
+        json.dump(new_config, f, indent=2, ensure_ascii=False, sort_keys=True)
+
+
+def copy_non_safetensors_files(input_dir, output_dir):
+    for filename in os.listdir(input_dir):
+        src_file_path = os.path.join(input_dir, filename)
+        if os.path.isfile(src_file_path) and not filename.endswith(".safetensors"):
+            dst_file_path = os.path.join(output_dir, filename)
+            shutil.copy2(src_file_path, dst_file_path)
+    print(f"All non-safetensors files have been copied to {output_dir}")
+
+
+def export_nextn_layer_parameters(input_dir, output_dir, nextn_layer_id):
+    prefix = f"model.layers.{nextn_layer_id}"
+    output_path = os.path.join(output_dir, "nextn_layer_parameters.safetensors")
+    params = {}
+    for filename in os.listdir(input_dir):
+        if not filename.endswith(".safetensors"):
+            continue
+
+        file_path = os.path.join(input_dir, filename)
+        print(f"Processing: {filename}")
+
+        try:
+            with safe_open(file_path, framework="pt") as f:
+                matching_keys = [k for k in f.keys() if k.startswith(prefix)]
+
+                if not matching_keys:
+                    print(f"  No parameters starting with '{prefix}' found")
+                    continue
+
+                for key in matching_keys:
+                    if "embed_tokens" in key or "shared_head.head" in key:
+                        continue
+                    new_key = key.replace(prefix, "model.layers.0")
+                    params[new_key] = f.get_tensor(key)
+
+        except Exception as e:
+            print(f"  Error processing {filename}: {str(e)}")
+
+    if params:
+        print(f"Saving {len(params)} parameters to {output_path}")
+        save_file(params, output_path)
+    else:
+        print("No matching parameters found.")
+
+    # Update safetensors index
+    index_path = os.path.join(output_dir, "model.safetensors.index.json")
+    print(f"Updating safetensors index to {index_path}")
+    index_data = {"weight_map": {}}
+    for key in params:
+        index_data["weight_map"][key] = "nextn_layer_parameters.safetensors"
+    with open(index_path, "w") as f:
+        json.dump(index_data, f, indent=4)
+
+    print("All done.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Export NextN layer parameters for DeepSeek-V3/R1"
+    )
+    parser.add_argument(
+        "--input-dir",
+        type=str,
+        required=True,
+        help="Input HF model directory.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        required=True,
+        help="Output nextn model directory.",
+    )
+    args = parser.parse_args()
+
+    config = AutoConfig.from_pretrained(args.input_dir, trust_remote_code=True)
+    assert config.num_nextn_predict_layers == 1, "Only 1 nextn layer is supported."
+    nextn_layer_id = get_nextn_layer_id(config)
+    os.makedirs(args.output_dir, exist_ok=True)
+    copy_non_safetensors_files(args.input_dir, args.output_dir)
+    update_and_save_config(config, args.output_dir)
+    export_nextn_layer_parameters(args.input_dir, args.output_dir, nextn_layer_id)

scripts/killall_sglang.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# DEPRECATED: This script will be migrated to python/sglang/cli/killall.py.
+# CI mode is already handled there. This script remains for local/non-CI usage.
+#
+# TODO: Migrate remaining modes (rocm, all, gpus) to killall.py and remove this file.
+#
+# Usage:
+#   ./killall_sglang.sh              - Kill SGLang processes only (NVIDIA mode)
+#   ./killall_sglang.sh rocm         - Kill SGLang processes only (ROCm mode)
+#   ./killall_sglang.sh all          - Kill all GPU processes (NVIDIA mode)
+#   ./killall_sglang.sh gpus 0,1,2,3 - Kill all processes on specific GPUs
+
+if [ "$1" = "rocm" ]; then
+    echo "Running in ROCm mode"
+
+    # Clean SGLang processes
+    pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt|sgl_diffusion::' | xargs -r kill -9
+
+elif [ "$1" = "gpus" ] && [ -n "$2" ]; then
+    # Kill all processes on specific GPUs only
+    echo "Killing all processes on GPUs: $2"
+
+    # Show current GPU status
+    nvidia-smi
+
+    # Build device file list from GPU IDs (e.g., "0,1,2,3" -> "/dev/nvidia0 /dev/nvidia1 ...")
+    devices=$(echo "$2" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed 's|^|/dev/nvidia|' | tr '\n' ' ')
+    echo "Targeting devices: $devices"
+
+    # Kill all processes using specified GPU devices
+    [ -n "$devices" ] && lsof $devices 2>/dev/null | awk 'NR>1 {print $2}' | sort -u | xargs -r kill -9 2>/dev/null
+
+    # Show GPU status after clean up
+    nvidia-smi
+
+else
+    # Show current GPU status
+    nvidia-smi
+
+    # Clean SGLang processes
+    pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt|sgl_diffusion::' | xargs -r kill -9
+
+    # Clean all GPU processes if "all" argument is provided
+    if [ "$1" = "all" ]; then
+        # Check if sudo is available
+        if command -v sudo >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y lsof
+        else
+            apt-get update
+            apt-get install -y lsof
+        fi
+        kill -9 $(nvidia-smi | sed -n '/Processes:/,$p' | grep "   [0-9]" | awk '{print $5}') 2>/dev/null
+        lsof /dev/nvidia* | awk '{print $2}' | xargs kill -9 2>/dev/null
+    fi
+
+    # Show GPU status after clean up
+    nvidia-smi
+fi

scripts/playground/bench_speculative.py

@@ -0,0 +1,319 @@
+"""
+Usage:
+# single GPU
+python3 bench_speculative.py --model-path meta-llama/Llama-2-7b-chat-hf --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B
+
+# multiple GPU
+python3 bench_speculative.py --model-path deepseek-ai/DeepSeek-V3 --speculative-draft-model-path lmsys/DeepSeek-V3-NextN --tp-size 8 --trust-remote-code --batch-size 1 4 8 16 32 --steps 0 1 2 --topk 0 1 2 4 --num_draft_tokens 0 2 4 8
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import time
+from types import SimpleNamespace
+from typing import List
+
+import numpy as np
+import requests
+from transformers import AutoTokenizer
+
+from sglang.bench_serving import benchmark, set_global_args
+from sglang.benchmark.datasets import DatasetRow
+from sglang.benchmark.datasets.mmmu import sample_mmmu_requests
+from sglang.srt.server_args import ServerArgs
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    kill_process_tree,
+    popen_launch_server,
+)
+
+
+def node0_print(msg):
+    if server_args.node_rank == 0:
+        print(msg)
+
+
+prompts = [
+    "Human: Give me a fully functional FastAPI server. Show the full, long python code without stop.\n\nAssistant:",
+    "Human: Imagine you are an experienced Ethereum developer tasked with creating a smart contract for a blockchain messenger. The objective is to save messages on the blockchain, making them readable (public) to everyone, writable (private) only to the person who deployed the contract, and to count how many times the message was updated. Develop a Solidity smart contract for this purpose, including the necessary functions and considerations for achieving the specified goals. Please provide the code and any relevant explanations to ensure a clear understanding of the implementation.\n\nAssistant:",
+    "Human: Write a travel blog post to Hawaii.\n\nAssistant:",
+    "Human: I want you to act as an English translator, spelling corrector and improver. I will speak to you in any language and you will detect the language, translate it and answer in the corrected and improved version of my text, in English. I want you to replace my simplified A0-level words and sentences with more beautiful and elegant, upper level English words and sentences. Keep the meaning same, but make them more literary. My first sentence is 'istanbulu cok seviyom burada olmak cok guzel'. Answer in more than 5000 words.\n\nAssistant:",
+    "Human: I want you to act as a storyteller. You will come up with entertaining stories that are engaging, imaginative and captivating for the audience. It can be fairy tales, educational stories or any other type of stories which has the potential to capture people's attention and imagination. Depending on the target audience, you may choose specific themes or topics for your storytelling session e.g., if it’s children then you can talk about animals; If it’s adults then history-based tales might engage them better etc. Answer in more than 5000 words. My first request is 'I need an interesting story on perseverance.'\n\nAssistant:",
+    "Human: Solve x^2 = -1. Think step-by-step. Give me a long detailed explanation. \n\nAssistant:",
+    "Human: Tell me about the president of the USA in wikipedia style.\n\nAssistant:",
+    "Human: Hello? Who are you? Write code, math, and poem to explanin yourself.\n\nAssistant:",
+]
+
+
+class FakeTokenizer:
+    def encode(self, text: str, add_special_tokens: bool = False):
+        return []
+
+
+def send_one_batch(base_url, num_prompts, batch_size, processor, is_multimodal):
+    # format: (prompt, input_len, output len). We set input_len as a dummy value 0.
+    if is_multimodal:
+        backend = "sglang-oai-chat"
+        api_url = f"{base_url}/v1/chat/completions"
+        input_requests = sample_mmmu_requests(
+            num_prompts,
+            processor,
+            backend=backend,
+            fixed_output_len=512,
+        )
+        tokenizer = processor.tokenizer
+    else:
+        padded_prompts = (prompts * ((num_prompts + len(prompts) - 1) // len(prompts)))[
+            :num_prompts
+        ]
+        input_requests: List[DatasetRow] = [
+            DatasetRow(p, 0, 512) for p in padded_prompts
+        ]
+        backend = "sglang"
+        api_url = f"{base_url}/generate"
+        tokenizer = processor
+
+    # We need to set some dummy values in order to call `benchmark` below.
+    args = SimpleNamespace(
+        disable_ignore_eos=False,
+        disable_stream=False,
+        return_logprob=False,
+        return_routed_experts=False,
+        plot_throughput=False,
+        backend=backend,
+        dataset_name="custom",
+        num_prompts=None,
+        sharegpt_output_len=None,
+        random_input_len=None,
+        random_output_len=None,
+        random_range_ratio=None,
+        output_file=None,
+        warmup_requests=1,
+        output_details=False,
+    )
+    set_global_args(args)
+
+    # Run benchmark
+    results = asyncio.run(
+        benchmark(
+            backend=backend,
+            api_url=api_url,
+            base_url=base_url,
+            model_id="default",
+            tokenizer=tokenizer,
+            input_requests=input_requests,
+            request_rate=float("inf"),
+            max_concurrency=batch_size,
+            disable_tqdm=False,
+            lora_names=None,
+            lora_request_distribution=None,
+            lora_zipf_alpha=None,
+            extra_request_body={},
+            profile=None,
+        )
+    )
+
+    assert results["completed"] == len(input_requests)
+    acc_length = results["accept_length"] or 1.0
+    avg_output_token = results["total_output_tokens"] / results["completed"]
+
+    server_info = requests.get(base_url + "/server_info").json()
+    # We use 20% percentile instead of median on purpose
+    step_time = np.percentile(
+        server_info["internal_states"][0]["step_time_dict"][str(batch_size)], 20
+    )
+    speed = 1 / step_time * acc_length
+
+    return (
+        round(acc_length, 3),
+        round(step_time, 5),
+        round(speed, 3),
+        avg_output_token,
+    )
+
+
+def main(args, server_args):
+    base_url = "http://127.0.0.1:20000"
+
+    configs = []
+    for batch_size in args.batch_size:
+        for steps in args.steps:
+            for topk in args.topk:
+                for num_draft_tokens in args.num_draft_tokens:
+                    if steps * topk + 1 < num_draft_tokens:
+                        continue
+
+                    if (steps == 0 or topk == 0 or num_draft_tokens == 0) and (
+                        steps + topk + num_draft_tokens != 0
+                    ):
+                        # steps == 0 and topk == 0 and num_draft_tokens == 0 is a special case for non-speculative decoding.
+                        continue
+
+                    configs.append((batch_size, steps, topk, num_draft_tokens))
+
+    for i in range(args.start, args.end or len(configs)):
+        batch_size, steps, topk, num_draft_tokens = configs[i]
+
+        node0_print(
+            f"Start {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}"
+        )
+
+        # Create an LLM.
+        if steps == 0:
+            other_args = []
+        else:
+            other_args = [
+                "--speculative-num-steps",
+                steps,
+                "--speculative-eagle-topk",
+                topk,
+                "--speculative-num-draft-tokens",
+                num_draft_tokens,
+            ]
+            if server_args.speculative_draft_model_path is not None:
+                other_args.extend(
+                    [
+                        "--speculative-draft-model-path",
+                        server_args.speculative_draft_model_path,
+                        "--speculative-algorithm",
+                        server_args.speculative_algorithm,
+                    ]
+                )
+
+        other_args.extend(
+            [
+                "--cuda-graph-max-bs",
+                batch_size,
+                "--mem-fraction-static",
+                server_args.mem_fraction_static,
+                "--tp-size",
+                server_args.tp_size,
+                "--max-running-requests",
+                batch_size,
+            ]
+        )
+
+        if server_args.trust_remote_code:
+            other_args.extend(
+                [
+                    "--trust-remote-code",
+                ]
+            )
+
+        if server_args.attention_backend:
+            other_args.extend(
+                [
+                    "--attention-backend",
+                    server_args.attention_backend,
+                ]
+            )
+
+        if server_args.quantization:
+            other_args.extend(
+                [
+                    "--quantization",
+                    server_args.quantization,
+                ]
+            )
+
+        process = popen_launch_server(
+            args.model_path,
+            base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=other_args,
+            env={
+                "SGLANG_RECORD_STEP_TIME": "1",
+                **os.environ,
+            },
+        )
+
+        if args.is_multimodal:
+            from transformers import AutoProcessor
+
+            processor = AutoProcessor.from_pretrained(
+                args.model_path, trust_remote_code=server_args.trust_remote_code
+            )
+        else:
+            processor = AutoTokenizer.from_pretrained(
+                args.model_path, trust_remote_code=server_args.trust_remote_code
+            )
+
+        try:
+            # Warmup
+            send_one_batch(
+                base_url, batch_size, batch_size, processor, args.is_multimodal
+            )
+
+            # Benchmark
+            acc_length, step_time, speed, completion_tokens = send_one_batch(
+                base_url,
+                max(args.num_prompts, batch_size),
+                batch_size,
+                processor,
+                args.is_multimodal,
+            )
+        finally:
+            kill_process_tree(process.pid)
+
+        node0_print(
+            f"Finish {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}, {speed=:.2f} token/s, step_time={step_time * 1000:.2f} ms"
+        )
+
+        record = {
+            "batch_size": batch_size,
+            "steps": steps,
+            "topk": topk,
+            "num_draft_tokens": num_draft_tokens,
+            "acc_length": acc_length,
+            "step_time": step_time,
+            "speed": speed,
+            "completion_tokens": completion_tokens,
+        }
+
+        with open(args.output, "a") as fout:
+            fout.write(json.dumps(record) + "\n")
+
+        # Wait for the server to shutdown
+        time.sleep(5)
+
+
+# The __main__ condition is necessary here because we use "spawn" to create subprocesses
+# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    ServerArgs.add_cli_args(parser)
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        nargs="+",
+        default=(1, 2, 4, 8, 16),
+    )
+    parser.add_argument(
+        "--steps",
+        type=int,
+        nargs="+",
+        default=(0, 1, 3, 5, 7),  # use (0, 1, 2, 3, 4) for large batch size
+    )
+    parser.add_argument(
+        "--topk",
+        type=int,
+        nargs="+",
+        default=(0, 1, 2, 4, 8),
+    )
+    parser.add_argument(
+        "--num_draft_tokens",
+        type=int,
+        nargs="+",
+        default=(0, 2, 4, 8, 16, 32),  # use (0, 2, 4, 8) for large batch size
+    )
+    parser.add_argument("--num-prompts", type=int, default=16)
+    parser.add_argument("--start", type=int, default=0)
+    parser.add_argument("--end", type=int)
+    parser.add_argument("--output", type=str, default="output.jsonl")
+    parser.add_argument("--is-multimodal", action="store_true", default=False)
+    args = parser.parse_args()
+    server_args: ServerArgs = ServerArgs.from_cli_args(args)
+
+    main(args, server_args)

scripts/playground/disaggregation/cli-logprob.py

@@ -0,0 +1,22 @@
+prompt = "The capital of france is "
+
+import json
+
+import requests
+
+response = requests.post(
+    "http://0.0.0.0:8000/generate",
+    json={
+        "text": prompt,
+        "sampling_params": {"temperature": 0},
+        "return_logprob": True,
+        "return_input_logprob": True,
+        "logprob_start_len": 0,
+    },
+)
+
+j = response.json()
+input_logprobs = j["meta_info"]["input_token_logprobs"]
+output_logprobs = j["meta_info"]["output_token_logprobs"]
+
+print(len(input_logprobs), len(output_logprobs))