sgl-project--sglang/.github/workflows/pr-test-rust.yml

name: PR Test (SMG)

on:
  push:
    branches: [ main ]
    paths:
      - "sgl-model-gateway/**"
      - ".github/workflows/pr-test-rust.yml"
      - "scripts/ci/cuda/ci_install_dependency.sh"
      - "scripts/ci/cuda/ci_install_gateway_dependencies.sh"
  pull_request:
    branches: [ main ]
    types: [opened, synchronize, reopened, labeled]
    paths:
      - "sgl-model-gateway/**"
      - ".github/workflows/pr-test-rust.yml"
      - "scripts/ci/cuda/ci_install_dependency.sh"
      - "scripts/ci/cuda/ci_install_gateway_dependencies.sh"
  workflow_dispatch:

concurrency:
  group: gateway-tests-${{ github.ref }}
  cancel-in-progress: true

env:
  SGLANG_IS_IN_CI: true

jobs:
  build-wheel:
    if: |
      github.event_name != 'pull_request' ||
      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
    # Pin to 22.04 so the wheel auditwheel-tags as manylinux_2_35; the
    # self-hosted GPU runners are Ubuntu 22.04 (glibc 2.35) and reject
    # manylinux_2_39 wheels produced on ubuntu-latest (Ubuntu 24.04).
    runs-on: ubuntu-22.04
    # sccache is only installed on the GitHub-hosted runners that run this
    # job and `unit-tests`; setting RUSTC_WRAPPER workflow-wide leaks it to
    # gateway-e2e on the self-hosted GPU runners (which don't have sccache),
    # so any pip-install that compiles a Rust extension would fail with
    # `could not execute process \`sccache rustc\``.
    env:
      RUSTC_WRAPPER: sccache
      SCCACHE_GHA_ENABLED: "true"
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install rust dependencies
        run: |
          bash scripts/ci/cuda/ci_install_gateway_dependencies.sh

      - name: Configure sccache
        uses: mozilla-actions/sccache-action@v0.0.9
        with:
          version: "v0.12.0"
          disable_annotations: true

      - name: Rust cache
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: sgl-model-gateway
          shared-key: "rust-cache"
          cache-all-crates: true
          cache-on-failure: true
          save-if: true

      - name: Build python binding
        run: |
          source "$HOME/.cargo/env"
          export RUSTC_WRAPPER=sccache
          cd sgl-model-gateway/bindings/python
          python3 -m pip install --upgrade pip maturin
          maturin build --profile ci --features vendored-openssl --out dist

      - name: List built wheel
        run: ls -lh sgl-model-gateway/bindings/python/dist/

      - name: Upload wheel artifact
        uses: actions/upload-artifact@v4
        with:
          name: smg-wheel
          path: sgl-model-gateway/bindings/python/dist/*.whl
          retention-days: 1

      - name: Test wheel install
        run: |
          pip install sgl-model-gateway/bindings/python/dist/*.whl
          python3 -c "import sglang_router; print('Python package: OK')"
          python3 -c "from sglang_router.sglang_router_rs import Router; print('Rust extension: OK')"
          python3 -m sglang_router.launch_router --help > /dev/null && echo "Entry point: OK"

  python-unit-tests:
    needs: build-wheel
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          path: sglang-repo

      - name: Move sgl-model-gateway folder to root
        run: |
          mv sglang-repo/sgl-model-gateway/* .
          rm -rf sglang-repo

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: Download wheel artifact
        uses: actions/download-artifact@v4
        with:
          name: smg-wheel
          path: dist/

      - name: Install wheel
        run: pip install dist/*.whl

      - name: Run Python unit tests
        run: |
          cd bindings/python
          python3 -m pip install pytest pytest-cov pytest-xdist
          pytest -q tests --cov=sglang_router --cov-config=.coveragerc --cov-report=term-missing --cov-fail-under=80

  unit-tests:
    if: |
      github.event_name != 'pull_request' ||
      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
    runs-on: ubuntu-latest
    env:
      RUSTC_WRAPPER: sccache
      SCCACHE_GHA_ENABLED: "true"
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          bash scripts/ci/cuda/ci_install_gateway_dependencies.sh

      - name: Configure sccache
        uses: mozilla-actions/sccache-action@v0.0.9
        with:
          version: "v0.12.0"
          disable_annotations: true

      - name: Rust cache
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: sgl-model-gateway
          shared-key: "rust-cache"
          cache-all-crates: true
          cache-on-failure: true
          save-if: true

      - name: Run lint
        run: |
          source "$HOME/.cargo/env"
          cd sgl-model-gateway/
          rustup component add clippy
          cargo clippy --all-targets --all-features -- -D warnings

      - name: Run fmt
        run: |
          source "$HOME/.cargo/env"
          cd sgl-model-gateway/
          rustup component add --toolchain nightly-x86_64-unknown-linux-gnu rustfmt
          rustup toolchain install nightly --profile minimal
          cargo +nightly fmt -- --check

      - name: Generate vision golden fixtures
        run: |
          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu

          pip install transformers pillow numpy scipy
          pip install transformers pillow numpy
          cd sgl-model-gateway/
          python scripts/generate_vision_golden.py

      - name: Run Rust tests
        timeout-minutes: 20
        run: |
          source "$HOME/.cargo/env"
          cd sgl-model-gateway/
          cargo test

      - name: Show sccache stats
        if: always()
        run: sccache --show-stats

  gateway-e2e:
    name: ${{ matrix.name }}
    needs: build-wheel
    if: |
      github.event_name != 'pull_request' ||
      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
    # The `responses` matrix entry is intentionally omitted. It needs
    # `docker run gvenzl/oracle-xe` + `docker run shoofio/brave-search-mcp-sse`
    # on the runner host, but the 2-/4-gpu-h100 runners are themselves
    # containers without a Docker daemon. Re-enable by adding back:
    #   - name: responses
    #     runner: 2-gpu-h100
    #     timeout: 45
    #     test_dirs: "e2e_test/responses"
    #     extra_deps: ""
    #     env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
    #     reruns: "--reruns 2 --reruns-delay 5"
    #     setup_oracle: true
    #     setup_brave: true
    #     parallel_opts: ""
    # plus the Oracle Instant Client / `gvenzl/oracle-xe` /
    # `shoofio/brave-search-mcp-sse` setup + cleanup steps (see commit
    # cf346bb15 for the exact step bodies) once a runner with
    # `docker.sock` (or binary-installed deps) is available.
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: benchmarks
            # 4 GPUs: test_pd_perf.py uses workers(prefill=2, decode=2) and
            # test_regular_perf.py uses workers(count=4) — both need tp*workers=4.
            runner: 4-gpu-h100
            timeout: 32
            test_dirs: "e2e_test/benchmarks"
            extra_deps: "genai-bench==0.0.3"
            env_vars: ""
            reruns: ""
            upload_benchmarks: true
            parallel_opts: ""  # No parallel for benchmarks (performance measurement)
          - name: e2e
            runner: 2-gpu-h100
            timeout: 45
            test_dirs: "e2e_test/router e2e_test/embeddings"
            extra_deps: ""
            env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
            reruns: "--reruns 2 --reruns-delay 5"
            # Run tests serially. pytest-parallel (unmaintained since 2019)
            # has buggy fixture-finalize handling under thread dispatch:
            # both class- and function-scoped fixture references leaked
            # between tests, leaving model_pool instances pinned at
            # _ref_count > 0 and deadlocking later tests that needed
            # eviction (50+ min hangs). On a 2-GPU runner with 5 distinct
            # model:mode combos in router+embeddings, the suite is
            # eviction-bound anyway, so the parallel speedup was illusory.
            parallel_opts: ""
          - name: chat-completions
            runner: 2-gpu-h100
            timeout: 45
            test_dirs: "e2e_test/chat_completions"
            extra_deps: ""
            env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
            reruns: "--reruns 2 --reruns-delay 5"
            parallel_opts: ""
          - name: chat-completions-4gpu
            runner: 4-gpu-h100
            timeout: 45
            # qwen-30b (tp=4) tests can't fit on the 2-gpu-h100 matrix entries —
            # they get skipped there by hooks.py. Run them here so coverage holds.
            test_dirs: "e2e_test/chat_completions/test_enable_thinking.py"
            extra_deps: ""
            env_vars: "SHOW_WORKER_LOGS=0 SHOW_ROUTER_LOGS=1"
            reruns: "--reruns 2 --reruns-delay 5"
            parallel_opts: ""
    runs-on: ${{ matrix.runner }}
    timeout-minutes: ${{ matrix.timeout }}
    # Self-hosted GPU runners are scarce; serialize per hardware type so
    # 2-gpu-h100 and 4-gpu-h100 each run one job at a time across all
    # in-flight PRs. Queue rather than cancel — different refs shouldn't
    # interrupt each other. Include matrix.name in the group so siblings
    # within the same run don't collide: GitHub keeps only one pending job
    # per concurrency group, so matrix entries sharing a group (e.g.
    # benchmarks + chat-completions-4gpu on 4-gpu-h100) would evict each
    # other regardless of cancel-in-progress: false.
    concurrency:
      group: pr-test-rust-${{ matrix.runner }}-${{ matrix.name }}
      cancel-in-progress: false
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install SGLang dependencies
        run: |
          bash scripts/ci/cuda/ci_install_dependency.sh

      - name: Download wheel artifact
        uses: actions/download-artifact@v4
        with:
          name: smg-wheel
          path: wheel/

      - name: Install wheel
        run: |
          pip uninstall -y sglang-router || true
          pip install wheel/*.whl

      - name: Install e2e test dependencies
        run: |
          python3 -m pip install pytest pytest-rerunfailures httpx openai grpcio grpcio-health-checking numpy
          if [ -n "${{ matrix.extra_deps }}" ]; then
            if echo "${{ matrix.extra_deps }}" | grep -q "genai-bench"; then
              # genai-bench's transitive deps (oci/locust) pull
              # transformers<5 which requires huggingface_hub<1.0 — that
              # downgrades the 1.11.0 ci_install_dependency.sh settled on
              # and breaks `kernels` (requires huggingface_hub>=1.3.0,<2.0).
              # Install --no-deps and supply the runtime deps explicitly.
              python3 -m pip --no-cache-dir install --no-deps ${{ matrix.extra_deps }}
              python3 -m pip --no-cache-dir install \
                locust click rich tenacity oci openpyxl gevent matplotlib
            else
              # Other extras with well-behaved transitive deps —
              # normal --upgrade install path.
              python3 -m pip --no-cache-dir install --upgrade ${{ matrix.extra_deps }}
            fi
          fi

      - name: Run E2E tests
        run: |
          python3 python/sglang/cli/killall.py
          cd sgl-model-gateway
          ${{ matrix.env_vars }} pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.test_dirs }} -s -vv -o log_cli=true --log-cli-level=INFO

      - name: Upload benchmark results
        if: matrix.upload_benchmarks && success()
        uses: actions/upload-artifact@v4
        with:
          name: genai-bench-results-all-policies
          path: sgl-model-gateway/benchmark_**/

  docker-build-test:
    if: |
      github.event_name != 'pull_request' ||
      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build Docker image (no push)
        uses: docker/build-push-action@v5
        with:
          context: .
          file: docker/gateway.Dockerfile
          push: false
          tags: sgl-model-gateway:test
          cache-from: type=gha
          cache-to: type=gha,mode=max

  k8s-integration:
    # Runs SMG against a kind cluster with fake worker pods to exercise the
    # K8s service discovery / reconciliation path. No GPU required (workers
    # are python:3.12-slim mocks); the h100 matrix runners are unsuitable
    # because they're containers without a Docker daemon.
    if: |
      github.event_name != 'pull_request' ||
      (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run-ci')) ||
      (github.event.action == 'labeled' && github.event.label.name == 'run-ci')
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install kind and kubectl
        run: |
          curl -fsSLo /tmp/kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
          chmod +x /tmp/kind && sudo mv /tmp/kind /usr/local/bin/kind
          KUBECTL_VERSION=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
          curl -fsSLo /tmp/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
          chmod +x /tmp/kubectl && sudo mv /tmp/kubectl /usr/local/bin/kubectl
          kind --version
          kubectl version --client

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build smg-gateway:test image
        uses: docker/build-push-action@v5
        with:
          context: sgl-model-gateway
          file: sgl-model-gateway/e2e_test/k8s_integration/Dockerfile.gateway
          tags: smg-gateway:test
          load: true
          cache-from: type=gha,scope=k8s-integration
          cache-to: type=gha,scope=k8s-integration,mode=max

      - name: Install Python test dependencies
        run: |
          python3 -m pip install --upgrade pip
          python3 -m pip install pytest httpx

      - name: Set up kind cluster and deploy
        env:
          SKIP_DOCKER_BUILD: "1"
        run: |
          cd sgl-model-gateway
          bash e2e_test/k8s_integration/setup.sh

      - name: Run K8s integration tests
        run: |
          cd sgl-model-gateway
          # confcutdir avoids loading the parent e2e_test/conftest.py, which
          # pulls in heavy infra deps (requests, sglang_router) that this job
          # intentionally doesn't install.
          pytest e2e_test/k8s_integration/ \
            --confcutdir=e2e_test/k8s_integration \
            -v -s -o log_cli=true --log-cli-level=INFO

      - name: Dump cluster state on failure
        if: failure()
        run: |
          kubectl --context kind-smg-test get all -A || true
          kubectl --context kind-smg-test -n smg-test describe pods || true
          kubectl --context kind-smg-test -n smg-test logs deploy/smg-gateway --tail=200 || true

      - name: Tear down kind cluster
        if: always()
        run: |
          cd sgl-model-gateway
          bash e2e_test/k8s_integration/setup.sh teardown || true

  finish:
    needs: [build-wheel, python-unit-tests, unit-tests, gateway-e2e, docker-build-test, k8s-integration]
    runs-on: ubuntu-latest
    steps:
      - name: Finish
        run: echo "This is an empty step to ensure that all jobs are completed."

  summarize-benchmarks:
    needs: gateway-e2e
    runs-on: ubuntu-latest
    if: success()

    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Download benchmark results
      uses: actions/download-artifact@v4
      with:
        name: genai-bench-results-all-policies

    - name: Create benchmark summary
      run: python3 sgl-model-gateway/e2e_test/benchmarks/summarize.py .