name: Rerun Test
run-name: ${{ inputs.pr_head_sha && format('[rerun-test] {0} {1}', inputs.test_command, inputs.pr_head_sha) || format('[rerun-test] {0}', inputs.test_command) }}

on:
  workflow_dispatch:
    inputs:
      mode:
        description: "Dispatch mode: cuda | multimodal_gen | cpu"
        required: true
        type: choice
        options:
          - cuda
          - multimodal_gen
          - cpu
      test_command:
        description: "Test command(s) to run, one per line (e.g. 'registered/core/test_srt_endpoint.py TestSRTEndpoint.test_simple_decode')"
        required: true
        type: string
      runs_on:
        description: "GHA runner label (cuda/multimodal_gen only; ignored for cpu)"
        required: false
        type: string
        default: ""
      install_script:
        description: "Install script path (cuda only). E.g. scripts/ci/cuda/ci_install_dependency.sh"
        required: false
        type: string
        default: ""
      install_timeout:
        description: "Install-step timeout minutes (cuda only)"
        required: false
        type: string
        default: "20"
      grace_blackwell:
        description: "Set GRACE_BLACKWELL for the install step (cuda only)"
        required: false
        type: string
        default: "0"
      rdma_devices:
        description: "SGLANG_CI_RDMA_ALL_DEVICES csv (cuda only; empty = unset)"
        required: false
        type: string
        default: ""
      pr_head_sha:
        description: "PR head SHA to checkout (for /rerun-test on fork PRs)"
        required: false
        type: string
        default: ""
      reply_comment_id:
        description: "Reply comment ID to write back result to"
        required: false
        type: string
        default: ""
      reply_marker:
        description: "Per-batch marker for locating the line in reply comment"
        required: false
        type: string
        default: ""

env:
  SGLANG_IS_IN_CI: true
  SGLANG_ENABLE_ASYNC_ASSERT: true
  SGLANG_CUDA_COREDUMP: "1"
  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
  # TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.
  FORCE_REBUILD_DEEPEP: '1'

permissions:
  actions: write
  contents: read
  issues: read

jobs:
  rerun-test-cuda:
    if: inputs.mode == 'cuda'
    runs-on: ${{ inputs.runs_on }}
    timeout-minutes: 120
    permissions:
      contents: read
      issues: write
    env:
      RUNNER_LABELS: ${{ inputs.runs_on }}
      SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.rdma_devices }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.pr_head_sha || github.sha }}

      - name: Mark runner picked up
        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
        continue-on-error: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
            source /etc/profile.d/sglang-ci.sh
          fi
          python3 scripts/ci/utils/update_rerun_test_status.py \
            --comment-id "${{ inputs.reply_comment_id }}" \
            --marker "${{ inputs.reply_marker }}" \
            --status running \
            --repo "${{ github.repository }}"

      - uses: ./.github/actions/check-maintenance

      - name: Install dependencies
        timeout-minutes: ${{ fromJson(inputs.install_timeout) }}
        env:
          GRACE_BLACKWELL: ${{ inputs.grace_blackwell }}
        run: |
          if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
            source /etc/profile.d/sglang-ci.sh
          fi
          # Fail fast on an empty install_script. `bash <empty>` is a silent
          # no-op that "succeeds" without installing sglang, so the test step
          # then dies with `ModuleNotFoundError: No module named 'sglang'`.
          # /rerun-test resolves install_script from runner_configs.yml; a manual
          # workflow_dispatch must pass it explicitly (it can't be derived from
          # runs_on, which is shared across configs, e.g. default vs deepep).
          if [ -z "${{ inputs.install_script }}" ]; then
            echo "::error::install_script is required for cuda mode (empty would silently skip installing sglang). Pass e.g. scripts/ci/cuda/ci_install_dependency.sh"
            exit 1
          fi
          bash ${{ inputs.install_script }}

      - name: Run test
        timeout-minutes: 60
        run: |
          if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
            source /etc/profile.d/sglang-ci.sh
          fi
          cmds=()
          while IFS= read -r cmd; do
            [ -z "$cmd" ] && continue
            cmds+=("$cmd")
          done <<< "${{ inputs.test_command }}"
          total=${#cmds[@]}
          suite_start=$SECONDS
          for idx in "${!cmds[@]}"; do
            i=$((idx + 1))
            cmd="${cmds[$idx]}"
            echo ""
            echo "."
            echo "Begin ($i/$total): python3 $cmd"
            echo "."
            file_start=$SECONDS
            # base-a perf tests (fwd_occupancy) need async assert off; match
            # _pr-test-stage.yml's base-a carve-out.
            test_file="${cmd%% *}"
            test_file="${test_file%%::*}"
            async_assert=true
            if grep -qE 'register_cuda_ci\(.*stage[[:space:]]*=[[:space:]]*"base-a"' "test/$test_file" 2>/dev/null; then
              async_assert=false
            fi
            (cd test/ && SGLANG_ENABLE_ASYNC_ASSERT=$async_assert python3 $cmd -f) || exit 1
            elapsed=$(( SECONDS - file_start ))
            echo "."
            echo "End ($i/$total): elapsed=${elapsed}s"
            echo "."
            echo ""
          done
          total_elapsed=$(( SECONDS - suite_start ))
          echo "All $total test(s) passed in ${total_elapsed}s"

      - uses: ./.github/actions/upload-cuda-coredumps
        if: failure()

  rerun-test-multimodal-gen:
    if: inputs.mode == 'multimodal_gen'
    runs-on: ${{ inputs.runs_on }}
    timeout-minutes: 120
    permissions:
      contents: read
      issues: write
    env:
      RUNNER_LABELS: ${{ inputs.runs_on }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.pr_head_sha || github.sha }}

      - name: Mark runner picked up
        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
        continue-on-error: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python3 scripts/ci/utils/update_rerun_test_status.py \
            --comment-id "${{ inputs.reply_comment_id }}" \
            --marker "${{ inputs.reply_marker }}" \
            --status running \
            --repo "${{ github.repository }}"

      - uses: ./.github/actions/check-maintenance

      - name: Install dependencies (diffusion)
        timeout-minutes: 20
        run: bash scripts/ci/cuda/ci_install_dependency.sh diffusion

      - name: Run test
        timeout-minutes: 60
        run: |
          cmds=()
          while IFS= read -r cmd; do
            [ -z "$cmd" ] && continue
            cmds+=("$cmd")
          done <<< "${{ inputs.test_command }}"
          total=${#cmds[@]}
          suite_start=$SECONDS
          for idx in "${!cmds[@]}"; do
            i=$((idx + 1))
            cmd="${cmds[$idx]}"
            echo ""
            echo "."
            echo "Begin ($i/$total): python3 -m pytest $cmd -x"
            echo "."
            file_start=$SECONDS
            python3 -m pytest $cmd -x || exit 1
            elapsed=$(( SECONDS - file_start ))
            echo "."
            echo "End ($i/$total): elapsed=${elapsed}s"
            echo "."
            echo ""
          done
          total_elapsed=$(( SECONDS - suite_start ))
          echo "All $total test(s) passed in ${total_elapsed}s"

      - uses: ./.github/actions/upload-cuda-coredumps
        if: failure()

  rerun-test-cpu:
    if: inputs.mode == 'cpu'
    runs-on: ubuntu-latest
    timeout-minutes: 120
    permissions:
      contents: read
      issues: write
    steps:
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
          df -h

      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.pr_head_sha || github.sha }}

      - name: Mark runner picked up
        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
        continue-on-error: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python3 scripts/ci/utils/update_rerun_test_status.py \
            --comment-id "${{ inputs.reply_comment_id }}" \
            --marker "${{ inputs.reply_marker }}" \
            --status running \
            --repo "${{ github.repository }}"

      - uses: ./.github/actions/check-maintenance

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install uv
        uses: astral-sh/setup-uv@v5

      # Needed by setuptools-rust to build the bundled native gRPC extension
      # (rust/sglang-grpc) when installing the main `sglang` wheel from source.
      - name: Install protoc + Rust toolchain
        timeout-minutes: 10
        run: bash scripts/ci/utils/install_rust_protoc.sh

      - name: Install dependencies
        timeout-minutes: 20
        env:
          UV_SYSTEM_PYTHON: "1"
        run: |
          uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow

      - name: Run test
        timeout-minutes: 60
        run: |
          cd test/
          cmds=()
          while IFS= read -r cmd; do
            [ -z "$cmd" ] && continue
            cmds+=("$cmd")
          done <<< "${{ inputs.test_command }}"
          total=${#cmds[@]}
          suite_start=$SECONDS
          for idx in "${!cmds[@]}"; do
            i=$((idx + 1))
            cmd="${cmds[$idx]}"
            echo ""
            echo "."
            echo "Begin ($i/$total): python3 $cmd"
            echo "."
            file_start=$SECONDS
            python3 $cmd -f || exit 1
            elapsed=$(( SECONDS - file_start ))
            echo "."
            echo "End ($i/$total): elapsed=${elapsed}s"
            echo "."
            echo ""
          done
          total_elapsed=$(( SECONDS - suite_start ))
          echo "All $total test(s) passed in ${total_elapsed}s"

  write-back-result:
    needs: [rerun-test-cuda, rerun-test-multimodal-gen, rerun-test-cpu]
    if: always() && inputs.reply_comment_id != '' && inputs.reply_marker != ''
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    concurrency:
      group: rerun-test-writeback-${{ inputs.reply_comment_id }}
      cancel-in-progress: false
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Write back result to reply comment
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [[ "${{ needs.rerun-test-cuda.result }}" == "success" \
             || "${{ needs.rerun-test-multimodal-gen.result }}" == "success" \
             || "${{ needs.rerun-test-cpu.result }}" == "success" ]]; then
            STATUS=success
          else
            STATUS=failure
          fi
          python3 scripts/ci/utils/update_rerun_test_status.py \
            --comment-id "${{ inputs.reply_comment_id }}" \
            --marker "${{ inputs.reply_marker }}" \
            --status "$STATUS" \
            --repo "${{ github.repository }}"