chore: import upstream snapshot with attribution

python/sglang/test/server_fixtures/default_fixture.py

@@ -0,0 +1,73 @@
+import logging
+import time
+from contextlib import contextmanager
+
+import requests
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def openai_api_env(api_key: str):
+    """Context manager to set OpenAI API environment variables."""
+    import os
+
+    original_api_key = os.environ.get("OPENAI_API_KEY")
+    original_base_url = os.environ.get("OPENAI_API_BASE")
+
+    os.environ["OPENAI_API_KEY"] = api_key
+    os.environ["OPENAI_API_BASE"] = f"{DEFAULT_URL_FOR_TEST}/v1"
+
+    try:
+        yield
+    finally:
+        if original_api_key is not None:
+            os.environ["OPENAI_API_KEY"] = original_api_key
+        else:
+            del os.environ["OPENAI_API_KEY"]
+
+        if original_base_url is not None:
+            os.environ["OPENAI_API_BASE"] = original_base_url
+        else:
+            del os.environ["OPENAI_API_BASE"]
+
+
+class DefaultServerBase(CustomTestCase):
+    model = None
+    base_url = DEFAULT_URL_FOR_TEST
+    timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    other_args: list[str] = []
+
+    # For OpenAI API settings
+    api_key = "sk-123456"
+
+    @classmethod
+    def setUpClass(cls):
+        assert cls.model is not None, "Please set cls.model in subclass"
+
+        # Set OpenAI API key and base URL environment variables.
+        # Needed for lmm-evals to work.
+        with openai_api_env(cls.api_key):
+            cls.process = popen_launch_server(
+                cls.model,
+                cls.base_url,
+                timeout=cls.timeout,
+                other_args=cls.other_args,
+            )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid, wait_timeout=60)
+        time.sleep(2)
+
+    @classmethod
+    def flush_cache(cls):
+        requests.post(cls.base_url + "/flush_cache")

python/sglang/test/server_fixtures/disaggregation_fixture.py

@@ -0,0 +1,458 @@
+import io
+import logging
+import os
+import shlex
+import time
+import warnings
+from typing import ClassVar, Optional
+from urllib.parse import urlparse
+
+from sglang.srt.environ import envs
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    is_in_ci,
+    popen_launch_pd_server,
+    popen_with_error_check,
+    start_subprocess_fail_fast_watcher,
+)
+from sglang.utils import wait_for_http_ready
+
+logger = logging.getLogger(__name__)
+
+
+class PDDisaggregationServerBase(CustomTestCase):
+    capture_per_side_logs: ClassVar[bool] = False
+    extra_prefill_env: ClassVar[dict[str, str]] = {}
+    extra_decode_env: ClassVar[dict[str, str]] = {}
+    _prefill_stdout_buf: ClassVar[Optional[io.StringIO]] = None
+    _prefill_stderr_buf: ClassVar[Optional[io.StringIO]] = None
+    _decode_stdout_buf: ClassVar[Optional[io.StringIO]] = None
+    _decode_stderr_buf: ClassVar[Optional[io.StringIO]] = None
+
+    @classmethod
+    def setUpClass(cls):
+        os.environ["MC_TCP_ENABLE_CONNECTION_POOL"] = "true"
+        parsed_url = urlparse(DEFAULT_URL_FOR_TEST)
+        cls.base_host = parsed_url.hostname
+        base_port = str(parsed_url.port)
+        cls.lb_port = base_port
+        cls.prefill_port = f"{int(base_port) + 100}"
+        cls.decode_port = f"{int(base_port) + 200}"
+        cls.bootstrap_port = f"{int(base_port) + 500}"
+        cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}"
+        cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}"
+        cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}"
+        cls.base_url = cls.lb_url
+        print(
+            f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=} {cls.bootstrap_port=}"
+        )
+        cls.process_lb, cls.process_decode, cls.process_prefill = None, None, None
+        if cls.capture_per_side_logs:
+            cls._prefill_stdout_buf = io.StringIO()
+            cls._prefill_stderr_buf = io.StringIO()
+            cls._decode_stdout_buf = io.StringIO()
+            cls._decode_stderr_buf = io.StringIO()
+        cls._fail_fast_stop = None
+
+        # config transfer backend and rdma devices
+        cls._mc_gid_index_set = False
+        if is_in_ci():
+            cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"]
+            ib_devices = get_rdma_devices_args()
+            cls.rdma_devices = ["--disaggregation-ib-device", ib_devices]
+            cls._mc_gid_index_set = _maybe_set_roce_gid_index(ib_devices)
+        else:
+            cls.transfer_backend = [
+                "--disaggregation-transfer-backend",
+                envs.SGLANG_TEST_PD_DISAGG_BACKEND.get(),
+            ]
+            cls.rdma_devices = [
+                "--disaggregation-ib-device",
+                envs.SGLANG_TEST_PD_DISAGG_DEVICES.get(),
+            ]
+            if cls.rdma_devices[1] is None:
+                cls.rdma_devices = []
+                msg = "No RDMA devices specified for disaggregation test, using default settings."
+                warnings.warn(msg)
+
+    # Subclasses can set these to customize server args
+    extra_prefill_args = []
+    extra_decode_args = []
+
+    @classmethod
+    def start_prefill(cls):
+        prefill_args = [
+            "--trust-remote-code",
+            "--disaggregation-mode",
+            "prefill",
+            "--disaggregation-bootstrap-port",
+            cls.bootstrap_port,
+            "--tp",
+            "1",
+        ] + list(cls.extra_prefill_args)
+        prefill_args += cls.transfer_backend + cls.rdma_devices
+        cls.process_prefill = popen_launch_pd_server(
+            cls.model,
+            cls.prefill_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=prefill_args,
+            env=dict(cls.extra_prefill_env),
+            return_stdout_stderr=(
+                (cls._prefill_stdout_buf, cls._prefill_stderr_buf)
+                if cls.capture_per_side_logs
+                else None
+            ),
+        )
+
+    @classmethod
+    def start_decode(cls):
+        decode_args = [
+            "--trust-remote-code",
+            "--disaggregation-mode",
+            "decode",
+            "--disaggregation-bootstrap-port",
+            cls.bootstrap_port,
+            "--tp",
+            "1",
+            "--base-gpu-id",
+            "1",
+        ] + list(cls.extra_decode_args)
+        decode_args += cls.transfer_backend + cls.rdma_devices
+        cls.process_decode = popen_launch_pd_server(
+            cls.model,
+            cls.decode_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=decode_args,
+            env=dict(cls.extra_decode_env),
+            return_stdout_stderr=(
+                (cls._decode_stdout_buf, cls._decode_stderr_buf)
+                if cls.capture_per_side_logs
+                else None
+            ),
+        )
+
+    @classmethod
+    def launch_all(cls):
+        """Start prefill, decode, wait for health, and launch LB."""
+        cls.start_prefill()
+        cls.start_decode()
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
+        cls.launch_lb()
+        cls._fail_fast_stop = start_subprocess_fail_fast_watcher(
+            [
+                ("prefill", cls.process_prefill),
+                ("decode", cls.process_decode),
+                ("lb", cls.process_lb),
+            ]
+        )
+
+    @classmethod
+    def launch_lb(cls):
+        lb_command = [
+            "python3",
+            "-m",
+            "sglang_router.launch_router",
+            "--pd-disaggregation",
+            "--mini-lb",  # FIXME: remove this
+            "--prefill",
+            cls.prefill_url,
+            "--decode",
+            cls.decode_url,
+            "--host",
+            cls.base_host,
+            "--port",
+            cls.lb_port,
+        ]
+        print("Starting load balancer:", shlex.join(lb_command))
+        cls.process_lb = popen_with_error_check(lb_command)
+        cls.wait_server_ready(cls.lb_url + "/health", process=cls.process_lb)
+
+    @classmethod
+    def wait_server_ready(
+        cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, process=None
+    ):
+        wait_for_http_ready(url=url, timeout=timeout, process=process)
+        print(f"Server {url} is ready")
+
+    @classmethod
+    def tearDownClass(cls):
+        # Stop the watcher BEFORE killing processes: kill_process_tree
+        # below makes them exit with a negative signal rc, which would
+        # otherwise trip the watcher and os._exit out of pytest mid-teardown.
+        if cls._fail_fast_stop is not None:
+            cls._fail_fast_stop.set()
+        os.environ.pop("MC_TCP_ENABLE_CONNECTION_POOL")
+        if getattr(cls, "_mc_gid_index_set", False):
+            os.environ.pop("MC_GID_INDEX", None)
+        for process in [cls.process_lb, cls.process_decode, cls.process_prefill]:
+            if process:
+                try:
+                    kill_process_tree(process.pid, wait_timeout=60)
+                except Exception as e:
+                    print(f"Error killing process {process.pid}: {e}")
+
+        if cls.capture_per_side_logs:
+            for buf in (
+                cls._prefill_stdout_buf,
+                cls._prefill_stderr_buf,
+                cls._decode_stdout_buf,
+                cls._decode_stderr_buf,
+            ):
+                if buf is not None:
+                    buf.close()
+            cls._prefill_stdout_buf = None
+            cls._prefill_stderr_buf = None
+            cls._decode_stdout_buf = None
+            cls._decode_stderr_buf = None
+
+        # wait for 5 seconds
+        time.sleep(5)
+
+
+def _get_available_ib_devices():
+    """Auto-detect available high-speed RDMA devices from sysfs.
+
+    Filters for devices that are:
+    1. Not Ethernet NICs (excludes devices with 'eth' in the name like mlx5_eth0)
+    2. Active (port state)
+    3. High-speed (rate >= 100 Gbps to exclude regular Ethernet NICs)
+    """
+    ib_sysfs_path = "/sys/class/infiniband"
+    if not os.path.isdir(ib_sysfs_path):
+        logger.warning("IB sysfs path %s does not exist", ib_sysfs_path)
+        return None
+
+    all_devices = sorted(os.listdir(ib_sysfs_path))
+    logger.warning("All IB devices in sysfs: %s", all_devices)
+
+    devices = []
+    for dev in all_devices:
+        # Check port 1 state and rate (most devices have single port)
+        port_path = os.path.join(ib_sysfs_path, dev, "ports", "1")
+        if not os.path.isdir(port_path):
+            logger.warning("Device %s: SKIPPED (no port 1)", dev)
+            continue
+
+        # Read state and rate for logging
+        state = "unknown"
+        rate = -1
+        state_file = os.path.join(port_path, "state")
+        rate_file = os.path.join(port_path, "rate")
+
+        try:
+            with open(state_file) as f:
+                state = f.read().strip()
+        except (OSError, IOError):
+            pass
+
+        try:
+            with open(rate_file) as f:
+                rate_str = f.read().strip()
+                rate = int(rate_str.split()[0])
+        except (OSError, IOError, ValueError, IndexError):
+            pass
+
+        # Log device properties for debugging
+        logger.warning(
+            "Device %s: state=%s, rate=%d Gbps, has_eth_in_name=%s",
+            dev,
+            state,
+            rate,
+            "eth" in dev.lower(),
+        )
+
+        # Skip devices with "eth" in the name - these are typically Ethernet NICs
+        # that don't work properly with RDMA (e.g., mlx5_eth0)
+        if "eth" in dev.lower():
+            logger.warning("Device %s: SKIPPED (contains 'eth' in name)", dev)
+            continue
+
+        # Check if port is active
+        # State format is like "4: ACTIVE" or just "ACTIVE"
+        if "ACTIVE" not in state.upper():
+            logger.warning("Device %s: SKIPPED (state=%s)", dev, state)
+            continue
+
+        # Check rate (filter out low-speed NICs like 10/25 Gbps Ethernet)
+        if rate >= 0 and rate < 100:  # Skip devices slower than 100 Gbps
+            logger.warning("Device %s: SKIPPED (rate=%d Gbps)", dev, rate)
+            continue
+
+        devices.append(dev)
+        logger.warning("Device %s: INCLUDED", dev)
+
+    logger.warning("Filtered IB devices: %s (count=%d)", devices, len(devices))
+    return devices if devices else None
+
+
+def get_rdma_devices_args():
+    def _parse_list_env(var_name: str):
+        val = os.getenv(var_name)
+        if not val:
+            return None
+        items = [x.strip() for x in val.split(",") if x.strip()]
+        return items or None
+
+    def _pick_default_pair(rdma_all_devices):
+        return [rdma_all_devices[0], rdma_all_devices[len(rdma_all_devices) // 2]]
+
+    # Priority: env var > auto-detect > hardcoded fallback
+    rdma_all_devices = (
+        _parse_list_env("SGLANG_CI_RDMA_ALL_DEVICES")
+        or _get_available_ib_devices()
+        or [f"mlx5_roce{i}" for i in range(8)]
+    )
+    logger.warning("Resolved rdma_all_devices=%s", rdma_all_devices)
+
+    n_rdma = len(rdma_all_devices)
+
+    # 1. Get visible GPU indices
+    cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES")
+    if not cuda_visible_devices:
+        warnings.warn("CUDA_VISIBLE_DEVICES is not set. Using default RDMA devices.")
+        return ",".join(_pick_default_pair(rdma_all_devices))
+
+    try:
+        # Convert to list of integers (handling possible spaces and empty strings)
+        gpu_indices = [
+            int(idx.strip()) for idx in cuda_visible_devices.split(",") if idx.strip()
+        ]
+        if not gpu_indices or len(gpu_indices) > 4:
+            return ",".join(_pick_default_pair(rdma_all_devices))
+    except ValueError:
+        warnings.warn(f"Invalid CUDA_VISIBLE_DEVICES format: {cuda_visible_devices}")
+        return ",".join(_pick_default_pair(rdma_all_devices))
+
+    # 2. Calculate base RDMA index group (each group of 4 GPUs uses consecutive devices)
+    base_rdma_group = (min(gpu_indices) // 4) * 4
+    for gpu_idx in gpu_indices:
+        if not (base_rdma_group <= gpu_idx < base_rdma_group + 4):
+            warnings.warn(
+                f"GPU index {gpu_idx} is outside expected group "
+                f"{base_rdma_group}-{base_rdma_group+3}"
+            )
+
+    # 3. Generate RDMA device names
+    # Detect total GPUs on the node (not just visible ones)
+    try:
+        import torch
+
+        total_gpus = torch.cuda.device_count()
+    except Exception:
+        total_gpus = 8  # Fallback to common 8-GPU setup
+
+    # Handle edge cases
+    if total_gpus == 0:
+        total_gpus = 8
+    if n_rdma > total_gpus:
+        logger.warning(
+            "More RDMA devices (%d) than GPUs (%d), using first and middle device",
+            n_rdma,
+            total_gpus,
+        )
+        return ",".join(_pick_default_pair(rdma_all_devices))
+
+    # Calculate how many GPUs share each RDMA device
+    gpus_per_rdma = max(1, total_gpus // n_rdma)
+    logger.warning(
+        "GPU-to-RDMA mapping: total_gpus=%d, n_rdma=%d, gpus_per_rdma=%d",
+        total_gpus,
+        n_rdma,
+        gpus_per_rdma,
+    )
+
+    rdma_devices = []
+    base_gpu = min(gpu_indices)
+    for gpu_idx in gpu_indices:
+        nic_index = min((gpu_idx - base_gpu) // gpus_per_rdma, n_rdma - 1)
+        rdma_devices.append(rdma_all_devices[nic_index])
+
+    if not rdma_devices:
+        return ",".join(_pick_default_pair(rdma_all_devices))
+
+    # Deduplicate while preserving order
+    return ",".join(dict.fromkeys(rdma_devices))
+
+
+_IB_SYSFS = "/sys/class/infiniband"
+
+
+def _roce_v2_gid_index(device: str):
+    """Return a RoCEv2 GID index for a device, preferring a global (routable)
+    GID over a link-local (fe80::) one, or None if the device has no RoCEv2 GID.
+    """
+    port = os.path.join(_IB_SYSFS, device, "ports", "1")
+    types_dir = os.path.join(port, "gid_attrs", "types")
+    try:
+        indices = sorted(int(x) for x in os.listdir(types_dir) if x.isdigit())
+    except OSError:
+        return None
+    fallback = None
+    for i in indices:
+        try:
+            with open(os.path.join(types_dir, str(i))) as f:
+                if f.read().strip() != "RoCE v2":
+                    continue
+        except OSError:
+            continue
+        if fallback is None:
+            fallback = i
+        try:
+            with open(os.path.join(port, "gids", str(i))) as f:
+                gid = f.read().strip()
+        except OSError:
+            gid = ""
+        # Prefer a global GID; link-local (fe80::) entries don't route between
+        # NICs on some fabrics.
+        if gid and not gid.lower().startswith("fe80"):
+            return i
+    return fallback
+
+
+def _detect_roce_gid_index(devices):
+    """Return a single RoCEv2 GID index shared by all `devices`, or None.
+
+    None when any device is InfiniBand (mooncake selects the GID automatically
+    there), when a device has no RoCEv2 GID, or when devices disagree on the
+    index — MC_GID_INDEX is a single global value, so a divergent set can't be
+    satisfied and is left to mooncake's own selection.
+    """
+    picked = None
+    for device in [d.strip() for d in devices if d.strip()]:
+        try:
+            with open(os.path.join(_IB_SYSFS, device, "ports", "1", "link_layer")) as f:
+                if f.read().strip() != "Ethernet":
+                    return None
+        except OSError:
+            return None
+        idx = _roce_v2_gid_index(device)
+        if idx is None:
+            return None
+        if picked is None:
+            picked = idx
+        elif picked != idx:
+            return None
+    return picked
+
+
+def _maybe_set_roce_gid_index(ib_devices) -> bool:
+    """Export MC_GID_INDEX for a RoCE fabric; return True if this call set it.
+
+    On RoCE-only hosts mooncake's automatic GID selection can come up empty
+    ("GID is NULL, please check your GID index by specifying MC_GID_INDEX"),
+    leaving the KV-transfer RDMA endpoint with no GID so every prefill->decode
+    transfer fails and PD accuracy collapses to 0. InfiniBand hosts don't need
+    this (auto GID works), and a user-provided MC_GID_INDEX is left untouched.
+    """
+    if not ib_devices or os.environ.get("MC_GID_INDEX"):
+        return False
+    gid_index = _detect_roce_gid_index(ib_devices.split(","))
+    if gid_index is None:
+        return False
+    os.environ["MC_GID_INDEX"] = str(gid_index)
+    logger.warning("RoCE fabric detected; set MC_GID_INDEX=%d for mooncake", gid_index)
+    return True

python/sglang/test/server_fixtures/dsa_mtp_fixture.py

@@ -0,0 +1,98 @@
+"""DSA model + MTP (EAGLE) speculative-decoding server fixture.
+
+Variants combine `DsaMtpServerBase` (server lifecycle) with
+`DsaMtpEvalConfigDefaults` (shared eval thresholds/params),
+`GSM8KMixin` and `SpecDecodingMixin`, then set `model` and per-variant
+overrides (`enable_dp_attention`, `mem_fraction_static`, `bs_1_speed_thres`).
+
+Example:
+    class TestDsv32DP(
+        DsaMtpServerBase,
+        DsaMtpEvalConfigDefaults,
+        GSM8KMixin,
+        SpecDecodingMixin,
+    ):
+        model = "deepseek-ai/DeepSeek-V3.2"
+        enable_dp_attention = True
+        bs_1_speed_thres = 90
+
+The base itself is NOT a runnable test (no `test_*` methods until a subclass
+mixes in the kits), so unittest discovery picks it up as empty.
+"""
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+
+class DsaMtpEvalConfigDefaults:
+    """Eval thresholds & params shared across DSA-MTP regression variants."""
+
+    # GSM8KMixin defaults.
+    gsm8k_accuracy_thres = 0.935
+    gsm8k_accept_length_thres = 3.7
+    gsm8k_num_questions = 500
+    gsm8k_num_threads = 500
+    gsm8k_num_shots = 20
+
+    # SpecDecodingMixin default; per-variant subclasses set `bs_1_speed_thres`.
+    accept_length_thres = 4.0
+
+
+class DsaMtpServerBase(CustomTestCase):
+    base_url = DEFAULT_URL_FOR_TEST
+
+    # Subclasses must set `model`; the others have sensible defaults.
+    model: str = ""
+    tp_size: int = 8
+    dp_size: int = 8
+    mem_fraction_static: float = 0.7
+    enable_dp_attention: bool = False
+    extra_server_args = ()
+
+    # EAGLE MTP config (fixed across DSA-MTP variants).
+    speculative_algorithm: str = "EAGLE"
+    speculative_num_steps: int = 5
+    speculative_eagle_topk: int = 1
+    speculative_num_draft_tokens: int = 6
+
+    @classmethod
+    def get_server_args(cls):
+        assert cls.model, f"{cls.__name__} must set `model`"
+        args = ["--trust-remote-code", "--tp", str(cls.tp_size)]
+        if cls.enable_dp_attention:
+            args += ["--dp", str(cls.dp_size), "--enable-dp-attention"]
+        args += [
+            "--speculative-algorithm",
+            cls.speculative_algorithm,
+            "--speculative-num-steps",
+            str(cls.speculative_num_steps),
+            "--speculative-eagle-topk",
+            str(cls.speculative_eagle_topk),
+            "--speculative-num-draft-tokens",
+            str(cls.speculative_num_draft_tokens),
+            "--mem-frac",
+            str(cls.mem_fraction_static),
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true, "num_threads": 64}',
+        ]
+        args += list(cls.extra_server_args)
+        return args
+
+    @classmethod
+    def setUpClass(cls):
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=cls.get_server_args(),
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        if hasattr(cls, "process") and cls.process:
+            kill_process_tree(cls.process.pid)

python/sglang/test/server_fixtures/eagle_fixture.py

@@ -0,0 +1,115 @@
+import json
+import random
+import time
+
+import requests
+
+from sglang.srt.utils.common import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_DRAFT_MODEL_EAGLE,
+    DEFAULT_TARGET_MODEL_EAGLE,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+PROMPTS = [
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nToday is a sunny day and I like[/INST]"
+    '[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nWhat are the mental triggers in Jeff Walker\'s Product Launch Formula and "Launch" book?[/INST]',
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nSummarize Russell Brunson's Perfect Webinar Script...[/INST]",
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nwho are you?[/INST]",
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nwhere are you from?[/INST]",
+]
+
+
+class EagleServerBase(CustomTestCase):
+    target_model = DEFAULT_TARGET_MODEL_EAGLE
+    draft_model = DEFAULT_DRAFT_MODEL_EAGLE
+    spec_algo = "EAGLE"
+    spec_steps = 5
+    spec_topk = 8
+    spec_tokens = 64
+    mem_fraction_static = 0.7
+    extra_args = []
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.process = popen_launch_server(
+            cls.target_model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=[
+                f"--speculative-algorithm={cls.spec_algo}",
+                f"--speculative-draft-model-path={cls.draft_model}",
+                f"--speculative-num-steps={cls.spec_steps}",
+                f"--speculative-eagle-topk={cls.spec_topk}",
+                f"--speculative-num-draft-tokens={cls.spec_tokens}",
+                f"--mem-fraction-static={cls.mem_fraction_static}",
+            ]
+            + cls.extra_args,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid, wait_timeout=60)
+
+    def send_request(self):
+        time.sleep(random.uniform(0, 2))
+        for prompt in PROMPTS:
+            url = self.base_url + "/generate"
+            data = {
+                "text": prompt,
+                "sampling_params": {
+                    "temperature": 0,
+                    "max_new_tokens": 1024,
+                },
+            }
+            response = requests.post(url, json=data)
+            assert response.status_code == 200
+
+    def send_requests_abort(self):
+        for prompt in PROMPTS:
+            try:
+                time.sleep(random.uniform(0, 2))
+                url = self.base_url + "/generate"
+                data = {
+                    "model": "base",
+                    "text": prompt,
+                    "sampling_params": {
+                        "temperature": 0,
+                        "max_new_tokens": 1024,
+                    },
+                }
+                # set timeout = 1s, mock disconnected
+                requests.post(url, json=data, timeout=1)
+            except Exception as e:
+                print(e)
+                pass
+
+    def run_decode(self, sampling_params):
+        return_logprob = True
+        top_logprobs_num = 5
+        return_text = True
+        n = 1
+
+        response = requests.post(
+            self.base_url + "/generate",
+            json={
+                "text": "Human: Write a travel blog post to Hawaii.\n\nAssistant:",
+                "sampling_params": {
+                    "max_new_tokens": 48,
+                    "n": n,
+                    "temperature": 0.7,
+                    **sampling_params,
+                },
+                "return_logprob": return_logprob,
+                "top_logprobs_num": top_logprobs_num,
+                "return_text_in_logprobs": return_text,
+                "logprob_start_len": 0,
+            },
+        )
+        self.assertEqual(response.status_code, 200)
+        print(json.dumps(response.json()))
+        print("=" * 100)

python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py

@@ -0,0 +1,102 @@
+"""Hybrid attention-backend (FA3 prefill + FlashInfer decode) test fixture.
+
+Variants combine `TestHybridAttnBackendBase` with their own
+`get_server_args()` / `accuracy_threshold` / `speculative_decode` knobs.
+
+Requires SM 90+ (H100); the base class wraps that in a `skipIf`.
+"""
+
+import unittest
+from types import SimpleNamespace
+
+import requests
+
+from sglang.srt.environ import envs
+from sglang.srt.utils import get_device_sm, kill_process_tree
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_MODEL_NAME_FOR_TEST,
+    DEFAULT_TARGET_MODEL_EAGLE,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+GSM_DATASET_PATH = None
+
+# Default server arguments shared across all hybrid-attn-backend tests
+DEFAULT_HYBRID_ATTN_SERVER_ARGS = [
+    "--trust-remote-code",
+    "--cuda-graph-max-bs-decode",
+    "8",
+    "--prefill-attention-backend",
+    "fa3",
+    "--decode-attention-backend",
+    "flashinfer",
+]
+
+
+@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher")
+class TestHybridAttnBackendBase(CustomTestCase):
+
+    model = DEFAULT_MODEL_NAME_FOR_TEST
+    base_url = DEFAULT_URL_FOR_TEST
+    accuracy_threshold = 0.65  # derived tests need to override this
+    speculative_decode = False
+    spec_decode_threshold = 2.2  # derived spec decoding tests need to override this
+    # Appended after DEFAULT_HYBRID_ATTN_SERVER_ARGS in get_server_args.
+    extra_args: list = []
+
+    @classmethod
+    def get_server_args(cls):
+        return DEFAULT_HYBRID_ATTN_SERVER_ARGS + list(cls.extra_args)
+
+    @classmethod
+    def setUpClass(cls):
+        # disable deep gemm precompile to make launch server faster
+        # please don't do this if you want to make your inference workload faster
+        with (
+            envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.override(False),
+            envs.SGLANG_ENABLE_JIT_DEEPGEMM.override(False),
+        ):
+            if cls.speculative_decode:
+                model = DEFAULT_TARGET_MODEL_EAGLE
+            else:
+                model = cls.model
+            cls.process = popen_launch_server(
+                model,
+                cls.base_url,
+                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                other_args=cls.get_server_args(),
+            )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        requests.get(self.base_url + "/flush_cache")
+
+        model = DEFAULT_TARGET_MODEL_EAGLE if self.speculative_decode else self.model
+        args = SimpleNamespace(
+            base_url=self.base_url,
+            model=model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
+        )
+        metrics = run_eval(args)
+        print(f"{metrics=}")
+
+        self.assertGreater(metrics["score"], self.accuracy_threshold)
+
+        if self.speculative_decode:
+            server_info = requests.get(self.base_url + "/server_info")
+            avg_spec_accept_length = server_info.json()["internal_states"][0][
+                "avg_spec_accept_length"
+            ]
+            print(f"{avg_spec_accept_length=}")
+            self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)

python/sglang/test/server_fixtures/mmmu_fixture.py

@@ -0,0 +1,71 @@
+import logging
+import os
+import time
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+logger = logging.getLogger(__name__)
+
+# Set default mem_fraction_static to 0.8
+DEFAULT_MEM_FRACTION_STATIC = 0.8
+
+
+class MMMUServerBase(CustomTestCase):
+    """Server fixture for MMMU VLM tests.
+
+    This fixture handles server lifecycle for single-model MMMU tests.
+    For multi-model tests that need to start/stop servers within test methods,
+    use MMMUMultiModelTestBase instead.
+    Set server_api_key = None to launch without auth when sharing the server
+    with mixins whose clients do not send API keys.
+    """
+
+    model = None
+    base_url = DEFAULT_URL_FOR_TEST
+    timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    other_args: list[str] = []
+    mem_fraction_static: float = DEFAULT_MEM_FRACTION_STATIC
+    server_api_key = "sk-123456"
+
+    @classmethod
+    def setUpClass(cls):
+        assert cls.model is not None, "Please set cls.model in subclass"
+
+        # Prepare environment variables
+        process_env = os.environ.copy()
+        process_env["SGLANG_USE_CUDA_IPC_TRANSPORT"] = "1"
+
+        # Build server args with MMMU-specific settings
+        server_args = [
+            "--trust-remote-code",
+            "--cuda-graph-max-bs-decode",
+            "64",
+            "--enable-multimodal",
+            "--mem-fraction-static",
+            str(cls.mem_fraction_static),
+            *cls.other_args,
+        ]
+
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=cls.timeout,
+            api_key=cls.server_api_key,
+            other_args=server_args,
+            env=process_env,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.process is not None and cls.process.poll() is None:
+            try:
+                kill_process_tree(cls.process.pid, wait_timeout=60)
+            except Exception as e:
+                logger.error(f"Error killing process: {e}")
+        time.sleep(2)

python/sglang/test/server_fixtures/ngram_fixture.py

@@ -0,0 +1,74 @@
+"""NGRAM speculative-decoding server fixture.
+
+Variants combine this base with `GSM8KMixin` and override `attention_backend`
+(required) plus optional `extra_args` to select a backend / pass extra flags.
+
+Example:
+    from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
+    from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
+
+    class TestNgramSpeculativeDecodingTriton(NgramServerBase, GSM8KMixin):
+        attention_backend = "triton"
+
+The base itself is NOT a runnable test (no `test_*` methods until a subclass
+mixes in GSM8KMixin), so unittest discovery picks it up as empty.
+"""
+
+from sglang.srt.environ import envs
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TARGET_MODEL_NGRAM,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+DEFAULT_NGRAM_SERVER_ARGS = [
+    "--trust-remote-code",
+    "--cuda-graph-max-bs-decode",
+    "8",
+    "--speculative-algorithm",
+    "NGRAM",
+    "--speculative-num-draft-tokens",
+    "16",
+    "--mem-fraction-static",
+    0.8,
+]
+
+
+class NgramServerBase(CustomTestCase):
+    model = DEFAULT_TARGET_MODEL_NGRAM
+    base_url = DEFAULT_URL_FOR_TEST
+    gsm8k_accuracy_thres = 0.79
+    gsm8k_accept_length_thres = 1.8
+
+    # Subclasses must set `attention_backend`; `extra_args` is optional.
+    attention_backend: str = ""
+    extra_args: list = []
+
+    @classmethod
+    def get_server_args(cls):
+        assert cls.attention_backend, f"{cls.__name__} must set `attention_backend`"
+        return (
+            DEFAULT_NGRAM_SERVER_ARGS
+            + ["--attention-backend", cls.attention_backend]
+            + list(cls.extra_args)
+        )
+
+    @classmethod
+    def setUpClass(cls):
+        # disable deep gemm precompile to make launch server faster
+        # please don't do this if you want to make your inference workload faster
+        envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
+        envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=cls.get_server_args(),
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)

python/sglang/test/server_fixtures/pcg_spec_fixture.py

@@ -0,0 +1,80 @@
+"""Piecewise CUDA Graph + speculative decoding test fixture.
+
+Each variant tests PCG coexisting with one speculative-decoding algorithm
+(EAGLE3 / NEXTN / STANDALONE / NGRAM). Variants differ widely on model /
+server args / thresholds, so the base only abstracts the common shape:
+  - launch a server with `server_args` (variant-supplied list)
+  - run gsm8k, assert `score > accuracy_threshold`
+  - read `avg_spec_accept_length` from /server_info, assert
+    `> speedup_threshold`
+
+Pure mixin (does NOT inherit `TestCase`), so unittest does not collect
+the base itself.
+"""
+
+from types import SimpleNamespace
+
+import requests
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    popen_launch_server,
+)
+
+
+class PCGSpecBase:
+    # Subclasses must set:
+    model: str = ""
+    server_args: list = []
+
+    # Optional knobs (variant defaults override):
+    timeout_mult: int = 2
+    server_env: dict = None  # passed to popen_launch_server `env=...`
+    accuracy_threshold: float = 0.70
+    speedup_threshold: float = 1.5
+    max_tokens: int = 512
+    thinking_mode: str = ""  # set to e.g. "qwen3" if needed
+
+    @classmethod
+    def setUpClass(cls):
+        assert (
+            cls.model and cls.server_args
+        ), f"{cls.__name__} must set `model` and `server_args`"
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        kwargs = dict(
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * cls.timeout_mult,
+            other_args=cls.server_args,
+        )
+        if cls.server_env:
+            kwargs["env"] = cls.server_env
+        cls.process = popen_launch_server(cls.model, cls.base_url, **kwargs)
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        eval_kwargs = dict(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            max_tokens=self.max_tokens,
+            num_examples=200,
+            num_threads=200,
+        )
+        if self.thinking_mode:
+            eval_kwargs["thinking_mode"] = self.thinking_mode
+        args = SimpleNamespace(**eval_kwargs)
+        metrics = run_eval(args)
+        print(metrics)
+        self.assertGreater(metrics["score"], self.accuracy_threshold)
+
+        server_info = requests.get(self.base_url + "/server_info").json()
+        avg_spec_accept_length = server_info["internal_states"][0][
+            "avg_spec_accept_length"
+        ]
+        print(f"{avg_spec_accept_length=}")
+        self.assertGreater(avg_spec_accept_length, self.speedup_threshold)

python/sglang/test/server_fixtures/spec_eagle_fixture.py

@@ -0,0 +1,243 @@
+"""Unified EAGLE/EAGLE3 speculative-decoding server fixture.
+
+A single popen-server base whose launch is fully described by class attributes,
+so concrete test classes only flip knobs (overlap on/off, model, topk, page size,
+backend, env overrides). Pair it with the kits in
+``sglang.test.kits.spec_server_kits`` to assemble test classes.
+
+The primary axis is ``disable_overlap``:
+  - ``False`` -> overlap scheduler
+  - ``True``  -> synchronous (non-overlap) scheduling
+"""
+
+import contextlib
+import random
+import time
+
+import requests
+
+from sglang.srt.environ import envs
+from sglang.srt.utils.common import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
+from sglang.test.test_utils import (
+    DEFAULT_DRAFT_MODEL_EAGLE,
+    DEFAULT_DRAFT_MODEL_EAGLE3,
+    DEFAULT_TARGET_MODEL_EAGLE,
+    DEFAULT_TARGET_MODEL_EAGLE3,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+# Chat-style prompts shared by send_request / send_requests_abort.
+PROMPTS = [
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nToday is a sunny day and I like[/INST]"
+    '[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nWhat are the mental triggers in Jeff Walker\'s Product Launch Formula and "Launch" book?[/INST]',
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nSummarize Russell Brunson's Perfect Webinar Script...[/INST]",
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nwho are you?[/INST]",
+    "[INST] <<SYS>>\\nYou are a helpful assistant.\\n<</SYS>>\\nwhere are you from?[/INST]",
+]
+
+
+class SpecEagleServerBase(CustomTestCase):
+    """Launch a single EAGLE/EAGLE3 spec server from class-attribute knobs."""
+
+    # -- model --
+    model = DEFAULT_TARGET_MODEL_EAGLE3
+    draft_model = DEFAULT_DRAFT_MODEL_EAGLE3
+    spec_algo = "EAGLE3"
+
+    # -- speculative config --
+    spec_steps = 5
+    spec_topk = 1
+    spec_tokens = 6
+
+    # -- runtime config --
+    page_size = 1
+    attention_backend = "flashinfer"
+    # Primary axis: False -> overlap scheduler; True -> synchronous (non-overlap).
+    disable_overlap = False
+    mem_fraction_static = 0.85
+    max_running_requests = 8
+    chunked_prefill_size = 128
+    # bf16 rather than fp16: fp16 activations can overflow (-> Inf -> NaN) on
+    # degenerate draft branches in verify and trip the CI NaN asserts.
+    dtype = "bfloat16"
+    cuda_graph_max_bs_decode = None
+    trust_remote_code = True
+    # Launch with --enable-return-hidden-states so SpecHiddenStatesKit can probe
+    # per-request hidden states; per-request gated, so other requests don't pay.
+    enable_return_hidden_states = False
+
+    # -- extras --
+    # env_overrides: (env_var_obj, value) pairs applied only around launch.
+    # Declare ONLY this class's own; _merged_env_overrides() unions them down the
+    # MRO (base first), so never restate a base's. Derived wins on a repeated env.
+    env_overrides = ()
+    extra_args = ()
+
+    @classmethod
+    def _launch_args(cls):
+        args = [
+            "--speculative-algorithm",
+            cls.spec_algo,
+            "--speculative-draft-model-path",
+            cls.draft_model,
+            "--speculative-num-steps",
+            str(cls.spec_steps),
+            "--speculative-eagle-topk",
+            str(cls.spec_topk),
+            "--speculative-num-draft-tokens",
+            str(cls.spec_tokens),
+            "--page-size",
+            str(cls.page_size),
+            "--attention-backend",
+            cls.attention_backend,
+            "--mem-fraction-static",
+            str(cls.mem_fraction_static),
+            "--max-running-requests",
+            str(cls.max_running_requests),
+            "--chunked-prefill-size",
+            str(cls.chunked_prefill_size),
+            "--dtype",
+            cls.dtype,
+        ]
+        if cls.disable_overlap:
+            args.append("--disable-overlap-schedule")
+        if cls.trust_remote_code:
+            args.append("--trust-remote-code")
+        if cls.enable_return_hidden_states:
+            args.append("--enable-return-hidden-states")
+        if cls.cuda_graph_max_bs_decode is not None:
+            args += ["--cuda-graph-max-bs-decode", str(cls.cuda_graph_max_bs_decode)]
+        args += [str(a) for a in cls.extra_args]
+        return args
+
+    @classmethod
+    def _merged_env_overrides(cls):
+        # Base first so a derived class wins for a repeated env var.
+        merged = []
+        for klass in reversed(cls.__mro__):
+            merged.extend(klass.__dict__.get("env_overrides", ()))
+        return merged
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        # Alias so kit methods can use either name.
+        cls.target_model = cls.model
+        cls._tokenizer = None
+        with contextlib.ExitStack() as stack:
+            stack.enter_context(
+                envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True)
+            )
+            for env_var, value in cls._merged_env_overrides():
+                stack.enter_context(env_var.override(value))
+            cls.process = popen_launch_server(
+                cls.model,
+                cls.base_url,
+                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                other_args=cls._launch_args(),
+            )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid, wait_timeout=60)
+
+    @property
+    def tokenizer(self):
+        if type(self)._tokenizer is None:
+            type(self)._tokenizer = get_tokenizer(self.model)
+        return type(self)._tokenizer
+
+    # -- helpers used by kits --
+
+    def send_request(self):
+        time.sleep(random.uniform(0, 2))
+        for prompt in PROMPTS:
+            url = self.base_url + "/generate"
+            data = {
+                "text": prompt,
+                "sampling_params": {
+                    "temperature": 0,
+                    "max_new_tokens": 1024,
+                },
+            }
+            response = requests.post(url, json=data)
+            assert response.status_code == 200
+
+    def send_requests_abort(self):
+        for prompt in PROMPTS:
+            try:
+                time.sleep(random.uniform(0, 2))
+                url = self.base_url + "/generate"
+                data = {
+                    "model": "base",
+                    "text": prompt,
+                    "sampling_params": {
+                        "temperature": 0,
+                        "max_new_tokens": 1024,
+                    },
+                }
+                # set timeout = 1s, mock disconnected
+                requests.post(url, json=data, timeout=1)
+            except Exception as e:
+                print(e)
+                pass
+
+    def run_decode(self, sampling_params):
+        response = requests.post(
+            self.base_url + "/generate",
+            json={
+                "text": "Human: Write a travel blog post to Hawaii.\n\nAssistant:",
+                "sampling_params": {
+                    "max_new_tokens": 48,
+                    "n": 1,
+                    "temperature": 0.7,
+                    **sampling_params,
+                },
+                "return_logprob": True,
+                "top_logprobs_num": 5,
+                "return_text_in_logprobs": True,
+                "logprob_start_len": 0,
+            },
+        )
+        self.assertEqual(response.status_code, 200)
+
+
+class Eagle3Base(SpecEagleServerBase):
+    """EAGLE3 (Llama-3.1) config preset, topk=1 / page_size=1 by default."""
+
+    model = DEFAULT_TARGET_MODEL_EAGLE3
+    draft_model = DEFAULT_DRAFT_MODEL_EAGLE3
+    spec_algo = "EAGLE3"
+    spec_steps = 5
+    spec_topk = 1
+    spec_tokens = 6
+    attention_backend = "flashinfer"
+    chunked_prefill_size = 1024
+    # EAGLE3 topk=1 accepts modestly; tune against CI if needed.
+    acc_length_thres = 1.6
+    batch_accept_len_thres = 1.3
+    gsm8k_score_thres = 0.7
+    gsm8k_accept_len_thres = 1.3
+
+
+class EagleLlama2Base(SpecEagleServerBase):
+    """EAGLE (Llama-2) config preset. topk=8 tree -> synchronous path; gsm8k is low."""
+
+    model = DEFAULT_TARGET_MODEL_EAGLE
+    draft_model = DEFAULT_DRAFT_MODEL_EAGLE
+    spec_algo = "EAGLE"
+    spec_steps = 5
+    spec_topk = 8
+    spec_tokens = 64
+    attention_backend = "flashinfer"
+    chunked_prefill_size = 128
+    mem_fraction_static = 0.7
+    gsm8k_score_thres = 0.20
+    acc_length_thres = 3.0
+    batch_accept_len_thres = 1.8
+    # topk>1 tree verify runs on the synchronous (non-overlap) path.
+    disable_overlap = True

python/sglang/test/server_fixtures/standalone_fixture.py

@@ -0,0 +1,118 @@
+"""STANDALONE speculative-decoding server fixture.
+
+Variants combine this base with `CustomTestCase` and override class
+attributes (`attention_backend`, plus optional `speculative_eagle_topk` /
+`speculative_num_draft_tokens` / `disable_overlap` /
+`enable_deterministic_inference`) to select a backend, deterministic mode,
+and the V1 / V2 spec engine.
+
+Pure mixin (does NOT inherit `TestCase`), so unittest does not collect
+the base itself.
+"""
+
+from types import SimpleNamespace
+
+import requests
+
+from sglang.srt.environ import envs
+from sglang.srt.utils import kill_process_tree
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_DRAFT_MODEL_STANDALONE,
+    DEFAULT_TARGET_MODEL_STANDALONE,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    popen_launch_server,
+)
+
+GSM_DATASET_PATH = None
+
+
+class StandaloneServerBase:
+    model = DEFAULT_TARGET_MODEL_STANDALONE
+    draft_model = DEFAULT_DRAFT_MODEL_STANDALONE
+    base_url = DEFAULT_URL_FOR_TEST
+    accuracy_threshold = 0.69
+    spec_decode_threshold = 3.6
+
+    # Subclasses set these:
+    attention_backend: str = ""
+    # Overlap defaults; synchronous subclasses override to (2, 7, True).
+    speculative_num_steps: int = 4
+    speculative_eagle_topk: int = 1
+    speculative_num_draft_tokens: int = 5
+    disable_overlap: bool = False
+    enable_deterministic_inference: bool = False
+
+    @classmethod
+    def get_server_args(cls):
+        assert cls.attention_backend, f"{cls.__name__} must set `attention_backend`"
+        args = [
+            "--trust-remote-code",
+            "--cuda-graph-max-bs-decode",
+            "8",
+            "--speculative-algorithm",
+            "STANDALONE",
+            "--speculative-draft-model-path",
+            DEFAULT_DRAFT_MODEL_STANDALONE,
+            "--speculative-num-steps",
+            str(cls.speculative_num_steps),
+            "--speculative-eagle-topk",
+            str(cls.speculative_eagle_topk),
+            "--speculative-num-draft-tokens",
+            str(cls.speculative_num_draft_tokens),
+            "--mem-fraction-static",
+            0.7,
+            "--attention-backend",
+            cls.attention_backend,
+        ]
+        if cls.enable_deterministic_inference:
+            args.append("--enable-deterministic-inference")
+        return args
+
+    @classmethod
+    def setUpClass(cls):
+        # disable deep gemm precompile to make launch server faster
+        # please don't do this if you want to make your inference workload faster
+        envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
+        envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
+        other_args = cls.get_server_args()
+        if cls.disable_overlap:
+            other_args = other_args + ["--disable-overlap-schedule"]
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=other_args,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        requests.get(self.base_url + "/flush_cache")
+
+        args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
+            num_shots=4,
+            gsm8k_data_path=GSM_DATASET_PATH,
+        )
+        metrics = run_eval(args)
+        print(f"{metrics=}")
+
+        metric_key = "score"
+        self.assertGreaterEqual(metrics[metric_key], self.accuracy_threshold)
+
+        server_info = requests.get(self.base_url + "/server_info")
+        avg_spec_accept_length = server_info.json()["internal_states"][0][
+            "avg_spec_accept_length"
+        ]
+        print(f"{avg_spec_accept_length=}")
+        self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)

python/sglang/test/server_fixtures/streaming_session_fixture.py

@@ -0,0 +1,435 @@
+"""Streaming-session test fixture.
+
+`TestStreamingSession` is the base class for all streaming-session tests
+(default config — Llama-3.1-8B, no spec). Variants in
+test_streaming_session.py and test_streaming_session_extra.py inherit
+it and only override `setUpClass`.
+
+Also exports:
+- ABORT_REPRO_* constants used by the basic file's abort-leak repro.
+- _abort_repro_run_all coroutine reused by the basic file.
+
+Lives under sglang.test.server_fixtures so siblings under test/registered
+can `import` it without sys.path hacks.
+"""
+
+import asyncio
+import json
+from typing import Any, Optional
+
+import aiohttp
+import requests
+
+from sglang.srt.environ import envs
+from sglang.srt.utils import kill_process_tree
+from sglang.srt.utils.hf_transformers_utils import get_tokenizer
+from sglang.test.test_utils import (
+    DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+LOGPROB_PROMPTS = [
+    "The quick brown fox jumps over the lazy dog.",
+    "Pack my box with five dozen liquor jugs.",
+    "How vexingly quick daft zebras jump.",
+    "Sphinx of black quartz judge my vow.",
+    "The five boxing wizards jump quickly.",
+]
+
+# Long enough to trigger chunked prefill at 200+ tokens per slice.
+LEAK_FILLER = (
+    "The quick brown fox jumps over the lazy dog. "
+    "Pack my box with five dozen liquor jugs. "
+    "How vexingly quick daft zebras jump. "
+    "Sphinx of black quartz, judge my vow. "
+    "The five boxing wizards jump quickly. "
+    "Jackdaws love my big sphinx of quartz. "
+    "A wizard's job is to vex chumps quickly in fog. "
+    "We promptly judged antique ivory buckles for the next prize. "
+) * 20
+
+ABORT_REPRO_CONTEXT_LEN = 512
+ABORT_REPRO_PAGE_SIZE = 256
+ABORT_REPRO_GEN_LEN = 4
+ABORT_REPRO_SESSIONS = 4
+ABORT_REPRO_WARMUP_TURNS = 1
+ABORT_REPRO_ROUNDS = 8
+ABORT_REPRO_STREAM_TOKENS = 16
+ABORT_REPRO_ABORT_TOKENS = 600
+ABORT_REPRO_NON_STREAMING_TOKENS = 16
+ABORT_REPRO_CHUNKED_PREFILL_SIZE = 4096
+
+CONCURRENT_LOGPROB_SESSIONS = 6
+CONCURRENT_LOGPROB_TURNS = 5
+CONCURRENT_LOGPROB_ROUNDS = 10
+
+STRESS_NUM_SESSIONS = 8
+STRESS_NUM_NON_STREAMING = 4
+STRESS_NUM_TURNS = 6
+STRESS_GEN_LEN = 16
+
+
+def _make_token_sized_ids(
+    tokenizer: Any, prefix: str, min_tokens: int, max_tokens: Optional[int] = None
+) -> list[int]:
+    text = prefix
+    chunk = " pack quartz wizard sphinx zebra fox " * 16
+    token_ids = tokenizer.encode(text)
+    while len(token_ids) < min_tokens:
+        text += chunk
+        token_ids = tokenizer.encode(text)
+    if max_tokens is not None:
+        token_ids = token_ids[:max_tokens]
+    return token_ids
+
+
+async def _abort_repro_generate(
+    base_url: str,
+    session: aiohttp.ClientSession,
+    input_ids: list[int],
+    max_new_tokens: int,
+    session_params: Optional[dict[str, Any]] = None,
+    expect_abort: bool = False,
+) -> Optional[dict[str, Any]]:
+    payload: dict[str, Any] = {
+        "input_ids": input_ids,
+        "sampling_params": {
+            "temperature": 0,
+            "max_new_tokens": max_new_tokens,
+            "no_stop_trim": True,
+            "skip_special_tokens": False,
+        },
+    }
+    if session_params:
+        payload["session_params"] = session_params
+
+    async with session.post(base_url + "/generate", json=payload) as resp:
+        text = await resp.text()
+        if expect_abort:
+            if resp.status == 200:
+                data = json.loads(text)
+                finish_reason = data.get("meta_info", {}).get("finish_reason", {})
+                assert finish_reason.get("type") == "abort", text
+                assert "maximum allowed length" in finish_reason.get(
+                    "message", ""
+                ) or "context length" in finish_reason.get("message", ""), text
+                return data
+            assert resp.status == 400, text
+            assert "maximum allowed length" in text or "context length" in text, text
+            return None
+
+        assert resp.status == 200, text
+        data = json.loads(text)
+        finish_reason = data.get("meta_info", {}).get("finish_reason", {})
+        assert finish_reason.get("type") != "abort", text
+        return data
+
+
+async def _abort_repro_run_all(base_url: str, tokenizer: Any) -> None:
+    timeout = aiohttp.ClientTimeout(total=300)
+    async with aiohttp.ClientSession(timeout=timeout) as http:
+        session_ids = []
+        for _ in range(ABORT_REPRO_SESSIONS):
+            async with http.post(
+                base_url + "/open_session",
+                json={"capacity_of_str_len": 50000, "streaming": True},
+            ) as resp:
+                assert resp.status == 200, await resp.text()
+                session_ids.append(await resp.json())
+
+        try:
+            for warmup_turn in range(ABORT_REPRO_WARMUP_TURNS):
+                warmup_tasks = []
+                for session_idx, session_id in enumerate(session_ids):
+                    input_ids = _make_token_sized_ids(
+                        tokenizer,
+                        prefix=f"[warmup={warmup_turn} session={session_idx}]",
+                        min_tokens=ABORT_REPRO_STREAM_TOKENS,
+                        max_tokens=ABORT_REPRO_STREAM_TOKENS + 8,
+                    )
+                    warmup_tasks.append(
+                        _abort_repro_generate(
+                            base_url,
+                            http,
+                            input_ids,
+                            ABORT_REPRO_GEN_LEN,
+                            session_params={"id": session_id, "rid": None},
+                        )
+                    )
+                await asyncio.gather(*warmup_tasks)
+
+            for round_idx in range(ABORT_REPRO_ROUNDS):
+                mixed_tasks = []
+                for session_idx, session_id in enumerate(session_ids):
+                    input_ids = _make_token_sized_ids(
+                        tokenizer,
+                        prefix=f"[round={round_idx} ok session={session_idx}]",
+                        min_tokens=ABORT_REPRO_STREAM_TOKENS,
+                        max_tokens=ABORT_REPRO_STREAM_TOKENS + 8,
+                    )
+                    mixed_tasks.append(
+                        _abort_repro_generate(
+                            base_url,
+                            http,
+                            input_ids,
+                            ABORT_REPRO_GEN_LEN,
+                            session_params={"id": session_id, "rid": None},
+                        )
+                    )
+
+                for ns_idx in range(2):
+                    input_ids = _make_token_sized_ids(
+                        tokenizer,
+                        prefix=f"[round={round_idx} ns={ns_idx}]",
+                        min_tokens=ABORT_REPRO_NON_STREAMING_TOKENS,
+                        max_tokens=ABORT_REPRO_NON_STREAMING_TOKENS + 8,
+                    )
+                    mixed_tasks.append(
+                        _abort_repro_generate(
+                            base_url,
+                            http,
+                            input_ids,
+                            ABORT_REPRO_GEN_LEN,
+                        )
+                    )
+                await asyncio.gather(*mixed_tasks)
+
+                abort_tasks = []
+                for session_idx, session_id in enumerate(session_ids):
+                    input_ids = _make_token_sized_ids(
+                        tokenizer,
+                        prefix=f"[round={round_idx} abort session={session_idx}]",
+                        min_tokens=ABORT_REPRO_ABORT_TOKENS,
+                    )
+                    abort_tasks.append(
+                        _abort_repro_generate(
+                            base_url,
+                            http,
+                            input_ids,
+                            ABORT_REPRO_GEN_LEN,
+                            session_params={"id": session_id, "rid": None},
+                            expect_abort=True,
+                        )
+                    )
+                await asyncio.gather(*abort_tasks)
+
+                recovery_tasks = []
+                for session_idx, session_id in enumerate(session_ids):
+                    input_ids = _make_token_sized_ids(
+                        tokenizer,
+                        prefix=f"[round={round_idx} recover session={session_idx}]",
+                        min_tokens=ABORT_REPRO_NON_STREAMING_TOKENS,
+                        max_tokens=ABORT_REPRO_NON_STREAMING_TOKENS + 8,
+                    )
+                    recovery_tasks.append(
+                        _abort_repro_generate(
+                            base_url,
+                            http,
+                            input_ids,
+                            ABORT_REPRO_GEN_LEN,
+                            session_params={"id": session_id, "rid": None},
+                        )
+                    )
+                recovery_results = await asyncio.gather(*recovery_tasks)
+                for result in recovery_results:
+                    assert result is not None
+                    assert result["meta_info"]["cached_tokens"] > 0, result
+
+                health = requests.get(base_url + "/health", timeout=10)
+                if health.status_code != 200:
+                    raise RuntimeError(
+                        f"server unhealthy after round={round_idx}: "
+                        f"{health.status_code} {health.text}"
+                    )
+        finally:
+            for session_id in session_ids:
+                async with http.post(
+                    base_url + "/close_session", json={"session_id": session_id}
+                ) as resp:
+                    assert resp.status == 200, await resp.text()
+
+
+async def _async_generate(
+    base_url: str,
+    session: aiohttp.ClientSession,
+    input_ids: list[int],
+    max_new_tokens: int = 8,
+    session_params: Optional[dict[str, Any]] = None,
+    return_logprob: bool = False,
+    logprob_start_len: Optional[int] = None,
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "input_ids": input_ids,
+        "sampling_params": {
+            "temperature": 0,
+            "max_new_tokens": max_new_tokens,
+            "no_stop_trim": True,
+            "skip_special_tokens": False,
+        },
+    }
+    if session_params:
+        payload["session_params"] = session_params
+    if return_logprob:
+        payload["return_logprob"] = True
+    if logprob_start_len is not None:
+        payload["logprob_start_len"] = logprob_start_len
+    timeout = aiohttp.ClientTimeout(total=300)
+    async with session.post(
+        base_url + "/generate", json=payload, timeout=timeout
+    ) as resp:
+        assert resp.status == 200, f"Generate failed: {await resp.text()}"
+        return await resp.json()
+
+
+async def _concurrent_logprob_run(base_url: str, tokenizer: Any, **gen_kwargs) -> None:
+    """N sessions per round, all requests fired simultaneously per turn so
+    the running batch has real concurrency (retract can actually kick one).
+    """
+    timeout = aiohttp.ClientTimeout(total=300)
+    async with aiohttp.ClientSession(timeout=timeout) as http:
+        for _ in range(CONCURRENT_LOGPROB_ROUNDS):
+            sids: list[str] = []
+            for _ in range(CONCURRENT_LOGPROB_SESSIONS):
+                async with http.post(
+                    base_url + "/open_session",
+                    json={"capacity_of_str_len": 50000, "streaming": True},
+                ) as resp:
+                    assert resp.status == 200
+                    sids.append(await resp.json())
+
+            rids: list[Optional[str]] = [None] * CONCURRENT_LOGPROB_SESSIONS
+            for turn in range(CONCURRENT_LOGPROB_TURNS):
+                tasks = []
+                for s in range(CONCURRENT_LOGPROB_SESSIONS):
+                    text = (
+                        f"S{s} T{turn}: "
+                        f"{LOGPROB_PROMPTS[turn % len(LOGPROB_PROMPTS)]}"
+                    )
+                    ids = tokenizer.encode(text)
+                    tasks.append(
+                        _async_generate(
+                            base_url,
+                            http,
+                            ids,
+                            session_params={"id": sids[s], "rid": rids[s]},
+                            **gen_kwargs,
+                        )
+                    )
+                results = await asyncio.gather(*tasks)
+                for s in range(CONCURRENT_LOGPROB_SESSIONS):
+                    rids[s] = results[s]["meta_info"]["id"]
+
+            for sid in sids:
+                async with http.post(
+                    base_url + "/close_session", json={"session_id": sid}
+                ) as resp:
+                    assert resp.status == 200
+
+
+async def _stress_run_all(base_url: str, tokenizer: Any) -> None:
+    """Streaming + non-streaming mixed batches under retract pressure.
+    Long prompts (~200+ tokens) trigger chunked prefill so retract can
+    interrupt mid-extend.
+    """
+    timeout = aiohttp.ClientTimeout(total=300)
+    async with aiohttp.ClientSession(timeout=timeout) as http:
+        sids: list[str] = []
+        for _ in range(STRESS_NUM_SESSIONS):
+            async with http.post(
+                base_url + "/open_session",
+                json={"capacity_of_str_len": 50000, "streaming": True},
+            ) as resp:
+                assert resp.status == 200
+                sids.append(await resp.json())
+
+        rids: list[Optional[str]] = [None] * STRESS_NUM_SESSIONS
+        for turn in range(STRESS_NUM_TURNS):
+            tasks = []
+            # Streaming requests — long prompts to trigger chunked prefill.
+            for s in range(STRESS_NUM_SESSIONS):
+                offset = (s * STRESS_NUM_TURNS + turn) * 200
+                text = (
+                    f"Session {s} turn {turn}: " f"{LEAK_FILLER[offset : offset + 800]}"
+                )
+                ids = tokenizer.encode(text)
+                tasks.append(
+                    _async_generate(
+                        base_url,
+                        http,
+                        ids,
+                        max_new_tokens=STRESS_GEN_LEN,
+                        session_params={"id": sids[s], "rid": rids[s]},
+                    )
+                )
+
+            # Non-streaming requests interleaved.
+            for ns in range(STRESS_NUM_NON_STREAMING):
+                text = (
+                    f"Non-streaming {ns} turn {turn}: "
+                    f"{LEAK_FILLER[ns * 100 : ns * 100 + 400]}"
+                )
+                ids = tokenizer.encode(text)
+                tasks.append(
+                    _async_generate(
+                        base_url,
+                        http,
+                        ids,
+                        max_new_tokens=STRESS_GEN_LEN,
+                    )
+                )
+
+            results = await asyncio.gather(*tasks)
+            for s in range(STRESS_NUM_SESSIONS):
+                rids[s] = results[s]["meta_info"]["id"]
+
+        for sid in sids:
+            async with http.post(
+                base_url + "/close_session", json={"session_id": sid}
+            ) as resp:
+                assert resp.status == 200
+
+
+class StreamingSessionServerBase(CustomTestCase):
+    """Minimal streaming-session server fixture.
+
+    Subclasses override class attrs to customize launch:
+    - `model`: defaults to the small model.
+    - `extra_args`: appended after `--enable-streaming-session` (set
+      `--chunked-prefill-size`, `--page-size`, spec args, etc. here).
+    - `env_overrides`: list of `(env_attr_name, value)` tuples; each is
+      pushed onto the `setUpClass` context stack so the env override is
+      live during `popen_launch_server` and torn down on
+      `tearDownClass`-time. `SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY=1`
+      is always applied on top of these.
+    """
+
+    model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
+    base_url = DEFAULT_URL_FOR_TEST
+    extra_args: list = []
+    env_overrides: list = []
+
+    @classmethod
+    def setUpClass(cls):
+        import contextlib
+
+        with contextlib.ExitStack() as stack:
+            stack.enter_context(
+                envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1)
+            )
+            stack.enter_context(envs.SGLANG_CHECK_KV_PAGE_INVARIANTS.override(True))
+            for name, val in cls.env_overrides:
+                stack.enter_context(getattr(envs, name).override(val))
+            cls.process = popen_launch_server(
+                cls.model,
+                cls.base_url,
+                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                other_args=["--enable-streaming-session"] + list(cls.extra_args),
+            )
+        cls.tokenizer = get_tokenizer(cls.model)
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)