sgl-project--sglang/python/sglang/test/kv_canary/pd_fixture.py

from __future__ import annotations

import uuid
from typing import ClassVar, Literal, Optional

from sglang.srt.kv_canary.config import CanaryMode
from sglang.test.kv_canary.mode_config import _MODE_CONFIGS, _ModeConfig
from sglang.test.kv_canary.utils import build_canary_server_args, post_parallel_generate
from sglang.test.kv_canary.violation_assert_mixin import CanaryViolationAssertMixin
from sglang.test.server_fixtures.disaggregation_fixture import (
    PDDisaggregationServerBase,
)

_SHORT_PROMPT_BODY = ("The quick brown fox jumps over the lazy dog. " * 8).strip()


class CanaryPDFixture(CanaryViolationAssertMixin, PDDisaggregationServerBase):
    capture_per_side_logs = True

    model_mode: ClassVar[Literal["mha", "swa", "dsv4"]]
    kv_canary_mode: ClassVar[CanaryMode] = CanaryMode.LOG
    extra_server_args: ClassVar[tuple[str, ...]] = (
        "--kv-canary-real-data",
        "partial",
        "--skip-server-warmup",
    )

    _cfg: ClassVar[Optional[_ModeConfig]] = None

    @classmethod
    def setUpClass(cls) -> None:
        super().setUpClass()
        cls._cfg = _MODE_CONFIGS[cls.model_mode]
        cls.model = cls._cfg.model_path

        canary_args = build_canary_server_args(
            kv_canary_mode=cls.kv_canary_mode,
            mode_cfg=cls._cfg,
            extra_server_args=cls.extra_server_args,
        )
        cls.extra_prefill_args = list(canary_args)
        cls.extra_decode_args = list(canary_args)
        if cls.model_mode == "swa":
            # SWA mode uses google/gemma-4-E2B-it, whose forward does a
            # ``positions += 1`` in-place. canary's WRITE/VERIFY require
            # forward_batch.positions to stay 0-indexed, so flip the gemma
            # path to out-of-place shift for these tests.
            cls.extra_prefill_env = {
                **cls.extra_prefill_env,
                "SGLANG_GEMMA_OUT_OF_PLACE_POSITION_MUTATION": "1",
            }
            cls.extra_decode_env = {
                **cls.extra_decode_env,
                "SGLANG_GEMMA_OUT_OF_PLACE_POSITION_MUTATION": "1",
            }
        cls.launch_all()

    def send_parallel_short_requests(
        self,
        n: int,
        *,
        assert_all_success: bool = True,
        max_new_tokens: int = 100,
        timeout: float = 60.0,
        distinct_prompts: bool = False,
    ) -> list[dict]:
        if distinct_prompts:
            # Diverge at the very first token (request index before the per-call
            # nonce) so requests share no radix-dedupable prefix beyond a
            # tokenizer-added BOS, and retries never hit earlier attempts' cache.
            nonce = uuid.uuid4().hex[:8]
            prompts = [f"{i} {nonce} {_SHORT_PROMPT_BODY}" for i in range(n)]
        else:
            prompts = [_SHORT_PROMPT_BODY] * n
        results = post_parallel_generate(
            url=self.lb_url + "/generate",
            prompts=prompts,
            max_new_tokens=max_new_tokens,
            timeout=timeout,
        )
        if assert_all_success:
            for result in results:
                self.assertEqual(result.get("status_code"), 200, result)
        return results

    def _captured_log_text(
        self, side: Optional[Literal["prefill", "decode"]] = None
    ) -> str:
        if side == "prefill":
            stdout_buf = type(self)._prefill_stdout_buf
            stderr_buf = type(self)._prefill_stderr_buf
        elif side == "decode":
            stdout_buf = type(self)._decode_stdout_buf
            stderr_buf = type(self)._decode_stderr_buf
        else:
            raise ValueError(f"Unsupported side={side!r}")
        stdout_text = stdout_buf.getvalue() if stdout_buf is not None else ""
        stderr_text = stderr_buf.getvalue() if stderr_buf is not None else ""
        return stdout_text + stderr_text