sgl-project--sglang/python/sglang/test/kv_canary/runner_test_base.py

from __future__ import annotations

from types import SimpleNamespace
from unittest.mock import patch

import torch

from sglang.jit_kernel.kv_canary.consts import RealKvHashMode
from sglang.jit_kernel.kv_canary.verify import CanaryLaunchTag
from sglang.srt.kv_canary import endpoint as endpoint_module
from sglang.srt.kv_canary.buffer_group import CanaryBufferGroup
from sglang.srt.kv_canary.capacities import CanaryLaunchCapacities
from sglang.srt.kv_canary.config import CanaryConfig, CanaryMode
from sglang.srt.kv_canary.perturb.config import PerturbConfig
from sglang.srt.kv_canary.runner import kernel_launcher as kernel_launcher_module
from sglang.srt.kv_canary.runner.canary_manager import CanaryManager
from sglang.test.kv_canary.fixtures import (
    DEFAULT_DEVICE,
    make_buffer_group,
    make_req_to_token_pool,
)
from sglang.test.test_utils import CustomTestCase


def make_config(
    *,
    mode: CanaryMode = CanaryMode.RAISE,
    ring_capacity: int = 1024,
    sweep_interval: int = 0,
    real_kv_hash_mode: RealKvHashMode = RealKvHashMode.NONE,
    enable_write_input_assert: bool = False,
    enable_verify_token_assert: bool = True,
    stats_print_every_n_steps: int = 100,
) -> CanaryConfig:
    return CanaryConfig(
        mode=mode,
        ring_capacity=ring_capacity,
        sweep_interval=sweep_interval,
        real_kv_hash_mode=real_kv_hash_mode,
        enable_write_input_assert=enable_write_input_assert,
        enable_verify_token_assert=enable_verify_token_assert,
        stats_print_every_n_steps=stats_print_every_n_steps,
    )


class RecordingEndpoint:
    def __init__(self, *, kernel_kind: CanaryLaunchTag) -> None:
        self.kernel_kind = kernel_kind
        self.calls: list[dict[str, object]] = []

    def launch_per_forward(self, **kwargs: object) -> None:
        self.calls.append(kwargs)


def make_perturb_config() -> PerturbConfig:
    """Build a PerturbConfig with every probability pinned to 0 so the
    perturb hooks do nothing during unit tests."""
    return PerturbConfig(
        req_to_token_prob=0.0,
        real_kv_used_prob=0.0,
        real_kv_unused_cache_prob=0.0,
        real_kv_post_forward_prob=0.0,
        target_group_kind=None,
        warmup_steps=0,
    )


def make_manager(
    *,
    device: torch.device,
    config: CanaryConfig | None = None,
    perturb_config: PerturbConfig | None = None,
    group: CanaryBufferGroup | None = None,
    req_pool: SimpleNamespace | None = None,
    per_forward_verify_capacity: int = 16,
    speculative_num_steps: int = 1,
) -> CanaryManager:
    if config is None:
        config = make_config()
    if perturb_config is None:
        perturb_config = make_perturb_config()
    if group is None:
        group = make_buffer_group(device=device)
    if req_pool is None:
        req_pool = make_req_to_token_pool(device=device, max_reqs=4, max_seq_len=8)
    return CanaryManager(
        config=config,
        perturb_config=perturb_config,
        buffer_groups=(group,),
        device=device,
        req_to_token_pool=req_pool,
        launch_capacities=CanaryLaunchCapacities(
            per_forward_verify_capacity=per_forward_verify_capacity,
            per_forward_write_req_capacity=2,
            per_forward_write_entry_capacity=8,
        ),
        speculative_num_steps=speculative_num_steps,
    )


class CanaryManagerTestCase(CustomTestCase):
    def setUp(self) -> None:
        self.device = DEFAULT_DEVICE
        # Stub plan/verify/write kernels so CPU runs don't need CUDA JIT.
        self._patchers = [
            patch.object(
                kernel_launcher_module,
                "launch_canary_plan_kernels",
                lambda **kwargs: None,
            ),
            patch.object(
                endpoint_module, "launch_canary_verify_kernel", lambda **kwargs: None
            ),
            patch.object(
                endpoint_module, "launch_canary_write_kernel", lambda **kwargs: None
            ),
        ]
        for patcher in self._patchers:
            patcher.start()
            self.addCleanup(patcher.stop)