lmcache--lmcache/tests/v1/test_vllm_kv_cache_groups.py

# SPDX-License-Identifier: Apache-2.0
# Standard
from dataclasses import dataclass, field

# Third Party
import pytest
import torch

# First Party
from lmcache.integration.vllm.kv_cache_groups import (
    create_engine_group_infos_from_vllm,
)
from lmcache.v1.multiprocess.group_view import (
    expand_engine_block_ids,
    get_engine_group_indices,
    num_engine_groups,
)

# Test doubles for the vLLM KV cache spec classes. Unit tests must run
# without vLLM installed; sliding-window specs are detected by class name,
# so the doubles share the vLLM class names.


@dataclass
class MockKVCacheSpec:
    block_size: int


@dataclass
class SlidingWindowSpec:
    block_size: int
    sliding_window: int


@dataclass
class SlidingWindowMLASpec(SlidingWindowSpec):
    pass


@dataclass
class FullAttentionSpec:
    block_size: int
    sliding_window: "int | None" = None


@dataclass
class MLAAttentionSpec:
    """Key-only, one-vector-per-token spec (an MLA index cache)."""

    block_size: int


@dataclass
class UniformTypeKVCacheSpecs:
    block_size: int
    kv_cache_specs: "dict[str, object]" = field(default_factory=dict)


@dataclass
class MockKVCacheGroup:
    layer_names: list[str]
    kv_cache_spec: object


@dataclass
class MockKVCacheConfig:
    kv_cache_groups: list[MockKVCacheGroup]


def _same_shape_caches(names: list[str]) -> dict[str, torch.Tensor]:
    return {n: torch.randn(2, 32, 16, 8, 64, dtype=torch.float16) for n in names}


def _mla_caches(names: list[str]) -> dict[str, torch.Tensor]:
    """Key-only rank-3 caches (num_blocks, block_size, head_size), MLA layout."""
    return {n: torch.randn(32, 16, 128, dtype=torch.bfloat16) for n in names}


def test_conversion_defaults_to_single_group_without_config():
    """No vLLM KV cache groups -> all layers fall into a single engine group."""
    spec = create_engine_group_infos_from_vllm(
        None, _same_shape_caches(["layer.0", "layer.1"])
    )

    assert num_engine_groups(spec) == 1
    assert [group.engine_group_id for group in spec] == [0]
    assert spec[0].layer_indices == (0, 1)


def test_conversion_preserves_engine_group_layers():
    """Two engine groups with identical tensor shape stay separate by group."""
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[
                MockKVCacheGroup(
                    ["layer.0", "layer.2"], MockKVCacheSpec(block_size=16)
                ),
                MockKVCacheGroup(
                    ["layer.1", "layer.3"], MockKVCacheSpec(block_size=16)
                ),
            ]
        ),
        _same_shape_caches(["layer.0", "layer.1", "layer.2", "layer.3"]),
    )

    assert num_engine_groups(spec) == 2
    assert get_engine_group_indices(spec, 4) == [0, 1, 0, 1]
    assert [group.tokens_per_block for group in spec] == [16, 16]


def test_conversion_splits_by_lmcache_layer_identity():
    """Layers split by both engine group and physical transfer identity."""
    caches = _same_shape_caches(["layer.0", "layer.1", "layer.2", "layer.3"])
    # layer.4 has a different head count -> distinct transfer identity.
    caches["layer.4"] = torch.randn(2, 32, 16, 16, 64, dtype=torch.float16)
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[
                MockKVCacheGroup(
                    ["layer.0", "layer.2", "layer.4"], MockKVCacheSpec(block_size=16)
                ),
                MockKVCacheGroup(
                    ["layer.1", "layer.3"], MockKVCacheSpec(block_size=16)
                ),
            ]
        ),
        caches,
    )

    assert [group.engine_group_id for group in spec] == [0, 1, 0]
    assert [group.layer_indices for group in spec] == [(0, 2), (1, 3), (4,)]
    assert expand_engine_block_ids(spec, [[10], [20]]) == [
        [10],
        [20],
        [10],
    ]


def test_conversion_resolves_sliding_window_size():
    """A SlidingWindowSpec group carries its window size in tokens;
    subclasses count too."""
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[
                MockKVCacheGroup(["layer.0"], FullAttentionSpec(block_size=16)),
                MockKVCacheGroup(
                    ["layer.1"], SlidingWindowSpec(block_size=16, sliding_window=64)
                ),
                MockKVCacheGroup(
                    ["layer.2"],
                    SlidingWindowMLASpec(block_size=16, sliding_window=128),
                ),
            ]
        ),
        _same_shape_caches(["layer.0", "layer.1", "layer.2"]),
    )

    assert [group.sw_size_tokens for group in spec] == [-1, 64, 128]


def test_conversion_ignores_full_attention_sliding_window():
    """SWA layers managed as full attention (hybrid allocator disabled) are
    not sliding window: vLLM allocates blocks for all tokens."""
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[
                MockKVCacheGroup(
                    ["layer.0", "layer.1"],
                    FullAttentionSpec(block_size=16, sliding_window=1024),
                ),
            ]
        ),
        _same_shape_caches(["layer.0", "layer.1"]),
    )

    assert [group.sw_size_tokens for group in spec] == [-1]


def test_conversion_defaults_sliding_window_for_non_sw_spec():
    """Groups whose spec is not a SlidingWindowSpec resolve to
    non-sliding-window."""
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[
                MockKVCacheGroup(["layer.0"], MockKVCacheSpec(block_size=16))
            ]
        ),
        _same_shape_caches(["layer.0"]),
    )

    assert [group.sw_size_tokens for group in spec] == [-1]


def test_conversion_uniform_type_specs_resolve_per_layer():
    """Inside a UniformTypeKVCacheSpecs group, per-layer specs decide the
    window. SW layers with a distinct transfer identity get their own group
    carrying the window size."""
    caches = _same_shape_caches(["layer.0", "layer.1"])
    # layer.1 has a different head count -> distinct transfer identity.
    caches["layer.1"] = torch.randn(2, 32, 16, 16, 64, dtype=torch.float16)
    uniform_spec = UniformTypeKVCacheSpecs(
        block_size=16,
        kv_cache_specs={
            "layer.0": FullAttentionSpec(block_size=16),
            "layer.1": SlidingWindowSpec(block_size=16, sliding_window=512),
        },
    )
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[MockKVCacheGroup(["layer.0", "layer.1"], uniform_spec)]
        ),
        caches,
    )

    assert [group.layer_indices for group in spec] == [(0,), (1,)]
    assert [group.sw_size_tokens for group in spec] == [-1, 512]


def test_conversion_mixed_window_layers_in_one_group_rejected():
    """Same-identity layers mixing different windows are inconsistent vLLM
    metadata and fail loudly."""
    uniform_spec = UniformTypeKVCacheSpecs(
        block_size=16,
        kv_cache_specs={
            "layer.0": FullAttentionSpec(block_size=16),
            "layer.1": SlidingWindowSpec(block_size=16, sliding_window=64),
        },
    )
    with pytest.raises(ValueError, match="different sliding window sizes"):
        create_engine_group_infos_from_vllm(
            MockKVCacheConfig(
                kv_cache_groups=[MockKVCacheGroup(["layer.0", "layer.1"], uniform_spec)]
            ),
            _same_shape_caches(["layer.0", "layer.1"]),
        )


def test_conversion_mixed_kv_and_mla_groups():
    """Mixed-format shape: a K+V FullAttentionSpec group plus a key-only MLA index
    group are detected per engine group and kept as separate LMCache groups with
    the correct membership and per-group token packing."""
    caches = {
        **_same_shape_caches(["main.0", "main.1"]),
        **_mla_caches(["idx.0", "idx.1"]),
    }
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[
                MockKVCacheGroup(
                    ["main.0", "main.1"], FullAttentionSpec(block_size=16)
                ),
                MockKVCacheGroup(["idx.0", "idx.1"], MLAAttentionSpec(block_size=128)),
            ]
        ),
        caches,
    )

    assert num_engine_groups(spec) == 2
    assert [group.engine_group_id for group in spec] == [0, 1]
    assert [group.layer_indices for group in spec] == [(0, 1), (2, 3)]
    assert [group.tokens_per_block for group in spec] == [16, 128]


def test_conversion_uniform_group_mixes_kv_and_mla_layouts():
    """vLLM can coalesce a rank-5 K+V group and a rank-3 key-only indexer group
    into ONE ``UniformTypeKVCacheSpecs`` group, not two. Detection must split it
    by layout so the indexer gets the rank-3 format instead of inheriting the
    K/V format; the two land in separate LMCache groups sharing one block-id
    space."""
    caches = {
        **_same_shape_caches(["main.0", "main.1"]),  # rank-5 K+V
        **_mla_caches(["idx.0", "idx.1"]),  # rank-3 indexer (key-only)
    }
    uniform_spec = UniformTypeKVCacheSpecs(
        block_size=128,
        kv_cache_specs={
            "main.0": FullAttentionSpec(block_size=128),
            "main.1": FullAttentionSpec(block_size=128),
            "idx.0": MLAAttentionSpec(block_size=128),
            "idx.1": MLAAttentionSpec(block_size=128),
        },
    )
    spec = create_engine_group_infos_from_vllm(
        MockKVCacheConfig(
            kv_cache_groups=[
                MockKVCacheGroup(["main.0", "main.1", "idx.0", "idx.1"], uniform_spec)
            ]
        ),
        caches,
    )

    # One vLLM engine group, but two LMCache groups split by per-layer format.
    assert num_engine_groups(spec) == 1
    assert [group.engine_group_id for group in spec] == [0, 0]
    assert [group.layer_indices for group in spec] == [(0, 1), (2, 3)]
    # Both LMCache groups share the unified block-id space (tokens_per_block).
    assert [group.tokens_per_block for group in spec] == [128, 128]


def test_group_layers_by_identity_uses_per_layer_format():
    """A per-layer Engine KV format gives the K+V layer kv_size=2 and the MLA
    layer kv_size=1, splitting them into separate identities -- the per-group
    distinction the single global format cannot express."""
    # First Party
    from lmcache.v1.kv_layer_groups import group_layers_by_identity
    import lmcache.c_ops as lmc_ops

    kv_caches = [
        torch.randn(2, 32, 16, 8, 64, dtype=torch.bfloat16),  # K+V (rank-5)
        torch.randn(32, 16, 128, dtype=torch.bfloat16),  # MLA key-only (rank-3)
    ]
    per_layer_format = [
        lmc_ops.EngineKVFormat.NL_X_TWO_NB_BS_NH_HS,
        lmc_ops.EngineKVFormat.NL_X_NB_BS_HS,
    ]
    groups = group_layers_by_identity(
        kv_caches,
        per_layer_format,
        per_layer_engine_group_idx=[0, 1],
    )

    kv_size_by_group = {
        identity.engine_group_idx: identity.kv_size for identity, _ in groups
    }
    num_heads_by_group = {
        identity.engine_group_idx: identity.num_heads for identity, _ in groups
    }
    assert kv_size_by_group == {0: 2, 1: 1}
    # The MLA group collapses heads to 1; the K+V group keeps its head count.
    assert num_heads_by_group == {0: 8, 1: 1}


def test_group_layers_by_identity_rejects_group_idx_length_mismatch():
    """per_layer_engine_group_idx must hold one entry per layer."""
    # First Party
    from lmcache.v1.kv_layer_groups import group_layers_by_identity
    import lmcache.c_ops as lmc_ops

    kv_caches = [torch.randn(2, 32, 16, 8, 64, dtype=torch.bfloat16)]
    # One layer (one format) but two engine-group ids.
    with pytest.raises(ValueError, match="per_layer_engine_group_idx"):
        group_layers_by_identity(
            kv_caches,
            [lmc_ops.EngineKVFormat.NL_X_TWO_NB_BS_NH_HS],
            per_layer_engine_group_idx=[0, 1],
        )