lmcache--lmcache/tests/v1/test_freeze_mode_integration.py

# SPDX-License-Identifier: Apache-2.0
# Standard
import random
import time

# Third Party
import requests
import torch

# First Party
from lmcache.utils import mock_up_broadcast_fn, mock_up_broadcast_object_fn
from lmcache.v1.cache_engine import LMCacheEngineBuilder
from lmcache.v1.config import LMCacheEngineConfig
from lmcache.v1.gpu_connector.mock_gpu_connector import MockGPUConnector
from lmcache.v1.internal_api_server.api_server import InternalAPIServer
from lmcache.v1.metadata import LMCacheMetadata
from tests.v1.utils import (
    MockAdapter,
    dumb_metadata,
    generate_kv_cache_paged_list_tensors,
    generate_tokens,
    get_available_port,
    recover_engine_states,
)


def test_freeze_with_real_cache_engine(autorelease_v1):
    """
    Integration test for freeze mode with real cache engine.

    This test verifies:
    1. Normal mode stores KV cache correctly to local_cpu backend
    2. Enable freeze mode via HTTP API
    3. Store operations are skipped in freeze mode (no new allocations)
    4. Only local_cpu backend is accessible in freeze mode
    5. Disable freeze mode and verify store works again
    """
    # Setup
    device = "cpu"
    instance_id = f"test_freeze_real_{random.getrandbits(64)}"
    chunk_size = 256
    num_tokens = 512
    num_blocks = 100
    block_size = 16
    dtype = torch.bfloat16
    kv_shape = (32, 2, chunk_size, 8, 128)

    # Create config with CPU backend for easy verification
    cfg = LMCacheEngineConfig.from_defaults(
        chunk_size=chunk_size,
        local_cpu=True,
    )

    # Enable Internal API Server
    cfg.internal_api_server_enabled = True
    # Get an available port for the API server
    # worker_id will be 0, so port_offset = 1 + 0 = 1
    # actual_port = port_start + port_offset, so port_start = actual_port - 1
    actual_port = get_available_port()
    cfg.internal_api_server_port_start = actual_port - 1
    cfg.internal_api_server_socket_path_prefix = None

    # Create mock GPU connector that works on CPU
    connector = MockGPUConnector(kv_shape=kv_shape)
    # Use explicit metadata with worker_id=0 to control port offset
    metadata = LMCacheMetadata(
        model_name="test_model",
        world_size=1,
        local_world_size=1,
        worker_id=0,
        local_worker_id=0,
        kv_dtype=torch.bfloat16,
        kv_shape=kv_shape,
    )

    # Create engine
    engine = autorelease_v1(
        LMCacheEngineBuilder.get_or_create(
            instance_id,
            cfg,
            metadata,
            connector,
            mock_up_broadcast_fn,
            mock_up_broadcast_object_fn,
        )
    )

    api_server = None
    try:
        # Create adapter and start API server
        adapter = MockAdapter(engine, cfg)
        api_server = InternalAPIServer(adapter)
        api_server.start()

        # Wait for server to start and become responsive
        max_retries = 40
        retry_delay = 0.5
        server_ready = False
        base_url = "http://localhost:%d" % actual_port

        # Wait for server to be responsive
        for i in range(max_retries):
            try:
                response = requests.get("%s/freeze/status" % base_url, timeout=1)
                if response.status_code in [200, 503]:
                    server_ready = True
                    break
            except requests.exceptions.RequestException:
                pass
            time.sleep(retry_delay)

        assert server_ready, "Server failed to respond after %d retries" % max_retries

        # ============================================
        # Step 1: Verify initial freeze mode is OFF
        # ============================================
        response = requests.get("%s/freeze/status" % base_url)
        assert response.status_code == 200
        assert response.json()["freeze"] is False

        # ============================================
        # Step 2: Normal mode - Store should work
        # ============================================
        tokens_1 = generate_tokens(num_tokens, device)
        kv_cache = generate_kv_cache_paged_list_tensors(
            num_blocks, device, block_size, dtype
        )
        slot_mapping_1 = random.sample(range(0, num_blocks * block_size), num_tokens)
        slot_mapping_1 = torch.tensor(slot_mapping_1, device=device)

        # Store data in normal mode
        engine.store(
            tokens=tokens_1,
            kvcaches=kv_cache,
            slot_mapping=slot_mapping_1,
        )
        recover_engine_states(engine)

        # Wait for async store to complete
        timeout = 5
        start_time = time.time()
        while engine.lookup(tokens_1) < num_tokens:
            if time.time() - start_time > timeout:
                raise TimeoutError("Store operation timed out in normal mode")
            time.sleep(0.1)

        # Verify data was stored
        stored_length = engine.lookup(tokens_1)
        assert stored_length == num_tokens, "Expected %d tokens stored, got %d" % (
            num_tokens,
            stored_length,
        )

        # ============================================
        # Step 3: Enable freeze mode via HTTP API
        # ============================================
        response = requests.put("%s/freeze/enable" % base_url)
        assert response.status_code == 200
        result = response.json()
        assert result["freeze"] is True
        assert result["status"] == "success"

        # Verify status
        response = requests.get("%s/freeze/status" % base_url)
        assert response.status_code == 200
        assert response.json()["freeze"] is True

        # ============================================
        # Step 4: Read-only mode - Store should be skipped
        # ============================================
        tokens_2 = generate_tokens(num_tokens, device)
        slot_mapping_2 = random.sample(range(0, num_blocks * block_size), num_tokens)
        slot_mapping_2 = torch.tensor(slot_mapping_2, device=device)

        # Try to store new data in freeze mode
        engine.store(
            tokens=tokens_2,
            kvcaches=kv_cache,
            slot_mapping=slot_mapping_2,
        )
        recover_engine_states(engine)

        # Wait a bit to ensure store would have completed if not skipped
        time.sleep(1)

        # Verify data was NOT stored (freeze mode should skip)
        stored_length_2 = engine.lookup(tokens_2)
        assert stored_length_2 == 0, (
            "Expected 0 tokens stored in freeze mode, got %d" % stored_length_2
        )

        # Original data should still be there
        assert engine.lookup(tokens_1) == num_tokens

        # ============================================
        # Step 5: Disable freeze mode via HTTP API
        # ============================================
        response = requests.put("%s/freeze/disable" % base_url)
        assert response.status_code == 200
        result = response.json()
        assert result["freeze"] is False
        assert result["status"] == "success"

        # ============================================
        # Step 6: Store should work again after disabling
        # ============================================
        tokens_3 = generate_tokens(num_tokens, device)
        slot_mapping_3 = random.sample(range(0, num_blocks * block_size), num_tokens)
        slot_mapping_3 = torch.tensor(slot_mapping_3, device=device)

        engine.store(
            tokens=tokens_3,
            kvcaches=kv_cache,
            slot_mapping=slot_mapping_3,
        )
        recover_engine_states(engine)

        # Wait for async store to complete
        start_time = time.time()
        while engine.lookup(tokens_3) < num_tokens:
            if time.time() - start_time > timeout:
                raise TimeoutError(
                    "Store operation timed out after disabling freeze mode"
                )
            time.sleep(0.1)

        # Verify data was stored
        stored_length_3 = engine.lookup(tokens_3)
        assert stored_length_3 == num_tokens, (
            "Expected %d tokens stored after disable, got %d"
            % (num_tokens, stored_length_3)
        )
    finally:
        # Cleanup
        if api_server:
            api_server.stop()
            # Give it some time to shut down
            time.sleep(0.5)
        LMCacheEngineBuilder.destroy(instance_id)


def test_freeze_direct_api(autorelease_v1):
    """
    Test freeze mode via direct engine API (without HTTP server).

    This test verifies:
    1. Freeze mode can be enabled/disabled via direct API
    2. Store operations are skipped in freeze mode
    3. No new memory allocations occur in freeze mode
    4. Only local_cpu backend is used for retrieval in freeze mode
    """
    # Setup
    device = "cpu"
    instance_id = "test_freeze_direct"
    chunk_size = 256
    num_tokens = 512
    num_blocks = 100
    block_size = 16
    dtype = torch.bfloat16
    kv_shape = (32, 2, chunk_size, 8, 128)

    cfg = LMCacheEngineConfig.from_defaults(
        chunk_size=chunk_size,
        local_cpu=True,
    )

    connector = MockGPUConnector(kv_shape=kv_shape)
    metadata = dumb_metadata(kv_shape)

    engine = autorelease_v1(
        LMCacheEngineBuilder.get_or_create(
            instance_id,
            cfg,
            metadata,
            connector,
            mock_up_broadcast_fn,
            mock_up_broadcast_object_fn,
        )
    )

    try:
        # Verify initial state
        assert engine.is_frozen() is False

        # Test set/get methods
        engine.freeze(True)
        assert engine.is_frozen() is True

        engine.freeze(False)
        assert engine.is_frozen() is False

        # Test store behavior with freeze mode
        tokens = generate_tokens(num_tokens, device)
        kv_cache = generate_kv_cache_paged_list_tensors(
            num_blocks, device, block_size, dtype
        )
        slot_mapping = random.sample(range(0, num_blocks * block_size), num_tokens)
        slot_mapping = torch.tensor(slot_mapping, device=device)

        # Store in normal mode
        engine.store(tokens=tokens, kvcaches=kv_cache, slot_mapping=slot_mapping)
        recover_engine_states(engine)

        timeout = 5
        start_time = time.time()
        while engine.lookup(tokens) < num_tokens:
            if time.time() - start_time > timeout:
                raise TimeoutError("Store timed out")
            time.sleep(0.1)

        assert engine.lookup(tokens) == num_tokens

        # Enable freeze mode
        engine.freeze(True)

        # Try to store new data
        new_tokens = generate_tokens(num_tokens, device)
        new_slot_mapping = random.sample(range(0, num_blocks * block_size), num_tokens)
        new_slot_mapping = torch.tensor(new_slot_mapping, device=device)

        engine.store(
            tokens=new_tokens, kvcaches=kv_cache, slot_mapping=new_slot_mapping
        )
        recover_engine_states(engine)

        time.sleep(1)

        # Should not be stored
        assert engine.lookup(new_tokens) == 0

    finally:
        LMCacheEngineBuilder.destroy(instance_id)