sgl-project--sglang/python/sglang/srt/layers/attention/linear/linear_metadata.py

from dataclasses import dataclass

import torch

from sglang.srt.layers.attention.mamba.mamba2_metadata import ForwardMetadata
from sglang.srt.model_executor.forward_batch_info import ForwardBatch


@dataclass(kw_only=True)
class BailingLinearMetadata(ForwardMetadata):
    num_prefills: int
    num_prefill_tokens: int
    num_decodes: int
    batch_size: int
    has_initial_states: torch.Tensor
    q_lengths: torch.Tensor

    @staticmethod
    def prepare_decode(
        query_start_loc: torch.Tensor,
        mamba_cache_indices: torch.Tensor,
        bs: int,
        seq_lens: torch.Tensor,
    ) -> "BailingLinearMetadata":
        """This path is run during CUDA graph capture, i.e. decode only, so `num_prefills` is 0"""
        return BailingLinearMetadata(
            batch_size=bs,
            query_start_loc=query_start_loc,
            mamba_cache_indices=mamba_cache_indices,
            num_decodes=seq_lens.shape[0],
            num_prefills=0,
            num_prefill_tokens=0,
            has_initial_states=torch.ones_like(seq_lens),
            q_lengths=query_start_loc.diff(),
        )

    @classmethod
    def prepare_mixed(
        cls,
        query_start_loc: torch.Tensor,
        mamba_cache_indices: torch.Tensor,
        forward_batch: ForwardBatch,
    ) -> "BailingLinearMetadata":
        """This path cannot run with CUDA graph, as it contains extend requests."""
        if forward_batch.extend_num_tokens is None:
            return cls.prepare_decode(
                query_start_loc=query_start_loc,
                mamba_cache_indices=mamba_cache_indices,
                bs=forward_batch.batch_size,
                seq_lens=forward_batch.seq_lens,
            )
        num_prefills = len(forward_batch.extend_seq_lens)
        num_prefill_tokens = forward_batch.extend_num_tokens
        num_decodes = len(forward_batch.seq_lens) - num_prefills
        context_lens_tensor = forward_batch.extend_prefix_lens
        assert context_lens_tensor is not None
        has_initial_states = context_lens_tensor > 0

        query_start_loc = query_start_loc[: num_prefills + 1]

        return BailingLinearMetadata(
            batch_size=forward_batch.batch_size,
            query_start_loc=query_start_loc,
            mamba_cache_indices=mamba_cache_indices,
            num_prefills=num_prefills,
            num_prefill_tokens=num_prefill_tokens,
            num_decodes=num_decodes,
            has_initial_states=has_initial_states,
            q_lengths=query_start_loc.diff(),
        )