# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations from collections.abc import Callable import torch def deepgemm_paged_mqa_logits_native( fp8_paged_mqa_logits_fn: Callable[..., torch.Tensor], q_fp8: torch.Tensor, kv_cache_fp8: torch.Tensor, weights: torch.Tensor, ctx_lens_2d: torch.Tensor, block_tables: torch.Tensor, schedule_metadata: torch.Tensor, max_seq_len: int, *, q_offset: int, B: int, next_n: int, ) -> torch.Tensor: # block_tables[::next_n] de-expands the caller's repeat_interleave without a # copy (DeepGEMM only checks `stride(1) == 1`). return fp8_paged_mqa_logits_fn( q_fp8[:q_offset].view(B, next_n, q_fp8.shape[1], q_fp8.shape[2]), kv_cache_fp8, weights[:q_offset], ctx_lens_2d, block_tables[::next_n], schedule_metadata, max_seq_len, clean_logits=False, ) def deepgemm_paged_mqa_logits_split( fp8_paged_mqa_logits_fn: Callable[..., torch.Tensor], q_fp8: torch.Tensor, kv_cache_fp8: torch.Tensor, weights: torch.Tensor, ctx_lens_2d: torch.Tensor, block_tables: torch.Tensor, schedule_metadata: torch.Tensor, max_seq_len: int, *, q_offset: int, ) -> torch.Tensor: q_fp8 = q_fp8.unsqueeze(1) return fp8_paged_mqa_logits_fn( q_fp8[:q_offset], kv_cache_fp8, weights[:q_offset], ctx_lens_2d, block_tables, schedule_metadata, max_seq_len, clean_logits=False, ) def aiter_paged_mqa_logits( q_fp8: torch.Tensor, kv_cache_fp8: torch.Tensor, weights: torch.Tensor, seq_lens: torch.Tensor, block_tables: torch.Tensor, max_seq_len: int, *, preshuffle: bool, kv_block_size: int, ) -> torch.Tensor: from aiter.ops.triton.pa_mqa_logits import deepgemm_fp8_paged_mqa_logits q_fp8 = q_fp8.unsqueeze(1) batch_size, next_n, _, _ = q_fp8.shape logits = torch.empty( (batch_size * next_n, max_seq_len), device=q_fp8.device, dtype=torch.float32, ) deepgemm_fp8_paged_mqa_logits( q_fp8, kv_cache_fp8, weights, logits, seq_lens, block_tables, max_seq_len, Preshuffle=preshuffle, KVBlockSize=kv_block_size, ) return logits def cutedsl_paged_mqa_logits( q_fp8: torch.Tensor, kv_cache_fp8: torch.Tensor, weights: torch.Tensor, ctx_lens_1d: torch.Tensor, block_tables: torch.Tensor, schedule_metadata: torch.Tensor | None, max_seq_len: int, *, q_offset: int, B: int, next_n: int, is_target_verify: bool, dsl_expand_factor: int, dsl_atom: int, blocksize: int, sm_count: int, get_paged_mqa_logits_metadata_fn: Callable[..., torch.Tensor], ) -> torch.Tensor: from sglang.jit_kernel.dsa.cutedsl_paged_mqa_logits import ( CuteDSLPagedMQALogitsRunner, ) dsl_atom_split = dsl_expand_factor > 1 and next_n == dsl_expand_factor * dsl_atom if is_target_verify and dsl_atom_split: exp_B = B * dsl_expand_factor q_dsl = q_fp8[:q_offset].view(exp_B, dsl_atom, q_fp8.shape[1], q_fp8.shape[2]) ctx_lens_1d = ctx_lens_1d.repeat_interleave(dsl_expand_factor) block_tables_dsl = block_tables[::next_n].repeat_interleave( dsl_expand_factor, dim=0 ) schedule_metadata = get_paged_mqa_logits_metadata_fn( ctx_lens_1d.unsqueeze(-1), blocksize, sm_count ) elif is_target_verify and next_n >= 2: # Native single-launch: one task per batch entry (the kernel iterates # next_n internally), so the schedule must be built from B-length # context lens, not the caller's [B, next_n] or per-token layout. q_dsl = q_fp8[:q_offset].view(B, next_n, q_fp8.shape[1], q_fp8.shape[2]) block_tables_dsl = block_tables[::next_n] schedule_metadata = get_paged_mqa_logits_metadata_fn( ctx_lens_1d.unsqueeze(-1), blocksize, sm_count ) else: q_dsl = q_fp8[:q_offset].unsqueeze(1) block_tables_dsl = block_tables[:B] return CuteDSLPagedMQALogitsRunner.forward( q_dsl, kv_cache_fp8.view(torch.uint8), weights[:q_offset], ctx_lens_1d, block_tables_dsl, schedule_metadata, max_seq_len, )