sgl-project--sglang/python/sglang/srt/layers/int4fp8_utils.py

"""
Common utilities for quark.
"""

import logging
from typing import Tuple

import torch

logger = logging.getLogger(__name__)


def quantize_fp8_scale_tensorwise(w: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
    FP8_MAX = 448.0
    scale = w.abs().amax().float() / FP8_MAX
    scaled = (w / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
    return scaled, scale


def quantize_int4_scale_columnwise(
    w: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
    S4_MAX = 7
    w_flat = w.reshape(-1, w.shape[-1]).float()
    scale = w_flat.abs().amax(axis=-1) / S4_MAX
    scaled = torch.round(w_flat / scale[:, None]).to(torch.int8).clamp(-S4_MAX, S4_MAX)
    return scaled.reshape(w.shape), scale.reshape(w.shape[:-1])


def pack_int4_to_int32(to_pack: torch.Tensor, reorder: bool = True) -> torch.Tensor:
    if to_pack.ndim > 2:
        raise ValueError(
            "Pack: Only supports tensors with dimensions not greater than 2."
        )

    if reorder:
        order_map = [0, 2, 4, 6, 1, 3, 5, 7]
    else:
        order_map = [0, 1, 2, 3, 4, 5, 6, 7]
    pack_num = 8
    if to_pack.ndim == 2:
        packed = torch.zeros(
            to_pack.shape[0],
            to_pack.shape[1] // pack_num,
            dtype=torch.int32,
            device=to_pack.device,
        )
        new_c = to_pack.shape[1] // pack_num
        for c in range(new_c):
            for i in range(pack_num):
                # Use -3 as an example, high_position is 11111111,cause bit_or generate errors, so we can't use int4 directly
                packed_col = to_pack[:, c * pack_num + order_map[i]].to(torch.int32)
                packed_col = packed_col & 0x0F
                packed[:, c] = torch.bitwise_or(
                    packed[:, c], torch.bitwise_left_shift(packed_col, i * 4)
                )
    elif to_pack.ndim == 0:
        packed = to_pack.to(torch.int32)
    else:
        packed = torch.zeros(
            to_pack.shape[0] // pack_num, dtype=torch.int32, device=to_pack.device
        )
        new_c = to_pack.shape[0] // pack_num
        for c in range(new_c):
            for i in range(pack_num):
                # Use -3 as an example, high_position is 11111111,cause bit_or generate errors, so we can't use int4 directly
                packed_col = to_pack[c * pack_num + order_map[i]]
                packed_col = packed_col & 0x0F
                packed[c] = torch.bitwise_or(
                    packed[c], torch.bitwise_left_shift(packed_col, i * 4)
                )

    return packed.view(torch.uint32)