lightseekorg--tokenspeed/tokenspeed-kernel/python/tokenspeed_kernel/numerics/quantize.py

# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import annotations

from typing import Any

import torch
from tokenspeed_kernel.numerics.inputs import (
    InputGenerator,
    set_benchmark_shapes,
    set_input_generator,
    set_standard_shapes,
)
from tokenspeed_kernel.numerics.tolerance import Tolerance, set_family_tolerance

# ---------------------------------------------------------------------------
# Tolerance
# ---------------------------------------------------------------------------
#
# Quantize kernels under test return ``qweight.float()`` — the fp8 values cast
# to fp32. Two correct implementations following the same round-to-nearest-even
# rule on the same group statistics produce *bit-identical* fp8 values, so the
# tolerance only needs to absorb a single fp8 ulp (~ 1/16 at magnitude 1) for
# values near the quantization boundary, where different scale-rounding paths
# could land one ulp apart.


def tolerance(
    dtype: torch.dtype,
    *,
    inputs: dict[str, Any] | None = None,
    **_: Any,
) -> Tolerance:
    # 1 fp8 e4m3 ulp at the values we actually compare. fp8_e4m3 has 3 mantissa
    # bits, so the relative gap between adjacent representable values is 2^-3
    # = 0.125. Allow that plus a small safety margin.
    return Tolerance(atol=0.2, rtol=0.2)


set_family_tolerance("quantize", tolerance)

# ---------------------------------------------------------------------------
# Input Generator
# ---------------------------------------------------------------------------


class QuantizeInputGenerator(InputGenerator):
    """Generates a 2D activation tensor [M, K] for fp8 quantize kernels."""

    def generate(self, M: int, K: int) -> dict[str, Any]:
        x = torch.randn(
            M, K, dtype=torch.float32, device=self.device, generator=self.rng
        ).to(self.dtype)
        return {"x": x}


set_input_generator("quantize", "fp8_token_group_128", QuantizeInputGenerator)
set_input_generator("quantize", "fp8_token", QuantizeInputGenerator)
set_input_generator("quantize", "fp8_tensor", QuantizeInputGenerator)

# ---------------------------------------------------------------------------
# Shape Presets
# ---------------------------------------------------------------------------
#
# K must be divisible by the group size (128). Cover (a) prefill batches, (b)
# decode (M = 1), and (c) DSv3 hidden / fused-A widths.

_QUANTIZE_STANDARD_SHAPES: list[dict[str, int]] = [
    {"M": 1, "K": 128},
    {"M": 1, "K": 7168},
    {"M": 8, "K": 128},
    {"M": 8, "K": 7168},
    {"M": 32, "K": 1536},
    {"M": 128, "K": 4096},
    {"M": 128, "K": 7168},
    {"M": 512, "K": 4096},
]

for _mode in ("fp8_token_group_128", "fp8_token", "fp8_tensor"):
    set_standard_shapes("quantize", _mode, _QUANTIZE_STANDARD_SHAPES)
    set_benchmark_shapes("quantize", _mode, _QUANTIZE_STANDARD_SHAPES)