chore: import upstream snapshot with attribution

tests/quantization/reference_mxfp4.py

@@ -0,0 +1,292 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import torch
+
+BFLOAT16_EXP_BIAS = 127
+BFLOAT16_MANTISSA_BITS = 7
+BFLOAT16_EXP_BITS = 8
+
+FLOAT16_EXP_BIAS = 15
+FLOAT16_MANTISSA_BITS = 10
+FLOAT16_EXP_BITS = 5
+
+FLOAT8_E8M0_MAX_EXP = 127
+FLOAT4_EXP_BIAS = 1
+FLOAT4_MANTISSA_BITS = 1
+
+FLOAT16_VAL_TO_ADD = 1 << (FLOAT16_MANTISSA_BITS - FLOAT4_MANTISSA_BITS - 1)
+FLOAT16_SIGN_EXPONENT_MASK = (
+    (1 << (FLOAT16_EXP_BITS + 1)) - 1
+) << FLOAT16_MANTISSA_BITS
+
+BFLOAT16_VAL_TO_ADD = 1 << (BFLOAT16_MANTISSA_BITS - FLOAT4_MANTISSA_BITS - 1)
+BFLOAT16_SIGN_EXPONENT_MASK = (
+    (1 << (BFLOAT16_EXP_BITS + 1)) - 1
+) << BFLOAT16_MANTISSA_BITS
+
+
+def e8m0_to_half(scale, half_dtype: torch.dtype):
+    assert scale.dtype == torch.uint8
+
+    scale_exp = scale.to(torch.int16) - 127
+
+    # This can be implemented with bitwise operations in a proper kernel.
+    scale_half = 2.0 ** (scale_exp.to(torch.float))
+
+    return scale_half.to(half_dtype)
+
+
+def upcast_fp4_to_fp16_or_bf16(
+    val, float_dtype: torch.dtype, half_exp_bias: int, half_mantissa_bits: int
+):
+    assert val.dtype == torch.uint8
+
+    unpacked = torch.zeros(
+        *val.shape[:-1], val.shape[-1] * 2, dtype=torch.uint8, device=val.device
+    )
+    unpacked[..., 1::2] = (val >> 4) & 0x0F  # Extract high 4 bits.
+    unpacked[..., ::2] = val & 0x0F  # Extract low 4 bits.
+
+    # Takes one float4 values represented as b0000xxxx,
+    # and converts it to the corresponding float16 value.
+
+    sign = unpacked >> 3
+
+    exp = (unpacked >> 1) & 3
+    new_mantissa = unpacked & 1
+
+    # if exp == 0 and new_mantissa == 0:
+    #     new_exp = 0
+    # else:
+    #     new_exp = exp - FLOAT4_EXP_BIAS + FLOAT16_EXP_BIAS
+
+    # int8_t works with float16, but may overflow with bfloat16.
+    new_exp = exp - FLOAT4_EXP_BIAS + half_exp_bias
+
+    # Cast b0000 to 0. in fp16/bf16.
+    new_exp = new_exp * torch.logical_or(exp > 0, new_mantissa > 0)
+
+    # Cast b0001 to 0.5 in fp16/bf16.
+    new_mantissa = torch.logical_and(new_mantissa, exp > 0)
+
+    new_mantissa = new_mantissa.to(torch.int32)
+    new_exp = new_exp.to(torch.int32)
+    sign = sign.to(torch.int32)
+
+    qdq_val = (
+        (sign << 15)
+        + (new_exp << half_mantissa_bits)
+        + (new_mantissa << (half_mantissa_bits - 1))
+    )
+
+    assert qdq_val.max() <= 65535
+    assert qdq_val.min() >= 0
+    qdq_val = qdq_val.to(torch.uint16)
+
+    result = qdq_val.view(float_dtype)
+
+    return result
+
+
+def dq_mxfp4_torch(
+    x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype
+) -> torch.Tensor:
+    assert x.dtype == torch.uint8
+    assert scale.dtype == torch.uint8
+
+    if float_dtype == torch.float16:
+        half_exp_bias = FLOAT16_EXP_BIAS
+        half_mantissa_bits = FLOAT16_MANTISSA_BITS
+    elif float_dtype == torch.bfloat16:
+        half_exp_bias = BFLOAT16_EXP_BIAS
+        half_mantissa_bits = BFLOAT16_MANTISSA_BITS
+
+    scale_half = e8m0_to_half(scale, half_dtype=float_dtype)
+
+    x_half = upcast_fp4_to_fp16_or_bf16(
+        x,
+        float_dtype=float_dtype,
+        half_exp_bias=half_exp_bias,
+        half_mantissa_bits=half_mantissa_bits,
+    )
+
+    x_half = x_half.reshape(*x_half.shape[:-1], -1, 32)
+    x_half = x_half * scale_half[..., None]
+    x_half = x_half.reshape(*x_half.shape[:-2], -1)
+
+    return x_half
+
+
+def fp16_to_fp4_simulate(
+    val, half_mantissa_bits: int, half_exp_bits: int, half_exp_bias: int
+):
+    # Casts an fp16/bf16 input to the restricted values of float4_e2m1,
+    # that is to say [0., 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0,
+    # -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0].
+
+    float_type = val.dtype
+
+    # "rshift_cuda" not implemented for 'UInt16'
+    val_view = val.view(torch.int16)  # .to(torch.int32)
+
+    exp = val_view >> half_mantissa_bits
+    exp = exp & ((1 << half_exp_bits) - 1)
+
+    exp = exp.view(torch.uint16).to(torch.int32)
+
+    sign = (val_view >> (half_mantissa_bits + half_exp_bits)) & 1
+
+    mantissa_last = (val_view >> (half_mantissa_bits - 1)) & 1
+
+    exp_unbias = exp - half_exp_bias
+    new_exp = exp_unbias + FLOAT4_EXP_BIAS
+
+    exp_shift = (new_exp <= 0) * (1 - new_exp)
+
+    # Typically 9.
+    # Take the min to prevent overflow on `uint16_t half`. This is the case for
+    # very small values, correctly mapped to `round_close`.
+    tail_bits = half_mantissa_bits - FLOAT4_MANTISSA_BITS + exp_shift
+    tail_bits[tail_bits >= 16] = 16
+
+    mantissa_plus_one = val_view & ((1 << (half_mantissa_bits + 1)) - 1)
+
+    half = 1 << (tail_bits - 1)
+
+    tail = mantissa_plus_one & ((1 << tail_bits) - 1)
+
+    round_close = tail < half  # round towards 0
+    round_away = tail > half  # round away from 0
+    tie = tail == half
+
+    new_mantissa_close = torch.zeros(val.shape, device=val.device, dtype=torch.bool)
+    new_exp_close = torch.zeros(val.shape, device=val.device, dtype=torch.uint16)
+
+    new_mantissa_away = torch.zeros(val.shape, device=val.device, dtype=torch.bool)
+    new_exp_away = torch.zeros(val.shape, device=val.device, dtype=torch.uint16)
+
+    new_exp_tie = torch.zeros(val.shape, device=val.device, dtype=torch.uint16)
+
+    # 1. round down
+    # if new_exp == 0: # case [0.5, 0.749999]
+    #     new_mantissa = 0
+    # elif new_exp < 0:  # case [0, 0.24999]
+    #     new_mantissa = 0
+    # else:
+    #     new_mantissa = mantissa_last
+
+    new_mantissa_close = (new_exp > 0) * mantissa_last
+    new_exp_close = exp
+
+    # # 2. round up
+    # if new_exp <= 0:  # case [0.250001, 0.499999] and [0.75001, 0.99999]
+    #     new_mantissa = 0
+    #     new_exp += 1
+    # elif mantissa_last == 0:
+    #     new_mantissa = 1
+    # else:
+    #     new_mantissa = 0
+    #     new_exp += 1
+
+    new_mantissa_away = torch.logical_and(new_exp > 0, mantissa_last == 0)
+    new_exp_away = exp + torch.logical_or(new_exp <= 0, mantissa_last == 1)
+
+    # # 3. tie
+    # 0.25 -> 0. (handled by `exp > (half_exp_bias - 2)`)
+    # 0.75 -> 1.
+    # 1.25 -> 1.
+    # 1.75 -> 2.
+    # 2.5 -> 2.
+    # 3.5 -> 4.
+    # 5. -> 4.
+    new_exp_tie = (exp > (half_exp_bias - 2)) * (exp + (mantissa_last == 1))
+
+    # Gather round up, round down and tie.
+    new_exp = (
+        round_away * new_exp_away + round_close * new_exp_close + tie * new_exp_tie
+    )
+
+    new_mantissa = round_away * new_mantissa_away + round_close * new_mantissa_close
+
+    # if new_exp > 3:
+    #     new_mantissa = 1
+    new_mantissa = new_mantissa + (new_exp > (2 + half_exp_bias)) * (new_mantissa == 0)
+
+    # Clamp the exponent to acceptable values.
+    new_exp = (new_exp >= (half_exp_bias - 2)) * torch.clamp(
+        new_exp, half_exp_bias - 2, half_exp_bias + 2
+    )
+
+    sign = sign.to(torch.int32)
+    new_mantissa = new_mantissa.to(torch.int32)
+
+    qdq_val = (
+        (sign << 15)
+        + (new_exp << half_mantissa_bits)
+        + (new_mantissa << (half_mantissa_bits - 1))
+    )
+
+    assert qdq_val.max() <= 65535
+    assert qdq_val.min() >= 0
+    assert qdq_val.dtype == torch.int32
+    qdq_val = qdq_val.to(torch.uint16)
+
+    result = qdq_val.view(float_type)
+    return result
+
+
+def qdq_mxfp4_torch(
+    x: torch.Tensor, scale_calculation_mode: str = "even"
+) -> torch.Tensor:
+    half_dtype = x.dtype
+
+    if half_dtype == torch.float16:
+        half_mantissa_bits = FLOAT16_MANTISSA_BITS
+        half_exp_bits = FLOAT16_EXP_BITS
+        half_exp_bias = FLOAT16_EXP_BIAS
+        val_to_add = FLOAT16_VAL_TO_ADD
+        sign_exponent_mask = FLOAT16_SIGN_EXPONENT_MASK
+    elif half_dtype == torch.bfloat16:
+        half_mantissa_bits = BFLOAT16_MANTISSA_BITS
+        half_exp_bits = BFLOAT16_EXP_BITS
+        half_exp_bias = BFLOAT16_EXP_BIAS
+        val_to_add = BFLOAT16_VAL_TO_ADD
+        sign_exponent_mask = BFLOAT16_SIGN_EXPONENT_MASK
+    else:
+        raise ValueError("not implemented")
+
+    x = x.reshape(*x.shape[:-1], -1, 32)
+
+    block_max = torch.max(torch.abs(x), dim=-1).values
+
+    block_max = block_max.view(torch.uint16).to(torch.int32)
+
+    block_max_uint = torch.bitwise_and(block_max + val_to_add, sign_exponent_mask)
+
+    assert block_max_uint.max() <= 65535
+    assert block_max_uint.min() >= 0
+    assert block_max_uint.dtype == torch.int32
+    block_max_uint = block_max_uint.to(torch.uint16)
+
+    block_max = block_max_uint.view(half_dtype)
+
+    scale_exp = (
+        FLOAT8_E8M0_MAX_EXP + torch.floor(torch.log2(block_max)).to(torch.int32) - 2
+    )
+
+    scale_exp = torch.clamp(scale_exp, 0, 2 * FLOAT8_E8M0_MAX_EXP)
+
+    scale = 2.0 ** (scale_exp - FLOAT8_E8M0_MAX_EXP)
+    scale = scale.to(half_dtype)
+
+    x = x / scale[..., None]
+
+    x_fp4 = fp16_to_fp4_simulate(
+        x,
+        half_exp_bits=half_exp_bits,
+        half_mantissa_bits=half_mantissa_bits,
+        half_exp_bias=half_exp_bias,
+    )
+
+    x_fp4 = x_fp4 * scale[..., None]
+    return x_fp4.reshape(*x_fp4.shape[:-2], -1)

tests/quantization/test_auto_awq.py

@@ -0,0 +1,231 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Tests for AutoAWQConfig behavior after unification.
+
+These tests verify the bug fixes for:
+1. CPU platform override conflict (auto_awq should not override on CPU)
+2. MoE fallback compatibility (full_config["quant_method"] should be "awq")
+3. Config attribute consistency
+4. End-to-end quantization method loading (auto_awq loads and runs correctly)
+
+Note: Tests that require importing the full auto_awq module (which has GPU-dependent
+imports) should use subprocess or be run in a GPU environment.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from tests.quantization.utils import is_quant_method_supported
+
+
+def _get_auto_awq_config_source() -> str:
+    """Read the AutoAWQConfig class source code for isolated testing."""
+    import inspect
+
+    import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
+
+    return inspect.getsource(auto_awq_module.AutoAWQConfig)
+
+
+class TestAutoAWQConfigFromConfig:
+    """Tests for AutoAWQConfig.from_config behavior.
+
+    These tests require GPU environment to import the full module.
+    They are skipped on non-GPU platforms.
+    """
+
+    def test_full_config_quant_method_is_awq_for_moe_fallback(self):
+        """full_config should have quant_method='awq' for MoE fallback compatibility.
+
+        MoeWNA16Config only accepts 'gptq' or 'awq' as linear_quant_method.
+        If full_config has 'auto_awq', the MoE fallback will fail.
+        """
+        from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
+
+        config = {
+            "w_bit": 4,
+            "q_group_size": 128,
+            "zero_point": True,
+            "lm_head": False,
+        }
+        awq_config = AutoAWQConfig.from_config(config)
+
+        # Verify quant_method is 'awq' for MoE fallback
+        assert awq_config.full_config["quant_method"] == "awq", (
+            f"Expected quant_method='awq', got {awq_config.full_config['quant_method']}"
+        )
+
+    def test_full_config_preserves_other_fields(self):
+        """full_config should preserve all original config fields."""
+        from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
+
+        config = {
+            "w_bit": 4,
+            "q_group_size": 128,
+            "zero_point": True,
+            "lm_head": False,
+            "custom_field": "custom_value",
+        }
+        awq_config = AutoAWQConfig.from_config(config)
+
+        assert awq_config.full_config["w_bit"] == 4
+        assert awq_config.full_config["q_group_size"] == 128
+        assert awq_config.full_config["zero_point"] is True
+        assert awq_config.full_config["lm_head"] is False
+        assert awq_config.full_config["custom_field"] == "custom_value"
+
+    def test_full_config_is_copy_not_original(self):
+        """full_config should be a copy, not the original dict."""
+        from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
+
+        config = {
+            "w_bit": 4,
+            "q_group_size": 128,
+            "zero_point": True,
+            "lm_head": False,
+        }
+        original_quant_method = config.get("quant_method")
+
+        AutoAWQConfig.from_config(config)
+
+        # Original config should not be modified
+        assert config.get("quant_method") == original_quant_method
+
+
+class TestAutoAWQConfigAttributes:
+    """Tests for AutoAWQConfig attribute consistency.
+
+    These tests require GPU environment to import the full module.
+    They are skipped on non-GPU platforms.
+    """
+
+    def test_config_attributes_match_input(self):
+        """Config attributes should match input values."""
+        from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
+
+        awq_config = AutoAWQConfig(
+            weight_bits=4,
+            group_size=128,
+            zero_point=True,
+            lm_head_quantized=False,
+            modules_to_not_convert=["lm_head"],
+        )
+
+        assert awq_config.weight_bits == 4
+        assert awq_config.group_size == 128
+        assert awq_config.zero_point is True
+        assert awq_config.lm_head_quantized is False
+        assert awq_config.modules_to_not_convert == ["lm_head"]
+
+    def test_pack_factor_for_4bit(self):
+        """Pack factor should be 8 for 4-bit quantization."""
+        from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
+
+        awq_config = AutoAWQConfig(
+            weight_bits=4,
+            group_size=128,
+            zero_point=True,
+            lm_head_quantized=False,
+        )
+
+        assert awq_config.pack_factor == 8  # 32 // 4
+
+
+class TestAutoAWQConfigOverrideLogic:
+    """Tests for override logic by parsing source code (no GPU import required)."""
+
+    def _get_auto_awq_source(self) -> str:
+        """Read the auto_awq.py source file."""
+        import inspect
+        import pathlib
+
+        import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
+
+        source_path = inspect.getfile(auto_awq_module)
+        return pathlib.Path(source_path).read_text()
+
+    def test_cpu_check_in_override_method(self):
+        """override_quantization_method should check current_platform.is_cpu()."""
+        source = self._get_auto_awq_source()
+
+        # Verify the CPU check exists in override method
+        assert "current_platform.is_cpu()" in source, (
+            "override_quantization_method should check is_cpu()"
+        )
+        assert "return None" in source, (
+            "override_quantization_method should return None on CPU"
+        )
+
+    def test_quant_method_normalization_in_from_config(self):
+        """from_config should normalize quant_method to 'awq' for MoE fallback."""
+        source = self._get_auto_awq_source()
+
+        # Verify the normalization exists
+        assert (
+            '"quant_method"] = "awq"' in source or "'quant_method'] = 'awq'" in source
+        ), "from_config should set quant_method='awq' in full_config"
+
+
+# =============================================================================
+# End-to-end integration tests (require GPU environment)
+# =============================================================================
+
+PROMPT = "On the surface of Mars, we found"
+
+# Small AWQ model for testing - using Qwen2 1.5B which has official AWQ checkpoint
+AWQ_MODELS = [
+    "Qwen/Qwen2-1.5B-Instruct-AWQ",
+]
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("auto_awq"),
+    reason="auto_awq is not supported on this GPU type.",
+)
+@pytest.mark.parametrize("model_id", AWQ_MODELS)
+def test_auto_awq_quantization_method(vllm_runner, model_id: str, monkeypatch):
+    """Test that quantization='auto_awq' loads and runs correctly."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    with vllm_runner(
+        model_id,
+        dtype=torch.float16,
+        quantization="auto_awq",
+        max_model_len=2048,
+        enforce_eager=True,
+    ) as llm:
+
+        def check_model(model):
+            from vllm.model_executor.layers.quantization.auto_awq import (
+                AutoAWQLinearMethod,
+                AutoAWQMarlinLinearMethod,
+            )
+
+            for name, submodule in model.named_modules():
+                if name == "model.layers.0.self_attn.qkv_proj":
+                    # Should use either AutoAWQLinearMethod (Triton) or
+                    # AutoAWQMarlinLinearMethod (Marlin) depending on hardware
+                    assert isinstance(
+                        submodule.quant_method,
+                        (AutoAWQLinearMethod, AutoAWQMarlinLinearMethod),
+                    ), (
+                        f"Expected AutoAWQLinearMethod or AutoAWQMarlinLinearMethod "
+                        f"for {name}, got {type(submodule.quant_method)}"
+                    )
+                    break
+
+        llm.apply_model(check_model)
+
+        outputs = llm.generate_greedy([PROMPT], max_tokens=8)
+        assert outputs
+        assert len(outputs[0][1]) > 0
+
+
+def test_auto_awq_config_get_name():
+    """Test that AutoAWQConfig.get_name() returns 'auto_awq'."""
+    from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
+
+    assert AutoAWQConfig.get_name() == "auto_awq"

tests/quantization/test_auto_gptq.py

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests that the auto_gptq quantization method works correctly.
+
+Run `pytest tests/quantization/test_auto_gptq.py -v -s`.
+"""
+
+import pytest
+import torch
+
+from tests.quantization.utils import is_quant_method_supported
+from vllm.model_executor.layers.quantization.auto_gptq import (
+    AutoGPTQConfig,
+    AutoGPTQLinearMethod,
+)
+
+PROMPT = "On the surface of Mars, we found"
+
+MODELS = [
+    "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
+]
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("auto_gptq"),
+    reason="auto_gptq is not supported on this GPU type.",
+)
+@pytest.mark.parametrize("model_id", MODELS)
+def test_auto_gptq_quantization_method(vllm_runner, model_id: str, monkeypatch):
+    """Test that quantization='auto_gptq' loads and runs correctly."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    with vllm_runner(
+        model_id,
+        dtype=torch.float16,
+        quantization="auto_gptq",
+        max_model_len=2048,
+        enforce_eager=True,
+    ) as llm:
+
+        def check_model(model):
+            for name, submodule in model.named_modules():
+                if name == "model.layers.0.self_attn.qkv_proj":
+                    assert isinstance(submodule.quant_method, AutoGPTQLinearMethod)
+                    break
+
+        llm.apply_model(check_model)
+
+        outputs = llm.generate_greedy([PROMPT], max_tokens=8)
+        assert outputs
+        assert len(outputs[0][1]) > 0
+
+
+def test_auto_gptq_config_get_name():
+    """Test that AutoGPTQConfig.get_name() returns 'auto_gptq'."""
+    assert AutoGPTQConfig.get_name() == "auto_gptq"

tests/quantization/test_auto_round.py

@@ -0,0 +1,774 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Test model set-up and inference for quantized HF models supported
+on the AutoRound.
+
+Validating the configuration and printing results for manual checking.
+
+Run `pytest tests/quantization/test_auto_round.py`.
+"""
+
+import pytest
+
+from vllm.model_executor.layers.fused_moe import RoutedExperts
+from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
+from vllm.model_executor.layers.quantization.inc import INCConfig
+from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
+from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
+from vllm.model_executor.layers.quantization.inc.schemes import (
+    INCWna16Scheme,
+    resolve_scheme,
+)
+from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
+    INCLinearScheme,
+)
+from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear import (
+    INCARKLinearMethod,
+    INCWNA16LinearScheme,
+    INCXPULinearMethod,
+)
+from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_scheme import (
+    _resolve_awq_moe,
+    _resolve_gptq_moe,
+)
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
+from vllm.platforms import current_platform
+
+MODELS = [
+    pytest.param(
+        "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
+        id="auto_round:auto_gptq",
+    ),
+    pytest.param(
+        "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound",
+        marks=pytest.mark.skipif(
+            not (current_platform.is_cuda() or current_platform.is_xpu()),
+            reason="AWQ AutoRound model only supports CUDA/XPU backend for now.",
+        ),
+        id="auto_round:auto_awq",
+    ),
+]
+
+
+@pytest.mark.skipif(
+    not (
+        current_platform.is_cpu()
+        or current_platform.is_xpu()
+        or current_platform.is_cuda()
+    ),
+    reason="Only supports CPU/XPU/CUDA backend.",
+)
+@pytest.mark.parametrize("model", MODELS)
+def test_auto_round_model(vllm_runner, model):
+    with vllm_runner(model) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
+
+    assert output
+    print(output[0][1])
+
+
+# ---------------------------------------------------------------------------
+# Unit tests for INCConfig and related classes
+# ---------------------------------------------------------------------------
+
+
+class DummyLayer:
+    pass
+
+
+class DummyFusedMoE:
+    pass
+
+
+def make_config(**overrides) -> INCConfig:
+    kwargs = {
+        "weight_bits": 4,
+        "group_size": 128,
+        "sym": True,
+        "packing_format": "auto_round:auto_gptq",
+        "block_name_to_quantize": None,
+        "extra_config": None,
+        "data_type": "int",
+        "backend": "auto",
+    }
+    kwargs.update(overrides)
+    return INCConfig(**kwargs)
+
+
+def make_layer_config(**overrides) -> INCLayerConfig:
+    kwargs = {
+        "bits": 4,
+        "group_size": 128,
+        "sym": True,
+        "packing_format": "auto_round:auto_gptq",
+        "backend": "auto",
+        "data_type": "int",
+        "quantized": True,
+    }
+    kwargs.update(overrides)
+    return INCLayerConfig(**kwargs)
+
+
+def test_inc_config_parser_exact_match() -> None:
+    config = make_config(
+        extra_config={
+            "layers.0.self_attn.q_proj": {
+                "bits": 8,
+                "group_size": 64,
+                "sym": False,
+            }
+        }
+    )
+
+    layer_config = config.config_parser.resolve(
+        DummyLayer(), "layers.0.self_attn.q_proj"
+    )
+
+    assert layer_config.bits == 8
+    assert layer_config.group_size == 64
+    assert layer_config.sym is False
+    assert layer_config.quantized is True
+
+
+def test_inc_model_prefix_early_exit() -> None:
+    """extra_config keys with model. prefix trigger early unquantized return."""
+    config = make_config(
+        extra_config={
+            "model.layers.1.mlp.gate_proj": {
+                "bits": 16,
+            },
+        }
+    )
+
+    # get_quant_method checks model. prefix for unquantized early-exit
+    result = config.get_quant_method(DummyLayer(), "layers.1.mlp.gate_proj")
+    assert isinstance(result, UnquantizedLinearMethod)
+
+
+def test_inc_config_parser_regex_match() -> None:
+    config = make_config(
+        extra_config={
+            r"layers\.\d+\.self_attn\.(q|k|v)_proj": {
+                "bits": 8,
+                "group_size": 64,
+                "sym": False,
+            }
+        }
+    )
+
+    layer_config = config.config_parser.resolve(
+        DummyLayer(), "layers.3.self_attn.q_proj"
+    )
+
+    assert layer_config.bits == 8
+    assert layer_config.group_size == 64
+    assert layer_config.sym is False
+
+
+def test_inc_config_parser_invalid_regex_ignored() -> None:
+    config = make_config(
+        extra_config={
+            "[invalid": {
+                "bits": 8,
+                "group_size": 64,
+                "sym": False,
+            }
+        }
+    )
+
+    layer_config = config.config_parser.resolve(
+        DummyLayer(), "layers.0.self_attn.q_proj"
+    )
+
+    assert layer_config.bits == 4
+    assert layer_config.group_size == 128
+    assert layer_config.sym is True
+
+
+def test_inc_config_parser_block_name_to_quantize_marks_unquantized() -> None:
+    config = make_config(block_name_to_quantize=["layers.1"])
+
+    layer_config = config.config_parser.resolve(
+        DummyLayer(), "layers.0.self_attn.q_proj"
+    )
+
+    assert layer_config.bits == 16
+    assert layer_config.group_size == -1
+    assert layer_config.sym is True
+    assert layer_config.quantized is False
+
+
+def test_inc_config_parser_parallel_lm_head_defaults_to_unquantized() -> None:
+    layer = object.__new__(ParallelLMHead)
+    config = make_config()
+
+    layer_config = config.config_parser.resolve(layer, "lm_head")
+
+    assert layer_config.quantized is False
+    assert layer_config.bits == 16
+
+
+def test_inc_config_parser_fused_moe_requires_consistent_configs() -> None:
+    config = make_config(
+        extra_config={
+            "layers.0.block_sparse_moe.experts.0.w1": {
+                "bits": 4,
+                "group_size": 128,
+                "sym": True,
+            },
+            "layers.0.block_sparse_moe.experts.0.w2": {
+                "bits": 8,
+                "group_size": 128,
+                "sym": True,
+            },
+        }
+    )
+
+    with pytest.raises(ValueError, match="requires consistent quant config"):
+        config.config_parser.resolve(DummyFusedMoE(), "layers.0.block_sparse_moe")
+
+
+def test_inc_config_parser_fused_module_requires_consistent_configs() -> None:
+    config = make_config(
+        extra_config={
+            "layers.0.self_attn.q_proj": {
+                "bits": 4,
+                "group_size": 128,
+                "sym": True,
+            },
+            "layers.0.self_attn.k_proj": {
+                "bits": 8,
+                "group_size": 128,
+                "sym": True,
+            },
+            "layers.0.self_attn.v_proj": {
+                "bits": 4,
+                "group_size": 128,
+                "sym": True,
+            },
+        }
+    )
+    config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
+
+    with pytest.raises(ValueError, match="requires consistent quant config"):
+        config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj")
+
+
+def test_inc_layer_config_mx_fp_helpers() -> None:
+    layer_config = INCLayerConfig(
+        bits=4,
+        group_size=32,
+        sym=True,
+        packing_format="",
+        backend="",
+        data_type="mx_fp",
+        quantized=True,
+    )
+
+    assert layer_config.is_mxfp4 is True
+    assert layer_config.is_mxfp8 is False
+
+
+def test_inc_resolve_scheme_selects_wna16() -> None:
+    layer_config = INCLayerConfig(
+        bits=4,
+        group_size=128,
+        sym=True,
+        packing_format="auto_round:auto_gptq",
+        backend="auto",
+        data_type="int",
+        quantized=True,
+    )
+
+    scheme = resolve_scheme(layer_config)
+
+    assert isinstance(scheme, INCWna16Scheme)
+
+
+class DummyLinearScheme(INCLinearScheme):
+    def __init__(self) -> None:
+        self.calls: list[tuple] = []
+
+    @classmethod
+    def get_min_capability(cls) -> int:
+        return 0
+
+    def create_weights(self, *args, **kwargs) -> None:
+        self.calls.append(("create_weights", args, kwargs))
+
+    def process_weights_after_loading(self, layer) -> None:
+        self.calls.append(("process_weights_after_loading", layer))
+
+    def apply_weights(self, layer, x, bias=None):
+        self.calls.append(("apply_weights", layer, x, bias))
+        return "applied"
+
+
+def test_inc_linear_method_delegates() -> None:
+    scheme = DummyLinearScheme()
+    method = INCLinearMethod(scheme)
+    layer = DummyLayer()
+
+    method.create_weights(
+        layer,
+        input_size_per_partition=1,
+        output_partition_sizes=[2],
+        input_size=1,
+        output_size=2,
+        params_dtype=None,
+    )
+    method.process_weights_after_loading(layer)
+    result = method.apply(layer, "x", "b")
+
+    assert result == "applied"
+    assert [call[0] for call in scheme.calls] == [
+        "create_weights",
+        "process_weights_after_loading",
+        "apply_weights",
+    ]
+
+
+def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
+    class DummyQuantLinear:
+        pass
+
+    monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
+        lambda: (True, None, object(), DummyQuantLinear),
+    )
+
+    method = INCWna16Scheme().get_linear_method(
+        make_config(),
+        object(),
+        "layer",
+        make_layer_config(),
+    )
+
+    assert isinstance(method, INCLinearMethod)
+    assert isinstance(method.scheme, INCARKLinearMethod)
+
+
+def test_wna16_xpu_falls_back_when_ark_unavailable(monkeypatch) -> None:
+    monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
+        lambda: (False, "missing", None, None),
+    )
+
+    method = INCWna16Scheme().get_linear_method(
+        make_config(),
+        object(),
+        "layer",
+        make_layer_config(),
+    )
+
+    assert isinstance(method, INCLinearMethod)
+    assert isinstance(method.scheme, INCXPULinearMethod)
+
+
+def test_wna16_cpu_gptq_prefers_ark_when_available(monkeypatch) -> None:
+    class DummyQuantLinear:
+        pass
+
+    monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
+        lambda: (True, None, object(), DummyQuantLinear),
+    )
+
+    method = INCWna16Scheme().get_linear_method(
+        make_config(),
+        object(),
+        "layer",
+        make_layer_config(),
+    )
+
+    assert isinstance(method, INCLinearMethod)
+    assert isinstance(method.scheme, INCARKLinearMethod)
+
+
+def test_wna16_cpu_gptq_raises_when_ark_and_marlin_unavailable(
+    monkeypatch,
+) -> None:
+    monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
+        lambda: (False, "missing", None, None),
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.check_marlin_supported",
+        lambda *args, **kwargs: False,
+    )
+
+    with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
+        INCWna16Scheme().get_linear_method(
+            make_config(),
+            object(),
+            "layer",
+            make_layer_config(),
+        )
+
+
+def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None:
+    captured = {}
+
+    class DummyMethod:
+        def __init__(self, cfg):
+            captured["cfg"] = cfg
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear."
+        "check_marlin_supported",
+        lambda *args, **kwargs: True,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQLinearMethod",
+        DummyMethod,
+    )
+
+    scheme = INCWNA16LinearScheme(make_layer_config())
+
+    assert isinstance(scheme.inner_method, DummyMethod)
+    assert isinstance(captured["cfg"], AutoGPTQConfig)
+    assert captured["cfg"].weight_bits == 4
+    assert captured["cfg"].group_size == 128
+    assert captured["cfg"].is_sym is True
+
+
+def test_wna16_linear_gptq_unsupported_config_raises() -> None:
+    with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
+        INCWNA16LinearScheme(make_layer_config(sym=False))
+
+
+def test_wna16_xpu_unsupported_config_still_raises(monkeypatch) -> None:
+    monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
+
+    with pytest.raises(NotImplementedError, match="unsupported config"):
+        INCWna16Scheme().get_linear_method(
+            make_config(sym=False),
+            object(),
+            "layer",
+            make_layer_config(sym=False),
+        )
+
+
+def test_inc_get_quant_method_unquantized_linear_returns_unquantized() -> None:
+    config = make_config(extra_config={"layer": {"bits": 16}})
+    layer = object.__new__(LinearBase)
+
+    method = config.get_quant_method(layer, "layer")
+
+    assert isinstance(method, UnquantizedLinearMethod)
+
+
+def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
+    monkeypatch,
+) -> None:
+    """Early-exit returns UnquantizedFusedMoEMethod for FusedMoE layers
+    when extra_config has bits >= 16."""
+    config = make_config(extra_config={"layer": {"bits": 16}})
+    layer = object.__new__(RoutedExperts)
+    layer.moe_config = None  # UnquantizedFusedMoEMethod accepts moe_config
+
+    class DummyUnquantizedFusedMoEMethod:
+        def __init__(self, moe_config) -> None:
+            self.moe_config = moe_config
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.inc.UnquantizedFusedMoEMethod",
+        DummyUnquantizedFusedMoEMethod,
+    )
+
+    method = config.get_quant_method(layer, "layer")
+
+    assert isinstance(method, DummyUnquantizedFusedMoEMethod)
+    assert method.moe_config is None
+
+
+def test_inc_get_quant_method_linear_uses_resolved_scheme(monkeypatch) -> None:
+    config = make_config()
+    layer = object.__new__(LinearBase)
+    sentinel = object()
+
+    class DummyScheme:
+        def get_linear_method(self, _config, _layer, _prefix, _layer_config):
+            return sentinel
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
+        lambda _layer_config: DummyScheme(),
+    )
+
+    method = config.get_quant_method(layer, "layer")
+
+    assert method is sentinel
+
+
+def test_inc_get_quant_method_moe_uses_resolved_scheme(monkeypatch) -> None:
+    config = make_config()
+    layer = object.__new__(RoutedExperts)
+    sentinel = object()
+
+    class DummyScheme:
+        def get_moe_method(self, _config, _layer, _prefix, _layer_config):
+            return sentinel
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
+        lambda _layer_config: DummyScheme(),
+    )
+
+    method = config.get_quant_method(layer, "layer")
+
+    assert method is sentinel
+
+
+def test_resolve_gptq_moe_falls_back_to_moe_wna16(monkeypatch) -> None:
+    captured = {}
+
+    class DummyMoeConfig:
+        pass
+
+    class DummyLayer:
+        moe_config = DummyMoeConfig()
+
+    class DummyBuiltConfig:
+        pass
+
+    built_config = DummyBuiltConfig()
+
+    class DummyMethod:
+        def __init__(self, cfg, moe):
+            captured["cfg"] = cfg
+            captured["moe"] = moe
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
+        lambda *args, **kwargs: False,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Config.from_config",
+        lambda cfg: captured.update({"from_config": cfg}) or built_config,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Method",
+        DummyMethod,
+    )
+
+    layer_config = INCLayerConfig(
+        bits=4,
+        group_size=128,
+        sym=True,
+        packing_format="auto_round:auto_gptq",
+        backend="auto",
+        data_type="int",
+        quantized=True,
+    )
+
+    _resolve_gptq_moe(DummyLayer(), layer_config)
+
+    assert captured["from_config"] == {
+        "quant_method": "gptq",
+        "bits": 4,
+        "group_size": 128,
+        "sym": True,
+        "lm_head": False,
+    }
+    assert captured["cfg"] is built_config
+    assert captured["moe"] is DummyLayer.moe_config
+
+
+def test_resolve_gptq_moe_uses_auto_gptq_when_supported(monkeypatch) -> None:
+    captured = {}
+
+    class DummyMoeConfig:
+        pass
+
+    class DummyLayer:
+        moe_config = DummyMoeConfig()
+
+    class DummyMethod:
+        def __init__(self, cfg, moe):
+            captured["cfg"] = cfg
+            captured["moe"] = moe
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
+        lambda *args, **kwargs: True,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.utils.marlin_utils."
+        "check_moe_marlin_supports_layer",
+        lambda *args, **kwargs: True,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQMoEMethod",
+        DummyMethod,
+    )
+
+    _resolve_gptq_moe(DummyLayer(), make_layer_config())
+
+    assert isinstance(captured["cfg"], AutoGPTQConfig)
+    assert captured["cfg"].weight_bits == 4
+    assert captured["cfg"].group_size == 128
+    assert captured["cfg"].is_sym is True
+    assert captured["moe"] is DummyLayer.moe_config
+
+
+def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None:
+    captured = {}
+
+    class DummyMoeConfig:
+        pass
+
+    class DummyLayer:
+        moe_config = DummyMoeConfig()
+
+    class DummyMethod:
+        def __init__(self, cfg, moe):
+            captured["cfg"] = cfg
+            captured["moe"] = moe
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
+        lambda *args, **kwargs: True,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.utils.marlin_utils.check_moe_marlin_supports_layer",
+        lambda *args, **kwargs: True,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.auto_awq.verify_marlin_supported",
+        lambda *args, **kwargs: None,
+    )
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.auto_awq.AutoAWQMoEMethod",
+        DummyMethod,
+    )
+
+    layer_config = INCLayerConfig(
+        bits=4,
+        group_size=128,
+        sym=False,
+        packing_format="auto_round:auto_awq",
+        backend="auto",
+        data_type="int",
+        quantized=True,
+    )
+
+    _resolve_awq_moe(DummyLayer(), layer_config)
+
+    assert captured["cfg"].weight_bits == 4
+    assert captured["cfg"].zero_point is True
+    assert captured["moe"] is DummyLayer.moe_config
+
+
+# ---------------------------------------------------------------------------
+# Tests for get_layer_config step 4 (fused QKV / packed_modules_mapping)
+# ---------------------------------------------------------------------------
+
+
+class TestGetLayerConfigFusedQKV:
+    """Tests for step-4 (fused QKV / packed_modules_mapping) logic.
+
+    Focused on preventing false-positive substring matches.
+    """
+
+    def test_exact_fusion_key_match(self):
+        """A layer whose name contains 'qkv' maps to its extra_config entry."""
+        config = make_config(
+            extra_config={
+                "model.layers.0.self_attn.qkv_proj": {"bits": 8},
+            }
+        )
+        config.packed_modules_mapping = {
+            "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+        }
+        bits, _, _ = config.get_layer_config(
+            DummyLayer(), "model.layers.0.self_attn.qkv_proj"
+        )
+        assert bits == 8
+
+    def test_false_substring_match_does_not_override(self):
+        """Regression test for the false-substring-match bug.
+
+        Scenario (Qwen3.6-35B-A3B VLM):
+        - packed_modules_mapping has "qkv" → ["qkv"] (from vision encoder).
+        - The GDN text-attention layer is named "in_proj_qkvz".
+        - "qkv" is a substring of "in_proj_qkvz", so old code would enter
+          step 4 and generate sub_name "in_proj_qkvz" (replacing "qkv" with
+          "qkv"). That name is NOT in extra_config, so get_config() falls
+          back to the global default (bits=4), even though correct is 16.
+        - Fix: skip the fusion key when none of the generated sub_names
+          actually exist in extra_config.
+        """
+        config = make_config(
+            extra_config={
+                "model.layers.0.in_proj_qkv": {"bits": 16},
+                "model.layers.0.in_proj_z": {"bits": 16},
+            }
+        )
+        config.packed_modules_mapping = {
+            "qkv": ["qkv"],
+        }
+        bits, _, _ = config.get_layer_config(
+            DummyLayer(), "model.layers.0.in_proj_qkvz"
+        )
+        # bits should be the global default (4) – no erroneous fusion match
+        assert bits == 4
+
+    def test_real_qkv_fusion_key_still_resolves(self):
+        """The true "qkv" fusion (vision encoder) still resolves correctly."""
+        config = make_config(
+            extra_config={
+                "vision_model.encoder.layers.0.self_attn.qkv": {"bits": 8},
+            }
+        )
+        config.packed_modules_mapping = {
+            "qkv": ["qkv"],
+        }
+        bits, _, _ = config.get_layer_config(
+            DummyLayer(), "vision_model.encoder.layers.0.self_attn.qkv"
+        )
+        assert bits == 8
+
+    def test_mixed_fp16_and_int4_fused_layer(self):
+        """All sub-keys must agree; inconsistent configs raise ValueError."""
+        config = make_config(
+            extra_config={
+                "model.layers.0.self_attn.q_proj": {"bits": 16},
+                "model.layers.0.self_attn.k_proj": {"bits": 4},
+                "model.layers.0.self_attn.v_proj": {"bits": 4},
+            }
+        )
+        config.packed_modules_mapping = {
+            "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+        }
+        with pytest.raises(ValueError, match="consistent quant config"):
+            config.get_layer_config(DummyLayer(), "model.layers.0.self_attn.qkv_proj")
+
+    def test_fusion_triggered_by_regex_configured_sub_name(self):
+        """Fusion step 4 is still triggered when sub_names match via regex.
+
+        Ensures the guard does not regress when extra_config uses regex
+        patterns instead of exact keys to configure sub-modules.
+        """
+        config = make_config(
+            extra_config={
+                r"model\.layers\.\d+\.self_attn\.(q|k|v)_proj": {"bits": 8},
+            }
+        )
+        config.packed_modules_mapping = {
+            "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+        }
+        bits, _, _ = config.get_layer_config(
+            DummyLayer(), "model.layers.0.self_attn.qkv_proj"
+        )
+        assert bits == 8

tests/quantization/test_blackwell_moe.py

@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+import os
+from typing import Any
+
+import pytest
+
+from tests.utils import RemoteOpenAIServer
+from vllm.platforms import current_platform
+
+if not current_platform.is_device_capability_family(100):
+    pytest.skip(
+        "This test only runs on Blackwell GPUs (SM10x).", allow_module_level=True
+    )
+
+
+@pytest.fixture(scope="module", autouse=True)
+def set_test_environment():
+    """Sets environment variables required for this test module."""
+    # Make sure TRTLLM attention is available
+    os.environ["VLLM_HAS_FLASHINFER_CUBIN"] = "1"
+    # Set compilation threads to 16 to speed up startup
+    os.environ["FLASHINFER_NVCC_THREADS"] = "16"
+
+
+# Override the backbone layers to 4 for faster startup
+HF_OVERRIDE_TEXT = {
+    "num_layers": 4,
+    "num_hidden_layers": 4,
+}
+HF_OVERRIDE_MM = {
+    "text_config": {"num_layers": 4, "num_hidden_layers": 4},
+}
+
+
+def can_initialize(
+    model: str,
+    hf_overrides: dict[str, Any] | None = None,
+    extra_args: list[str] | None = None,
+):
+    # Server arguments
+    extra_args = extra_args if extra_args is not None else []
+    server_args = [
+        "--max-model-len",
+        "2048",
+        "--max-num-batched-tokens",
+        "256",
+        "--load-format",
+        "dummy",
+        "--trust-remote-code",
+        "--limit-mm-per-prompt",
+        json.dumps({"image": 0}),
+        *extra_args,
+    ]
+
+    # Launch server and make a simple request
+    with RemoteOpenAIServer(
+        model,
+        server_args,
+        max_wait_seconds=1500,  # Due to FlashInfer compile
+        override_hf_configs=hf_overrides,
+    ) as server:
+        client = server.get_client()
+        # Make a simple request to verify the server works
+        completion = client.completions.create(
+            model=model,
+            prompt=["Hello, World!"],
+            temperature=0,
+            max_tokens=2,
+        )
+        print(completion)
+        assert completion.choices[0].text is not None
+
+
+## Llama4 ##
+
+
+@pytest.mark.skip(
+    reason=(
+        "RuntimeError: run_moe() Expected a value of type "
+        "'Optional[List[Tensor]]' for argument '_9' but instead found type "
+        "'list'."
+    )
+)
+def test_llama4_fp8_tensor_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
+        hf_overrides=HF_OVERRIDE_MM,
+        extra_args=["--moe-backend=flashinfer_cutlass"],
+    )
+
+
+def test_llama4_fp8_tensor_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
+        hf_overrides=HF_OVERRIDE_MM,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )
+
+
+def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
+        hf_overrides=HF_OVERRIDE_MM,
+        extra_args=["--moe-backend=flashinfer_cutlass"],
+    )
+
+
+def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",
+        hf_overrides=HF_OVERRIDE_MM,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )
+
+
+## DeepSeekV3 ##
+
+
+def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "deepseek-ai/DeepSeek-V3.1",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=deep_gemm"],
+    )
+
+
+def test_deepseek_fp8_block_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "deepseek-ai/DeepSeek-V3.1",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=triton"],
+    )
+
+
+@pytest.mark.skip(
+    reason=(
+        "Known issue: lack of kernel support. "
+        "Expected failure: assert self.block_quant is None"
+    )
+)
+def test_deepseek_fp8_block_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "deepseek-ai/DeepSeek-V3.1",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_cutlass"],
+    )
+
+
+def test_deepseek_fp8_block_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "deepseek-ai/DeepSeek-V3.1",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )
+
+
+def test_deepseek_nvfp4_moe_flashinfer_vllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/DeepSeek-R1-0528-FP4-v2",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=cutlass"],
+    )
+
+
+def test_deepseek_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/DeepSeek-R1-0528-FP4-v2",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_cutlass"],
+    )
+
+
+def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/DeepSeek-R1-0528-FP4-v2",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )
+
+
+## GPT-OSS ##
+
+
+def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "openai/gpt-oss-20b",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )
+
+
+def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "openai/gpt-oss-20b",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=[
+            "--moe-backend",
+            "flashinfer_cutlass",
+            "--quantization-config.moe.activation",
+            "mxfp8",
+        ],
+    )
+
+
+def test_gptoss_mxfp4mxfp8_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "openai/gpt-oss-20b",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=[
+            "--quantization-config.moe.activation",
+            "mxfp8",
+        ],
+    )
+
+
+def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "openai/gpt-oss-20b",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--enforce-eager"],
+    )
+
+
+## Qwen3 Next ##
+
+
+@pytest.mark.skip(
+    reason=(
+        "FLASHINFER TRTLLM MoE has a bug with all negative router logits "
+        "for models with RENORMALIZE. This will be re-enabled once the "
+        "issue is fixed in flashinfer."
+    )
+)
+def test_qwen3_next_bf16_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "Qwen/Qwen3-Next-80B-A3B-Instruct",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )
+
+
+## NemoTron ##
+
+
+def test_nemotron_fp8_moe_flashinfer_throughput(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_cutlass"],
+    )
+
+
+@pytest.mark.skip(
+    reason=(
+        "FP8 MoE backend FLASHINFER_TRTLLM does not support the "
+        "deployment configuration since kernel does not support "
+        "no act_and_mul MLP layer."
+    )
+)
+def test_nemotron_fp8_moe_flashinfer_latency(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )
+
+
+@pytest.mark.skip(
+    reason=(
+        "FP8 MoE backend TRITON does not support the "
+        "deployment configuration since kernel does not support "
+        "no act_and_mul MLP layer."
+    )
+)
+def test_nemotron_fp8_moe_vllm_triton(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=triton"],
+    )
+
+
+def test_nemotron_fp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_cutlass"],
+    )
+
+
+def test_nemotron_fp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
+    can_initialize(
+        "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
+        hf_overrides=HF_OVERRIDE_TEXT,
+        extra_args=["--moe-backend=flashinfer_trtllm"],
+    )

tests/quantization/test_compressed_tensors.py

@@ -0,0 +1,962 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Test model set-up and weight loading for llmcompressor-quantized models.
+
+Run `pytest tests/quantization/test_compressed_tensors.py`.
+"""
+
+from contextlib import contextmanager
+from unittest.mock import Mock
+
+import pytest
+import torch
+from compressed_tensors.quantization import (
+    ActivationOrdering,
+    QuantizationArgs,
+    QuantizationStrategy,
+    QuantizationType,
+)
+
+from tests.models.utils import check_logprobs_close
+from vllm.model_executor.kernels.linear import (
+    Fp8BlockScaledMMLinearKernel,
+)
+from vllm.model_executor.layers.fused_moe import UnquantizedFusedMoEMethod
+from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import (  # noqa: E501
+    CompressedTensorsConfig,
+    CompressedTensorsLinearMethod,
+    CompressedTensorsW4A4Fp4,
+    CompressedTensorsW4A4Mxfp4,
+    CompressedTensorsW4A8Fp8,
+    CompressedTensorsW8A8Fp8,
+    CompressedTensorsW8A8Int8,
+    CompressedTensorsW8A8Mxfp8,
+    CompressedTensorsW8A16Fp8,
+    CompressedTensorsWNA8O8Int,
+    CompressedTensorsWNA16,
+)
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
+    find_matched_target,
+)
+from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
+from vllm.platforms import current_platform
+from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
+
+# AITER only supports per-channel-per-channel INT8 gemm
+# and per-tensor-per-tensor INT8 GEMM.
+# It does not support mix precision MM and mix quantization scheme.
+ROCM_AITER_SUPPORTED_INT8_MODEL = [
+    "neuralmagic/Llama-3.2-1B-quantized.w8a8",
+    "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",
+]
+
+# TritonInt8ScaledMMLinearKernel only supports symmetric quantization.
+ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL = [
+    "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
+    "nm-testing/tinyllama-oneshot-w8-channel-a8-tensor",
+    "neuralmagic/Llama-3.2-1B-quantized.w8a8",
+    "nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2",
+    "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",
+]
+
+
+@pytest.fixture(scope="function", autouse=True)
+def enable_pickle(monkeypatch):
+    """`LLM.apply_model` requires pickling a function."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+
+@pytest.mark.parametrize(
+    "model_args",
+    [
+        (
+            "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change",
+            "tensor",
+            QuantizationType.INT,
+            2560,
+            True,
+        ),
+        (
+            "nm-testing/asym-w8w8-int8-static-per-tensor-tiny-llama",
+            "tensor",
+            QuantizationType.INT,
+            2560,
+            False,
+        ),
+    ],
+)
+def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
+    model_path, strategy, quant_type, shape_0, is_symmetric = model_args
+
+    if (
+        current_platform.is_rocm()
+        and model_path not in ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL
+    ):
+        pytest.skip(f"Skip model {model_path} as it is not supported on ROCm.")
+
+    with vllm_runner(model_path, enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            o_proj = layer.self_attn.o_proj
+            gate_up_proj = layer.mlp.gate_up_proj
+            down_proj = layer.mlp.down_proj
+
+            # assert zp for symmetric and asymmetric cases
+            def zp_valid(zp: torch.Tensor | None):
+                if is_symmetric:
+                    return zp is None
+
+                return zp is not None and zp.dtype is torch.int32
+
+            assert zp_valid(qkv_proj.input_zero_point)
+            assert zp_valid(o_proj.input_zero_point)
+            assert zp_valid(gate_up_proj.input_zero_point)
+            assert zp_valid(down_proj.input_zero_point)
+
+            assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(gate_up_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(down_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
+
+            assert qkv_proj.scheme.strategy == strategy
+            assert qkv_proj.scheme.is_static_input_scheme
+            expected_type = torch.int8
+
+            assert qkv_proj.weight.dtype is expected_type
+            assert o_proj.weight.dtype is expected_type
+            assert gate_up_proj.weight.dtype is expected_type
+
+            if qkv_proj.scheme.strategy == "tensor":
+                # Make sure it is a channelwise buffer
+                # After running process_weights_after_loading
+                assert len(qkv_proj.weight_scale.shape) == 2
+                assert qkv_proj.weight_scale.shape[0] == shape_0
+                assert qkv_proj.weight_scale.shape[1] == 1
+            assert qkv_proj.weight_scale.dtype is torch.float32
+            assert qkv_proj.input_scale.dtype is torch.float32
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        assert output
+
+
+@pytest.mark.parametrize(
+    "model_path",
+    [
+        "neuralmagic/Llama-3.2-1B-quantized.w8a8",
+    ],
+)
+@pytest.mark.parametrize("max_tokens", [4])
+@pytest.mark.parametrize("num_logprobs", [10])
+@pytest.mark.parametrize(
+    "use_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+def test_compressed_tensors_w8a8_logprobs(
+    hf_runner,
+    vllm_runner,
+    example_prompts,
+    model_path,
+    max_tokens,
+    num_logprobs,
+    use_aiter,
+    monkeypatch,
+):
+    if (
+        current_platform.is_rocm()
+        and model_path not in ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL
+    ):
+        pytest.skip(f"Skip model {model_path} as it is not supported on ROCm.")
+
+    if use_aiter:
+        if model_path not in ROCM_AITER_SUPPORTED_INT8_MODEL:
+            pytest.skip(f"Skip model {model_path} as it is not support by aiter.")
+        # this will enable VLLM_ROCM_USE_AITER_LINEAR
+        monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
+
+    dtype = "bfloat16"
+
+    # skip language translation prompt for the static per tensor models
+    if model_path in (
+        "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Sym",
+        "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Static-Per-Tensor-Asym",
+    ):
+        example_prompts = example_prompts[0:-1]
+
+    with hf_runner(model_path, dtype=dtype) as hf_model:
+        hf_outputs = hf_model.generate_greedy_logprobs_limit(
+            example_prompts, max_tokens, num_logprobs
+        )
+
+    with vllm_runner(model_path, dtype=dtype, enforce_eager=True) as vllm_model:
+        vllm_outputs = vllm_model.generate_greedy_logprobs(
+            example_prompts, max_tokens, num_logprobs
+        )
+
+    check_logprobs_close(
+        outputs_0_lst=hf_outputs,
+        outputs_1_lst=vllm_outputs,
+        name_0="hf",
+        name_1="vllm",
+    )
+
+    if current_platform.is_rocm():
+        torch.accelerator.synchronize()
+
+
+def test_compressed_tensors_no_enforce_eager(vllm_runner):
+    model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
+    with vllm_runner(model_path) as llm:
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+@pytest.mark.parametrize(
+    "model_args",
+    [
+        ("nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2", "tensor"),
+        (
+            "nm-testing/tinyllama-oneshot-w8a8-channel-dynamic-token-v2",
+            "channel",
+        ),
+    ],
+)
+@pytest.mark.parametrize(
+    "use_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+def test_compressed_tensors_w8a8_dynamic_per_token(
+    vllm_runner,
+    model_args,
+    use_aiter,
+    monkeypatch,
+):
+    model_path, strategy = model_args
+
+    if (
+        current_platform.is_rocm()
+        and model_path not in ROCM_TRITON_SCALED_MM_SUPPORTED_INT8_MODEL
+    ):
+        pytest.skip(f"Skip model {model_path} as it is not supported on ROCm.")
+
+    if use_aiter:
+        if model_path not in ROCM_AITER_SUPPORTED_INT8_MODEL:
+            pytest.skip(f"Skip model {model_path} as it is not support by aiter.")
+        # this will enable VLLM_ROCM_USE_AITER_LINEAR
+        monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
+
+    with vllm_runner(model_path, enforce_eager=True, dtype=torch.float16) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+
+            assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
+            assert not qkv_proj.scheme.is_static_input_scheme
+            assert qkv_proj.scheme.strategy == strategy
+            assert qkv_proj.weight.dtype is torch.int8
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        assert output
+
+
+@pytest.mark.parametrize(
+    "wNa16_args",
+    [
+        (
+            "nm-testing/tinyllama-oneshot-w4a16-channel-v2",
+            "channel",
+            None,
+            8,
+            True,
+            False,
+        ),
+        (
+            "nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
+            "group",
+            128,
+            8,
+            False,
+            True,
+        ),
+    ],
+)
+@pytest.mark.skipif(
+    not current_platform.is_cuda(), reason="The tests are skipped on non-CUDA platform."
+)
+def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
+    model, strategy, group, pack_factor, symmetric, has_g_idx = wNa16_args
+    with vllm_runner(model, enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
+
+            assert qkv_proj.scheme.strategy == strategy
+            assert qkv_proj.scheme.group_size == (-1 if group is None else group)
+
+            assert qkv_proj.scheme.pack_factor == pack_factor
+            assert qkv_proj.scheme.symmetric == symmetric
+            assert qkv_proj.scheme.has_g_idx == has_g_idx
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+def test_compressed_tensors_fp8(vllm_runner):
+    model_path = "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
+    with vllm_runner(model_path, enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+
+            assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(
+                qkv_proj.scheme,
+                (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8),
+            )
+
+            assert qkv_proj.input_scale.dtype is torch.float32
+
+            if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8):
+                assert len(qkv_proj.input_scale.shape) == 0
+                assert qkv_proj.weight.dtype is current_platform.fp8_dtype()
+                assert qkv_proj.weight_scale.dtype is torch.float32
+                assert len(qkv_proj.weight_scale.shape) == 0
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
+)
+def test_compressed_tensors_kv_cache_fp8_per_tensor(vllm_runner):
+    model_path = "nm-testing/TinyLlama-1.1B-Chat-v1.0-kvcache-fp8-tensor"
+    with vllm_runner(model_path) as llm:
+        output = llm.generate_greedy("Hello world!", max_tokens=4)
+        assert output
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
+)
+def test_compressed_tensors_kv_cache_fp8_per_attn_head(vllm_runner):
+    model_path = "nm-testing/TinyLlama-1.1B-Chat-v1.0-kvcache-fp8-attn_head"
+    try:
+        fa_version = get_flash_attn_version()
+    except Exception:
+        pytest.skip("This test requires FlashAttention backend.")
+    if fa_version is None or fa_version < 3:
+        pytest.skip("This test requires FlashAttention version >= 3.")
+
+    with vllm_runner(model_path, attention_config={"backend": "FLASH_ATTN"}) as llm:
+        output = llm.generate_greedy("Hello world!", max_tokens=4)
+        assert output
+
+
+@contextmanager
+def _nvfp4_marlin_error_context(model, capfd):
+    is_rocm_and_unsupported = (
+        model == "nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4A16"
+        and current_platform.is_rocm()
+    )
+
+    if is_rocm_and_unsupported:
+        expected_error = (
+            "ValueError: Forced NVFP4 kernel MarlinNvFp4LinearKernel is not "
+            "supported: Marlin FP4 not available"
+        )
+        with pytest.raises(RuntimeError, match="Engine core initialization failed"):
+            yield
+
+        captured = capfd.readouterr()
+        assert expected_error in captured.out + captured.err
+    else:
+        yield
+
+
+@pytest.mark.parametrize(
+    "args",
+    [
+        ("nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4A16", True),
+        ("nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4", False),
+    ],
+)
+def test_compressed_tensors_nvfp4(vllm_runner, args, capfd):
+    model, use_a16 = args
+    with (
+        _nvfp4_marlin_error_context(model, capfd),
+        vllm_runner(model, enforce_eager=True) as llm,
+    ):
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(qkv_proj.scheme, CompressedTensorsW4A4Fp4)
+            assert qkv_proj.scheme.use_a16 == use_a16
+            assert qkv_proj.scheme.group_size == 16
+
+        llm.apply_model(check_model)
+        output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        print(output)
+        assert output
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda() or not current_platform.has_device_capability(90),
+    reason="W4A8 FP8 is not yet supported on this GPU type.",
+)
+@pytest.mark.parametrize(
+    "args",
+    [("czhu-cohere/TinyLlama-1.1B-Chat-v1.0-W4A8-e2e", CompressedTensorsW4A8Fp8)],
+)
+def test_compressed_tensors_w4a8_fp8(vllm_runner, args):
+    model, scheme = args
+    with vllm_runner(model, enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            o_proj = layer.self_attn.o_proj
+            gate_up_proj = layer.mlp.gate_up_proj
+            down_proj = layer.mlp.down_proj
+
+            for proj in (qkv_proj, o_proj, gate_up_proj, down_proj):
+                assert isinstance(proj.quant_method, CompressedTensorsLinearMethod)
+                assert isinstance(proj.scheme, scheme)
+
+                assert proj.weight_packed.dtype is torch.int32
+                assert proj.weight_scale.dtype is torch.float8_e4m3fn
+                assert proj.weight_chan_scale.dtype is torch.float32
+                assert proj.scheme.group_size == 128
+
+        llm.apply_model(check_model)
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        print(output)
+        assert output
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
+)
+@pytest.mark.parametrize(
+    "model,prompt,exp_perplexity",
+    [
+        (
+            "nm-testing/Llama-3.2-1B-Instruct-spinquantR1R2R4-w4a16",
+            "Flat is better than nested.\nSparse is better than dense.",
+            150.0,
+        ),
+        (
+            "nm-testing/Llama-3.2-1B-Instruct-quip-w4a16",
+            "Flat is better than nested.\nSparse is better than dense.",
+            150.0,
+        ),
+    ],
+)
+def test_compressed_tensors_transforms_perplexity(
+    vllm_runner, model, prompt, exp_perplexity
+):
+    with vllm_runner(model, enforce_eager=True) as llm:
+        perplexity = llm.generate_prompt_perplexity([prompt])[0]
+        print(perplexity)
+        assert perplexity <= exp_perplexity
+
+
+def test_compressed_tensors_fp8_block_enabled(vllm_runner):
+    model_path = "RedHatAI/Qwen3-0.6B-FP8-BLOCK"
+    with vllm_runner(model_path, enforce_eager=True) as llm:
+        fp8_dtype = current_platform.fp8_dtype()
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8)
+            assert isinstance(qkv_proj.scheme.fp8_linear, Fp8BlockScaledMMLinearKernel)
+
+            assert qkv_proj.weight.dtype is fp8_dtype
+            assert qkv_proj.weight_scale.dtype is torch.float32
+            assert len(qkv_proj.weight.shape) == 2
+            assert len(qkv_proj.weight_scale.shape) == 2
+
+            input_quant_op = qkv_proj.scheme.fp8_linear.quant_fp8
+            assert isinstance(input_quant_op, QuantFP8)
+            assert input_quant_op._forward_method in (
+                input_quant_op.forward_cuda,
+                input_quant_op.forward_hip,
+                input_quant_op.forward_xpu,
+            )
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda(),
+    reason="This test is not for non-CUDA platforms",
+)
+def test_compressed_tensors_moe_ignore_with_model(vllm_runner):
+    """
+    Integration test for MoE layer ignore functionality with a real model.
+
+    This test would verify that when loading a compressed-tensors quantized
+    MoE model where some MoE layers are in the ignore list, those layers
+    use UnquantizedFusedMoEMethod while non-ignored layers use the
+    quantized method.
+
+    Expected model structure:
+    - Compressed-tensors quantized MoE model (e.g., Mixtral-based)
+    - Config with ignore list containing specific MoE layers
+    - Multiple MoE layers where some are quantized and some are not
+    """
+
+    # model_path = "nm-testing/tinysmokeqwen3moe-W4A16-first-only" # CT 12.3
+    model_path = "nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable"  # CT 12.2
+
+    with vllm_runner(model_path, enforce_eager=True) as llm:
+
+        def check_model(model):
+            from vllm.model_executor.layers.fused_moe import MoERunner
+            from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import (  # noqa: E501
+                CompressedTensorsMoEMethod,
+            )
+
+            # Check layer 0 MoE (should be quantized)
+            layer_quantized = model.model.layers[0].mlp.experts
+            assert isinstance(layer_quantized, MoERunner)
+            assert isinstance(layer_quantized._quant_method, CompressedTensorsMoEMethod)
+
+            # Check layer 10 MoE (should be unquantized + ignored)
+            layer_unquantized = model.model.layers[3].mlp.experts
+            assert isinstance(layer_unquantized, MoERunner)
+            assert isinstance(
+                layer_unquantized._quant_method, UnquantizedFusedMoEMethod
+            )
+
+        llm.apply_model(check_model)
+
+        # Verify the model can generate output
+        output = llm.generate_greedy("Hello, my name is", max_tokens=4)
+        assert output
+
+
+def test_w4a16_moe_torch_compile(vllm_runner):
+    """Regression test: MoE quant_config must be initialized inside the
+    moe_forward custom op, not just in forward_native which is compiled by
+    Dynamo (attribute mutations are not replayed at runtime).
+
+    Without the fix in _moe_forward/_moe_forward_shared, this hits:
+        AssertionError: Hidden size mismatch 2048 != 1024
+    because use_int4_w4a16 is False (moe_quant_config stays None).
+    """
+    model_path = "nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable"
+
+    with vllm_runner(
+        model_path,
+        enforce_eager=False,
+        max_model_len=256,
+        compilation_config={
+            "cudagraph_mode": "NONE",
+        },
+    ) as llm:
+        output = llm.generate_greedy("Hi", max_tokens=1)
+        assert output
+
+
+def _make_ct_config(*, target: str = "Linear") -> CompressedTensorsConfig:
+    """Build a minimal CompressedTensorsConfig with INT8 channel quant."""
+    weight_quant = QuantizationArgs(
+        num_bits=8,
+        type=QuantizationType.INT,
+        strategy=QuantizationStrategy.CHANNEL,
+        symmetric=True,
+        dynamic=False,
+    )
+    return CompressedTensorsConfig(
+        target_scheme_map={
+            target: {
+                "weights": weight_quant,
+                "input_activations": None,
+                "format": "pack-quantized",
+            }
+        },
+        ignore=[],
+        quant_format="pack-quantized",
+    )
+
+
+def test_get_quant_method_returns_linear_method_for_parallel_lm_head():
+    """ParallelLMHead whose name matches a target must get a quantised method."""
+    config = _make_ct_config(target="re:.*lm_head")
+    mock_lm_head = Mock(spec=ParallelLMHead)
+    mock_lm_head.__class__ = ParallelLMHead
+
+    method = config.get_quant_method(mock_lm_head, prefix="model.lm_head")
+
+    assert isinstance(method, CompressedTensorsLinearMethod), (
+        f"Expected CompressedTensorsLinearMethod, got {type(method).__name__}"
+    )
+
+
+def test_get_quant_method_returns_none_for_ignored_parallel_lm_head():
+    """ParallelLMHead on the ignore list should be left unquantized (None)."""
+    config = _make_ct_config(target="re:.*lm_head")
+    config.ignore = ["re:.*lm_head"]
+    mock_lm_head = Mock(spec=ParallelLMHead)
+    mock_lm_head.__class__ = ParallelLMHead
+
+    method = config.get_quant_method(mock_lm_head, prefix="model.lm_head")
+
+    assert method is None, (
+        f"Expected None for ignored ParallelLMHead, got {type(method).__name__}"
+    )
+
+
+def test_get_quant_method_returns_none_for_unmatched_parallel_lm_head():
+    """ParallelLMHead with target='Linear' (typical real model) must not crash.
+
+    Most compressed-tensors models only target 'Linear'. ParallelLMHead does
+    not match that target, so get_quant_method should return None (unquantized)
+    instead of raising ValueError.
+    """
+    config = _make_ct_config(target="Linear")
+    mock_lm_head = Mock(spec=ParallelLMHead)
+    mock_lm_head.__class__ = ParallelLMHead
+
+    method = config.get_quant_method(mock_lm_head, prefix="model.lm_head")
+
+    assert method is None, (
+        f"Expected None for unmatched ParallelLMHead, got {type(method).__name__}"
+    )
+
+
+def test_find_matched_target_returns_none_on_no_match():
+    result = find_matched_target(
+        layer_name="model.layers.0.self_attn.qkv_proj",
+        module=Mock(spec=torch.nn.Linear),
+        targets=["no_match_target"],
+    )
+    assert result is None
+
+
+def test_get_scheme_dict_returns_none_on_no_match():
+    config = _make_ct_config(target="matched_layer")
+    result = config.get_scheme_dict(
+        layer=Mock(spec=torch.nn.Linear),
+        layer_name="model.layers.0.unmatched_layer",
+    )
+    assert result is None
+
+
+# Test constants for activation quantization
+_STATIC_SYM_INT8_ACT = QuantizationArgs(
+    num_bits=8,
+    type=QuantizationType.INT,
+    strategy=QuantizationStrategy.TENSOR.value,
+    symmetric=True,
+    dynamic=False,
+)
+
+_STATIC_ASYM_INT8_ACT = QuantizationArgs(
+    num_bits=8,
+    type=QuantizationType.INT,
+    strategy=QuantizationStrategy.TENSOR.value,
+    symmetric=False,
+    dynamic=False,
+)
+
+_DYNAMIC_INT8_ACT = QuantizationArgs(
+    num_bits=8,
+    type=QuantizationType.INT,
+    strategy=QuantizationStrategy.TOKEN.value,
+    symmetric=True,
+    dynamic=True,
+)
+
+
+@pytest.mark.parametrize(
+    "weight_bits,weight_strategy,input_act,output_act,format,expected_scheme",
+    [
+        # W8A8 int-quantized -> W8A8Int8 (regression test for #46389)
+        pytest.param(
+            8,
+            QuantizationStrategy.CHANNEL.value,
+            _STATIC_SYM_INT8_ACT,
+            None,
+            "int-quantized",
+            CompressedTensorsW8A8Int8,
+            id="w8a8_channel_static_sym",
+        ),
+        pytest.param(
+            8,
+            QuantizationStrategy.CHANNEL.value,
+            _STATIC_ASYM_INT8_ACT,
+            None,
+            "int-quantized",
+            CompressedTensorsW8A8Int8,
+            id="w8a8_channel_static_asym",
+        ),
+        pytest.param(
+            8,
+            QuantizationStrategy.TENSOR.value,
+            _STATIC_SYM_INT8_ACT,
+            None,
+            "int-quantized",
+            CompressedTensorsW8A8Int8,
+            id="w8a8_tensor_static",
+        ),
+        pytest.param(
+            8,
+            QuantizationStrategy.CHANNEL.value,
+            _DYNAMIC_INT8_ACT,
+            None,
+            "int-quantized",
+            CompressedTensorsW8A8Int8,
+            id="w8a8_channel_dynamic",
+        ),
+        # W8A8O8 int-quantized -> WNA8O8Int (both input and output)
+        pytest.param(
+            8,
+            QuantizationStrategy.CHANNEL.value,
+            _STATIC_SYM_INT8_ACT,
+            _STATIC_SYM_INT8_ACT,
+            "int-quantized",
+            CompressedTensorsWNA8O8Int,
+            id="w8a8o8_channel",
+        ),
+        pytest.param(
+            4,
+            QuantizationStrategy.GROUP.value,
+            _STATIC_SYM_INT8_ACT,
+            _STATIC_SYM_INT8_ACT,
+            "int-quantized",
+            CompressedTensorsWNA8O8Int,
+            id="w4a8o8_group",
+        ),
+        # Weight-only pack-quantized -> WNA16
+        pytest.param(
+            8,
+            QuantizationStrategy.CHANNEL.value,
+            None,
+            None,
+            "pack-quantized",
+            CompressedTensorsWNA16,
+            id="w8_pack",
+        ),
+        pytest.param(
+            4,
+            QuantizationStrategy.GROUP.value,
+            None,
+            None,
+            "pack-quantized",
+            CompressedTensorsWNA16,
+            id="w4_pack",
+        ),
+        pytest.param(
+            2,
+            QuantizationStrategy.GROUP.value,
+            None,
+            None,
+            "pack-quantized",
+            CompressedTensorsWNA16,
+            id="w2_pack",
+        ),
+        pytest.param(
+            3,
+            QuantizationStrategy.GROUP.value,
+            None,
+            None,
+            "pack-quantized",
+            CompressedTensorsWNA16,
+            id="w3_pack",
+        ),
+        pytest.param(
+            5,
+            QuantizationStrategy.GROUP.value,
+            None,
+            None,
+            "pack-quantized",
+            CompressedTensorsWNA16,
+            id="w5_pack",
+        ),
+        pytest.param(
+            6,
+            QuantizationStrategy.GROUP.value,
+            None,
+            None,
+            "pack-quantized",
+            CompressedTensorsWNA16,
+            id="w6_pack",
+        ),
+        pytest.param(
+            7,
+            QuantizationStrategy.GROUP.value,
+            None,
+            None,
+            "pack-quantized",
+            CompressedTensorsWNA16,
+            id="w7_pack",
+        ),
+    ],
+)
+def test_scheme_selection(
+    weight_bits, weight_strategy, input_act, output_act, format, expected_scheme
+):
+    """Test that _get_scheme_from_parts selects the correct scheme.
+
+    This parametrized test verifies scheme selection for various combinations
+    of weight bits, quantization strategies, input/output activations, and
+    compression formats.
+
+    Key regression test: W8A8 int-quantized models with channel-wise weights
+    should use W8A8Int8 (true int8 gemm), not WNA8O8Int (fake-quant).
+    WNA8O8Int should only match when BOTH input and output activations are
+    present.
+    """
+    weight_quant = QuantizationArgs(
+        num_bits=weight_bits,
+        type=QuantizationType.INT,
+        strategy=weight_strategy,
+        symmetric=True,
+        dynamic=False,
+        group_size=128 if weight_strategy == QuantizationStrategy.GROUP.value else None,
+    )
+
+    config = CompressedTensorsConfig(
+        target_scheme_map={},
+        ignore=[],
+        quant_format=format,
+    )
+
+    scheme = config._get_scheme_from_parts(
+        weight_quant=weight_quant,
+        input_quant=input_act,
+        output_quant=output_act,
+        format=format,
+    )
+
+    assert isinstance(scheme, expected_scheme), (
+        f"Expected {expected_scheme.__name__} for "
+        f"W{weight_bits} {weight_strategy} + "
+        f"input_act={input_act} + output_act={output_act} + "
+        f"format={format}, got {type(scheme).__name__}"
+    )
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda() or not current_platform.has_device_capability(75),
+    reason="MXFP8 requires Turing (sm_75+) or newer.",
+)
+def test_compressed_tensors_mxfp8_moe_setup(vllm_runner):
+    """Verify MXFP8 scheme, dtypes, and generation for a MoE model."""
+    model_path = "AliEdalati97/Qwen3-30B-A3B-MXFP8"
+    with vllm_runner(
+        model_path,
+        enforce_eager=True,
+        load_format="dummy",
+        hf_overrides={"num_hidden_layers": 4},
+    ) as llm:
+
+        def check_model(model):
+            from vllm.model_executor.layers.fused_moe import MoERunner
+            from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_w8a8_mxfp8 import (  # noqa: E501
+                CompressedTensorsW8A8Mxfp8MoEMethod,
+            )
+
+            layer = model.model.layers[0]
+
+            qkv = layer.self_attn.qkv_proj
+            assert isinstance(qkv.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(qkv.scheme, CompressedTensorsW8A8Mxfp8)
+
+            experts = layer.mlp.experts
+            assert isinstance(experts, MoERunner)
+            assert isinstance(
+                experts._quant_method, CompressedTensorsW8A8Mxfp8MoEMethod
+            )
+
+        llm.apply_model(check_model)
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+@pytest.mark.parametrize(
+    "actorder,group_size,part,full,expected",
+    [
+        # actorder="group" with real grouping: must load full-K w2 scales and,
+        # when sharded (part != full), report is_k_full=False.
+        (ActivationOrdering.GROUP, 32, 64, 128, (True, 128, False)),
+        # actorder="group" but unsharded (part == full): full scales, k_full.
+        (ActivationOrdering.GROUP, 32, 128, 128, (True, 128, True)),
+        # actorder="group" with channel-wise (group_size == -1): no full load.
+        (ActivationOrdering.GROUP, -1, 64, 128, (False, 64, False)),
+        # "static"/"weight" reorder at quant time -> shard normally + k_full.
+        # Regression: static actorder under TP must keep is_k_full=True so the
+        # Marlin kernel never gets the invalid (group_size=16, is_k_full=0).
+        ("static", 32, 64, 128, (False, 64, True)),
+        ("weight", 32, 64, 128, (False, 64, True)),
+        (None, 32, 64, 128, (False, 64, True)),
+    ],
+)
+def test_wna16_marlin_moe_w2_scale_sharding(actorder, group_size, part, full, expected):
+    from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16_marlin import (  # noqa: E501
+        CompressedTensorsWNA16MarlinMoEMethod,
+    )
+
+    result = CompressedTensorsWNA16MarlinMoEMethod._w2_scale_sharding(
+        actorder, group_size, part, full
+    )
+    assert result == expected
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda() or not current_platform.has_device_capability(80),
+    reason="MXFP4 requires ampere or newer",
+)
+def test_compressed_tensors_mxfp4(vllm_runner):
+    model_path = "nm-testing/TinyLlama-1.1B-Chat-v1.0-MXFP4"
+    with vllm_runner(model_path, enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            o_proj = layer.self_attn.o_proj
+            gate_up_proj = layer.mlp.gate_up_proj
+            down_proj = layer.mlp.down_proj
+
+            for proj in (qkv_proj, o_proj, gate_up_proj, down_proj):
+                assert isinstance(proj.quant_method, CompressedTensorsLinearMethod)
+                assert isinstance(proj.scheme, CompressedTensorsW4A4Mxfp4)
+
+                # Verify group size
+                assert proj.scheme.group_size == 32
+
+        llm.apply_model(check_model)
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output

tests/quantization/test_configs.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests whether Marlin models can be loaded from the autogptq config.
+
+Run `pytest tests/quantization/test_configs.py --forked`.
+"""
+
+from dataclasses import dataclass
+
+import pytest
+
+from vllm.config import ModelConfig
+from vllm.platforms import current_platform
+
+
+@dataclass
+class ModelPair:
+    model_marlin: str
+    model_gptq: str
+
+
+# Model Id // Quantization Arg // Expected Type
+MODEL_ARG_EXPTYPES = [
+    # AUTOGPTQ
+    # compat: autogptq <=0.7.1 is_marlin_format: bool
+    # Model Serialized in Exllama Format.
+    ("TheBloke/Llama-2-7B-Chat-GPTQ", None, "auto_gptq"),
+    (
+        "TheBloke/Llama-2-7B-Chat-GPTQ",
+        "marlin",
+        "auto_gptq" if current_platform.is_cuda_alike() else "ERROR",
+    ),
+    ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "auto_gptq"),
+    ("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"),
+    # compat: autogptq >=0.8.0 use checkpoint_format: str
+    # Model Serialized in Exllama Format.
+    ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "auto_gptq"),
+    (
+        "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
+        "marlin",
+        "auto_gptq" if current_platform.is_cuda_alike() else "ERROR",
+    ),
+    ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
+    ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
+    # AUTOAWQ
+    # AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models
+    # when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq"
+    (
+        "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
+        None,
+        "auto_awq",
+    ),
+    ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"),
+    (
+        "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
+        "marlin",
+        "auto_awq" if current_platform.is_cuda_alike() else "ERROR",
+    ),
+    ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
+]
+
+
+@pytest.mark.parametrize("model_arg_exptype", MODEL_ARG_EXPTYPES)
+def test_auto_gptq(model_arg_exptype: tuple[str, None, str]) -> None:
+    model_path, quantization_arg, expected_type = model_arg_exptype
+
+    try:
+        model_config = ModelConfig(model_path, quantization=quantization_arg)
+        found_quantization_type = model_config.quantization
+    except ValueError:
+        found_quantization_type = "ERROR"
+
+    assert found_quantization_type == expected_type, (
+        f"Expected quant_type == {expected_type} for {model_path}, "
+        f"but found {found_quantization_type} "
+        f"for no --quantization {quantization_arg} case"
+    )

tests/quantization/test_cpu_offload.py

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Expanded quantized model tests for CPU offloading
+# Base tests: tests/basic_correctness/test_cpu_offload.py
+
+import pytest
+
+from tests.quantization.utils import is_quant_method_supported
+
+from ..utils import compare_two_settings
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="fp8 is not supported on this GPU type.",
+)
+def test_cpu_offload_fp8():
+    # Test loading a quantized checkpoint
+    compare_two_settings(
+        "neuralmagic/Qwen2-1.5B-Instruct-FP8",
+        ["--enforce_eager"],
+        ["--enforce_eager", "--cpu-offload-gb", "1"],
+        max_wait_seconds=480,
+    )
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("gptq_marlin"),
+    reason="gptq_marlin is not supported on this GPU type.",
+)
+def test_cpu_offload_gptq(monkeypatch):
+    # This quant method is sensitive to dummy weights, so we force real weights
+    monkeypatch.setenv("VLLM_TEST_FORCE_LOAD_FORMAT", "auto")
+    # Test GPTQ Marlin
+    compare_two_settings(
+        "Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4",
+        ["--enforce_eager"],
+        ["--enforce_eager", "--cpu-offload-gb", "1"],
+        max_wait_seconds=480,
+    )
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("awq_marlin"),
+    reason="awq_marlin is not supported on this GPU type.",
+)
+def test_cpu_offload_awq(monkeypatch):
+    # This quant method is sensitive to dummy weights, so we force real weights
+    monkeypatch.setenv("VLLM_TEST_FORCE_LOAD_FORMAT", "auto")
+    # Test AWQ Marlin
+    compare_two_settings(
+        "Qwen/Qwen2-1.5B-Instruct-AWQ",
+        ["--enforce_eager"],
+        ["--enforce_eager", "--cpu-offload-gb", "1"],
+        max_wait_seconds=480,
+    )
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("gptq_marlin"),
+    reason="gptq_marlin is not supported on this GPU type.",
+)
+def test_cpu_offload_compressed_tensors(monkeypatch):
+    # This quant method is sensitive to dummy weights, so we force real weights
+    monkeypatch.setenv("VLLM_TEST_FORCE_LOAD_FORMAT", "auto")
+    # Test wNa16
+    compare_two_settings(
+        "nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16",
+        ["--enforce_eager"],
+        ["--enforce_eager", "--cpu-offload-gb", "1"],
+        max_wait_seconds=480,
+        include_seeded_sampling=False,
+    )

tests/quantization/test_cpu_w8a8.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from vllm.platforms import current_platform
+
+if not current_platform.is_cpu():
+    pytest.skip("skipping CPU-only tests", allow_module_level=True)
+
+MODELS = [
+    "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8",  # INT8 W8A8 MoE
+]
+DTYPE = ["bfloat16"]
+
+
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("dtype", DTYPE)
+def test_cpu_w8a8(vllm_runner, model, dtype):
+    with vllm_runner(model, dtype=dtype) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
+    assert output
+    print(output)

tests/quantization/test_cpu_wna16.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from vllm.platforms import current_platform
+
+if not current_platform.is_cpu():
+    pytest.skip("skipping CPU-only tests", allow_module_level=True)
+
+MODELS = [
+    "TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ",
+    "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",  # with g_idx
+    "Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4",  # without g_idx
+    "RedHatAI/Qwen3-1.7B-quantized.w4a16",  # with zp
+    "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
+    "Qwen/Qwen3-0.6B-FP8",  # FP8 W8A16 block-quantized linear
+    "Qwen/Qwen3-30B-A3B-FP8",  # FP8 W8A16 block-quantized MoE
+    "openai/gpt-oss-20b",  # MXFP4 W4A16
+    "QuixiAI/Qwen3-30B-A3B-AWQ",  # AWQ W4A16 MoE
+    "Qwen/Qwen3-30B-A3B-GPTQ-Int4",  # GPTQ W4A16 MoE
+    "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",  # compressed-tensors W4A16 MoE
+]
+DTYPE = ["bfloat16"]
+
+
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("dtype", DTYPE)
+def test_cpu_quant(vllm_runner, model, dtype):
+    with vllm_runner(model, dtype=dtype) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
+    assert output
+    print(output)

tests/quantization/test_cutlass_w4a16.py

@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for Cutlass W4A16 (Machete) kernel on Hopper.
+
+Verifies that W4A16 quantized models loaded through vllm select the
+MacheteLinearKernel on sm_90 GPUs, that weights are correctly repacked,
+and that inference produces valid output.
+
+Run `pytest tests/quantization/test_cutlass_w4a16.py`.
+"""
+
+import pytest
+import torch
+
+from vllm.platforms import current_platform
+
+if not current_platform.has_device_capability(90) or current_platform.is_rocm():
+    pytest.skip(
+        "Machete W4A16 requires Hopper (sm_90).",
+        allow_module_level=True,
+    )
+
+from vllm.model_executor.kernels.linear import (
+    MPLinearLayerConfig,
+    choose_mp_linear_kernel,
+)
+from vllm.model_executor.kernels.linear.mixed_precision import (
+    MacheteLinearKernel,
+)
+from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import (  # noqa: E501
+    CompressedTensorsLinearMethod,
+    CompressedTensorsWNA16,
+)
+from vllm.scalar_type import scalar_types
+
+
+@pytest.fixture(scope="function", autouse=True)
+def enable_pickle(monkeypatch):
+    """`LLM.apply_model` requires pickling a function."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+
+@pytest.mark.parametrize(
+    "act_type,weight_type,group_size,zero_points",
+    [
+        (torch.float16, scalar_types.uint4b8, 128, False),
+        (torch.bfloat16, scalar_types.uint4b8, 128, False),
+        (torch.float16, scalar_types.uint4, 128, True),
+        (torch.float16, scalar_types.uint4b8, -1, False),
+    ],
+    ids=[
+        "fp16-gptq-g128",
+        "bf16-gptq-g128",
+        "fp16-awq-g128",
+        "fp16-channelwise",
+    ],
+)
+def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points):
+    """Verify choose_mp_linear_kernel picks MacheteLinearKernel."""
+    config = MPLinearLayerConfig(
+        full_weight_shape=(4096, 4096),
+        partition_weight_shape=(4096, 4096),
+        act_type=act_type,
+        weight_type=weight_type,
+        group_size=group_size,
+        zero_points=zero_points,
+        has_g_idx=False,
+    )
+    kernel = choose_mp_linear_kernel(config)
+    assert kernel is MacheteLinearKernel, (
+        f"Expected MacheteLinearKernel, got {kernel.__name__}"
+    )
+
+
+@pytest.mark.parametrize(
+    "full_shape,part_shape,weight_type,group_size,has_g_idx,expected_reason",
+    [
+        ((4096, 4096), (2048, 4096), scalar_types.uint4b8, 128, True, "Act reordering"),
+        (
+            (4096, 4096),
+            (4096, 4096),
+            scalar_types.float6_e3m2f,
+            128,
+            False,
+            "Quant type",
+        ),
+        ((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, False, "Group size"),
+    ],
+    ids=["partitioned-g_idx", "unsupported-quant-type", "unsupported-group-size"],
+)
+def test_machete_rejects_invalid_config(
+    full_shape, part_shape, weight_type, group_size, has_g_idx, expected_reason
+):
+    """Verify Machete rejects unsupported configurations."""
+    config = MPLinearLayerConfig(
+        full_weight_shape=full_shape,
+        partition_weight_shape=part_shape,
+        act_type=torch.float16,
+        weight_type=weight_type,
+        group_size=group_size,
+        zero_points=False,
+        has_g_idx=has_g_idx,
+    )
+    can_impl, reason = MacheteLinearKernel.can_implement(config)
+    assert not can_impl
+    assert expected_reason in reason
+
+
+def test_kernel_selection_with_disabled_machete(monkeypatch):
+    """Verify kernel selection falls back when Machete is disabled."""
+    monkeypatch.setattr("vllm.envs.VLLM_DISABLED_KERNELS", ["MacheteLinearKernel"])
+
+    config = MPLinearLayerConfig(
+        full_weight_shape=(4096, 4096),
+        partition_weight_shape=(4096, 4096),
+        act_type=torch.float16,
+        weight_type=scalar_types.uint4b8,
+        group_size=128,
+        zero_points=False,
+        has_g_idx=False,
+    )
+    kernel = choose_mp_linear_kernel(config)
+    assert kernel is not MacheteLinearKernel, "MacheteLinearKernel should be disabled"
+
+
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "nm-testing/tinyllama-oneshot-w4a16-channel-v2",
+        "nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
+    ],
+)
+def test_w4a16_machete_e2e(vllm_runner, model_name):
+    """Load a W4A16 model, verify Machete kernel is used, and generate."""
+    with vllm_runner(model_name, enforce_eager=True, gpu_memory_utilization=0.5) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+            qkv_proj = layer.self_attn.qkv_proj
+
+            assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
+            assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
+            assert isinstance(qkv_proj.scheme.kernel, MacheteLinearKernel), (
+                f"Expected MacheteLinearKernel on Hopper, "
+                f"got {type(qkv_proj.scheme.kernel).__name__}"
+            )
+
+            assert hasattr(qkv_proj, "weight_packed")
+            assert hasattr(qkv_proj, "weight_scale")
+            assert qkv_proj.weight_packed.dtype == torch.int32
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=10)
+        assert output
+        assert len(output[0][1]) > 0
+
+
+def test_w4a16_machete_bfloat16_deterministic(vllm_runner):
+    """Verify Machete works with bf16 activations and is deterministic."""
+    model_name = "nm-testing/tinyllama-oneshot-w4a16-channel-v2"
+    prompt = "The capital of France is"
+
+    with vllm_runner(
+        model_name,
+        enforce_eager=True,
+        dtype="bfloat16",
+        gpu_memory_utilization=0.5,
+    ) as llm:
+
+        def check_kernel_type(model):
+            layer = model.model.layers[0]
+            scheme = layer.self_attn.qkv_proj.scheme
+            assert isinstance(scheme.kernel, MacheteLinearKernel), (
+                f"Expected MacheteLinearKernel with bf16, "
+                f"got {type(scheme.kernel).__name__}"
+            )
+
+        llm.apply_model(check_kernel_type)
+
+        out1 = llm.generate_greedy(prompt, max_tokens=10)
+        out2 = llm.generate_greedy(prompt, max_tokens=10)
+        assert out1[0][1] == out2[0][1], (
+            f"Non-deterministic: '{out1[0][1]}' vs '{out2[0][1]}'"
+        )

tests/quantization/test_experts_int8.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# flake8: noqa
+"""Tests experts_int8 quantization startup and generation,
+doesn't test correctness
+"""
+
+import pytest
+
+from tests.quantization.utils import is_quant_method_supported
+
+from ..models.registry import HF_EXAMPLE_MODELS
+
+MODELS = ["ai21labs/Jamba-tiny-random", "pfnet/plamo-2-1b"]
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("experts_int8"),
+    reason="ExpertsInt8 is not supported on this GPU type.",
+)
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+@pytest.mark.parametrize("max_tokens", [4])
+def test_model_experts_int8_startup(
+    hf_runner,
+    vllm_runner,
+    example_prompts,
+    model: str,
+    dtype: str,
+    max_tokens: int,
+) -> None:
+    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+    model_info.check_transformers_version(on_fail="skip")
+
+    with vllm_runner(
+        model,
+        dtype=dtype,
+        enforce_eager=True,
+        quantization="experts_int8",
+    ) as vllm_model:
+        vllm_model.generate_greedy(example_prompts, max_tokens)

tests/quantization/test_fp8.py

@@ -0,0 +1,445 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests whether FP8 computation is enabled correctly.
+
+Run `pytest tests/quantization/test_fp8.py --forked`.
+"""
+
+import logging
+
+import pytest
+import regex as re
+import torch
+
+from tests.quantization.utils import is_quant_method_supported
+from vllm import _custom_ops as ops
+from vllm.config.model import ModelConfig
+from vllm.model_executor.kernels.linear.scaled_mm import (
+    MarlinFP8ScaledMMLinearKernel,
+)
+from vllm.model_executor.layers.fused_moe import FusedMoE
+from vllm.model_executor.layers.quantization.fp8 import (
+    Fp8Config,
+    Fp8KVCacheMethod,
+    Fp8LinearMethod,
+    Fp8MoEMethod,
+)
+from vllm.model_executor.layers.quantization.online.fp8 import (
+    Fp8PerTensorOnlineLinearMethod,
+)
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+from vllm.platforms import current_platform
+
+DEVICE_TYPE = current_platform.device_type
+
+MODELS = [
+    "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
+    # The checkpoint below was removed from the HF.
+    # TODO: add a small replacement checkpoint.
+    pytest.param(
+        "nm-testing/Qwen2-0.5B-Instruct-FP8-SkipQKV",
+        marks=pytest.mark.skip(reason="Checkpoint removed from HF."),
+    ),
+]
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+@pytest.mark.parametrize("model_id", MODELS)
+@pytest.mark.parametrize(
+    "force_marlin", [True, False] if current_platform.is_cuda() else [False]
+)
+@pytest.mark.parametrize(
+    "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+def test_model_load_and_run(
+    vllm_runner, model_id: str, force_marlin: bool, use_rocm_aiter: bool, monkeypatch
+) -> None:
+    if use_rocm_aiter:
+        monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
+
+    if force_marlin:
+        monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
+
+    with vllm_runner(model_id, enforce_eager=True) as llm:
+        # note: this does not test accuracy, just that we can run through
+        # see lm-eval tests for accuracy
+        outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        print(outputs[0][1])
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
+@pytest.mark.parametrize(
+    "force_marlin", [True, False] if current_platform.is_cuda() else [False]
+)
+@pytest.mark.parametrize(
+    "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+def test_online_quantization(
+    vllm_runner,
+    kv_cache_dtype: str,
+    force_marlin: bool,
+    use_rocm_aiter: bool,
+    monkeypatch,
+) -> None:
+    if use_rocm_aiter:
+        monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
+
+    # `LLM.apply_model` requires pickling a function.
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    if force_marlin:
+        monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")
+
+    with vllm_runner(
+        "facebook/opt-125m",
+        quantization="fp8",
+        enforce_eager=True,
+        kv_cache_dtype=kv_cache_dtype,
+    ) as llm:
+
+        def check_model(model):
+            fc1 = model.model.decoder.layers[0].fc1
+            assert isinstance(fc1.quant_method, Fp8PerTensorOnlineLinearMethod)
+            if kv_cache_dtype == "fp8":
+                attn = model.model.decoder.layers[0].self_attn.attn
+                assert isinstance(attn.quant_method, Fp8KVCacheMethod)
+                assert attn._k_scale == 1.0
+                assert attn._v_scale == 1.0
+
+            if current_platform.is_cuda() or current_platform.is_xpu():
+                if current_platform.supports_fp8() and not force_marlin:
+                    # For GPUs with hardware support, we keep weights in fp8
+                    assert fc1.weight.dtype == torch.float8_e4m3fn
+                    assert not isinstance(
+                        fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
+                    )
+                else:
+                    # For GPUs without hardware support, we pack the fp8 weights
+                    # for weight-only quantization using Marlin kernels
+                    assert fc1.weight.dtype == torch.int32
+                    assert isinstance(
+                        fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel
+                    )
+            elif current_platform.is_rocm():
+                if current_platform.supports_fp8() and not force_marlin:
+                    # For GPUs with hardware support, we keep weights in fp8
+                    assert fc1.weight.dtype == current_platform.fp8_dtype()
+                else:  # unsupported ROCm platform
+                    pytest.skip(
+                        "Skip `test_load_fp16_model`. "
+                        "It only runs on ROCm platform with FP8 compute."
+                        " e.g. MI300X and above."
+                    )
+            else:  # unsupported platform
+                pytest.skip(
+                    "Skip `test_load_fp16_model`. "
+                    "It only runs on CUDA and ROCm platform."
+                )
+
+        llm.apply_model(check_model)
+
+        outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        print(outputs[0][1])
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+def test_online_quant_peak_mem(
+    vllm_runner,
+    caplog_mp_spawn,
+    monkeypatch,
+) -> None:
+    # Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
+    # 1. it covers both Linear and MoE paths
+    # 2. it is already used by other tests in CI, so adding it here
+    #    does not increase disk space for CI runners
+    # I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
+    # which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
+    # 1.3 GiB fp8), but could not as adding one more model makes CI
+    # run out of disk space.
+    model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
+
+    # Force spawn to ensure caplog_mp_spawn works consistently
+    # (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
+    monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+    with (
+        caplog_mp_spawn(logging.DEBUG) as log_holder,
+        vllm_runner(
+            model_name,
+            quantization="fp8",
+            enforce_eager=True,
+        ) as llm,
+    ):
+        outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
+        print(outputs[0][1])
+
+    log_text = log_holder.text
+
+    # Parse memory usage from captured logs
+    model_memory_gib = None
+    peak_memory_gib = None
+    for line in log_text.splitlines():
+        if model_memory_gib is None:
+            match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
+            if match:
+                model_memory_gib = float(match.group(1))
+        if peak_memory_gib is None:
+            match = re.search(
+                r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
+            )
+            if match:
+                peak_memory_gib = float(match.group(1))
+
+    assert model_memory_gib is not None, "Could not find model loading memory log"
+    assert peak_memory_gib is not None, "Could not find peak memory log"
+    print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
+    print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
+
+    # model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant
+    # uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB)
+    expected_model_memory_gib = 6.7
+
+    # for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
+    # GiB, which is 1.36x above model_memory_gib. A slightly higher number is
+    # expected as when we load and quantize weights in a streaming fashion we
+    # need to have individual weights in bf16 + fp8 alive at the same time.
+    expected_peak_memory_gib = expected_model_memory_gib * 1.4
+
+    assert model_memory_gib < expected_model_memory_gib, (
+        f"{model_memory_gib=} higher than {expected_model_memory_gib}"
+    )
+    assert peak_memory_gib < expected_peak_memory_gib, (
+        f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
+    )
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+def test_online_quant_load_format_dummy(
+    vllm_runner,
+    monkeypatch,
+    caplog,
+) -> None:
+    with vllm_runner(
+        "ibm-granite/granite-3.0-1b-a400m-base",
+        quantization="fp8",
+        enforce_eager=True,
+        load_format="dummy",
+    ) as llm:
+        outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
+        print(outputs[0][1])
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
+def test_scaled_fp8_quant(dtype) -> None:
+    def quantize_ref(tensor, inv_scale):
+        # The reference implementation that fully aligns to
+        # the kernel being tested.
+        finfo = torch.finfo(current_platform.fp8_dtype())
+        scale = inv_scale.reciprocal()
+        qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max)
+        qweight = qweight.to(current_platform.fp8_dtype())
+        return qweight
+
+    def per_tensor_dequantize(tensor, inv_scale, dtype):
+        fake_qweight = tensor.to(dtype)
+        dq_weight = fake_qweight * inv_scale
+        return dq_weight
+
+    # Note that we use a shape % 4 != 0 to cover edge cases,
+    # because scaled_fp8_quant is vectorized by 4.
+    x = (torch.randn(size=(11, 11), device=DEVICE_TYPE) * 13).to(dtype)
+
+    # Dynamic quantization
+    ref_y, inv_scale = ops.scaled_fp8_quant(x, None)
+    ref_y = per_tensor_dequantize(ref_y, inv_scale, dtype)
+
+    # Reference dynamic quantization
+    y = quantize_ref(x, inv_scale)
+    torch.testing.assert_close(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
+
+    # Static quantization
+    y, _ = ops.scaled_fp8_quant(x, inv_scale)
+    torch.testing.assert_close(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
+
+    # Padding
+    y, _ = ops.scaled_fp8_quant(x, inv_scale, num_token_padding=17)
+    assert y.shape[0] == 17
+    torch.testing.assert_close(
+        ref_y,
+        per_tensor_dequantize(torch.narrow(y, 0, 0, x.shape[0]), inv_scale, dtype),
+    )
+
+    # non-contiguous input with padding
+    m, n, padded_stride = 975, 512, 576
+    padded_tensor = (torch.randn(size=(m, padded_stride), device=DEVICE_TYPE) * 13).to(
+        dtype
+    )
+    x_nc = padded_tensor[:, :n]  # shape (m, n) with stride (padded_stride, 1)
+
+    assert not x_nc.is_contiguous()
+    assert x_nc.stride(0) == padded_stride
+
+    # dynamic quantization
+    ref_y_nc, inv_scale_nc = ops.scaled_fp8_quant(x_nc, None)
+    ref_y_nc = per_tensor_dequantize(ref_y_nc, inv_scale_nc, dtype)
+
+    # reference dynamic quantization
+    y_nc = quantize_ref(x_nc, inv_scale_nc)
+    torch.testing.assert_close(
+        ref_y_nc, per_tensor_dequantize(y_nc, inv_scale_nc, dtype)
+    )
+
+    # static quantization
+    y_nc, _ = ops.scaled_fp8_quant(x_nc, inv_scale_nc)
+    torch.testing.assert_close(
+        ref_y_nc, per_tensor_dequantize(y_nc, inv_scale_nc, dtype)
+    )
+
+    # padding after non-contiguous input quantization
+    y_nc_pad, _ = ops.scaled_fp8_quant(x_nc, inv_scale_nc, num_token_padding=m + 10)
+    assert y_nc_pad.shape[0] == m + 10
+    torch.testing.assert_close(
+        ref_y_nc,
+        per_tensor_dequantize(
+            torch.narrow(y_nc_pad, 0, 0, x_nc.shape[0]), inv_scale_nc, dtype
+        ),
+    )
+
+
+@pytest.mark.skipif(
+    current_platform.is_fp8_fnuz(),
+    reason="FP8 e4m3fn weight reloading is not supported on e4m3fnuz platforms",
+)
+@pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])
+# FP8 weight reloading does not support online quantization
+@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True])  # skip False
+@pytest.mark.parametrize("weight_block_size", [None, [1, 1]])
+# any postprocessing that is applied to the weights such as padding and repacking
+# (excluding device sharding) must also be applied to the reloaded weights
+#
+# this is the case for marlin as well as per-tensor Fp8MoEMethod
+@pytest.mark.parametrize("use_marlin", [False])  # skip True
+def test_fp8_reloading(
+    default_vllm_config,
+    method_cls,
+    is_checkpoint_fp8_serialized,
+    weight_block_size,
+    use_marlin,
+    dist_init,
+    monkeypatch,
+):
+    # NOTE(rob): this test fails when using DeepGEMM because the
+    # shapes are invalid. Previously the test was passing because
+    # we set fp8_backend to None, which sidestepped the issue.
+    monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
+
+    if is_checkpoint_fp8_serialized is False:
+        pytest.skip("FP8 weight reloading does not support online quantization")
+
+    if method_cls is Fp8MoEMethod and weight_block_size is None:
+        pytest.skip(
+            "FP8 Tensor weight reloading does not support fusing w13_weight_scale. "
+            "If this is your use case, consider using a restore function like #26327"
+        )
+
+    # Set model config as model_config.dtype is required in Fp8LinearMethod.
+    default_vllm_config.model_config = ModelConfig()
+    with torch.device(f"{DEVICE_TYPE}:0"):
+        config = Fp8Config(
+            is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,
+            weight_block_size=weight_block_size,
+        )
+
+        if method_cls is Fp8LinearMethod:
+            layer = torch.nn.Linear(1, 1)
+            method = method_cls(config)
+            method.create_weights(
+                layer=layer,
+                input_size_per_partition=1,
+                output_partition_sizes=[1],
+                input_size=1,
+                output_size=1,
+                params_dtype=torch.bfloat16,
+                weight_loader=default_weight_loader,
+            )
+            method.use_marlin = use_marlin
+
+        else:
+            layer = FusedMoE(
+                num_experts=1,
+                top_k=1,
+                hidden_size=1,
+                intermediate_size=1,
+            )
+            layer = layer.routed_experts
+            method = method_cls(config, layer)
+            method.create_weights(
+                layer=layer,
+                num_experts=1,
+                hidden_size=1,
+                intermediate_size_per_partition=1,
+                params_dtype=torch.bfloat16,
+                weight_loader=default_weight_loader,
+            )
+
+    # capture weights format during loading
+    original_metadata = [
+        (name, param.shape, getattr(param, "weight_loader", default_weight_loader))
+        for name, param in layer.named_parameters()
+    ]
+
+    # test loading
+    for name, shape, _ in original_metadata:
+        param = getattr(layer, name)
+        weight_loader = getattr(param, "weight_loader", default_weight_loader)
+        weight_loader(param, torch.zeros(shape))  # cannot use empty
+
+    method.process_weights_after_loading(layer)
+
+    # test reloading works after loading
+    for name, shape, _ in original_metadata:
+        param = getattr(layer, name)
+        weight_loader = getattr(param, "weight_loader", default_weight_loader)
+        weight_loader(param, torch.zeros(shape))  # cannot use empty
+
+    method.process_weights_after_loading(layer)
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+def test_kv_cache_dtype_skip_layers(vllm_runner, monkeypatch):
+    """Test that kv_cache_dtype_skip_layers skips quantization for specified layers."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    with vllm_runner(
+        "facebook/opt-125m",
+        kv_cache_dtype="fp8",
+        kv_cache_dtype_skip_layers=["0", "2"],
+        enforce_eager=True,
+    ) as llm:
+
+        def check_layers(model):
+            for i, layer in enumerate(model.model.decoder.layers):
+                expected = "auto" if str(i) in ["0", "2"] else "fp8"
+                assert layer.self_attn.attn.kv_cache_dtype == expected
+
+        llm.apply_model(check_layers)

tests/quantization/test_fp8_per_channel.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for FP8 per-channel online quantization.
+
+Per-output-channel weight scale + dynamic per-token activation scale.
+bf16/fp16 checkpoints are quantized at load time with one fp32 scale per
+output channel for weights and one fp32 scale per token for activations
+(computed dynamically inside the kernel). Run via
+`pytest tests/quantization/test_fp8_per_channel.py --forked`.
+"""
+
+import pytest
+import torch
+
+from tests.quantization.utils import is_quant_method_supported
+from vllm import _custom_ops as ops
+from vllm.config.quantization import (
+    _ONLINE_SHORTHANDS,
+    QUANT_KEY_NAMES,
+    QuantizationConfigArgs,
+)
+from vllm.model_executor.layers.quantization.online.base import (
+    _ONLINE_LINEAR_METHODS,
+    _ONLINE_MOE_METHODS,
+)
+from vllm.model_executor.layers.quantization.online.fp8 import (
+    Fp8PtpcOnlineLinearMethod,
+    Fp8PtpcOnlineMoEMethod,
+)
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
+    kFp8StaticChannelSym,
+)
+from vllm.platforms import current_platform
+
+
+def test_fp8_per_channel_shorthand_registered() -> None:
+    """The `fp8_per_channel` CLI shorthand must resolve to a config that
+    dispatches the per-channel methods. Guards against regressions in
+    `_ONLINE_SHORTHANDS` / `_ONLINE_LINEAR_METHODS` / `_ONLINE_MOE_METHODS`
+    drifting out of sync.
+    """
+    args = _ONLINE_SHORTHANDS["fp8_per_channel"]
+    assert isinstance(args, QuantizationConfigArgs)
+    assert args.linear is not None
+    assert args.moe is not None
+    assert args.linear.weight is kFp8StaticChannelSym
+    assert args.moe.weight is kFp8StaticChannelSym
+
+    assert _ONLINE_LINEAR_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineLinearMethod
+    assert _ONLINE_MOE_METHODS[kFp8StaticChannelSym] is Fp8PtpcOnlineMoEMethod
+
+    assert QUANT_KEY_NAMES["fp8_per_channel_static"] is kFp8StaticChannelSym
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+def test_scaled_fp8_quant_per_channel_shape() -> None:
+    """Verify the kernel call per-channel quant depends on: passing a 2D
+    weight to `ops.scaled_fp8_quant` with `use_per_token_if_dynamic=True`
+    yields one scale per output row -- a [N, 1] fp32 tensor.
+    """
+    x = (torch.randn(size=(96, 256), device="cuda") * 13).to(torch.bfloat16)
+    y, s = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
+    assert y.shape == (96, 256)
+    assert y.dtype == current_platform.fp8_dtype()
+    assert s.shape == (96, 1)
+    assert s.dtype == torch.float32
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+def test_fp8_per_channel_online_quantization(
+    vllm_runner,
+    monkeypatch,
+) -> None:
+    """End-to-end smoke: load `facebook/opt-125m` bf16 with
+    `quantization='fp8_per_channel'`, check a dense Linear is wrapped by
+    `Fp8PtpcOnlineLinearMethod`, its weights are fp8 with per-channel
+    scales (shape `[N, 1]`), and a short greedy generation works.
+    """
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    with vllm_runner(
+        "facebook/opt-125m",
+        quantization="fp8_per_channel",
+        enforce_eager=True,
+    ) as llm:
+
+        def check_model(model):
+            fc1 = model.model.decoder.layers[0].fc1
+            assert isinstance(fc1.quant_method, Fp8PtpcOnlineLinearMethod)
+            assert fc1.weight.dtype == current_platform.fp8_dtype()
+            assert fc1.weight_scale.ndim == 2
+            assert fc1.weight_scale.shape[-1] == 1
+            assert fc1.input_scale is None
+
+        llm.apply_model(check_model)
+        outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        print(outputs[0][1])

tests/quantization/test_gfx950_moe.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Tests for MXFP4 MoE oracle backend selection on mi355x (GFX950).
+
+These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms.
+"""
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.fused_moe.config import (
+    FusedMoEConfig,
+    FusedMoEParallelConfig,
+    RoutingMethodType,
+)
+from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
+    Mxfp4MoeBackend,
+    select_mxfp4_moe_backend,
+)
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
+    kMxfp4Dynamic,
+)
+from vllm.platforms import current_platform
+
+ROCM_AVAILABLE = current_platform.is_rocm()
+ROCM_GFX950 = False
+ROCM_AITER_AVAILABLE = False
+
+if ROCM_AVAILABLE:
+    from vllm._aiter_ops import rocm_aiter_ops
+    from vllm.platforms.rocm import on_gfx950
+
+    ROCM_GFX950 = on_gfx950()
+    ROCM_AITER_AVAILABLE = rocm_aiter_ops.is_fused_moe_enabled()
+
+
+def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
+    from vllm.model_executor.layers.fused_moe.activation import MoEActivation
+
+    return FusedMoEConfig(
+        num_experts=8,
+        experts_per_token=2,
+        hidden_dim=256,
+        intermediate_size=256,
+        num_local_experts=8,
+        num_logical_experts=8,
+        moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
+        activation=MoEActivation.SILU,
+        in_dtype=torch.bfloat16,
+        device="cuda",
+        routing_method=RoutingMethodType.Renormalize,
+        moe_backend=moe_backend,
+    )
+
+
+@pytest.fixture
+def mxfp4_oracle_config():
+    """Stub the config the oracle reads (``model_config.quantization_config``)
+    so backend dispatch resolves without a real model / user override."""
+    from unittest.mock import patch
+
+    with patch(
+        "vllm.model_executor.layers.fused_moe.oracle.mxfp4.get_current_vllm_config"
+    ) as mock_get_config:
+        mock_get_config.return_value.model_config.quantization_config = None
+        yield
+
+
+@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
+@pytest.mark.skipif(not ROCM_AITER_AVAILABLE, reason="Requires AITER enabled")
+def test_w4a4_dispatches_to_aiter(mxfp4_oracle_config):
+    """With AITER enabled + GFX950, W4A4 selects AITER_MXFP4_MXFP4."""
+    config = _make_w4a4_moe_config()
+    backend, experts_cls = select_mxfp4_moe_backend(
+        config, activation_key=kMxfp4Dynamic
+    )
+    assert backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4
+    assert experts_cls is not None
+
+
+@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
+@pytest.mark.skipif(
+    ROCM_AITER_AVAILABLE,
+    reason="Test requires AITER disabled (unset VLLM_ROCM_USE_AITER)",
+)
+def test_w4a4_falls_back_to_triton_unfused_without_aiter(mxfp4_oracle_config):
+    """Without AITER and no --moe-backend, ROCm falls back to TRITON_UNFUSED."""
+    config = _make_w4a4_moe_config()
+    backend, experts_cls = select_mxfp4_moe_backend(
+        config, activation_key=kMxfp4Dynamic
+    )
+    assert backend == Mxfp4MoeBackend.TRITON_UNFUSED
+    assert experts_cls is not None
+
+
+@pytest.mark.skipif(not ROCM_GFX950, reason="Requires GFX950 (mi355x)")
+def test_w4a4_dispatches_to_emulation_with_moe_backend(mxfp4_oracle_config):
+    """With --moe-backend emulation, W4A4 selects EMULATION."""
+    config = _make_w4a4_moe_config(moe_backend="emulation")
+    backend, experts_cls = select_mxfp4_moe_backend(
+        config, activation_key=kMxfp4Dynamic
+    )
+    assert backend == Mxfp4MoeBackend.EMULATION
+    assert experts_cls is not None

tests/quantization/test_gptq_dynamic.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests whether gptq models with dynamic quantized can be loaded.
+
+Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
+
+Note: Only symmetric GPTQ models are supported after consolidation to Marlin.
+"""
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
+from vllm.model_executor.layers.quantization.utils.gptq_utils import (
+    get_dynamic_override,
+)
+
+PROMPT = "On the surface of Mars, we found"
+
+# The first layer is quantized using bits=4, group_size=128
+# The second layer is quantized using bits=8, group_size=32
+# All other layers (layer index >= 2) are not quantized
+# Note: Only symmetric models are supported with Marlin kernels
+MODELS = [
+    "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
+]
+
+
+@pytest.mark.parametrize("model_id", MODELS)
+def test_gptq_with_dynamic(vllm_runner, model_id: str, monkeypatch):
+    # `LLM.apply_model` requires pickling a function.
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    linear_method_cls = AutoGPTQLinearMethod
+
+    with vllm_runner(
+        model_id, dtype=torch.float16, max_model_len=2048, enforce_eager=True
+    ) as llm:
+
+        def check_model(model):
+            for name, submodule in model.named_modules():
+                if name == "lm_head":
+                    assert isinstance(submodule.quant_method, linear_method_cls)
+                elif name == "model.layers.0.self_attn.qkv_proj":
+                    # The first layer is quantized using bits=4, group_size=128
+                    # desc_act=True
+                    assert isinstance(submodule.quant_method, linear_method_cls)
+                    config = submodule.quant_method.quant_config
+                    assert config.weight_bits == 4
+                    assert config.group_size == 128
+                    assert config.desc_act
+                elif name == "model.layers.1.self_attn.qkv_proj":
+                    # The second layer is quantized using bits=8, group_size=32
+                    # desc_act=False
+                    assert isinstance(submodule.quant_method, linear_method_cls)
+                    config = submodule.quant_method.quant_config
+                    assert (
+                        get_dynamic_override(config, layer_name=name, key="bits") == 8
+                    )
+                    assert (
+                        get_dynamic_override(config, layer_name=name, key="group_size")
+                        == 32
+                    )
+                    assert not get_dynamic_override(
+                        config, layer_name=name, key="desc_act"
+                    )
+                elif (
+                    name == "model.layers.2.self_attn.qkv_proj"
+                    or name == "model.layers.2.mlp.gate_up_proj"
+                ):
+                    # All other layers (layer index >= 2) are not quantized
+                    assert isinstance(submodule.quant_method, UnquantizedLinearMethod)
+
+        llm.apply_model(check_model)

tests/quantization/test_gptq_v2.py

@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests whether vllm correctly load and run gptq_v2 format checkpoints.
+
+Run `pytest tests/quantization/test_gptq_v2.py --forked`.
+
+Note: 2/3-bit GPTQ models are no longer supported after the consolidation
+to Marlin kernels. Only 4/8-bit symmetric GPTQ models are supported.
+"""
+
+import pytest
+import torch
+from transformers import AutoTokenizer
+
+from vllm import SamplingParams
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
+
+# A dummy small model quantized by GPTQModel, stored in GPTQ v2 format
+# Note: This is a 2-bit model which is no longer supported with Marlin kernels
+MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"]
+
+# Generate multiple sequences for testing, because an 1.7B 2-bit model
+# cannot always generate normal texts.
+N_SEQ = 5
+
+
+@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
+@pytest.mark.parametrize("model_id", MODELS)
+def test_model_load(vllm_runner, model_id, monkeypatch):
+    # `LLM.apply_model` requires pickling a function.
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:
+
+        def check_model(model_id):
+            for name, submodule in model_id.named_modules():
+                # Could check more modules if necessary
+                if name == "model_id.layers.0.self_attn.qkv_proj":
+                    assert isinstance(submodule.quant_method, AutoGPTQLinearMethod)
+                    # Just break since currently we only check 1 module
+                    break
+
+        # Check if gptq_v2 format is correctly loaded
+        llm.apply_model(check_model)
+
+
+@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
+@pytest.mark.parametrize("model_id", MODELS)
+def test_model_inference(vllm_runner, model_id):
+    # Prepare prompt to test the model's generation result.
+    prompt = "What is the meaning of life?"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt},
+    ]
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+        enable_thinking=False,  # If thinking model, set it to false
+    )
+    sampling_params = SamplingParams(
+        n=N_SEQ,
+        max_tokens=128,
+        temperature=0.7,
+        top_p=0.8,
+        top_k=20,
+        min_p=0,
+        presence_penalty=2,
+    )
+
+    with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:
+        # Generate a response to verify inference correctness
+        output = llm.generate(text, sampling_params)
+
+    # Make sure the output exists
+    assert output
+    assert output[0][1]
+    assert len(output[0][1]) == N_SEQ
+
+    def has_normal_char_distribution(texts, min_len):
+        for text in texts:
+            # Response too short
+            if len(text) < min_len:
+                return False
+
+            # Basic ratio checks
+            letters = sum(c.isalpha() for c in text)
+            spaces = sum(c.isspace() for c in text)
+            total = len(text)
+
+            letter_ratio = letters / total
+            space_ratio = spaces / total
+
+            # At least 1 normal text should exist within output sequences
+            # Normal text should be mostly letters with reasonable spacing
+            # Some magic numbers, could be adjusted
+            if 0.5 <= letter_ratio <= 0.9 and 0.01 <= space_ratio <= 0.3:
+                return True
+        # No sequence contains normal text, output might be broken
+        return False
+
+    # Apply some simple checks for giberish output
+    # Print the output sequences if failed
+    assert has_normal_char_distribution(output[0][1], 5), output[0][1]

tests/quantization/test_lm_head.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests whether gptq models with quantized lm_head can be loaded.
+
+Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
+"""
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
+from vllm.model_executor.layers.vocab_parallel_embedding import (
+    UnquantizedEmbeddingMethod,
+)
+
+PROMPT = "On the surface of Mars, we found"
+
+MODELS_QUANT = [
+    ("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head", True),
+    ("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
+]
+
+
+@pytest.mark.parametrize("model_id, lm_head_quantized", MODELS_QUANT)
+def test_lm_head(
+    vllm_runner,
+    model_id: str,
+    lm_head_quantized: bool,
+    monkeypatch,
+) -> None:
+    # `LLM.apply_model` requires pickling a function.
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+    with vllm_runner(
+        model_id, dtype=torch.float16, max_model_len=2048, enforce_eager=True
+    ) as vllm_model:
+
+        def check_model(model):
+            lm_head_layer = model.lm_head
+            if lm_head_quantized:
+                assert isinstance(
+                    lm_head_layer.quant_method,
+                    AutoGPTQLinearMethod,
+                )
+            else:
+                assert isinstance(
+                    lm_head_layer.quant_method, UnquantizedEmbeddingMethod
+                )
+
+        vllm_model.apply_model(check_model)
+
+        print(vllm_model.generate_greedy(["Hello my name is"], max_tokens=4)[0][1])

tests/quantization/test_mixed_precision.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Test quark-quantized {MXFP4, FP8} mixed precision models.
+
+Run `pytest tests/quantization/test_mixed_precision.py`.
+
+"""
+
+import importlib
+import importlib.metadata
+import importlib.util
+from dataclasses import dataclass
+
+import lm_eval
+import pytest
+from packaging import version
+
+from tests.utils import (
+    multi_gpu_only,
+)
+
+QUARK_MXFP4_AVAILABLE = importlib.util.find_spec("quark") is not None and version.parse(
+    importlib.metadata.version("amd-quark")
+) >= version.parse("0.8.99")
+
+
+@dataclass
+class ModelCase:
+    model_id: str
+    tp: int
+
+
+@dataclass
+class EvaluationConfig:
+    model_name: str
+
+    def get_model_args(self) -> str:
+        return (
+            f"pretrained={self.model_name},"
+            "tensor_parallel_size=4,dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=False"
+        )
+
+
+TEST_CONFIGS = {
+    # Mixed-precision (AMP) model
+    # - Demonstrates end-to-end pipeline functionality
+    "amd/Qwen3-8B-WMXFP4FP8-AMXFP4FP8-AMP-KVFP8": {"arc_challenge": 0.52, "mmlu": 0.72},
+    # Non-mixed-precision (PTQ) model
+    # - Reference for pipeline compatibility verification -> No conflicts or breakings
+    "amd/Llama-2-70b-chat-hf_FP8_MLPerf_V2": {
+        "arc_challenge": 0.53,
+        "mmlu": 0.61,
+    },
+}
+
+
+@pytest.mark.parametrize("model_name, accuracy_numbers", TEST_CONFIGS.items())
+@pytest.mark.skipif(not QUARK_MXFP4_AVAILABLE, reason="amd-quark>=0.9 is not available")
+@multi_gpu_only(num_gpus=4)
+def test_mixed_precision_model_accuracies(model_name: str, accuracy_numbers: dict):
+    results = lm_eval.simple_evaluate(
+        model="vllm",
+        model_args=EvaluationConfig(model_name).get_model_args(),
+        tasks=list(accuracy_numbers.keys()),
+        batch_size=8,
+    )
+
+    rtol = 0.05
+
+    for task, expect_accuracy in accuracy_numbers.items():
+        measured_accuracy = results["results"][task]["acc,none"]
+        assert (
+            measured_accuracy - rtol < expect_accuracy
+            and measured_accuracy + rtol > expect_accuracy
+        ), f"Expected: {expect_accuracy} |  Measured: {measured_accuracy}"

tests/quantization/test_modelopt.py

@@ -0,0 +1,607 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Test ModelOpt quantization method setup and weight loading.
+
+Run `pytest tests/quantization/test_modelopt.py`.
+"""
+
+import os
+from typing import Any, NoReturn
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+import torch
+
+from tests.quantization.utils import is_quant_method_supported
+from vllm.config.model import ModelConfig
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
+from vllm.model_executor.layers.quantization.modelopt import (
+    ModelOptFp8Config,
+    ModelOptMixedPrecisionConfig,
+    ModelOptMxFp8Config,
+    ModelOptNvFp4Config,
+    ModelOptNvFp4LinearMethod,
+)
+from vllm.model_executor.layers.vocab_parallel_embedding import (
+    ParallelLMHead,
+    VocabParallelEmbedding,
+)
+from vllm.platforms import current_platform
+
+
+@pytest.fixture(scope="function", autouse=True)
+def enable_pickle(monkeypatch):
+    """`LLM.apply_model` requires pickling a function."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+
+def _skip(msg: str) -> NoReturn:
+    pytest.skip(msg)
+    raise RuntimeError(msg)
+
+
+def _snapshot_download_or_skip(model_id: str) -> str:
+    try:
+        from huggingface_hub import snapshot_download
+    except Exception as e:  # pragma: no cover
+        _skip(f"huggingface_hub is required to download {model_id}: {e}")
+
+    try:
+        return snapshot_download(
+            repo_id=model_id,
+            repo_type="model",
+            # These checkpoints are already small; download full repo for simplicity.
+            allow_patterns=["*"],
+        )
+    except Exception as e:
+        _skip(f"Failed to download {model_id} from the HF Hub: {e}")
+
+
+def _mock_lm_head() -> Mock:
+    lm_head = Mock(spec=ParallelLMHead)
+    lm_head.__class__ = ParallelLMHead
+    return lm_head
+
+
+def _mixed_precision_config(quantized_layers: dict) -> ModelOptMixedPrecisionConfig:
+    return ModelOptMixedPrecisionConfig(
+        kv_cache_quant_method=None,
+        exclude_modules=[],
+        quantized_layers=quantized_layers,
+        fp8_config=ModelOptFp8Config(
+            quant_method="FP8",
+            is_checkpoint_fp8_serialized=True,
+            kv_cache_quant_method=None,
+            exclude_modules=[],
+        ),
+        nvfp4_config=ModelOptNvFp4Config(
+            is_checkpoint_nvfp4_serialized=True,
+            kv_cache_quant_algo=None,
+            exclude_modules=[],
+        ),
+        w4a16_nvfp4_config=ModelOptNvFp4Config(
+            quant_method="W4A16_NVFP4",
+            is_checkpoint_nvfp4_serialized=True,
+            kv_cache_quant_algo=None,
+            exclude_modules=[],
+        ),
+        mxfp8_config=ModelOptMxFp8Config(
+            is_checkpoint_mxfp8_serialized=True,
+            kv_cache_quant_algo=None,
+            exclude_modules=[],
+        ),
+    )
+
+
+def test_modelopt_nvfp4_quantizes_parallel_lm_head():
+    config = ModelOptNvFp4Config(
+        is_checkpoint_nvfp4_serialized=True,
+        kv_cache_quant_algo=None,
+        exclude_modules=[],
+    )
+
+    with patch(
+        "vllm.model_executor.layers.quantization.modelopt.init_nvfp4_linear_kernel"
+    ):
+        method = config.get_quant_method(_mock_lm_head(), prefix="lm_head")
+
+    assert isinstance(method, ModelOptNvFp4LinearMethod)
+
+
+def test_modelopt_nvfp4_leaves_excluded_parallel_lm_head_unquantized():
+    config = ModelOptNvFp4Config(
+        is_checkpoint_nvfp4_serialized=True,
+        kv_cache_quant_algo=None,
+        exclude_modules=["lm_head"],
+    )
+
+    method = config.get_quant_method(_mock_lm_head(), prefix="lm_head")
+
+    assert isinstance(method, UnquantizedLinearMethod)
+
+
+def test_modelopt_mixed_precision_quantizes_parallel_lm_head():
+    config = _mixed_precision_config(
+        {"lm_head": {"quant_algo": "NVFP4", "group_size": 16}}
+    )
+
+    with patch(
+        "vllm.model_executor.layers.quantization.modelopt.init_nvfp4_linear_kernel"
+    ):
+        method = config.get_quant_method(_mock_lm_head(), prefix="lm_head")
+
+    assert isinstance(method, ModelOptNvFp4LinearMethod)
+
+
+def test_modelopt_mixed_precision_resolves_declared_packed_projection():
+    config = _mixed_precision_config(
+        {
+            "model.layers.0.self_attn.q_proj": {"quant_algo": "MXFP8"},
+            "model.layers.0.self_attn.k_proj": {"quant_algo": "MXFP8"},
+            "model.layers.0.self_attn.v_proj": {"quant_algo": "MXFP8"},
+        }
+    )
+    config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
+
+    assert config._resolve_quant_algo("model.layers.0.self_attn.qkv_proj") == "MXFP8"
+
+
+def test_modelopt_mixed_precision_does_not_quantize_unlisted_fused_sibling():
+    config = _mixed_precision_config(
+        {
+            "model.layers.0.linear_attn.in_proj_qkv": {"quant_algo": "FP8"},
+            "model.layers.0.linear_attn.in_proj_z": {"quant_algo": "FP8"},
+            "model.layers.0.linear_attn.out_proj": {"quant_algo": "FP8"},
+        }
+    )
+    config.packed_modules_mapping = {
+        "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
+        "in_proj_ba": ["in_proj_b", "in_proj_a"],
+    }
+
+    assert (
+        config._resolve_quant_algo("model.layers.0.linear_attn.in_proj_qkvz") == "FP8"
+    )
+    assert config._resolve_quant_algo("model.layers.0.linear_attn.in_proj_ba") is None
+
+
+def test_modelopt_mixed_precision_infers_fused_gate_up_projection():
+    from vllm.model_executor.layers.linear import LinearBase
+
+    config = _mixed_precision_config(
+        {
+            "model.layers.0.mlp.gate_proj": {"quant_algo": "NVFP4"},
+            "model.layers.0.mlp.up_proj": {"quant_algo": "NVFP4"},
+        }
+    )
+
+    fake_layer = MagicMock(spec=LinearBase)
+    with patch(
+        "vllm.model_executor.layers.quantization.modelopt.init_nvfp4_linear_kernel"
+    ):
+        method = config.get_quant_method(fake_layer, "model.layers.0.mlp.gate_up_proj")
+
+    assert isinstance(method, ModelOptNvFp4LinearMethod)
+
+
+@pytest.mark.parametrize(
+    ("quantized_prefix", "missing_prefix"),
+    [
+        ("model.layers.0.mlp.gate_proj", "model.layers.0.mlp.down_proj"),
+        ("model.layers.0.self_attn.o_proj", "model.layers.0.self_attn.qkv_proj"),
+    ],
+)
+def test_modelopt_mixed_precision_does_not_infer_missing_sibling_linear(
+    quantized_prefix, missing_prefix
+):
+    from vllm.model_executor.layers.linear import LinearBase
+
+    config = _mixed_precision_config(
+        {
+            quantized_prefix: {"quant_algo": "NVFP4"},
+        }
+    )
+
+    fake_layer = MagicMock(spec=LinearBase)
+    method = config.get_quant_method(fake_layer, missing_prefix)
+
+    assert isinstance(method, UnquantizedLinearMethod)
+
+
+def test_vocab_parallel_embedding_weight_loader_accepts_scalar_scale():
+    holder = Mock()
+    scale = torch.nn.Parameter(torch.empty(1))
+    loaded_scale = torch.tensor(2.0)
+
+    VocabParallelEmbedding.weight_loader(holder, scale, loaded_scale)
+
+    assert torch.equal(scale, loaded_scale.reshape(1))
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("modelopt"),
+    reason="ModelOpt FP8 is not supported on this GPU type.",
+)
+def test_modelopt_fp8_checkpoint_setup(default_vllm_config, vllm_runner):
+    """Test ModelOpt FP8 checkpoint loading and structure validation."""
+    # TODO: provide a small publicly available test checkpoint
+    model_path = (
+        "/home/scratch.omniml_data_1/zhiyu/ckpts/test_ckpts/"
+        "TinyLlama-1.1B-Chat-v1.0-fp8-0710"
+    )
+
+    # Skip test if checkpoint doesn't exist
+    if not os.path.exists(model_path):
+        pytest.skip(
+            f"Test checkpoint not found at {model_path}. "
+            "This test requires a local ModelOpt FP8 checkpoint."
+        )
+
+    # Set model config as model_config.dtype is required in ModelOptFp8LinearMethod.
+    default_vllm_config.model_config = ModelConfig()
+    with vllm_runner(model_path, quantization="modelopt", enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            o_proj = layer.self_attn.o_proj
+            gate_up_proj = layer.mlp.gate_up_proj
+            down_proj = layer.mlp.down_proj
+
+            # Check that ModelOpt quantization method is properly applied
+            from vllm.model_executor.layers.quantization.modelopt import (
+                ModelOptFp8LinearMethod,
+            )
+
+            assert isinstance(qkv_proj.quant_method, ModelOptFp8LinearMethod)
+            assert isinstance(o_proj.quant_method, ModelOptFp8LinearMethod)
+            assert isinstance(gate_up_proj.quant_method, ModelOptFp8LinearMethod)
+            assert isinstance(down_proj.quant_method, ModelOptFp8LinearMethod)
+
+            # Check weight dtype is FP8
+            assert qkv_proj.weight.dtype == torch.float8_e4m3fn
+            assert o_proj.weight.dtype == torch.float8_e4m3fn
+            assert gate_up_proj.weight.dtype == torch.float8_e4m3fn
+            assert down_proj.weight.dtype == torch.float8_e4m3fn
+
+            # Check scales are present and have correct dtype
+            assert hasattr(qkv_proj, "weight_scale")
+            assert hasattr(qkv_proj, "input_scale")
+            assert qkv_proj.weight_scale.dtype == torch.float32
+            assert qkv_proj.input_scale.dtype == torch.float32
+
+            assert hasattr(o_proj, "weight_scale")
+            assert hasattr(o_proj, "input_scale")
+            assert o_proj.weight_scale.dtype == torch.float32
+            assert o_proj.input_scale.dtype == torch.float32
+
+            assert hasattr(gate_up_proj, "weight_scale")
+            assert hasattr(gate_up_proj, "input_scale")
+            assert gate_up_proj.weight_scale.dtype == torch.float32
+            assert gate_up_proj.input_scale.dtype == torch.float32
+
+            assert hasattr(down_proj, "weight_scale")
+            assert hasattr(down_proj, "input_scale")
+            assert down_proj.weight_scale.dtype == torch.float32
+            assert down_proj.input_scale.dtype == torch.float32
+
+        llm.apply_model(check_model)
+
+        # Run a simple generation test to ensure the model works
+        output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        assert output
+        print(f"ModelOpt FP8 output: {output}")
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("modelopt"),
+    reason="ModelOpt FP8 is not supported on this GPU type.",
+)
+def test_modelopt_fp8_pc_pt_checkpoint_setup(default_vllm_config, vllm_runner):
+    """Test ModelOpt FP8_PER_CHANNEL_PER_TOKEN checkpoint setup."""
+    model_id = "CedricHwang/qwen2.5-0.5b-modelopt-fp8-pc-pt"
+    model_path = _snapshot_download_or_skip(model_id)
+
+    # Set model config as model_config.dtype is required in ModelOptFp8LinearMethod.
+    default_vllm_config.model_config = ModelConfig()
+    with vllm_runner(model_path, quantization="modelopt", enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            o_proj = layer.self_attn.o_proj
+            gate_up_proj = layer.mlp.gate_up_proj
+            down_proj = layer.mlp.down_proj
+
+            from vllm.model_executor.layers.quantization.modelopt import (
+                ModelOptFp8PcPtLinearMethod,
+            )
+
+            assert isinstance(qkv_proj.quant_method, ModelOptFp8PcPtLinearMethod)
+            assert isinstance(o_proj.quant_method, ModelOptFp8PcPtLinearMethod)
+            assert isinstance(gate_up_proj.quant_method, ModelOptFp8PcPtLinearMethod)
+            assert isinstance(down_proj.quant_method, ModelOptFp8PcPtLinearMethod)
+
+            fp8_dtype = current_platform.fp8_dtype()
+            assert qkv_proj.weight.dtype == fp8_dtype
+            assert o_proj.weight.dtype == fp8_dtype
+            assert gate_up_proj.weight.dtype == fp8_dtype
+            assert down_proj.weight.dtype == fp8_dtype
+
+            # Per-channel scales; activations are dynamically scaled per token.
+            assert hasattr(qkv_proj, "weight_scale")
+            assert qkv_proj.weight_scale.dtype == torch.float32
+            assert qkv_proj.weight_scale.dim() == 1
+            assert not hasattr(qkv_proj, "input_scale")
+
+            assert hasattr(o_proj, "weight_scale")
+            assert o_proj.weight_scale.dtype == torch.float32
+            assert o_proj.weight_scale.dim() == 1
+            assert not hasattr(o_proj, "input_scale")
+
+            assert hasattr(gate_up_proj, "weight_scale")
+            assert gate_up_proj.weight_scale.dtype == torch.float32
+            assert gate_up_proj.weight_scale.dim() == 1
+            assert not hasattr(gate_up_proj, "input_scale")
+
+            assert hasattr(down_proj, "weight_scale")
+            assert down_proj.weight_scale.dtype == torch.float32
+            assert down_proj.weight_scale.dim() == 1
+            assert not hasattr(down_proj, "input_scale")
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        assert output
+        print(f"ModelOpt FP8_PER_CHANNEL_PER_TOKEN output: {output}")
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("modelopt"),
+    reason="ModelOpt FP8 is not supported on this GPU type.",
+)
+def test_modelopt_fp8_pb_wo_checkpoint_setup(default_vllm_config, vllm_runner):
+    """Test ModelOpt FP8_PB_WO checkpoint setup."""
+    model_id = "CedricHwang/qwen2.5-0.5b-modelopt-fp8-pb-wo"
+    model_path = _snapshot_download_or_skip(model_id)
+
+    # Set model config as model_config.dtype is required in ModelOptFp8LinearMethod.
+    default_vllm_config.model_config = ModelConfig()
+    with vllm_runner(model_path, quantization="modelopt", enforce_eager=True) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+            o_proj = layer.self_attn.o_proj
+            gate_up_proj = layer.mlp.gate_up_proj
+            down_proj = layer.mlp.down_proj
+
+            from vllm.model_executor.layers.quantization.modelopt import (
+                ModelOptFp8PbWoLinearMethod,
+            )
+
+            assert isinstance(qkv_proj.quant_method, ModelOptFp8PbWoLinearMethod)
+            assert isinstance(o_proj.quant_method, ModelOptFp8PbWoLinearMethod)
+            assert isinstance(gate_up_proj.quant_method, ModelOptFp8PbWoLinearMethod)
+            assert isinstance(down_proj.quant_method, ModelOptFp8PbWoLinearMethod)
+
+            assert qkv_proj.weight.dtype == torch.float8_e4m3fn
+            assert o_proj.weight.dtype == torch.float8_e4m3fn
+            assert gate_up_proj.weight.dtype == torch.float8_e4m3fn
+            assert down_proj.weight.dtype == torch.float8_e4m3fn
+
+            # Block scales; should be materialized as a 2D [out_blk, in_blk] tensor.
+            assert hasattr(qkv_proj, "weight_scale")
+            assert qkv_proj.weight_scale.dtype == torch.float32
+            assert qkv_proj.weight_scale.dim() == 2
+
+            assert hasattr(o_proj, "weight_scale")
+            assert o_proj.weight_scale.dtype == torch.float32
+            assert o_proj.weight_scale.dim() == 2
+
+            assert hasattr(gate_up_proj, "weight_scale")
+            assert gate_up_proj.weight_scale.dtype == torch.float32
+            assert gate_up_proj.weight_scale.dim() == 2
+
+            assert hasattr(down_proj, "weight_scale")
+            assert down_proj.weight_scale.dtype == torch.float32
+            assert down_proj.weight_scale.dim() == 2
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        assert output
+        print(f"ModelOpt FP8_PB_WO output: {output}")
+
+
+def test_modelopt_nvfp4_config_dispatches_w4a4_method():
+    """``quant_method="NVFP4"`` (W4A4 default) routes to the existing
+    ``ModelOptNvFp4LinearMethod``."""
+    from vllm.model_executor.layers.quantization.modelopt import (
+        ModelOptNvFp4Config,
+        ModelOptNvFp4LinearMethod,
+    )
+
+    config = ModelOptNvFp4Config(
+        quant_method="NVFP4",
+        is_checkpoint_nvfp4_serialized=True,
+        kv_cache_quant_algo=None,
+        exclude_modules=[],
+    )
+    assert config.LinearMethodCls is ModelOptNvFp4LinearMethod
+    assert config.quant_method == "NVFP4"
+
+
+def test_modelopt_nvfp4_config_dispatches_w4a16_method():
+    """``quant_method="W4A16_NVFP4"`` routes to the new
+    ``ModelOptNvFp4W4A16LinearMethod`` instead of the W4A4 sibling.
+
+    Mirrors the FP8 dispatch precedent (``ModelOptFp8Config`` selects
+    one of three FP8 LinearMethods on ``quant_method``); a regression
+    here would mean a W4A16 NVFP4 checkpoint silently loaded under the
+    W4A4 method, which would try to register an ``input_scale`` runtime
+    parameter and (more importantly) call the cutlass W4A4 NVFP4 GEMM
+    instead of FP4 Marlin.
+    """
+    from vllm.model_executor.layers.quantization.modelopt import (
+        ModelOptNvFp4Config,
+        ModelOptNvFp4LinearMethod,
+        ModelOptNvFp4W4A16LinearMethod,
+    )
+
+    config = ModelOptNvFp4Config(
+        quant_method="W4A16_NVFP4",
+        is_checkpoint_nvfp4_serialized=True,
+        kv_cache_quant_algo=None,
+        exclude_modules=[],
+    )
+    assert config.LinearMethodCls is ModelOptNvFp4W4A16LinearMethod
+    assert config.LinearMethodCls is not ModelOptNvFp4LinearMethod
+    assert config.quant_method == "W4A16_NVFP4"
+
+
+@pytest.mark.parametrize(
+    "quant_method, expected_use_a16, act_key_is_none",
+    [
+        ("NVFP4", False, False),  # W4A4 default
+        ("W4A16_NVFP4", True, True),  # native W4A16 ckpt
+    ],
+)
+def test_modelopt_nvfp4_moe_dispatches_to_marlin_when_w4a16(
+    quant_method, expected_use_a16, act_key_is_none
+):
+    """``ModelOptNvFp4FusedMoE``: when the ckpt's ``quant_method`` is
+    ``W4A16_NVFP4``, the MoE class must pass ``activation_key=None`` to
+    ``select_nvfp4_moe_backend``. That filters out every W4A4 backend
+    (their ``_supports_quant_scheme`` requires
+    ``(kNvfp4Static, kNvfp4Dynamic)`` exactly); Marlin survives because
+    it only checks ``weight_key``. A regression here would mean a W4A16
+    ckpt silently went to the cutlass W4A4 path.
+    """
+    from vllm.model_executor.layers.quantization.modelopt import (
+        ModelOptNvFp4Config,
+        ModelOptNvFp4FusedMoE,
+    )
+    from vllm.model_executor.layers.quantization.utils.quant_utils import (
+        kNvfp4Dynamic,
+        kNvfp4Static,
+    )
+
+    config = ModelOptNvFp4Config(
+        quant_method=quant_method,
+        is_checkpoint_nvfp4_serialized=True,
+        kv_cache_quant_algo=None,
+        exclude_modules=[],
+        group_size=16,
+    )
+
+    mock_select = MagicMock(return_value=(MagicMock(), MagicMock()))
+    with (
+        patch(
+            "vllm.model_executor.layers.quantization.modelopt.select_nvfp4_moe_backend",
+            mock_select,
+        ),
+        patch(
+            "vllm.model_executor.layers.quantization.modelopt."
+            "is_global_sf_supported_for_nvfp4_backend",
+            return_value=False,
+        ),
+    ):
+        moe = ModelOptNvFp4FusedMoE(config, MagicMock())
+
+    assert moe.use_a16 is expected_use_a16
+    _, kwargs = mock_select.call_args
+    assert kwargs["weight_key"] is kNvfp4Static
+    if act_key_is_none:
+        assert kwargs["activation_key"] is None
+    else:
+        assert kwargs["activation_key"] is kNvfp4Dynamic
+
+
+@pytest.mark.parametrize(
+    "per_layer_algo, expected_linear_cls_name",
+    [
+        ("NVFP4", "ModelOptNvFp4LinearMethod"),
+        ("W4A16_NVFP4", "ModelOptNvFp4W4A16LinearMethod"),
+    ],
+)
+def test_modelopt_mixed_precision_dispatches_w4a16_layer(
+    per_layer_algo, expected_linear_cls_name
+):
+    """``ModelOptMixedPrecisionConfig.get_quant_method`` must route a Linear
+    layer to the right LinearMethod based on its per-layer ``quant_algo``
+    entry in ``quantized_layers``. Verifies the new ``W4A16_NVFP4`` branch
+    coexists with the existing ``NVFP4`` branch without regression. A
+    regression here would mean a W4A16 layer in a mixed-precision ckpt
+    silently fell through to ``UnquantizedLinearMethod``.
+
+    NOTE: FP8 dispatch (the third branch of get_quant_method) is not
+    covered here because ``ModelOptFp8LinearMethod.__init__`` reads
+    ``get_current_vllm_config().model_config.dtype``, which requires a
+    fully constructed ``ModelConfig`` (real model path). FP8 routing in
+    mixed-precision is exercised by the existing integration tests
+    above that use the ``vllm_runner`` fixture (e.g.
+    ``test_modelopt_fp8_checkpoint_setup``). Our PR doesn't change the
+    FP8 branch, so this isn't a coverage gap.
+    """
+    from vllm.model_executor.layers.linear import LinearBase
+    from vllm.model_executor.layers.quantization import modelopt as m
+
+    if (
+        expected_linear_cls_name == "ModelOptNvFp4W4A16LinearMethod"
+        and current_platform.is_rocm()
+    ):
+        pytest.skip("ModelOptNvFp4W4A16LinearMethod is not supported with rocm")
+
+    hf_quant_config: dict[str, Any] = {
+        "quantization": {
+            "quant_algo": "MIXED_PRECISION",
+            "kv_cache_quant_algo": None,
+            "exclude_modules": [],
+            "group_size": 16,
+            "quantized_layers": {
+                "model.layers.0.fake_proj": {"quant_algo": per_layer_algo},
+            },
+        }
+    }
+    config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
+
+    fake_layer = MagicMock(spec=LinearBase)
+    method = config.get_quant_method(fake_layer, "model.layers.0.fake_proj")
+
+    expected_cls = getattr(m, expected_linear_cls_name)
+    assert isinstance(method, expected_cls), (
+        f"Expected {expected_linear_cls_name}, got {type(method).__name__}"
+    )
+
+
+def test_modelopt_mixed_precision_builds_w4a16_sibling_config():
+    """Sanity: ``ModelOptMixedPrecisionConfig._from_config`` builds **two**
+    NVFP4 sub-configs — one for W4A4 (default) and one tagged
+    ``quant_method='W4A16_NVFP4'`` — so per-layer dispatch can hand
+    Marlin-bound layers the right config without re-instantiating it on
+    every call.
+    """
+    from vllm.model_executor.layers.quantization import modelopt as m
+
+    hf_quant_config: dict[str, Any] = {
+        "quantization": {
+            "quant_algo": "MIXED_PRECISION",
+            "kv_cache_quant_algo": None,
+            "exclude_modules": [],
+            "group_size": 16,
+            "quantized_layers": {
+                "model.layers.0.a": {"quant_algo": "NVFP4"},
+                "model.layers.0.b": {"quant_algo": "W4A16_NVFP4"},
+            },
+        }
+    }
+    config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
+
+    assert config.nvfp4_config.quant_method == "NVFP4"
+    assert config.nvfp4_config.LinearMethodCls is m.ModelOptNvFp4LinearMethod
+    assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4"
+    assert config.w4a16_nvfp4_config.LinearMethodCls is m.ModelOptNvFp4W4A16LinearMethod

tests/quantization/test_moe_wna16.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.fused_moe.activation import MoEActivation
+from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Method
+from vllm.platforms import current_platform
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
+def test_moe_wna16_apply_passes_layer_activation(monkeypatch):
+    captured_kwargs = {}
+
+    def fake_fused_experts(*args, **kwargs):
+        captured_kwargs.update(kwargs)
+        return torch.empty(1, 2)
+
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.fused_moe.fused_experts",
+        fake_fused_experts,
+    )
+
+    method = object.__new__(MoeWNA16Method)
+    method.moe = SimpleNamespace(disable_inplace=False)
+    method.moe_quant_config = object()
+    layer = SimpleNamespace(
+        w13_qweight=torch.empty(1, 2),
+        w2_qweight=torch.empty(1, 2),
+        activation=MoEActivation.GELU_TANH,
+        apply_router_weight_on_input=False,
+        global_num_experts=1,
+        expert_map=None,
+    )
+
+    output = method.apply(
+        layer,
+        x=torch.empty(1, 2),
+        topk_weights=torch.empty(1, 1),
+        topk_ids=torch.empty(1, 1, dtype=torch.int32),
+        shared_experts=None,
+        shared_experts_input=None,
+    )
+
+    assert output.shape == (1, 2)
+    assert captured_kwargs["activation"] is MoEActivation.GELU_TANH

tests/quantization/test_online.py

@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests online quantization."""
+
+import pytest
+import torch
+
+from tests.quantization.utils import (
+    _test_online_quant_peak_mem_impl,
+    is_quant_method_supported,
+)
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
+from vllm.model_executor.layers.quantization.online.fp8 import (
+    Fp8PerBlockOnlineLinearMethod,
+    Fp8PerBlockOnlineMoEMethod,
+    Fp8PerTensorOnlineLinearMethod,
+    Fp8PerTensorOnlineMoEMethod,
+)
+from vllm.platforms import current_platform
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+@pytest.mark.parametrize(
+    "quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls",
+    [
+        # simple case - quantization='fp8_per_tensor'
+        (
+            "fp8_per_tensor",
+            None,
+            Fp8PerTensorOnlineLinearMethod,
+            Fp8PerTensorOnlineMoEMethod,
+        ),
+        # simple case - quantization='fp8_per_block'
+        (
+            "fp8_per_block",
+            None,
+            Fp8PerBlockOnlineLinearMethod,
+            Fp8PerBlockOnlineMoEMethod,
+        ),
+        # quantization='online' with per-layer-kind overrides
+        (
+            "online",
+            {
+                "linear": "fp8_per_block",
+                "moe": "fp8_per_tensor",
+            },
+            Fp8PerBlockOnlineLinearMethod,
+            Fp8PerTensorOnlineMoEMethod,
+        ),
+        # ignore with direct layer name
+        (
+            "fp8_per_tensor",
+            # qkv_proj is fused from q_proj/k_proj/v_proj, so currently the
+            # ignore regex must match the unfused shard names
+            # TODO(future PR): also make 're:.*qkv_proj.*' work
+            {"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]},
+            Fp8PerTensorOnlineLinearMethod,
+            Fp8PerTensorOnlineMoEMethod,
+        ),
+    ],
+)
+@pytest.mark.parametrize(
+    "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+def test_online_quantization(
+    vllm_runner,
+    quant_scheme: str,
+    online_quant_args: dict | None,
+    expected_linear_cls,
+    expected_moe_cls,
+    use_rocm_aiter: bool,
+    monkeypatch,
+) -> None:
+    """
+    Tests that online quantization frontend configuration works -
+    selecting quant schemes, overriding quant schemes by type, ignoring
+    layers.
+
+    Does not test performance, peak memory usage, etc.
+    """
+
+    if use_rocm_aiter:
+        monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
+
+    # `LLM.apply_model` requires pickling a function.
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    # a tiny model with both dense and MoE layers
+    model_name = "ibm-granite/granite-3.0-1b-a400m-base"
+
+    runner_kwargs = dict(
+        quantization=quant_scheme,
+        enforce_eager=True,
+    )
+    if online_quant_args is not None:
+        runner_kwargs["quantization_config"] = online_quant_args
+
+    with vllm_runner(
+        model_name,
+        **runner_kwargs,
+    ) as llm:
+
+        def check_model(model):
+            # checks further down in the test case are hardcoded for this
+            # model
+            assert model_name == "ibm-granite/granite-3.0-1b-a400m-base"
+
+            o_proj = model.model.layers[0].self_attn.o_proj
+            moe = model.model.layers[0].block_sparse_moe.experts
+
+            # o_proj and moe in layer 0 are always quantized (never ignored)
+            # because of how we craft the test case inputs
+            assert isinstance(o_proj.quant_method, expected_linear_cls)
+            if moe is not None:
+                assert isinstance(moe._quant_method, expected_moe_cls)
+
+            if current_platform.is_cuda():
+                assert o_proj.weight.dtype == torch.float8_e4m3fn
+            elif current_platform.is_rocm():
+                assert o_proj.weight.dtype == current_platform.fp8_dtype()
+            else:
+                pytest.skip("Only runs on CUDA and ROCm.")
+
+            # Verify ignored layers are unquantized.
+            if isinstance(online_quant_args, dict) and "ignore" in online_quant_args:
+                # only .*1.self_attn_o_proj is skipped
+                for layer_idx in range(len(model.model.layers)):
+                    o_proj = model.model.layers[layer_idx].self_attn.o_proj
+                    if layer_idx == 1:
+                        assert isinstance(o_proj.quant_method, UnquantizedLinearMethod)
+                    else:
+                        assert isinstance(o_proj.quant_method, expected_linear_cls)
+
+                # every .*self_attn.qkv_proj is skipped
+                for layer_idx in range(len(model.model.layers)):
+                    qkv_proj = model.model.layers[layer_idx].self_attn.qkv_proj
+                    assert isinstance(qkv_proj.quant_method, UnquantizedLinearMethod)
+
+        llm.apply_model(check_model)
+
+        outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)
+        print(outputs[0][1])
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+def test_online_quant_peak_mem(
+    vllm_runner,
+    caplog_mp_spawn,
+    monkeypatch,
+) -> None:
+    _test_online_quant_peak_mem_impl(
+        "fp8_per_tensor", vllm_runner, caplog_mp_spawn, monkeypatch
+    )
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("fp8"),
+    reason="FP8 is not supported on this GPU type.",
+)
+def test_online_quant_load_format_dummy(
+    vllm_runner,
+    monkeypatch,
+    caplog,
+) -> None:
+    with vllm_runner(
+        "ibm-granite/granite-3.0-1b-a400m-base",
+        quantization="fp8_per_tensor",
+        enforce_eager=True,
+        load_format="dummy",
+    ) as llm:
+        outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
+        print(outputs[0][1])

tests/quantization/test_per_token_kv_cache.py

@@ -0,0 +1,723 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for per-token-head KV cache quantization (INT4, INT8 and FP8).
+
+Covers:
+- Per-token-head Triton reshape-and-cache kernel
+- Round-trip quantize/dequantize accuracy
+- process_weights_after_loading early-return path
+- End-to-end integration with Triton unified attention kernel
+
+Run: pytest tests/quantization/test_per_token_kv_cache.py -v -s
+"""
+
+import random
+from dataclasses import dataclass
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
+    get_fp8_min_max,
+)
+from vllm.platforms import current_platform
+from vllm.utils.torch_utils import set_random_seed
+from vllm.v1.attention.ops.int4_per_token_head import single_rht
+from vllm.v1.kv_cache_interface import KVQuantMode, is_quantized_kv_cache
+
+DEVICE_TYPE = current_platform.device_type
+
+# Skip entire module if no CUDA/ROCm GPU available
+pytestmark = [
+    pytest.mark.skipif(
+        current_platform.is_cpu(),
+        reason="Per-token-head KV cache tests require GPU.",
+    ),
+]
+
+# ---------------------------------------------------------------------------
+# Test parameters
+# ---------------------------------------------------------------------------
+NUM_TOKENS = [1, 7, 42]
+NUM_KV_HEADS = [1, 4, 8]
+HEAD_SIZES = [64, 128]
+BLOCK_SIZES = [16]
+SEEDS = [0]
+
+# Platform-dependent FP8 dtype and range
+FP8_DTYPE = current_platform.fp8_dtype()
+FP8_MIN, FP8_MAX = get_fp8_min_max()
+
+
+# ---------------------------------------------------------------------------
+# Per-dtype quantization config
+# ---------------------------------------------------------------------------
+@dataclass(frozen=True)
+class QuantConfig:
+    """Quantization parameters for a given cache dtype."""
+
+    cache_dtype: torch.dtype  # torch.int8 or FP8_DTYPE
+    kv_cache_dtype_str: str  # "int8_per_token_head" or "fp8_per_token_head"
+    quant_max: float
+    quant_min: float
+    kv_quant_mode: KVQuantMode
+    # INT8 rounds explicitly; FP8 relies on dtype cast rounding.
+    rounds_before_store: bool
+
+
+INT8_CONFIG = QuantConfig(
+    cache_dtype=torch.int8,
+    kv_cache_dtype_str="int8_per_token_head",
+    quant_max=127.0,
+    quant_min=-128.0,
+    kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD,
+    rounds_before_store=True,
+)
+FP8_CONFIG = QuantConfig(
+    cache_dtype=FP8_DTYPE,
+    kv_cache_dtype_str="fp8_per_token_head",
+    quant_max=FP8_MAX,
+    quant_min=FP8_MIN,
+    kv_quant_mode=KVQuantMode.FP8_PER_TOKEN_HEAD,
+    rounds_before_store=False,
+)
+INT4_CONFIG = QuantConfig(
+    cache_dtype=torch.uint8,
+    kv_cache_dtype_str="int4_per_token_head",
+    quant_max=7.0,
+    quant_min=-8.0,
+    kv_quant_mode=KVQuantMode.INT4_PER_TOKEN_HEAD,
+    # Unused for int4 (handled by its own rint path); kept for the dataclass.
+    rounds_before_store=False,
+)
+QUANT_CONFIGS = [INT4_CONFIG, INT8_CONFIG, FP8_CONFIG]
+
+
+@pytest.fixture(params=QUANT_CONFIGS, ids=["int4", "int8", "fp8"])
+def qcfg(request) -> QuantConfig:
+    return request.param
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+def _quantize_per_token_head_ref(
+    data: torch.Tensor,  # [num_tokens, num_heads, head_size]
+    cfg: QuantConfig,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """Reference per-token-head quantization (one scale per token per head).
+
+    Returns (quantized, scales) where scales is [num_tokens, num_heads].
+    """
+    absmax = data.float().abs().amax(dim=2)  # [num_tokens, num_heads]
+    scales = (absmax / cfg.quant_max).clamp(min=1e-6)
+    scaled = data.float() * (1.0 / scales[:, :, None])
+    if cfg.rounds_before_store:
+        q = scaled.round().clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
+    else:
+        q = scaled.clamp(cfg.quant_min, cfg.quant_max).to(cfg.cache_dtype)
+    return q, scales
+
+
+# ===========================================================================
+# 1. is_quantized_kv_cache / get_kv_quant_mode
+# ===========================================================================
+class TestIsQuantizedKvCache:
+    def test_fp8_variants(self):
+        assert is_quantized_kv_cache("fp8")
+        assert is_quantized_kv_cache("fp8_e4m3")
+        assert is_quantized_kv_cache("fp8_e5m2")
+
+    def test_int4_per_token_head(self):
+        assert is_quantized_kv_cache("int4_per_token_head")
+
+    def test_int8_per_token_head(self):
+        assert is_quantized_kv_cache("int8_per_token_head")
+
+    def test_fp8_per_token_head(self):
+        assert is_quantized_kv_cache("fp8_per_token_head")
+
+    def test_auto(self):
+        assert not is_quantized_kv_cache("auto")
+
+    def test_bfloat16(self):
+        assert not is_quantized_kv_cache("bfloat16")
+
+    def test_kv_quant_mode_int4(self):
+        from vllm.v1.kv_cache_interface import get_kv_quant_mode
+
+        assert (
+            get_kv_quant_mode("int4_per_token_head") == KVQuantMode.INT4_PER_TOKEN_HEAD
+        )
+
+    def test_kv_quant_mode_int8(self):
+        from vllm.v1.kv_cache_interface import get_kv_quant_mode
+
+        assert (
+            get_kv_quant_mode("int8_per_token_head") == KVQuantMode.INT8_PER_TOKEN_HEAD
+        )
+
+    def test_kv_quant_mode_fp8(self):
+        from vllm.v1.kv_cache_interface import get_kv_quant_mode
+
+        assert get_kv_quant_mode("fp8_per_token_head") == KVQuantMode.FP8_PER_TOKEN_HEAD
+
+
+# ===========================================================================
+# 2. Triton per-token-head kernel (reshape-and-cache)
+# ===========================================================================
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
+@pytest.mark.parametrize("num_heads", NUM_KV_HEADS)
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
+@pytest.mark.parametrize("seed", SEEDS)
+@torch.inference_mode()
+def test_reshape_and_cache_per_token_head(
+    qcfg: QuantConfig,
+    num_tokens: int,
+    num_heads: int,
+    head_size: int,
+    block_size: int,
+    seed: int,
+):
+    """Test triton_reshape_and_cache_flash_per_token_head_quant kernel."""
+    from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
+        triton_reshape_and_cache_flash_per_token_head_quant,
+    )
+
+    set_random_seed(seed)
+    torch.set_default_device(DEVICE_TYPE)
+
+    num_blocks = (num_tokens + block_size - 1) // block_size + 4
+    is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
+    cache_head_size = head_size // 2 if is_int4 else head_size
+
+    key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
+    value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
+
+    key_cache = torch.zeros(
+        num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
+    )
+    value_cache = torch.zeros(
+        num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
+    )
+    k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
+    v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
+
+    num_slots = block_size * num_blocks
+    slot_mapping = torch.tensor(
+        random.sample(range(num_slots), num_tokens), dtype=torch.long
+    )
+
+    triton_reshape_and_cache_flash_per_token_head_quant(
+        key,
+        value,
+        key_cache,
+        value_cache,
+        k_scale_cache,
+        v_scale_cache,
+        slot_mapping,
+        kv_quant_mode=qcfg.kv_quant_mode,
+    )
+
+    # INT4 (RHT + asymmetric), INT8/FP8 have different dequant paths.  Only
+    # INT8/FP8 can be compared to a PyTorch reference.
+    if not is_int4:
+        ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, qcfg)
+        ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, qcfg)
+
+    for i, slot in enumerate(slot_mapping.tolist()):
+        blk = slot // block_size
+        off = slot % block_size
+
+        if is_int4:
+            # Coarser quantization → wider tolerance.
+            deq_atol = deq_rtol = 0.5
+            for label, data, cache, sc in [
+                ("key", key, key_cache, k_scale_cache),
+                ("val", value, value_cache, v_scale_cache),
+            ]:
+                packed_scale = sc[blk, off]  # [num_heads] float32
+                scale_bits = packed_scale.view(torch.int32)
+                zp = (scale_bits & 0xF).to(torch.float32)
+                clean_scale = (scale_bits & -16).view(torch.float32)
+
+                packed = cache[blk, off]
+                lo = (packed & 0xF).to(torch.float32)
+                hi = ((packed >> 4) & 0xF).to(torch.float32)
+                full = torch.zeros(num_heads, head_size, dtype=torch.float32)
+                full[:, 0::2] = lo
+                full[:, 1::2] = hi
+                # Asymmetric dequant in RHT domain, then IRHT/d → original
+                deq_rht = (full - zp[:, None]) * clean_scale[:, None]
+                deq = single_rht(deq_rht, inverse=True) / head_size
+                ref_deq = data[i].float()
+                torch.testing.assert_close(deq, ref_deq, atol=deq_atol, rtol=deq_rtol)
+        else:
+            actual_k_scale = k_scale_cache[blk, off]  # [num_heads]
+            k_deq = key_cache[blk, off].float() * actual_k_scale[:, None]
+            k_ref_deq = key[i].float()
+            torch.testing.assert_close(
+                k_deq,
+                k_ref_deq,
+                atol=0.1,
+                rtol=0.1,
+            )
+            actual_v_scale = v_scale_cache[blk, off]  # [num_heads]
+            v_deq = value_cache[blk, off].float() * actual_v_scale[:, None]
+            v_ref_deq = value[i].float()
+            torch.testing.assert_close(
+                v_deq,
+                v_ref_deq,
+                atol=0.1,
+                rtol=0.1,
+            )
+            # Per-head scales: [num_heads]
+            torch.testing.assert_close(
+                k_scale_cache[blk, off], ref_k_scales[i], atol=1e-4, rtol=1e-3
+            )
+            torch.testing.assert_close(
+                v_scale_cache[blk, off], ref_v_scales[i], atol=1e-4, rtol=1e-3
+            )
+
+
+# ===========================================================================
+# 3. Per-token-head round-trip accuracy (quantize -> dequantize)
+# ===========================================================================
+@pytest.mark.parametrize("num_tokens", [1, 16])
+@pytest.mark.parametrize("num_heads", [4])
+@pytest.mark.parametrize("head_size", [128])
+@pytest.mark.parametrize("block_size", [16])
+@torch.inference_mode()
+def test_per_token_head_round_trip_accuracy(
+    qcfg: QuantConfig,
+    num_tokens: int,
+    num_heads: int,
+    head_size: int,
+    block_size: int,
+):
+    """Verify per-token-head round-trip: kernel dequant matches reference.
+
+    INT8: round-to-nearest before int8 store.
+    FP8: hardware cast (clamp then cast).
+    """
+    from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
+        triton_reshape_and_cache_flash_per_token_head_quant,
+    )
+
+    torch.set_default_device(DEVICE_TYPE)
+    set_random_seed(42)
+
+    is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
+    num_blocks = (num_tokens + block_size - 1) // block_size + 2
+    cache_head_size = head_size // 2 if is_int4 else head_size
+
+    key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
+    value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16) * 0.5
+
+    key_cache = torch.zeros(
+        num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
+    )
+    value_cache = torch.zeros(
+        num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
+    )
+    k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
+    v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
+
+    slot_mapping = torch.arange(num_tokens, dtype=torch.long)
+
+    triton_reshape_and_cache_flash_per_token_head_quant(
+        key,
+        value,
+        key_cache,
+        value_cache,
+        k_scale_cache,
+        v_scale_cache,
+        slot_mapping,
+        kv_quant_mode=qcfg.kv_quant_mode,
+    )
+
+    rt_atol = 0.5 if is_int4 else 0.1
+
+    for i in range(num_tokens):
+        blk = i // block_size
+        off = i % block_size
+
+        for label, data, cache, sc in [
+            ("key", key, key_cache, k_scale_cache),
+            ("val", value, value_cache, v_scale_cache),
+        ]:
+            for h in range(num_heads):
+                orig = data[i, h].float()
+                actual_sc = sc[blk, off, h]
+                if is_int4:
+                    sc_bits = actual_sc.view(torch.int32)
+                    zp = (sc_bits & 0xF).to(torch.float32)
+                    clean_sc = (sc_bits & -16).view(torch.float32)
+                    packed = cache[blk, off, h]
+                    lo = (packed & 0xF).to(torch.float32)
+                    hi = ((packed >> 4) & 0xF).to(torch.float32)
+                    full = torch.zeros(head_size)
+                    full[0::2] = lo
+                    full[1::2] = hi
+                    deq_rht = (full - zp) * clean_sc
+                    actual_deq = (
+                        single_rht(deq_rht.unsqueeze(0), inverse=True).squeeze(0)
+                        / head_size
+                    )
+                else:
+                    actual_deq = cache[blk, off, h].float() * actual_sc
+                torch.testing.assert_close(
+                    actual_deq,
+                    orig,
+                    atol=rt_atol,
+                    rtol=rt_atol,
+                )
+
+
+@torch.inference_mode()
+def test_int8_per_token_head_raw_cache_matches_round_reference():
+    """INT8 cache writes should match round-to-nearest quantization exactly."""
+    from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
+        triton_reshape_and_cache_flash_per_token_head_quant,
+    )
+
+    torch.set_default_device(DEVICE_TYPE)
+
+    head_size = 8
+    block_size = 4
+
+    key = torch.tensor(
+        [[[-127.0, -2.6, -2.4, -1.6, -1.4, -0.6, -0.4, 127.0]]],
+        dtype=torch.bfloat16,
+    )
+    value = -key
+
+    key_cache = torch.zeros(1, block_size, 1, head_size, dtype=torch.int8)
+    value_cache = torch.zeros_like(key_cache)
+    k_scale_cache = torch.ones(1, block_size, 1, dtype=torch.float32)
+    v_scale_cache = torch.ones_like(k_scale_cache)
+    slot_mapping = torch.tensor([2], dtype=torch.long)
+
+    triton_reshape_and_cache_flash_per_token_head_quant(
+        key,
+        value,
+        key_cache,
+        value_cache,
+        k_scale_cache,
+        v_scale_cache,
+        slot_mapping,
+        kv_quant_mode=INT8_CONFIG.kv_quant_mode,
+    )
+
+    ref_k_quant, ref_k_scales = _quantize_per_token_head_ref(key, INT8_CONFIG)
+    ref_v_quant, ref_v_scales = _quantize_per_token_head_ref(value, INT8_CONFIG)
+
+    slot = slot_mapping.item()
+    blk = slot // block_size
+    off = slot % block_size
+    assert torch.equal(key_cache[blk, off], ref_k_quant[0])
+    assert torch.equal(value_cache[blk, off], ref_v_quant[0])
+    torch.testing.assert_close(k_scale_cache[blk, off], ref_k_scales[0])
+    torch.testing.assert_close(v_scale_cache[blk, off], ref_v_scales[0])
+
+
+# ===========================================================================
+# 4. Negative slot mapping (padding tokens should be skipped)
+# ===========================================================================
+@torch.inference_mode()
+def test_per_token_head_negative_slot_skipped(qcfg: QuantConfig):
+    """Tokens with slot_mapping=-1 should leave the cache unchanged."""
+    from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
+        triton_reshape_and_cache_flash_per_token_head_quant,
+    )
+
+    torch.set_default_device(DEVICE_TYPE)
+    num_tokens = 4
+    num_heads = 2
+    head_size = 64
+    block_size = 16
+    num_blocks = 2
+    is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
+    cache_head_size = head_size // 2 if is_int4 else head_size
+
+    key = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
+    value = torch.randn(num_tokens, num_heads, head_size, dtype=torch.bfloat16)
+
+    key_cache = torch.zeros(
+        num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
+    )
+    value_cache = torch.zeros(
+        num_blocks, block_size, num_heads, cache_head_size, dtype=qcfg.cache_dtype
+    )
+    k_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
+    v_scale_cache = torch.ones(num_blocks, block_size, num_heads, dtype=torch.float32)
+
+    slot_mapping = torch.tensor([0, -1, 1, -1], dtype=torch.long)
+
+    key_cache_before = key_cache.clone()
+    val_cache_before = value_cache.clone()
+
+    triton_reshape_and_cache_flash_per_token_head_quant(
+        key,
+        value,
+        key_cache,
+        value_cache,
+        k_scale_cache,
+        v_scale_cache,
+        slot_mapping,
+        kv_quant_mode=qcfg.kv_quant_mode,
+    )
+
+    # Slots 0 and 1 should have been written (tokens 0 and 2)
+    assert not torch.equal(key_cache[0, 0], key_cache_before[0, 0])
+    assert not torch.equal(key_cache[0, 1], key_cache_before[0, 1])
+    assert not torch.equal(value_cache[0, 0], val_cache_before[0, 0])
+
+    # All other slots should be unchanged
+    assert torch.equal(key_cache[0, 2:], key_cache_before[0, 2:])
+    assert torch.equal(key_cache[1], key_cache_before[1])
+    assert torch.equal(value_cache[0, 2:], val_cache_before[0, 2:])
+
+
+# ===========================================================================
+# 5. process_weights_after_loading -- per-token-head early return
+# ===========================================================================
+@pytest.mark.parametrize(
+    "kv_cache_dtype",
+    ["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
+)
+def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str):
+    """Per-token-head should set _k_scale=1.0, _v_scale=1.0
+    and delete checkpoint attrs."""
+    from vllm.model_executor.layers.quantization.kv_cache import (
+        BaseKVCacheMethod,
+    )
+
+    layer = MagicMock()
+    layer.kv_cache_dtype = kv_cache_dtype
+    layer.calculate_kv_scales = False
+    layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
+    layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
+    layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
+    layer.prob_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
+    layer._k_scale = torch.tensor(0.0)
+    layer._v_scale = torch.tensor(0.0)
+    layer._k_scale_float = 0.0
+    layer._v_scale_float = 0.0
+
+    method = BaseKVCacheMethod.__new__(BaseKVCacheMethod)
+    method.quant_config = MagicMock()
+    method.process_weights_after_loading(layer)
+
+    assert layer._k_scale_float == 1.0
+    assert layer._v_scale_float == 1.0
+    assert not hasattr(layer, "k_scale")
+    assert not hasattr(layer, "v_scale")
+    assert not hasattr(layer, "q_scale")
+    assert not hasattr(layer, "prob_scale")
+
+
+# ===========================================================================
+# 6. Triton unified_attention -- per-token-head scale cache (INT4/INT8/FP8)
+# ===========================================================================
+@pytest.mark.parametrize(
+    "seq_lens",
+    [
+        [(1, 128)],
+        [(1, 64), (1, 32)],
+    ],
+)
+@pytest.mark.parametrize("num_heads", [(4, 4)])
+@pytest.mark.parametrize("head_size", [128])
+@pytest.mark.parametrize("block_size", [16])
+@torch.inference_mode()
+def test_triton_unified_attention_per_token_head_scale(
+    qcfg: QuantConfig,
+    seq_lens: list[tuple[int, int]],
+    num_heads: tuple[int, int],
+    head_size: int,
+    block_size: int,
+):
+    """End-to-end: quantized KV with per-token-head scale caches."""
+    from vllm.utils.math_utils import next_power_of_2
+    from vllm.v1.attention.ops.triton_unified_attention import unified_attention
+
+    torch.set_default_device(DEVICE_TYPE)
+    set_random_seed(0)
+
+    is_int4 = qcfg.kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD
+
+    num_seqs = len(seq_lens)
+    query_lens = [s[0] for s in seq_lens]
+    kv_lens = [s[1] for s in seq_lens]
+    num_query_heads, num_kv_heads = num_heads
+    max_query_len = max(query_lens)
+    max_kv_len = max(kv_lens)
+    scale = head_size**-0.5
+    num_blocks = 2048
+
+    query = torch.randn(
+        sum(query_lens), num_query_heads, head_size, dtype=torch.bfloat16
+    )
+
+    key_cache_bf16 = torch.randn(
+        num_blocks, block_size, num_kv_heads, head_size, dtype=torch.bfloat16
+    )
+    value_cache_bf16 = torch.randn_like(key_cache_bf16)
+
+    if is_int4:
+        # Asymmetric quantization reference (matches the Triton kernel).
+        kf = key_cache_bf16.float()
+        vf = value_cache_bf16.float()
+        k_min = kf.amin(dim=-1)
+        k_max = kf.amax(dim=-1)
+        v_min = vf.amin(dim=-1)
+        v_max = vf.amax(dim=-1)
+        k_scale_cache = ((k_max - k_min) / 15.0).clamp(min=1e-6).to(torch.float32)
+        v_scale_cache = ((v_max - v_min) / 15.0).clamp(min=1e-6).to(torch.float32)
+        k_zp = (-k_min / k_scale_cache).round().clamp(0, 15)
+        v_zp = (-v_min / v_scale_cache).round().clamp(0, 15)
+
+        key_cache_q_full = (
+            (kf / k_scale_cache[..., None] + k_zp[..., None]).round().clamp(0, 15)
+        )
+        value_cache_q_full = (
+            (vf / v_scale_cache[..., None] + v_zp[..., None]).round().clamp(0, 15)
+        )
+
+        # Dequantized reference: x_hat = (q - zp) * scale
+        key_cache_deq = (key_cache_q_full - k_zp[..., None]) * k_scale_cache[..., None]
+        value_cache_deq = (value_cache_q_full - v_zp[..., None]) * v_scale_cache[
+            ..., None
+        ]
+
+        # Pack two uint4 values into one byte
+        def _pack_int4(data_float):
+            u = data_float.to(torch.uint8)
+            lo = u[..., 0::2]
+            hi = u[..., 1::2]
+            return (lo & 0xF) | ((hi & 0xF) << 4)
+
+        key_cache_q = _pack_int4(key_cache_q_full)
+        value_cache_q = _pack_int4(value_cache_q_full)
+
+        # Steganography: pack zp into low 4 bits of scale
+        k_zp_int = k_zp.to(torch.int32)
+        k_bits = k_scale_cache.view(torch.int32)
+        k_scale_cache = ((k_bits & -16) | (k_zp_int & 0xF)).view(torch.float32)
+        v_zp_int = v_zp.to(torch.int32)
+        v_bits = v_scale_cache.view(torch.int32)
+        v_scale_cache = ((v_bits & -16) | (v_zp_int & 0xF)).view(torch.float32)
+    else:
+        # Symmetric quantization for int8/fp8.
+        k_absmax = key_cache_bf16.float().abs().amax(dim=-1)
+        v_absmax = value_cache_bf16.float().abs().amax(dim=-1)
+        k_scale_cache = (k_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
+        v_scale_cache = (v_absmax / qcfg.quant_max).clamp(min=1e-6).to(torch.float32)
+        scaled_k = key_cache_bf16.float() / k_scale_cache[:, :, :, None]
+        scaled_v = value_cache_bf16.float() / v_scale_cache[:, :, :, None]
+
+        key_cache_q_full = scaled_k.round().clamp(qcfg.quant_min, qcfg.quant_max)
+        value_cache_q_full = scaled_v.round().clamp(qcfg.quant_min, qcfg.quant_max)
+
+        key_cache_deq = key_cache_q_full * k_scale_cache[:, :, :, None]
+        value_cache_deq = value_cache_q_full * v_scale_cache[:, :, :, None]
+
+    if not is_int4 and qcfg.rounds_before_store:
+        key_cache_q = key_cache_q_full.to(qcfg.cache_dtype)
+        value_cache_q = value_cache_q_full.to(qcfg.cache_dtype)
+    elif not is_int4:
+        key_cache_q = scaled_k.clamp(qcfg.quant_min, qcfg.quant_max).to(
+            qcfg.cache_dtype
+        )
+        value_cache_q = scaled_v.clamp(qcfg.quant_min, qcfg.quant_max).to(
+            qcfg.cache_dtype
+        )
+
+    cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
+        dim=0, dtype=torch.int32
+    )
+    kv_lens_t = torch.tensor(kv_lens, dtype=torch.int32)
+
+    max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
+    block_tables = torch.randint(
+        0, num_blocks, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
+    )
+
+    head_size_padded = next_power_of_2(head_size)
+    seq_threshold_3D = 0
+    num_par_softmax_segments = 16
+    softmax_segm_output = torch.empty(
+        (seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
+        dtype=torch.float32,
+    )
+    softmax_segm_max = torch.empty(
+        (seq_threshold_3D, num_query_heads, num_par_softmax_segments),
+        dtype=torch.float32,
+    )
+    softmax_segm_expsum = torch.empty(
+        (seq_threshold_3D, num_query_heads, num_par_softmax_segments),
+        dtype=torch.float32,
+    )
+
+    output_q = torch.empty_like(query)
+    unified_attention(
+        q=query,
+        k=key_cache_q,
+        v=value_cache_q,
+        out=output_q,
+        cu_seqlens_q=cu_query_lens,
+        seqused_k=kv_lens_t,
+        max_seqlen_q=max_query_len,
+        max_seqlen_k=max_kv_len,
+        softmax_scale=scale,
+        causal=True,
+        window_size=(-1, -1),
+        block_table=block_tables,
+        softcap=0,
+        q_descale=None,
+        k_descale=None,
+        v_descale=None,
+        seq_threshold_3D=seq_threshold_3D,
+        num_par_softmax_segments=num_par_softmax_segments,
+        softmax_segm_output=softmax_segm_output,
+        softmax_segm_max=softmax_segm_max,
+        softmax_segm_expsum=softmax_segm_expsum,
+        kv_quant_mode=qcfg.kv_quant_mode,
+        k_scale_cache=k_scale_cache,
+        v_scale_cache=v_scale_cache,
+    )
+
+    output_ref = torch.empty_like(query)
+    unified_attention(
+        q=query,
+        k=key_cache_deq.to(torch.bfloat16),
+        v=value_cache_deq.to(torch.bfloat16),
+        out=output_ref,
+        cu_seqlens_q=cu_query_lens,
+        seqused_k=kv_lens_t,
+        max_seqlen_q=max_query_len,
+        max_seqlen_k=max_kv_len,
+        softmax_scale=scale,
+        causal=True,
+        window_size=(-1, -1),
+        block_table=block_tables,
+        softcap=0,
+        q_descale=None,
+        k_descale=None,
+        v_descale=None,
+        seq_threshold_3D=seq_threshold_3D,
+        num_par_softmax_segments=num_par_softmax_segments,
+        softmax_segm_output=softmax_segm_output,
+        softmax_segm_max=softmax_segm_max,
+        softmax_segm_expsum=softmax_segm_expsum,
+    )
+
+    # Coarser quantization → wider tolerance.
+    if is_int4:
+        atol, rtol = 0.5, 0.5
+    else:
+        atol, rtol = 5e-2, 5e-2
+    torch.testing.assert_close(output_q, output_ref, atol=atol, rtol=rtol)

tests/quantization/test_quantization_config_args.py

@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for QuantizationConfigArgs parsing."""
+
+import pytest
+
+from vllm.config.quantization import (
+    QUANT_KEY_NAMES,
+    QuantizationConfigArgs,
+    QuantSpec,
+    resolve_quantization_config,
+)
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
+    kFp8Dynamic128Sym,
+    kFp8DynamicTokenSym,
+    kFp8Static128BlockSym,
+    kFp8StaticTensorSym,
+    kInt8StaticChannelSym,
+    kMxfp8Dynamic,
+)
+
+# ---- QuantSpec ------------------------------------------------------------
+
+
+def test_quant_spec_resolves_string_to_quant_key():
+    spec = QuantSpec(weight="mxfp8", activation="fp8_per_token")
+    assert spec.weight == kMxfp8Dynamic
+    assert spec.activation == kFp8DynamicTokenSym
+
+
+def test_quant_spec_accepts_quant_key_directly():
+    spec = QuantSpec(weight=kFp8StaticTensorSym)
+    assert spec.weight is kFp8StaticTensorSym
+    assert spec.activation is None
+
+
+def test_quant_spec_rejects_unknown_name():
+    with pytest.raises(ValueError, match="unknown quantization name"):
+        QuantSpec(weight="not_a_real_format")
+
+
+# ---- QuantizationConfigArgs string shorthand on linear/moe ----------------
+
+
+def test_args_linear_string_resolves_via_quant_key_names():
+    # A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=<key>).
+    args = QuantizationConfigArgs(linear="fp8_per_block_static")
+    assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
+    assert args.moe is None
+
+
+def test_args_moe_string_resolves_via_online_shorthand():
+    # An online-shorthand name pulls the matching slot from _ONLINE_SHORTHANDS
+    # (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the
+    # same per-layer-kind spec the `--quantization fp8_per_block` shorthand
+    # would).
+    args = QuantizationConfigArgs(moe="fp8_per_block")
+    assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
+
+
+def test_args_string_shorthand_missing_slot_raises():
+    # int8_per_channel_weight_only sets only `moe`; using it on `linear`
+    # has no defined spec and should raise rather than silently no-op.
+    with pytest.raises(ValueError, match="does not define a linear spec"):
+        QuantizationConfigArgs(linear="int8_per_channel_weight_only")
+
+
+def test_args_accepts_dict_form():
+    args = QuantizationConfigArgs(moe={"activation": "mxfp8"})
+    assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
+
+
+# ---- resolve_quantization_config -----------------------------------------
+
+
+def test_resolve_shorthand_only_populates_both_slots():
+    args = resolve_quantization_config("fp8_per_block", None)
+    assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
+    assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
+
+
+def test_resolve_int8_shorthand_leaves_linear_unset():
+    # int8_per_channel_weight_only is MoE-only; linear stays None so that
+    # OnlineQuantizationConfig leaves Linear layers in full precision.
+    args = resolve_quantization_config("int8_per_channel_weight_only", None)
+    assert args.linear is None
+    assert args.moe == QuantSpec(weight=kInt8StaticChannelSym)
+
+
+def test_resolve_quantization_config_only():
+    # When only `quantization_config` is given (e.g. for an already-quantized
+    # checkpoint that needs an activation override), it's returned as-is.
+    args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}})
+    assert args.linear is None
+    assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
+
+
+def test_resolve_merges_explicit_over_shorthand():
+    # Explicit linear in quantization_config wins; moe falls back to the
+    # shorthand's slot.
+    args = resolve_quantization_config(
+        "fp8_per_tensor",
+        {"linear": "fp8_per_block"},
+    )
+    assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
+    assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
+
+
+def test_resolve_rejects_quantization_config_with_non_shorthand_quant():
+    # If --quantization names something other than an online shorthand,
+    # quantization_config is not allowed via this path (checkpoint quant
+    # paths read it directly off ModelConfig instead).
+    with pytest.raises(ValueError, match="quantization_config is only supported"):
+        resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
+
+
+# ---- QUANT_KEY_NAMES coverage --------------------------------------------
+
+
+def test_quant_key_names_round_trip():
+    # Every advertised name should round-trip through QuantSpec without error
+    # and produce the same QuantKey it maps to.
+    for name, expected in QUANT_KEY_NAMES.items():
+        assert QuantSpec(weight=name).weight == expected, name
+        assert QuantSpec(activation=name).activation == expected, name
+
+
+def test_static_block_weight_paired_with_dynamic_block_activation():
+    # The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic
+    # activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly
+    # rewire the kernel dispatch.
+    spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
+    assert spec.weight == kFp8Static128BlockSym
+    assert spec.activation == kFp8Dynamic128Sym

tests/quantization/test_quark.py

@@ -0,0 +1,535 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Test model set-up and weight loading for quark-quantized models.
+
+Run `pytest tests/quantization/test_quark.py`.
+
+See also `tests/kernels/moe/test_ocp_mx_moe.py`.
+"""
+
+import importlib.metadata
+from dataclasses import dataclass
+from importlib.util import find_spec
+
+import huggingface_hub
+import lm_eval
+import pytest
+import torch
+from packaging import version
+
+from vllm.model_executor.layers.quantization.quark.quark import (  # noqa: E501
+    QuarkLinearMethod,
+    QuarkW8A8Fp8,
+    QuarkW8A8Int8,
+)
+from vllm.model_executor.layers.quantization.quark.quark_moe import (  # noqa: E501
+    QuarkW8A8Int8MoEMethod,
+)
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
+    is_layer_skipped,
+)
+from vllm.platforms import current_platform
+
+if current_platform.is_rocm():
+    from vllm.platforms.rocm import on_gfx950
+else:
+
+    def on_gfx950() -> bool:
+        return False
+
+
+from .reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
+
+# Minimum amd-quark version for MXFP4/OCP_MX tests (single source of truth).
+QUARK_MXFP4_MIN_VERSION = "0.8.99"
+
+QUARK_MXFP4_AVAILABLE = find_spec("quark") is not None and version.parse(
+    importlib.metadata.version("amd-quark")
+) >= version.parse(QUARK_MXFP4_MIN_VERSION)
+
+DEVICE_TYPE = current_platform.device_type
+
+if QUARK_MXFP4_AVAILABLE:
+    from quark.torch.export.nn.modules.realquantizer import StaticScaledRealQuantizer
+    from quark.torch.kernel import mx as mx_kernel
+    from quark.torch.quantization.config.config import FP4PerGroupSpec
+
+try:
+    huggingface_hub.list_repo_refs(
+        "amd/Llama-3.3-70B-Instruct-WMXFP4-AMXFP4-KVFP8-Scale-UINT8-SQ"
+    )
+    HF_HUB_AMD_ORG_ACCESS = True
+except huggingface_hub.errors.RepositoryNotFoundError:
+    HF_HUB_AMD_ORG_ACCESS = False
+
+
+@pytest.fixture(scope="function", autouse=True)
+def enable_pickle(monkeypatch):
+    """`LLM.apply_model` requires pickling a function."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+
+@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
+@pytest.mark.parametrize("tp", [1])
+def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp):
+    model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
+    with vllm_runner(
+        model_path,
+        enforce_eager=True,
+        kv_cache_dtype=kv_cache_dtype,
+        tensor_parallel_size=tp,
+    ) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+
+            assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
+            assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
+
+            if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
+                assert len(qkv_proj.input_scale.shape) == 0
+                assert qkv_proj.weight.dtype is current_platform.fp8_dtype()
+                assert len(qkv_proj.weight_scale.shape) == 0
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+@pytest.mark.parametrize("tp", [1])
+def test_quark_fp8_w_per_channel_a_per_token(vllm_runner, tp):
+    model_path = "amd/Qwen2.5-1.5B-Instruct-ptpc-Quark-ts"
+    with vllm_runner(model_path, enforce_eager=True, tensor_parallel_size=tp) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+
+            assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
+            assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
+
+            if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
+                assert qkv_proj.weight.dtype is current_platform.fp8_dtype()
+                assert qkv_proj.weight_scale.shape[0] == qkv_proj.weight.shape[1]
+                assert qkv_proj.weight_scale.shape[1] == 1
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+@pytest.mark.parametrize("tp", [1])
+def test_quark_int8_w_per_tensor_a_per_tensor(vllm_runner, tp):
+    model_path = "amd/Llama-3.1-8B-Instruct-w-int8-a-int8-sym-test"
+    with vllm_runner(model_path, enforce_eager=True, tensor_parallel_size=tp) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+
+            qkv_proj = layer.self_attn.qkv_proj
+
+            assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
+            assert isinstance(qkv_proj.scheme, QuarkW8A8Int8)
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=4)
+        assert output
+
+
+@pytest.mark.parametrize("tp", [1])
+def test_quark_int8_w8a8_moe(vllm_runner, tp):
+    """Test W8A8 INT8 MoE quantization with a tiny Qwen3 MoE model."""
+    model_path = "nameistoken/tiny-qwen3-moe-w8a8-int8-quark"
+    with vllm_runner(
+        model_path,
+        enforce_eager=True,
+        tensor_parallel_size=tp,
+        gpu_memory_utilization=0.1,
+    ) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+            # MoE experts should use QuarkW8A8Int8MoEMethod
+            moe = layer.mlp.experts
+            assert isinstance(moe._quant_method, QuarkW8A8Int8MoEMethod), (
+                f"Expected QuarkW8A8Int8MoEMethod, got {type(moe._quant_method)}"
+            )
+            # Non-MoE linear layers should use QuarkW8A8Int8
+            qkv_proj = layer.self_attn.qkv_proj
+            assert isinstance(qkv_proj.scheme, QuarkW8A8Int8)
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello", max_tokens=4)
+        assert output
+
+
+def test_quark_fp8_parity(vllm_runner):
+    quark_model_id = "amd-quark/llama-tiny-fp8-quark-quant-method"
+    fp8_model_id = "amd-quark/llama-tiny-fp8-quant-method"
+
+    llm_kwargs = {
+        "tensor_parallel_size": 1,
+        "enforce_eager": True,
+        "gpu_memory_utilization": 0.1,
+    }
+    with (
+        vllm_runner(quark_model_id, **llm_kwargs) as quark_handle,
+        vllm_runner(fp8_model_id, **llm_kwargs) as fp8_handle,
+    ):
+
+        def get_state_dict(model):
+            return {k: v.cpu() for k, v in model.state_dict().items()}
+
+        (quark_state_dict,) = quark_handle.apply_model(get_state_dict)
+        (fp8_state_dict,) = fp8_handle.apply_model(get_state_dict)
+
+    assert fp8_state_dict.keys() == quark_state_dict.keys()
+
+    for key in fp8_state_dict:
+        assert torch.equal(fp8_state_dict[key], quark_state_dict[key])
+
+
+@dataclass
+class AccuracyTestConfig:
+    model_name: str
+    excepted_value: float
+
+    def get_model_args(
+        self,
+        tp_size: int,
+        model_max_len: int | None = None,
+        kwargs: dict | None = None,
+    ) -> dict:
+        if kwargs is None:
+            kwargs = {}
+
+        model_args = {
+            "pretrained": self.model_name,
+            "dtype": "auto",
+            "add_bos_token": True,
+            "tensor_parallel_size": tp_size,
+            "gpu_memory_utilization": 0.7,
+            **kwargs,
+        }
+        if model_max_len is not None:
+            model_args["max_model_len"] = model_max_len
+
+        return model_args
+
+
+GSM8K_ACCURACY_CONFIGS = [
+    # Private model.
+    AccuracyTestConfig(
+        model_name="amd/DeepSeek-R1-WMXFP4-AMXFP4-Scale-UINT8-MoE-Quant",
+        excepted_value=0.96,
+    ),
+]
+
+WIKITEXT_ACCURACY_CONFIGS = [
+    AccuracyTestConfig(
+        model_name="fxmarty/qwen1.5_moe_a2.7b_chat_w_fp4_a_fp6_e2m3",
+        excepted_value=11.3,
+    ),
+    AccuracyTestConfig(
+        model_name="fxmarty/qwen1.5_moe_a2.7b_chat_w_fp6_e3m2_a_fp6_e3m2",
+        excepted_value=10.6,
+    ),
+    AccuracyTestConfig(
+        model_name="fxmarty/qwen_1.5-moe-a2.7b-mxfp4", excepted_value=12.4
+    ),
+]
+
+
+@pytest.mark.skipif(
+    not QUARK_MXFP4_AVAILABLE,
+    reason=f"amd-quark>={QUARK_MXFP4_MIN_VERSION} is not available",
+)
+@pytest.mark.parametrize(
+    "config",
+    [pytest.param(val, id=f"config:{val}") for val in WIKITEXT_ACCURACY_CONFIGS],
+)
+@pytest.mark.parametrize(
+    "tp_size", [pytest.param(val, id=f"tp_size:{val}") for val in [1, 2]]
+)
+def test_ocp_mx_wikitext_correctness(config: AccuracyTestConfig, tp_size: int):
+    device_count = torch.accelerator.device_count()
+    if device_count < tp_size:
+        pytest.skip(f"This test requires >={tp_size} gpus, got only {device_count}")
+
+    task = "wikitext"
+    rtol = 0.1
+
+    # Smaller cudagraph_capture_sizes to speed up the test.
+    results = lm_eval.simple_evaluate(
+        model="vllm",
+        model_args=config.get_model_args(
+            tp_size=tp_size, kwargs={"cudagraph_capture_sizes": [16]}
+        ),
+        tasks=task,
+        batch_size=64,
+    )
+
+    EXPECTED_VALUE = config.excepted_value
+    measured_value = results["results"][task]["word_perplexity,none"]
+    assert (
+        measured_value < EXPECTED_VALUE + rtol
+        and measured_value > EXPECTED_VALUE - rtol
+    ), f"Expected: {EXPECTED_VALUE} |  Measured: {measured_value}"
+
+
+@pytest.mark.skipif(
+    not QUARK_MXFP4_AVAILABLE,
+    reason=f"amd-quark>={QUARK_MXFP4_MIN_VERSION} is not available",
+)
+@pytest.mark.parametrize("tp_size", [1, 2])
+def test_nvfp4_wikitext_correctness(tp_size: int):
+    device_count = torch.accelerator.device_count()
+    if device_count < tp_size:
+        pytest.skip(f"This test requires >={tp_size} gpus, got only {device_count}")
+
+    # NOTE: expected_value from nvidia/Qwen3-30B-A3B-NVFP4
+    expected_value = 11.2391
+
+    model_name = "amd-quark/Qwen3-30B-A3B-nvfp4-quark"
+    task = "wikitext"
+
+    rtol = 0.25
+
+    config = AccuracyTestConfig(
+        model_name=model_name,
+        excepted_value=expected_value,
+    )
+
+    model_args = config.get_model_args(
+        tp_size=tp_size,
+        kwargs={
+            "cudagraph_capture_sizes": [16],
+        },
+    )
+    model_args.pop("add_bos_token")
+
+    # Smaller cudagraph_capture_sizes to speed up the test.
+    results = lm_eval.simple_evaluate(
+        model="vllm",
+        model_args=model_args,
+        tasks=task,
+        batch_size=64,
+    )
+
+    EXPECTED_VALUE = config.excepted_value
+    measured_value = results["results"][task]["word_perplexity,none"]
+    assert (
+        measured_value < EXPECTED_VALUE + rtol
+        and measured_value > EXPECTED_VALUE - rtol
+    ), f"Expected: {EXPECTED_VALUE} |  Measured: {measured_value}"
+
+
+@pytest.mark.parametrize("config", GSM8K_ACCURACY_CONFIGS)
+@pytest.mark.skipif(
+    not QUARK_MXFP4_AVAILABLE,
+    reason=f"amd-quark>={QUARK_MXFP4_MIN_VERSION} is not available",
+)
+@pytest.mark.skipif(
+    not HF_HUB_AMD_ORG_ACCESS,
+    reason="Read access to huggingface.co/amd is required for this test.",
+)
+def test_mxfp4_gsm8k_correctness(config: AccuracyTestConfig):
+    device_count = torch.accelerator.device_count()
+    if device_count < 8:
+        pytest.skip(f"This test requires >=8 gpus, got only {device_count}")
+
+    task = "gsm8k"
+    rtol = 0.03
+
+    results = lm_eval.simple_evaluate(
+        model="vllm",
+        model_args=config.get_model_args(tp_size=8, model_max_len=38768),
+        tasks=task,
+        batch_size=64,
+        num_fewshot=8,
+    )
+
+    EXPECTED_VALUE = config.excepted_value
+    measured_value = results["results"][task]["exact_match,strict-match"]
+    assert (
+        measured_value - rtol < EXPECTED_VALUE
+        and measured_value + rtol > EXPECTED_VALUE
+    ), f"Expected: {EXPECTED_VALUE} |  Measured: {measured_value}"
+
+
+@pytest.mark.skipif(
+    not QUARK_MXFP4_AVAILABLE,
+    reason=f"amd-quark>={QUARK_MXFP4_MIN_VERSION} is not available",
+)
+@pytest.mark.parametrize("float_dtype", [torch.bfloat16, torch.float16])
+@pytest.mark.parametrize("scalings", [[2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]])
+def test_mxfp4_fused_qdq_match_quark(float_dtype: torch.dtype, scalings: list[int]):
+    torch.manual_seed(0)
+
+    hidden_size = 64 * 32
+    inp = (torch.rand(1, hidden_size, dtype=float_dtype, device=DEVICE_TYPE) - 0.5) * 2
+    for i in range(hidden_size // 32):
+        inp[:, i * 32 : (i + 1) * 32] = (
+            inp[:, i * 32 : (i + 1) * 32] * scalings[i % len(scalings)]
+        )
+
+    inp_kernel = inp.clone()
+    inp_kernel_clone = inp_kernel.clone()
+
+    res_hip = mx_kernel.qdq_mxfp4_hip(inp_kernel_clone, "even")
+    res_torch = qdq_mxfp4_torch(inp_kernel, "even")
+
+    for i in range(hidden_size // 32):
+        assert torch.all(torch.isfinite(res_hip[:, i * 32 : (i + 1) * 32]))
+        assert torch.all(torch.isfinite(res_torch[:, i * 32 : (i + 1) * 32]))
+
+        torch.testing.assert_close(
+            res_hip[:, i * 32 : (i + 1) * 32], res_torch[:, i * 32 : (i + 1) * 32]
+        )
+
+
+@pytest.mark.skipif(
+    not QUARK_MXFP4_AVAILABLE,
+    reason=f"amd-quark>={QUARK_MXFP4_MIN_VERSION} is not available",
+)
+@pytest.mark.parametrize("float_dtype", [torch.bfloat16, torch.float16])
+@pytest.mark.parametrize("scalings", [[2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]])
+def test_mxfp4_dequant_kernel_match_quark(
+    float_dtype: torch.dtype, scalings: list[int]
+):
+    qspec = FP4PerGroupSpec(
+        ch_axis=-1,
+        group_size=32,
+        scale_format="e8m0",
+        scale_calculation_mode="even",
+        is_dynamic=False,
+    ).to_quantization_spec()
+
+    weight_quantizer = StaticScaledRealQuantizer(
+        qspec=qspec,
+        quantizer=None,
+        reorder=False,
+        real_quantized=True,
+        float_dtype=float_dtype,
+        device=DEVICE_TYPE,
+    )
+
+    observer = qspec.observer_cls(qspec, device=DEVICE_TYPE)
+
+    hidden_size = 512
+    shape = (11008, hidden_size)
+
+    w = (torch.rand(shape, device=DEVICE_TYPE, dtype=float_dtype) - 0.5) * 2
+
+    # Make it so that different groups have different scales.
+    for i in range(hidden_size // 32):
+        w[:, i * 32 : (i + 1) * 32] = (
+            w[:, i * 32 : (i + 1) * 32] * scalings[i % len(scalings)]
+        )
+
+    observer(w)
+    scale, _ = observer._calculate_qparams()
+    weight_quantizer.scale = scale
+
+    w_mxfp4 = weight_quantizer.to_real_quantize_params(w).to(DEVICE_TYPE)
+    weight_quantizer.maybe_convert_and_transpose_scale()
+
+    scale = weight_quantizer.scale
+
+    out_hip = mx_kernel.dq_mxfp4_hip(w_mxfp4, scale, float_dtype)
+
+    out_torch = dq_mxfp4_torch(w_mxfp4, scale, float_dtype)
+
+    assert torch.equal(out_hip, out_torch)
+
+
+# Unit tests for ``is_layer_skipped`` fused-name handling.
+
+FUSED_MAPPING = {
+    "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+    "gate_up_proj": ["gate_proj", "up_proj"],
+}
+
+
+def test_fused_name_listed_directly_is_skipped():
+    # Regression for Step-3.5-Flash-FP8: the checkpoint lists the fused
+    # name (``qkv_proj``) directly in ``modules_to_not_convert``. When a
+    # ``packed_modules_mapping`` is registered on the model, the fused
+    # match must still win over per-shard expansion.
+    ignored = ["model.layers.0.self_attn.qkv_proj"]
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=ignored,
+        fused_mapping=FUSED_MAPPING,
+    )
+    assert is_layer_skipped(
+        prefix="model.layers.0.mlp.gate_up_proj",
+        ignored_layers=["model.layers.0.mlp.gate_up_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_unfused_shards_listed_is_skipped():
+    # Quark INT8 style: per-shard names listed; all shards present means
+    # the fused layer is skipped via expansion.
+    ignored = [
+        "model.layers.0.self_attn.q_proj",
+        "model.layers.0.self_attn.k_proj",
+        "model.layers.0.self_attn.v_proj",
+    ]
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=ignored,
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_partial_shards_raises():
+    # Only some shards listed -> ambiguous, must raise. Fused name is
+    # not in ignored_layers, so we fall through to per-shard expansion.
+    ignored = ["model.layers.0.self_attn.q_proj"]
+    with pytest.raises(ValueError):
+        is_layer_skipped(
+            prefix="model.layers.0.self_attn.qkv_proj",
+            ignored_layers=ignored,
+            fused_mapping=FUSED_MAPPING,
+        )
+
+
+def test_not_skipped_when_nothing_listed():
+    assert not is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=["model.layers.0.mlp.gate_up_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_non_fused_layer_unaffected():
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.o_proj",
+        ignored_layers=["model.layers.0.self_attn.o_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+    assert not is_layer_skipped(
+        prefix="model.layers.0.self_attn.o_proj",
+        ignored_layers=["model.layers.1.self_attn.o_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_substr_match_on_fused_name():
+    # skip_with_substr=True path: fused-name substring match should also
+    # short-circuit before shard expansion.
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=["self_attn.qkv_proj"],
+        fused_mapping=FUSED_MAPPING,
+        skip_with_substr=True,
+    )

tests/quantization/test_register_quantization_config.py

@@ -0,0 +1,146 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests register custom quantization config.
+
+See https://github.com/vllm-project/vllm/issues/11926 for more details.
+
+Run `pytest tests/quantization/test_register_quantization_config.py`.
+"""
+
+import logging
+from typing import Any
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from vllm.model_executor.layers.linear import (
+    LinearBase,  # noqa: E501
+    UnquantizedLinearMethod,
+)
+from vllm.model_executor.layers.quantization import (
+    QuantizationMethods,
+    get_quantization_config,
+    register_quantization_config,
+)
+from vllm.model_executor.layers.quantization.base_config import (
+    QuantizationConfig,  # noqa: E501
+)
+
+
+class FakeQuantLinearMethod(UnquantizedLinearMethod):
+    """Fake quantization linear method for per-token dynamic quantization."""
+
+    def __init__(self, num_bits: int = 8) -> None:
+        """Initialize the quantization method."""
+        super().__init__()
+        self.num_bits = num_bits
+
+    def apply(
+        self,
+        layer: torch.nn.Module,
+        x: torch.Tensor,
+        bias: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        """Perform fake quantization before the linear layer."""
+
+        # Calculate the scales dynamically
+        max_val = torch.amax(x, dim=(0, -1), keepdims=True)
+        min_val = torch.amin(x, dim=(0, -1), keepdims=True)
+        scales = (max_val - min_val) / (2**self.num_bits - 1)
+
+        # Fake quantize the input
+        quant_x = torch.clamp(
+            torch.round(x / scales),
+            -(2 ** (self.num_bits - 1)),
+            2 ** (self.num_bits - 1) - 1,
+        )
+        dequant_x = quant_x * scales
+
+        return F.linear(dequant_x, layer.weight, bias)
+
+
+@register_quantization_config("custom_quant")
+class CustomQuantConfig(QuantizationConfig):
+    """Custom quantization config for per-token dynamic fake quantization."""
+
+    def __init__(self, num_bits: int = 8) -> None:
+        """Initialize the quantization config."""
+        super().__init__()
+        self.num_bits = num_bits
+
+    def get_name(self) -> QuantizationMethods:
+        """Name of the quantization method."""
+        return "custom_quant"
+
+    def get_supported_act_dtypes(self) -> list[torch.dtype]:
+        """List of supported activation dtypes."""
+        return [torch.float16, torch.bfloat16]
+
+    @classmethod
+    def get_min_capability(cls) -> int:
+        """Minimum GPU capability to support the quantization method."""
+        return -1
+
+    @staticmethod
+    def get_config_filenames() -> list[str]:
+        """List of filenames to search for in the model directory."""
+        return []
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> "CustomQuantConfig":
+        """Create a config class from the model's quantization config."""
+        return CustomQuantConfig(num_bits=config.get("num_bits", 8))
+
+    def get_quant_method(
+        self, layer: torch.nn.Module, prefix: str
+    ) -> FakeQuantLinearMethod | None:
+        """Get the quantize method to use for the quantized layer."""
+        if isinstance(layer, LinearBase):
+            return FakeQuantLinearMethod(num_bits=self.num_bits)
+        return None
+
+
+def test_register_quantization_config(caplog_vllm):
+    """Test register custom quantization config."""
+
+    # The quantization method `custom_quant` should be registered.
+    assert get_quantization_config("custom_quant") == CustomQuantConfig
+
+    # The quantization method `custom_quant` is already exists,
+    # should raise a debug message when re-registering it.
+    with caplog_vllm.at_level(logging.DEBUG, logger="vllm"):
+        register_quantization_config("custom_quant")(CustomQuantConfig)
+
+    assert any(
+        "The quantization method 'custom_quant' already exists" in message
+        for message in caplog_vllm.messages
+    ), "Expected a debug message when re-registering custom_quant"
+
+
+@pytest.mark.parametrize(
+    argnames="model",
+    argvalues=[
+        "meta-llama/Llama-3.2-1B-Instruct",
+    ],
+)
+def test_custom_quant(vllm_runner, model, monkeypatch):
+    """Test infer with the custom quantization method."""
+    # `LLM.apply_model` requires pickling a function.
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    with vllm_runner(
+        model_name=model, quantization="custom_quant", enforce_eager=True
+    ) as llm:
+
+        def check_model(model):
+            layer = model.model.layers[0]
+            qkv_proj = layer.self_attn.qkv_proj
+
+            # Check the quantization method is FakeQuantLinearMethod
+            assert isinstance(qkv_proj.quant_method, FakeQuantLinearMethod)
+
+        llm.apply_model(check_model)
+
+        output = llm.generate_greedy("Hello my name is", max_tokens=1)
+        assert output

tests/quantization/test_torchao.py

@@ -0,0 +1,402 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import importlib.util
+
+import pytest
+import torch
+
+from vllm.model_executor.model_loader import get_model_loader
+from vllm.platforms import current_platform
+
+DEVICE_TYPE = current_platform.device_type
+DTYPE = ["bfloat16"]
+
+TORCHAO_AVAILABLE = importlib.util.find_spec("torchao") is not None
+
+
+@pytest.mark.skipif(
+    current_platform.is_rocm() and current_platform.is_fp8_fnuz(),
+    reason="Only fp8_fnuz supported on CDNA3 architecture",
+)
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+def test_pre_quantized_model(vllm_runner):
+    with vllm_runner(
+        "torchao-testing/opt-125m-Float8WeightOnlyConfig-v2-0.15.0",
+        quantization="torchao",
+        dtype="bfloat16",
+        enforce_eager=True,
+    ) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+    assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+@pytest.mark.parametrize(
+    "pt_load_map_location",
+    [
+        f"{DEVICE_TYPE}:0",
+        # {"": "cuda"},
+    ],
+)
+def test_opt_125m_int8wo_model_loading_with_params(vllm_runner, pt_load_map_location):
+    torch._dynamo.reset()
+    model_name = "jerryzh168/opt-125m-int8wo-partial-quant"
+    with vllm_runner(
+        model_name=model_name,
+        quantization="torchao",
+        dtype="bfloat16",
+        pt_load_map_location=pt_load_map_location,
+        enforce_eager=True,
+    ) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+        assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+def test_qwenvl_int8wo_model_loading_with_params(vllm_runner):
+    torch._dynamo.reset()
+    model_name = "mobicham/Qwen2.5-VL-3B-Instruct_int8wo_ao"
+    with vllm_runner(
+        model_name=model_name,
+        quantization="torchao",
+        dtype="bfloat16",
+        pt_load_map_location=f"{DEVICE_TYPE}:0",
+        enforce_eager=True,
+    ) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+        assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+@pytest.mark.skip(
+    reason="since torchao nightly is only compatible with torch nightly"
+    "currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
+    "torchao tests that requires newer versions (0.14.0.dev+) for now"
+)
+def test_opt_125m_awq_int4wo_model_loading_with_params(vllm_runner):
+    torch._dynamo.reset()
+    model_name = "torchao-testing/opt-125m-AWQConfig-Int4WeightOnlyConfig-v2-0.14.0.dev"
+    with vllm_runner(
+        model_name=model_name,
+        quantization="torchao",
+        dtype="bfloat16",
+        pt_load_map_location=f"{DEVICE_TYPE}:0",
+    ) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+        assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+def test_online_quant_config_dict_json(vllm_runner, enable_pickle):
+    """Testing online quantization, load_weights integration point,
+    with config dict serialized to json string
+    """
+    torch._dynamo.reset()
+    model_name = "facebook/opt-125m"
+
+    import json
+
+    from torchao.core.config import config_to_dict
+    from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
+
+    torchao_quant_config = Float8DynamicActivationFloat8WeightConfig(
+        granularity=PerRow()
+    )
+    hf_overrides = {
+        "quantization_config_dict_json": json.dumps(
+            config_to_dict(torchao_quant_config)
+        )
+    }
+    with vllm_runner(
+        model_name=model_name,
+        dtype="bfloat16",
+        pt_load_map_location=f"{DEVICE_TYPE}:0",
+        quantization="torchao",
+        hf_overrides=hf_overrides,
+        enforce_eager=True,
+    ) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+        load_config = llm.llm.llm_engine.vllm_config.load_config
+        model_config = llm.llm.llm_engine.vllm_config.model_config
+
+        def load_weights(model):
+            model_loader = get_model_loader(load_config)
+            weights_iterator = model_loader.get_all_weights(model_config, model)
+            model.load_weights(weights_iterator)
+
+        llm.apply_model(load_weights)
+
+        reload_output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+        assert output[0][0] == reload_output[0][0]
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+def test_online_quant_config_file(vllm_runner):
+    """Testing on the fly quantization, load_weights integration point,
+    with config file
+    """
+    torch._dynamo.reset()
+    model_name = "facebook/opt-125m"
+    import json
+    from tempfile import NamedTemporaryFile
+
+    from torchao.core.config import config_to_dict
+    from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
+
+    config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
+
+    with NamedTemporaryFile(mode="w", delete=False) as f:
+        f.write(json.dumps(config_to_dict(config)))
+        # close the file to save it
+        f.close()
+        config_file_name = str(f.name)
+
+        hf_overrides = {"quantization_config_file": config_file_name}
+        with vllm_runner(
+            model_name=model_name,
+            dtype="bfloat16",
+            pt_load_map_location=f"{DEVICE_TYPE}:0",
+            quantization="torchao",
+            hf_overrides=hf_overrides,
+            enforce_eager=True,
+        ) as llm:
+            output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+            assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+def test_reload_weights():
+    import json
+
+    from torchao.core.config import config_to_dict
+    from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
+
+    from vllm import LLM, SamplingParams
+
+    torchao_quant_config = Float8DynamicActivationFloat8WeightConfig(
+        granularity=PerRow()
+    )
+
+    hf_overrides = {
+        "quantization_config_dict_json": json.dumps(
+            config_to_dict(torchao_quant_config)
+        )
+    }
+
+    llm = LLM(
+        model="Qwen/Qwen3-0.6B",
+        dtype="bfloat16",
+        load_format="dummy",
+        enforce_eager=True,
+        quantization="torchao",
+        hf_overrides=hf_overrides,
+    )
+    # Update load format from `dummy` to `auto`
+    llm.collective_rpc(
+        "update_config", args=({"load_config": {"load_format": "auto"}},)
+    )
+    # Now reload real weights inplace
+    llm.collective_rpc("reload_weights")
+    prompts = [
+        "Hello, my name is",
+        "The president of the United States is",
+        "The capital of France is",
+        "The future of AI is",
+    ]
+    # Create a sampling params object.
+    sampling_params = SamplingParams(temperature=0, top_p=0.95)
+    outputs = llm.generate(prompts, sampling_params)
+    # make sure it runs
+    for output in outputs:
+        generated_text = output.outputs[0].text
+        assert generated_text
+        # can also uncomment locally to make sure the generated
+        # output makes sense
+        # prompt = output.prompt
+        # print(f"Prompt:    {prompt!r}")
+        # print(f"Output:    {generated_text!r}")
+        # print("-" * 60)
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+@pytest.mark.skip(
+    reason="since torchao nightly is only compatible with torch nightly"
+    "currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
+    "torchao tests that requires newer versions (0.15.0.dev+) for now"
+)
+def test_safetensors_model_loading_with_params(vllm_runner):
+    torch._dynamo.reset()
+    # using this model to test safetensors loading with file sharding
+    model_name = "torchao-testing/Qwen3-8B-INT4-0.15.0dev-safetensors"
+    with vllm_runner(model_name=model_name, dtype="bfloat16") as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+        assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+@pytest.mark.skip(
+    reason="since torchao nightly is only compatible with torch nightly"
+    "currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
+    "torchao tests that requires newer versions (0.14.0.dev+) for now"
+)
+def test_opt_125m_module_fqn_to_config_regex_model(vllm_runner):
+    torch._dynamo.reset()
+    model_name = "torchao-testing/opt-125m-ModuleFqnToConfig-v1-regex-0.14.0.dev"
+    with vllm_runner(
+        model_name=model_name, dtype="bfloat16", pt_load_map_location=f"{DEVICE_TYPE}:0"
+    ) as llm:
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+    assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+@pytest.mark.skip(
+    reason="since torchao nightly is only compatible with torch nightly"
+    "currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
+    "torchao tests that requires newer versions (0.14.0.dev+) for now"
+)
+def test_opt_125m_int4wo_model_running_preshuffled_kernel(vllm_runner, monkeypatch):
+    """We load a model with Int4Tensor (plain format) linear weights
+    and verify that the weight is updated to Int4PreshuffledTensor
+    after loading in vllm
+    """
+    from torchao.quantization import Int4PreshuffledTensor
+    from torchao.utils import _is_fbgemm_gpu_genai_available, is_sm_at_least_90
+
+    torch._dynamo.reset()
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+    model_name = "torchao-testing/opt-125m-Int4WeightOnlyConfig-v2-0.14.0.dev"
+    # Note: using enforce_eager=True because the `bf16i4bf16_shuffled` doesn't
+    # have meta kernel implemented yet, can remove this flag after that is implemented
+    with vllm_runner(
+        model_name=model_name,
+        quantization="torchao",
+        dtype="bfloat16",
+        pt_load_map_location=f"{DEVICE_TYPE}:0",
+        enforce_eager=True,
+    ) as llm:
+
+        def has_int4_preshuffled_tensor_weight(model):
+            return isinstance(
+                model.model.decoder.layers[0].self_attn.qkv_proj.weight,
+                Int4PreshuffledTensor,
+            )
+
+        def get_weight_attrs(model):
+            weight = model.model.decoder.layers[0].self_attn.qkv_proj.weight
+            return [
+                weight.requires_grad,
+                weight.input_dim,
+                weight.output_dim,
+                hasattr(weight, "weight_loader"),
+            ]
+
+        llm_engine = llm.get_llm().llm_engine
+        has_int4_preshuffled_tensor = any(
+            llm_engine.apply_model(has_int4_preshuffled_tensor_weight)
+        )
+        weight_attrs = llm_engine.apply_model(get_weight_attrs)[0]
+
+        # making sure we are using Int4PreshuffledTensor on H100 GPU, when
+        # fbgemm_gpu_genai
+        # library is installed, otherwise it should be using Int4Tensor
+        if _is_fbgemm_gpu_genai_available() and is_sm_at_least_90():
+            assert has_int4_preshuffled_tensor
+        else:
+            assert not has_int4_preshuffled_tensor
+
+        assert weight_attrs == [False, 1, 0, True]
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=32)
+
+        assert output
+
+
+@pytest.mark.skipif(not TORCHAO_AVAILABLE, reason="torchao is not available")
+@pytest.mark.skip(
+    reason="since torchao nightly is only compatible with torch nightly"
+    "currently https://github.com/pytorch/ao/issues/2919, we'll have to skip "
+    "torchao tests that requires newer versions (0.14.0.dev+) for now"
+)
+def test_opt_125m_int4wo_model_running_preshuffled_kernel_online_quant(
+    vllm_runner, monkeypatch
+):
+    """We load a bf16 model and online quantize the model to int4, then verify that
+    the weights are updated to Int4PreshuffledTensor after online quantization
+    """
+    from torchao.quantization import Int4PreshuffledTensor
+    from torchao.utils import _is_fbgemm_gpu_genai_available, is_sm_at_least_90
+
+    torch._dynamo.reset()
+    model_name = "facebook/opt-125m"
+
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    import json
+
+    from torchao.core.config import config_to_dict
+    from torchao.quantization import Int4WeightOnlyConfig
+
+    torchao_quant_config = Int4WeightOnlyConfig(
+        group_size=128, int4_packing_format="plain"
+    )
+    hf_overrides = {
+        "quantization_config_dict_json": json.dumps(
+            config_to_dict(torchao_quant_config)
+        )
+    }
+
+    # Note: using enforce_eager=True because the `bf16i4bf16_shuffled` doesn't
+    # have meta kernel implemented yet, can remove this flag after that is implemented
+    with vllm_runner(
+        model_name=model_name,
+        quantization="torchao",
+        dtype="bfloat16",
+        pt_load_map_location=f"{DEVICE_TYPE}:0",
+        hf_overrides=hf_overrides,
+        enforce_eager=True,
+    ) as llm:
+
+        def has_int4_preshuffled_tensor_weight(model):
+            return isinstance(
+                model.model.decoder.layers[0].self_attn.qkv_proj.weight,
+                Int4PreshuffledTensor,
+            )
+
+        def get_weight_attrs(model):
+            weight = model.model.decoder.layers[0].self_attn.qkv_proj.weight
+            return [
+                weight.requires_grad,
+                weight.input_dim,
+                weight.output_dim,
+                hasattr(weight, "weight_loader"),
+            ]
+
+        llm_engine = llm.get_llm().llm_engine
+        has_int4_preshuffled_tensor = any(
+            llm_engine.apply_model(has_int4_preshuffled_tensor_weight)
+        )
+        weight_attrs = llm_engine.apply_model(get_weight_attrs)[0]
+
+        # making sure we are using Int4PreshuffledTensor on H100 GPU, when
+        # fbgemm_gpu_genai
+        # library is installed, otherwise it should be using Int4Tensor
+        if _is_fbgemm_gpu_genai_available() and is_sm_at_least_90():
+            assert has_int4_preshuffled_tensor
+        else:
+            assert not has_int4_preshuffled_tensor
+
+        assert weight_attrs == [False, 1, 0, True]
+        output = llm.generate_greedy(["The capital of France is"], max_tokens=4)
+
+        assert output
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
+    align_trtllm_fp4_moe_hidden_dim_for_fi,
+)
+
+
+def test_align_trtllm_fp4_moe_hidden_dim_noop():
+    w13 = torch.arange(2 * 8 * 256, dtype=torch.uint8).reshape(2, 8, 256)
+    w13_scale = torch.arange(2 * 8 * 32, dtype=torch.uint8).reshape(2, 8, 32)
+    w2 = torch.arange(2 * 512 * 4, dtype=torch.uint8).reshape(2, 512, 4)
+    w2_scale = torch.arange(2 * 512 * 1, dtype=torch.uint8).reshape(2, 512, 1)
+
+    out_w13, out_w13_scale, out_w2, out_w2_scale, padded_hidden = (
+        align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
+    )
+
+    assert padded_hidden == 512
+    assert out_w13 is w13
+    assert out_w13_scale is w13_scale
+    assert out_w2 is w2
+    assert out_w2_scale is w2_scale
+
+
+def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
+    hidden_dim = 2688
+    padded_hidden_dim = 2816
+
+    w13 = torch.arange(2 * 12 * (hidden_dim // 2), dtype=torch.uint8).reshape(
+        2, 12, hidden_dim // 2
+    )
+    w13_scale = torch.arange(2 * 12 * (hidden_dim // 16), dtype=torch.uint8).reshape(
+        2, 12, hidden_dim // 16
+    )
+
+    w2 = torch.arange(2 * hidden_dim * 6, dtype=torch.uint8).reshape(2, hidden_dim, 6)
+    w2_scale = torch.arange(2 * hidden_dim * 2, dtype=torch.uint8).reshape(
+        2, hidden_dim, 2
+    )
+
+    out_w13, out_w13_scale, out_w2, out_w2_scale, out_hidden_dim = (
+        align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
+    )
+
+    assert out_hidden_dim == padded_hidden_dim
+    assert out_w13.shape == (2, 12, padded_hidden_dim // 2)
+    assert out_w13_scale.shape == (2, 12, padded_hidden_dim // 16)
+    assert out_w2.shape == (2, padded_hidden_dim, 6)
+    assert out_w2_scale.shape == (2, padded_hidden_dim, 2)
+
+    torch.testing.assert_close(out_w13[:, :, : hidden_dim // 2], w13)
+    torch.testing.assert_close(out_w13_scale[:, :, : hidden_dim // 16], w13_scale)
+    torch.testing.assert_close(out_w2[:, :hidden_dim, :], w2)
+    torch.testing.assert_close(out_w2_scale[:, :hidden_dim, :], w2_scale)
+
+    assert torch.count_nonzero(out_w13[:, :, hidden_dim // 2 :]) == 0
+    assert torch.count_nonzero(out_w13_scale[:, :, hidden_dim // 16 :]) == 0
+    assert torch.count_nonzero(out_w2[:, hidden_dim:, :]) == 0
+    assert torch.count_nonzero(out_w2_scale[:, hidden_dim:, :]) == 0

tests/quantization/test_turboquant.py

@@ -0,0 +1,749 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for TurboQuant KV-cache quantization.
+
+Run: .venv/bin/python -m pytest tests/quantization/test_turboquant.py -v
+"""
+
+import math
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.quantization.turboquant.centroids import (
+    get_centroids,
+    solve_lloyd_max,
+)
+from vllm.model_executor.layers.quantization.turboquant.config import (
+    TQ_PRESETS,
+    TurboQuantConfig,
+)
+from vllm.platforms import current_platform
+from vllm.utils.math_utils import next_power_of_2
+
+# ============================================================================
+# Helpers
+# ============================================================================
+
+ALL_PRESETS = list(TQ_PRESETS.keys())
+
+
+def _assert_strictly_sorted(seq, name="sequence"):
+    for i in range(len(seq) - 1):
+        assert seq[i] < seq[i + 1], f"{name} not sorted at index {i}"
+
+
+def _is_power_of_2(n: int) -> bool:
+    return n > 0 and next_power_of_2(n) == n
+
+
+# Expected concrete values for each preset at head_dim=128.
+# fmt: off
+PRESET_EXPECTED = {
+    "turboquant_k8v4": dict(
+        key_fp8=True,  key_quant_bits=8,
+        key_mse_bits=0, value_quant_bits=4,
+        mse_bits=4, n_centroids=16, centroid_bits=4,
+        norm_correction=False,
+        key_packed_size=128, value_packed_size=68,
+        slot_size=196, slot_size_aligned=196,
+    ),
+    "turboquant_4bit_nc": dict(
+        key_fp8=False, key_quant_bits=4,
+        key_mse_bits=4, value_quant_bits=4,
+        mse_bits=4, n_centroids=16, centroid_bits=4,
+        norm_correction=True,
+        key_packed_size=66, value_packed_size=68,
+        slot_size=134, slot_size_aligned=134,
+    ),
+    "turboquant_k3v4_nc": dict(
+        key_fp8=False, key_quant_bits=3,
+        key_mse_bits=3, value_quant_bits=4,
+        mse_bits=3, n_centroids=8, centroid_bits=3,
+        norm_correction=True,
+        key_packed_size=50, value_packed_size=68,
+        slot_size=118, slot_size_aligned=118,
+    ),
+    "turboquant_3bit_nc": dict(
+        key_fp8=False, key_quant_bits=3,
+        key_mse_bits=3, value_quant_bits=3,
+        mse_bits=3, n_centroids=8, centroid_bits=3,
+        norm_correction=True,
+        key_packed_size=50, value_packed_size=52,
+        slot_size=102, slot_size_aligned=102,
+    ),
+}
+# fmt: on
+
+
+# ============================================================================
+# Config tests (CPU-only, no dependencies beyond config.py)
+# ============================================================================
+
+
+class TestTurboQuantConfig:
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_preset_parses(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        assert isinstance(cfg, TurboQuantConfig)
+
+    def test_invalid_preset_raises(self):
+        with pytest.raises(ValueError, match="Unknown TurboQuant"):
+            TurboQuantConfig.from_cache_dtype("turboquant_invalid", head_dim=128)
+
+    # ---- Per-preset concrete value checks (table-driven) ----
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_key_mode(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        exp = PRESET_EXPECTED[preset]
+        assert cfg.key_fp8 is exp["key_fp8"]
+        assert cfg.key_quant_bits == exp["key_quant_bits"]
+        assert cfg.key_mse_bits == exp["key_mse_bits"]
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_value_mode(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        exp = PRESET_EXPECTED[preset]
+        assert cfg.value_quant_bits == exp["value_quant_bits"]
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_bits_and_centroids(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        exp = PRESET_EXPECTED[preset]
+        assert cfg.mse_bits == exp["mse_bits"]
+        assert cfg.n_centroids == exp["n_centroids"]
+        assert cfg.centroid_bits == exp["centroid_bits"]
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_norm_correction(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        assert cfg.norm_correction is PRESET_EXPECTED[preset]["norm_correction"]
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_packed_sizes(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        exp = PRESET_EXPECTED[preset]
+        assert cfg.key_packed_size == exp["key_packed_size"]
+        assert cfg.value_packed_size == exp["value_packed_size"]
+        assert cfg.slot_size == exp["slot_size"]
+        assert cfg.slot_size_aligned == exp["slot_size_aligned"]
+
+    # ---- Cross-preset structural invariants ----
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_slot_equals_key_plus_value(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        assert cfg.slot_size == cfg.key_packed_size + cfg.value_packed_size
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_padded_slot_is_even(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        assert cfg.slot_size_aligned >= cfg.slot_size
+        assert cfg.slot_size_aligned % 2 == 0, (
+            f"slot_size_aligned={cfg.slot_size_aligned} is not even"
+        )
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_key_value_packed_sizes_positive(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        assert cfg.key_packed_size > 0
+        assert cfg.value_packed_size > 0
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_n_centroids_is_2_to_mse_bits(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        assert cfg.n_centroids == 2**cfg.mse_bits
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_centroid_bits_always_positive(self, preset):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        assert cfg.centroid_bits > 0
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    def test_mse_key_or_fp8_exclusive(self, preset):
+        """Each preset is either FP8 keys or MSE keys, never both."""
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        if cfg.key_fp8:
+            assert cfg.key_mse_bits == 0
+            assert cfg.key_quant_bits == 8
+        else:
+            assert cfg.key_mse_bits > 0
+            assert cfg.key_quant_bits in (3, 4)
+
+    @pytest.mark.parametrize("preset", ALL_PRESETS)
+    @pytest.mark.parametrize("head_dim", [64, 96, 128, 256])
+    def test_all_presets_all_head_dims(self, preset, head_dim):
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=head_dim)
+        assert cfg.head_dim == head_dim
+        assert cfg.slot_size == cfg.key_packed_size + cfg.value_packed_size
+        assert cfg.slot_size_aligned >= cfg.slot_size
+        assert cfg.slot_size_aligned % 2 == 0
+
+    # ---- Boundary skip layers ----
+
+    @staticmethod
+    def _dense_model_config(num_layers):
+        from types import SimpleNamespace
+
+        return SimpleNamespace(
+            is_hybrid=False,
+            hf_text_config=SimpleNamespace(num_hidden_layers=num_layers),
+        )
+
+    def test_boundary_skip_layers_basic(self):
+        mc = self._dense_model_config(32)
+        layers = TurboQuantConfig.get_boundary_skip_layers(mc)
+        assert layers == ["0", "1", "30", "31"]
+
+    def test_boundary_skip_layers_zero(self):
+        mc = self._dense_model_config(32)
+        assert TurboQuantConfig.get_boundary_skip_layers(mc, 0) == []
+
+    def test_boundary_skip_layers_small_model(self):
+        mc = self._dense_model_config(4)
+        layers = TurboQuantConfig.get_boundary_skip_layers(mc)
+        assert layers == ["0", "1", "2", "3"]
+
+    def test_boundary_skip_layers_cap_at_half(self):
+        mc = self._dense_model_config(8)
+        layers = TurboQuantConfig.get_boundary_skip_layers(mc, 10)
+        assert len(layers) == 8
+
+
+class TestHybridAttentionIndices:
+    """Regression tests for boundary protection on hybrid models.
+
+    Hybrid models (attention + Mamba / linear-attention) identify KV-carrying
+    layers via layer_types / layers_block_type / attn_type_list. The helper
+    must return the *global* layer indices of the full-attention layers so
+    that kv_cache_dtype_skip_layers matches what extract_layer_index(prefix)
+    reports on the Attention layers at runtime.
+    """
+
+    @staticmethod
+    def _fake_model_config(text_cfg=None, hf_cfg=None):
+        from types import SimpleNamespace
+
+        return SimpleNamespace(
+            hf_text_config=text_cfg if text_cfg is not None else SimpleNamespace(),
+            hf_config=hf_cfg if hf_cfg is not None else SimpleNamespace(),
+        )
+
+    def test_layer_types_full_attention(self):
+        from vllm.model_executor.layers.quantization.turboquant.config import (
+            _get_full_attention_layer_indices,
+        )
+
+        cfg = type("C", (), {})()
+        cfg.layer_types = [
+            "linear_attention",
+            "linear_attention",
+            "full_attention",
+            "linear_attention",
+            "full_attention",
+            "full_attention",
+        ]
+        mc = self._fake_model_config(text_cfg=cfg)
+        assert _get_full_attention_layer_indices(mc) == [2, 4, 5]
+
+    def test_layers_block_type_jamba(self):
+        from vllm.model_executor.layers.quantization.turboquant.config import (
+            _get_full_attention_layer_indices,
+        )
+
+        cfg = type("C", (), {})()
+        cfg.layers_block_type = ["mamba", "attention", "mamba", "attention"]
+        mc = self._fake_model_config(text_cfg=cfg)
+        assert _get_full_attention_layer_indices(mc) == [1, 3]
+
+    def test_attn_type_list_minimax(self):
+        from vllm.model_executor.layers.quantization.turboquant.config import (
+            _get_full_attention_layer_indices,
+        )
+
+        hf = type("C", (), {})()
+        hf.attn_type_list = [0, 1, 0, 1, 1]
+        mc = self._fake_model_config(hf_cfg=hf)
+        assert _get_full_attention_layer_indices(mc) == [1, 3, 4]
+
+    def test_no_hybrid_hints_returns_empty(self):
+        from vllm.model_executor.layers.quantization.turboquant.config import (
+            _get_full_attention_layer_indices,
+        )
+
+        mc = self._fake_model_config()
+        assert _get_full_attention_layer_indices(mc) == []
+
+
+class TestTurboQuantWorkspaceReservation:
+    @staticmethod
+    def _fake_vllm_config(
+        *,
+        max_num_seqs: int = 16,
+        max_num_batched_tokens: int = 4096,
+        enable_chunked_prefill: bool = True,
+        max_model_len: int = 8192,
+        dtype: torch.dtype = torch.float16,
+        max_num_kv_splits: int = 4,
+    ):
+        return SimpleNamespace(
+            scheduler_config=SimpleNamespace(
+                max_num_seqs=max_num_seqs,
+                max_num_batched_tokens=max_num_batched_tokens,
+                enable_chunked_prefill=enable_chunked_prefill,
+            ),
+            model_config=SimpleNamespace(
+                max_model_len=max_model_len,
+                dtype=dtype,
+                get_num_attention_heads=lambda parallel_config: 8,
+            ),
+            parallel_config=SimpleNamespace(
+                tensor_parallel_size=2,
+                decode_context_parallel_size=1,
+            ),
+            attention_config=SimpleNamespace(
+                tq_max_kv_splits_for_cuda_graph=max_num_kv_splits
+            ),
+        )
+
+    @staticmethod
+    def _fake_kv_cache_spec():
+        from vllm.v1.kv_cache_interface import TQFullAttentionSpec
+
+        return TQFullAttentionSpec(
+            block_size=32,
+            num_kv_heads=4,
+            head_size=128,
+            head_size_v=128,
+            dtype=torch.uint8,
+            tq_slot_size=102,
+        )
+
+    def test_metadata_builder_reserves_decode_and_continuation_prefill_workspace(
+        self, monkeypatch
+    ):
+        from vllm.v1.attention.backends import turboquant_attn
+
+        calls = []
+
+        class FakeWorkspaceManager:
+            def get_simultaneous(self, *shapes_and_dtypes):
+                calls.append(shapes_and_dtypes)
+
+        monkeypatch.setattr(
+            turboquant_attn,
+            "current_workspace_manager",
+            lambda: FakeWorkspaceManager(),
+        )
+        monkeypatch.setattr(
+            turboquant_attn,
+            "is_workspace_manager_initialized",
+            lambda: True,
+        )
+
+        turboquant_attn.TurboQuantMetadataBuilder(
+            kv_cache_spec=self._fake_kv_cache_spec(),
+            layer_names=["layers.0.self_attn.attn"],
+            vllm_config=self._fake_vllm_config(),
+            device=torch.device("cuda"),
+        )
+
+        assert calls == [
+            (
+                ((16, 8, 4, 129), torch.float32),
+                ((16, 8, 128), torch.float16),
+                ((16, 8), torch.float32),
+            ),
+            (
+                ((1, 4, 8192, 128), torch.float16),
+                ((1, 4, 8192, 128), torch.float16),
+            ),
+        ]
+
+    def test_metadata_builder_skips_continuation_prefill_when_disabled(
+        self, monkeypatch
+    ):
+        from vllm.v1.attention.backends import turboquant_attn
+
+        calls = []
+
+        class FakeWorkspaceManager:
+            def get_simultaneous(self, *shapes_and_dtypes):
+                calls.append(shapes_and_dtypes)
+
+        monkeypatch.setattr(
+            turboquant_attn,
+            "current_workspace_manager",
+            lambda: FakeWorkspaceManager(),
+        )
+        monkeypatch.setattr(
+            turboquant_attn,
+            "is_workspace_manager_initialized",
+            lambda: True,
+        )
+
+        turboquant_attn.TurboQuantMetadataBuilder(
+            kv_cache_spec=self._fake_kv_cache_spec(),
+            layer_names=["layers.0.self_attn.attn"],
+            vllm_config=self._fake_vllm_config(enable_chunked_prefill=False),
+            device=torch.device("cuda"),
+        )
+
+        assert calls == [
+            (
+                ((16, 8, 4, 129), torch.float32),
+                ((16, 8, 128), torch.float16),
+                ((16, 8), torch.float32),
+            )
+        ]
+
+
+# ============================================================================
+# Centroids tests (CPU-only)
+# ============================================================================
+
+
+class TestCentroids:
+    @pytest.mark.parametrize("bits,expected_n", [(2, 4), (3, 8), (4, 16)])
+    def test_centroids_shape(self, bits, expected_n):
+        c = get_centroids(128, bits)
+        assert c.shape == (expected_n,)
+
+    @pytest.mark.parametrize("bits", [2, 3, 4])
+    def test_centroids_sorted(self, bits):
+        _assert_strictly_sorted(get_centroids(128, bits), "centroids")
+
+    def test_centroids_cached(self):
+        c1 = get_centroids(128, 3)
+        c2 = get_centroids(128, 3)
+        assert c1 is c2, "get_centroids should return cached object"
+
+    def test_centroids_different_dims_not_identical(self):
+        c64 = get_centroids(64, 3)
+        c128 = get_centroids(128, 3)
+        assert not torch.equal(c64, c128)
+
+    @pytest.mark.parametrize("bits", [2, 3, 4])
+    def test_centroids_symmetric_around_zero(self, bits):
+        """N(0, 1/d) is symmetric, so centroids should be ~symmetric."""
+        c = get_centroids(128, bits)
+        assert abs(c.mean().item()) < 0.01, "Centroids not centered near 0"
+        assert abs(c[0].item() + c[-1].item()) < 0.01
+
+    @pytest.mark.parametrize("bits", [2, 3, 4])
+    def test_centroids_within_4sigma(self, bits):
+        """All centroids should be within ~4 sigma of N(0, 1/d)."""
+        sigma = math.sqrt(1.0 / 128)
+        c = get_centroids(128, bits)
+        for i, val in enumerate(c):
+            assert abs(val.item()) < 4 * sigma, (
+                f"Centroid {i}={val:.6f} outside 4*sigma={4 * sigma:.6f}"
+            )
+
+
+class TestLloydMax:
+    @pytest.mark.parametrize("bits,expected_n", [(2, 4), (3, 8), (4, 16)])
+    def test_solve_shapes(self, bits, expected_n):
+        centroids, boundaries = solve_lloyd_max(128, bits)
+        assert centroids.shape == (expected_n,)
+        assert boundaries.shape == (expected_n - 1,)
+
+    @pytest.mark.parametrize("bits", [2, 3, 4])
+    def test_centroids_sorted(self, bits):
+        centroids, _ = solve_lloyd_max(128, bits)
+        _assert_strictly_sorted(centroids, "centroids")
+
+    @pytest.mark.parametrize("bits", [2, 3, 4])
+    def test_boundaries_sorted(self, bits):
+        _, boundaries = solve_lloyd_max(128, bits)
+        _assert_strictly_sorted(boundaries, "boundaries")
+
+    @pytest.mark.parametrize("bits", [2, 3, 4])
+    def test_boundaries_between_centroids(self, bits):
+        """Each boundary must lie between its adjacent centroids."""
+        centroids, boundaries = solve_lloyd_max(128, bits)
+        for i in range(len(boundaries)):
+            assert centroids[i] < boundaries[i] < centroids[i + 1], (
+                f"Boundary {i}={boundaries[i]:.6f} not between "
+                f"c[{i}]={centroids[i]:.6f} and c[{i + 1}]={centroids[i + 1]:.6f}"
+            )
+
+    @pytest.mark.parametrize("bits", [2, 3, 4])
+    def test_boundaries_are_midpoints(self, bits):
+        """Lloyd-Max boundaries are midpoints of adjacent centroids."""
+        centroids, boundaries = solve_lloyd_max(128, bits)
+        for i in range(len(boundaries)):
+            expected = (centroids[i] + centroids[i + 1]) / 2.0
+            assert abs(boundaries[i].item() - expected.item()) < 1e-6
+
+    def test_solve_deterministic(self):
+        c1, b1 = solve_lloyd_max(128, 3)
+        c2, b2 = solve_lloyd_max(128, 3)
+        assert torch.equal(c1, c2)
+        assert torch.equal(b1, b2)
+
+    def test_solve_dtype_float32(self):
+        centroids, boundaries = solve_lloyd_max(128, 3)
+        assert centroids.dtype == torch.float32
+        assert boundaries.dtype == torch.float32
+
+    @pytest.mark.parametrize("bits", [3, 4])
+    def test_centroids_match_scipy_reference(self, bits):
+        """Verify _trapz(n=200) centroids match scipy.integrate.quad reference.
+
+        This ensures our scipy-free trapezoid integration doesn't silently
+        drift from the published Lloyd-Max quality.
+        """
+        pytest.importorskip("scipy")
+        from scipy.integrate import quad
+
+        d = 128
+        sigma2 = 1.0 / d
+        sigma = math.sqrt(sigma2)
+
+        def pdf(x):
+            return (1.0 / math.sqrt(2 * math.pi * sigma2)) * math.exp(
+                -x * x / (2 * sigma2)
+            )
+
+        n_levels = 2**bits
+        lo, hi = -3.5 * sigma, 3.5 * sigma
+        ref_centroids = [lo + (hi - lo) * (i + 0.5) / n_levels for i in range(n_levels)]
+        for _ in range(200):
+            boundaries = [
+                (ref_centroids[i] + ref_centroids[i + 1]) / 2.0
+                for i in range(n_levels - 1)
+            ]
+            edges = [lo * 3] + boundaries + [hi * 3]
+            new_centroids = []
+            for i in range(n_levels):
+                a, b = edges[i], edges[i + 1]
+                num, _ = quad(lambda x: x * pdf(x), a, b)
+                den, _ = quad(pdf, a, b)
+                new_centroids.append(num / den if den > 1e-15 else ref_centroids[i])
+            if (
+                max(abs(new_centroids[i] - ref_centroids[i]) for i in range(n_levels))
+                < 1e-10
+            ):
+                break
+            ref_centroids = new_centroids
+
+        # Compare our _trapz centroids against scipy reference
+        our_centroids, _ = solve_lloyd_max(d, bits)
+        ref_t = torch.tensor(ref_centroids, dtype=torch.float32)
+        max_err = (our_centroids - ref_t).abs().max().item()
+        # _trapz(n=200) has ~O(h^2) error vs adaptive quad; 1e-3 is tight
+        # enough to catch regression while allowing trapezoid approximation.
+        assert max_err < 1e-3, (
+            f"d={d}, bits={bits}: max centroid error vs scipy = {max_err:.2e}"
+        )
+
+
+# ============================================================================
+# Rotation matrix tests (GPU required)
+# ============================================================================
+
+GPGPU_AVAILABLE = torch.cuda.is_available() or torch.xpu.is_available()
+DEVICE_TYPE = current_platform.device_type
+
+
+def generate_rotation_matrix(d: int, seed: int, device: str = "cpu") -> torch.Tensor:
+    """Haar-distributed random orthogonal matrix via QR (test/benchmark only)."""
+    gen = torch.Generator(device="cpu")
+    gen.manual_seed(seed)
+    G = torch.randn(d, d, generator=gen, device="cpu", dtype=torch.float32)
+    # torch.linalg.qr on CPU requires LAPACK, which some torch wheels
+    # (ROCm) ship without. Run QR on accelerator instead
+    qr_device = "cuda" if torch.cuda.is_available() else "cpu"
+    Q, R = torch.linalg.qr(G.to(qr_device))
+    diag_sign = torch.sign(torch.diag(R))
+    diag_sign[diag_sign == 0] = 1.0
+    Q = Q * diag_sign.unsqueeze(0)
+    return Q.to(device)
+
+
+@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
+class TestRotationMatrix:
+    """Tests for the QR-based rotation (standalone benchmarks only)."""
+
+    @pytest.mark.parametrize("dim", [64, 96, 128, 256])
+    def test_rotation_matrix_shape_and_orthogonal(self, dim):
+        Pi = generate_rotation_matrix(dim, seed=42, device=DEVICE_TYPE)
+        assert Pi.shape == (dim, dim)
+        eye = Pi @ Pi.T
+        assert torch.allclose(eye, torch.eye(dim, device=DEVICE_TYPE), atol=1e-5), (
+            f"Pi not orthogonal for dim={dim}"
+        )
+
+    def test_rotation_matrix_deterministic(self):
+        Pi1 = generate_rotation_matrix(128, seed=42)
+        Pi2 = generate_rotation_matrix(128, seed=42)
+        assert torch.equal(Pi1, Pi2)
+
+    def test_rotation_matrix_different_seeds(self):
+        Pi1 = generate_rotation_matrix(128, seed=42)
+        Pi2 = generate_rotation_matrix(128, seed=99)
+        assert not torch.equal(Pi1, Pi2)
+
+    def test_rotation_matrix_det_is_pm1(self):
+        """Orthogonal matrix determinant must be +1 or -1."""
+        Pi = generate_rotation_matrix(128, seed=42, device=DEVICE_TYPE)
+        det = torch.linalg.det(Pi)
+        assert abs(abs(det.item()) - 1.0) < 1e-4
+
+
+# ============================================================================
+# Hadamard rotation tests (serving path: _build_hadamard)
+# ============================================================================
+
+
+def _build_hadamard(d: int, device: str = "cpu") -> torch.Tensor:
+    """Reproduce the serving-path Hadamard construction."""
+    H = torch.tensor([[1.0]])
+    while H.shape[0] < d:
+        H = torch.cat([torch.cat([H, H], 1), torch.cat([H, -H], 1)], 0)
+    return (H / math.sqrt(d)).to(torch.device(device))
+
+
+@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
+class TestHadamardRotation:
+    """Tests for the Hadamard rotation used in serving."""
+
+    @pytest.mark.parametrize("dim", [64, 128, 256])
+    def test_hadamard_orthonormal(self, dim):
+        """H must be orthonormal: H @ H^T = I."""
+        H = _build_hadamard(dim, DEVICE_TYPE)
+        eye = H @ H.T
+        assert torch.allclose(eye, torch.eye(dim, device=DEVICE_TYPE), atol=1e-5), (
+            f"Hadamard not orthonormal for dim={dim}"
+        )
+
+    @pytest.mark.parametrize("dim", [64, 128, 256])
+    def test_hadamard_symmetric(self, dim):
+        """Sylvester Hadamard must be symmetric: H = H^T."""
+        H = _build_hadamard(dim, DEVICE_TYPE)
+        assert torch.allclose(H, H.T, atol=1e-6), (
+            f"Hadamard not symmetric for dim={dim}"
+        )
+
+
+# ============================================================================
+# Store → Decode round-trip test (GPU + Triton required)
+# ============================================================================
+
+
+@pytest.mark.skipif(not GPGPU_AVAILABLE, reason="GPGPU not available")
+class TestStoreDecodeRoundTrip:
+    """End-to-end: store KV into TQ cache, decode, compare vs fp16 ref."""
+
+    @pytest.mark.parametrize(
+        "preset",
+        ["turboquant_k8v4", "turboquant_4bit_nc"],
+    )
+    def test_single_token_roundtrip(self, preset):
+        """Store 1 token, decode with query=key, check attention output.
+
+        For a single token with query=key, attention output should equal
+        the value (softmax over single key = 1.0). Quantization error
+        means we check cosine similarity rather than exact equality.
+        """
+        from vllm.model_executor.layers.quantization.turboquant.centroids import (
+            solve_lloyd_max,
+        )
+        from vllm.v1.attention.ops.triton_turboquant_decode import (
+            triton_turboquant_decode_attention,
+        )
+        from vllm.v1.attention.ops.triton_turboquant_store import (
+            triton_turboquant_store,
+        )
+
+        cfg = TurboQuantConfig.from_cache_dtype(preset, head_dim=128)
+        D = 128
+        Hk = 4  # num_kv_heads
+        Hq = 4  # num_q_heads (no GQA for simplicity)
+        B = 1  # single token
+        block_size = 16
+        num_blocks = 1
+
+        device = torch.device(DEVICE_TYPE)
+
+        # Pure Hadamard rotation (symmetric: H = H^T, so Pi = PiT = H)
+        H = _build_hadamard(D, DEVICE_TYPE)
+        PiT = H
+        Pi = H
+
+        # Generate centroids
+        centroids, _ = solve_lloyd_max(D, cfg.centroid_bits)
+        centroids = centroids.float().to(device)
+        c_sorted, _ = centroids.sort()
+        midpoints = ((c_sorted[:-1] + c_sorted[1:]) / 2).to(device)
+
+        # Random K, V
+        torch.manual_seed(123)
+        key = torch.randn(B, Hk, D, device=device, dtype=torch.float16)
+        value = torch.randn(B, Hk, D, device=device, dtype=torch.float16)
+
+        # Allocate KV cache
+        padded_slot = cfg.slot_size_aligned
+        kv_cache = torch.zeros(
+            num_blocks,
+            block_size,
+            Hk,
+            padded_slot,
+            device=device,
+            dtype=torch.uint8,
+        )
+        slot_mapping = torch.tensor([0], device=device, dtype=torch.int32)
+
+        # Store
+        triton_turboquant_store(
+            key,
+            value,
+            kv_cache,
+            slot_mapping,
+            PiT,
+            midpoints,
+            mse_bits=cfg.key_mse_bits,
+            key_packed_size=cfg.key_packed_size,
+            value_quant_bits=cfg.effective_value_quant_bits,
+            key_fp8=cfg.key_fp8,
+        )
+
+        # Decode: use key as query so attention = softmax([1]) * V = V
+        query = key.expand(B, Hq, D).contiguous().to(torch.float16)
+        block_table = torch.tensor([[0]], device=device, dtype=torch.int32)
+        seq_lens = torch.tensor([1], device=device, dtype=torch.int32)
+
+        output = triton_turboquant_decode_attention(
+            query=query,
+            kv_cache=kv_cache,
+            block_table=block_table,
+            seq_lens=seq_lens,
+            Pi=Pi,
+            centroids=centroids,
+            scale=1.0 / math.sqrt(D),
+            mse_bits=cfg.key_mse_bits,
+            key_packed_size=cfg.key_packed_size,
+            value_quant_bits=cfg.effective_value_quant_bits,
+            key_fp8=cfg.key_fp8,
+            norm_correction=cfg.norm_correction,
+            PiT=PiT,
+            max_num_kv_splits=4,
+        )
+
+        # With single KV, output should approximate the stored value.
+        # Check per-head cosine similarity > threshold.
+        out_fp32 = output.float()
+        val_fp32 = value.expand(B, Hq, D).float()
+        for h in range(Hq):
+            cos_sim = torch.nn.functional.cosine_similarity(
+                out_fp32[0, h].unsqueeze(0),
+                val_fp32[0, h].unsqueeze(0),
+            ).item()
+            # FP8 keys should be very accurate; MSE keys have more error
+            threshold = 0.95 if cfg.key_fp8 else 0.85
+            assert cos_sim > threshold, (
+                f"Preset {preset} head {h}: cosine_sim={cos_sim:.4f} < {threshold}"
+            )

tests/quantization/utils.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import logging
+
+import regex as re
+
+from vllm.model_executor.layers.quantization import get_quantization_config
+from vllm.platforms import current_platform
+
+
+def is_quant_method_supported(quant_method: str) -> bool:
+    # Currently, all quantization methods require Nvidia or AMD GPUs
+    if not (current_platform.is_cuda() or current_platform.is_rocm()):
+        return False
+
+    try:
+        current_platform.verify_quantization(quant_method)
+    except ValueError:
+        return False
+
+    capability = current_platform.get_device_capability()
+    assert capability is not None
+
+    min_capability = get_quantization_config(quant_method).get_min_capability()
+
+    return capability.to_int() >= min_capability
+
+
+def _test_online_quant_peak_mem_impl(
+    quantization_arg_value,
+    vllm_runner,
+    caplog_mp_spawn,
+    monkeypatch,
+) -> None:
+    # Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:
+    # 1. it covers both Linear and MoE paths
+    # 2. it is already used by other tests in CI, so adding it here
+    #    does not increase disk space for CI runners
+    # I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`
+    # which I think is the smallest MoE model in vLLM (2.5 GiB bf16,
+    # 1.3 GiB fp8), but could not as adding one more model makes CI
+    # run out of disk space.
+    model_name = "allenai/OLMoE-1B-7B-0125-Instruct"
+
+    # Force spawn to ensure caplog_mp_spawn works consistently
+    # (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)
+    monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+    with (
+        caplog_mp_spawn(logging.DEBUG) as log_holder,
+        vllm_runner(
+            model_name,
+            quantization=quantization_arg_value,
+            enforce_eager=True,
+        ) as llm,
+    ):
+        outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)
+        print(outputs[0][1])
+
+    log_text = log_holder.text
+
+    # Parse memory usage from captured logs
+    model_memory_gib = None
+    peak_memory_gib = None
+    for line in log_text.splitlines():
+        if model_memory_gib is None:
+            match = re.search(r"Model loading took ([\d.]+) GiB memory", line)
+            if match:
+                model_memory_gib = float(match.group(1))
+        if peak_memory_gib is None:
+            match = re.search(
+                r"Peak GPU memory after loading weights: ([\d.]+) GiB", line
+            )
+            if match:
+                peak_memory_gib = float(match.group(1))
+
+    assert model_memory_gib is not None, "Could not find model loading memory log"
+    assert peak_memory_gib is not None, "Could not find peak memory log"
+    print(f"GPU memory used after loading weights: {model_memory_gib} GiB")
+    print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")
+
+    expected_model_memory_gib = 6.7
+
+    # for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06
+    # GiB on CUDA, which is 1.36x above model_memory_gib. A slightly higher
+    # number is expected as when we load and quantize weights in a streaming
+    # fashion we need to have individual weights in bf16 + fp8 alive at the
+    # same time.
+    expected_peak_memory_gib = expected_model_memory_gib * 1.4
+
+    assert model_memory_gib < expected_model_memory_gib, (
+        f"{model_memory_gib=} higher than {expected_model_memory_gib}"
+    )
+    assert peak_memory_gib < expected_peak_memory_gib, (
+        f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"
+    )