wehub-resource-sync
142 lines
5.2 KiB
Python
142 lines
5.2 KiB
Python
"""Tests for non-GGUF quantization inference."""
|
|
|
|
from whichllm.engine.quantization import (
|
|
effective_quant_type,
|
|
estimate_weight_bytes,
|
|
infer_non_gguf_quant_type,
|
|
)
|
|
from whichllm.engine.vram import estimate_vram
|
|
from whichllm.models.types import ModelInfo
|
|
|
|
|
|
def _make_model(model_id: str, params: int = 14_000_000_000) -> ModelInfo:
|
|
return ModelInfo(
|
|
id=model_id,
|
|
family_id=model_id,
|
|
name=model_id.split("/")[-1],
|
|
parameter_count=params,
|
|
)
|
|
|
|
|
|
def test_infer_non_gguf_awq():
|
|
model = _make_model("Qwen/Qwen2.5-14B-Instruct-AWQ")
|
|
assert infer_non_gguf_quant_type(model.id) == "AWQ"
|
|
assert effective_quant_type(model, None) == "AWQ"
|
|
|
|
|
|
def test_estimate_weight_bytes_for_awq():
|
|
model = _make_model("Qwen/Qwen2.5-14B-Instruct-AWQ", params=10_000_000_000)
|
|
assert estimate_weight_bytes(model, None) == 5_000_000_000
|
|
|
|
|
|
def test_awq_vram_is_lower_than_fp16_fallback():
|
|
awq = _make_model("Qwen/Qwen2.5-14B-Instruct-AWQ")
|
|
fp16 = _make_model("Qwen/Qwen2.5-14B-Instruct")
|
|
assert estimate_vram(awq, None, context_length=4096) < estimate_vram(
|
|
fp16, None, context_length=4096
|
|
)
|
|
|
|
|
|
def test_infer_mxfp4():
|
|
model = _make_model("openai/gpt-oss-20b-MXFP4")
|
|
assert infer_non_gguf_quant_type(model.id) == "MXFP4"
|
|
assert effective_quant_type(model, None) == "MXFP4"
|
|
|
|
|
|
def test_infer_nvfp4():
|
|
model = _make_model("nvidia/Llama-3.1-8B-Instruct-NVFP4")
|
|
assert infer_non_gguf_quant_type(model.id) == "NVFP4"
|
|
assert effective_quant_type(model, None) == "NVFP4"
|
|
|
|
|
|
def test_fp4_patterns_do_not_false_match_plain_ids():
|
|
# A bare id with no fp4 token must not be mislabeled as a microscaling float.
|
|
plain = _make_model("meta-llama/Llama-3.1-8B-Instruct")
|
|
assert infer_non_gguf_quant_type(plain.id) == "FP16"
|
|
|
|
|
|
def test_estimate_weight_bytes_for_fp4_formats():
|
|
mxfp4 = _make_model("openai/gpt-oss-20b-MXFP4", params=20_000_000_000)
|
|
nvfp4 = _make_model("nvidia/model-NVFP4", params=20_000_000_000)
|
|
assert estimate_weight_bytes(mxfp4, None) == int(20_000_000_000 * 0.53125)
|
|
assert estimate_weight_bytes(nvfp4, None) == int(20_000_000_000 * 0.5625)
|
|
|
|
|
|
def test_fp4_vram_is_lower_than_fp16_fallback():
|
|
mxfp4 = _make_model("openai/gpt-oss-20b-MXFP4")
|
|
fp16 = _make_model("openai/gpt-oss-20b")
|
|
assert estimate_vram(mxfp4, None, context_length=4096) < estimate_vram(
|
|
fp16, None, context_length=4096
|
|
)
|
|
|
|
|
|
def test_extract_quant_type_parses_fp4_gguf_filenames():
|
|
from whichllm.models.fetcher import _extract_quant_type
|
|
|
|
assert _extract_quant_type("gpt-oss-20b-MXFP4.gguf") == "MXFP4"
|
|
assert _extract_quant_type("model.NVFP4.gguf") == "NVFP4"
|
|
|
|
|
|
def test_extract_quant_type_canonicalizes_full_precision_aliases():
|
|
# llama.cpp publishes full-precision GGUFs as *-FP16/*-FP32; the byte and
|
|
# penalty tables key these as F16/F32, so the extractor must canonicalize.
|
|
from whichllm.models.fetcher import _extract_quant_type
|
|
|
|
assert _extract_quant_type("Meta-Llama-3-8B-FP16.gguf") == "F16"
|
|
assert _extract_quant_type("model.FP32.gguf") == "F32"
|
|
# Canonical spellings still pass through unchanged.
|
|
assert _extract_quant_type("model-F16.gguf") == "F16"
|
|
assert _extract_quant_type("model.BF16.gguf") == "BF16"
|
|
|
|
|
|
def test_extract_quant_type_recognizes_ternary_gguf():
|
|
# BitNet-class ternary GGUFs (TQ1_0/TQ2_0) are fully priced in the tables
|
|
# but were previously extracted as "unknown" and dropped at fetch.
|
|
from whichllm.models.fetcher import _extract_quant_type
|
|
|
|
assert _extract_quant_type("BitNet-b1.58-2B-4T-TQ1_0.gguf") == "TQ1_0"
|
|
assert _extract_quant_type("model.TQ2_0.gguf") == "TQ2_0"
|
|
|
|
|
|
def test_estimate_gguf_size_does_not_undersize_fp16():
|
|
# An FP16 GGUF must size at full precision (2.0 bytes/weight), not collapse
|
|
# to the Q4_K_M 0.5625 default that an unrecognized token falls back to.
|
|
from whichllm.models.fetcher import _estimate_gguf_size, _extract_quant_type
|
|
|
|
params = 7_000_000_000
|
|
quant = _extract_quant_type("model-FP16.gguf")
|
|
size = _estimate_gguf_size(params, quant)
|
|
assert size == params * 2 # 14 GB, not the ~3.94 GB default
|
|
assert size == _estimate_gguf_size(params, "F16")
|
|
|
|
|
|
def test_extract_quant_type_keys_resolve_in_byte_table():
|
|
# Drift guard: every quant the extractor surfaces from a real GGUF filename
|
|
# must resolve in QUANT_BYTES_PER_WEIGHT, otherwise it is silently mis-sized
|
|
# by the default or dropped at fetch. Keeps the extractor and tables aligned.
|
|
from whichllm.data.quantization import QUANT_BYTES_PER_WEIGHT
|
|
from whichllm.models.fetcher import _extract_quant_type
|
|
|
|
filenames = [
|
|
"model-Q4_K_M.gguf",
|
|
"model-Q8_0.gguf",
|
|
"model-Q6_K.gguf",
|
|
"model-IQ4_NL.gguf",
|
|
"model-IQ3_XXS.gguf",
|
|
"model-TQ1_0.gguf",
|
|
"model-TQ2_0.gguf",
|
|
"model-F16.gguf",
|
|
"model-FP16.gguf",
|
|
"model-BF16.gguf",
|
|
"model-F32.gguf",
|
|
"model-FP32.gguf",
|
|
"model-MXFP4.gguf",
|
|
"model-NVFP4.gguf",
|
|
]
|
|
for fname in filenames:
|
|
quant = _extract_quant_type(fname)
|
|
assert quant != "unknown", f"{fname} not recognized by extractor"
|
|
assert quant in QUANT_BYTES_PER_WEIGHT, (
|
|
f"{fname} -> {quant!r} missing from QUANT_BYTES_PER_WEIGHT"
|
|
)
|