sgl-project--sglang/python/sglang/srt/utils/weight_checker.py

import hashlib
import logging
import time
from typing import Dict, Iterable, NamedTuple, Optional, Set

import torch
import torch.distributed as dist
from pydantic import BaseModel, ConfigDict

from sglang.srt.managers.mm_utils import tensor_hash
from sglang.srt.utils.weight_checker_comparator import (
    CHUNK_NUMEL,
    ComparableWeight,
    RawComparable,
    compare_weights,
    select_comparable_weight,
)

logger = logging.getLogger(__name__)


class _StrictBaseModel(BaseModel):
    model_config = ConfigDict(extra="forbid")


class ParallelismInfo(_StrictBaseModel):
    tp_rank: int
    tp_size: int
    dp_rank: int
    dp_size: int
    pp_rank: int
    pp_size: int
    rank: int
    size: int


class ChecksumInfo(_StrictBaseModel):
    checksums: Dict[str, str]
    per_gpu_checksum: str
    parallelism_info: ParallelismInfo


class CheckEntry(NamedTuple):
    name: str
    should_compare: bool
    comparable: ComparableWeight


class QuantizedWeight(NamedTuple):
    comparable_cls: type[ComparableWeight]
    scale_name: str


_NON_PERSISTENT_BUFFER_PATTERNS = (
    "cos_sin_cache",
    "inv_freq",
    "freqs_cis",
    "_weight_fp32",
)


def _is_non_persistent_buffer_name(name: str) -> bool:
    return any(pat in name for pat in _NON_PERSISTENT_BUFFER_PATTERNS)


class WeightChecker:
    def __init__(self, model_runner):
        self._model_runner = model_runner
        self._snapshot_tensors = None

    def handle(self, action: str, allow_quant_error: bool = False) -> Optional[Dict]:
        logger.info(
            f"[WeightChecker] handle action={action} allow_quant_error={allow_quant_error}"
        )
        if action == "snapshot":
            return self._snapshot()
        elif action == "reset_tensors":
            return self._reset_tensors()
        elif action == "compare":
            return self._compare(allow_quant_error=allow_quant_error)
        elif action == "checksum":
            return self._compute_checksum()
        else:
            raise Exception(f"Unsupported {action=}")

    def _snapshot(self):
        named_tensors = [
            (name, param.data.detach().cpu()) for name, param in self._model_state()
        ]
        self._snapshot_tensors = dict(named_tensors)
        assert len(self._snapshot_tensors) == len(
            named_tensors
        ), f"should not have duplicated tensor name"

    def _reset_tensors(self):
        for name, param in self._model_state():
            if _is_non_persistent_buffer_name(name):
                continue
            param.copy_(_random_like(param))

    def _compare(self, allow_quant_error: bool = False):
        assert self._snapshot_tensors is not None

        quantized_set = _build_quantized_set(self._model_runner.model)
        skip_compare_names = {
            name
            for name, param in self._model_state()
            if getattr(param, "_skip_weight_check", False)
        }
        _check_tensors(
            expect_tensors=_build_check_entries(
                self._snapshot_tensors, skip_compare_names, quantized_set
            ),
            actual_tensors=_build_check_entries(
                dict(self._model_state()), skip_compare_names, quantized_set
            ),
            allow_quant_error=allow_quant_error,
        )

    def _compute_checksum(self) -> Dict:
        torch.cuda.synchronize()
        start = time.perf_counter()

        quantized_set = _build_quantized_set(self._model_runner.model)
        skip_compare_names = {
            name
            for name, param in self._model_state()
            if getattr(param, "_skip_weight_check", False)
        }

        # Hash the dequantized weight so two (qweight, scale) pairs with the same
        # bf16 hash equal.
        checksums = {}
        for name, should_compare, comparable in _build_check_entries(
            dict(self._model_state()), skip_compare_names, quantized_set
        ):
            if should_compare:
                checksums[name] = _hash_tensor(comparable.dequantize().data)

        h = hashlib.sha256()
        for name in sorted(checksums):
            h.update(name.encode())
            h.update(checksums[name].encode())
        overall = h.hexdigest()

        torch.cuda.synchronize()
        elapsed = time.perf_counter() - start
        logger.info(
            f"[WeightChecker] checksum computed for {len(checksums)} tensors in {elapsed:.3f}s"
        )

        info = ChecksumInfo(
            checksums=checksums,
            per_gpu_checksum=overall,
            parallelism_info=self._parallelism_info(),
        )
        return info.model_dump()

    def _parallelism_info(self) -> ParallelismInfo:
        mr = self._model_runner
        return ParallelismInfo(
            tp_rank=mr.tp_rank,
            tp_size=mr.tp_size,
            dp_rank=mr.dp_rank if mr.dp_rank is not None else 0,
            dp_size=mr.dp_size,
            pp_rank=mr.pp_rank,
            pp_size=mr.pp_size,
            rank=dist.get_rank() if dist.is_initialized() else 0,
            size=dist.get_world_size() if dist.is_initialized() else 1,
        )

    def _model_state(self):
        yield from self._model_runner.model.named_parameters()
        yield from self._model_runner.model.named_buffers()


def _hash_tensor(t: torch.Tensor) -> str:
    return f"{tensor_hash(t):016x}"


def _check_tensors(
    expect_tensors: Iterable[CheckEntry],
    actual_tensors: Iterable[CheckEntry],
    allow_quant_error: bool = False,
):
    good_names = []
    error_messages = []
    info_messages = []

    for (expect_name, should_compare, expect_comparable), (
        actual_name,
        actual_should_compare,
        actual_comparable,
    ) in zip(expect_tensors, actual_tensors, strict=True):
        assert expect_name == actual_name, f"{expect_name=} {actual_name=}"
        assert (
            should_compare == actual_should_compare
        ), f"{should_compare=} {actual_should_compare=}"
        name = expect_name

        try:
            equal, max_abs_err, mean_abs_err, num_exceed = compare_weights(
                expect_comparable, actual_comparable
            )
        except Exception as e:
            e.add_note(
                f"when handling {name=} expect={expect_comparable!r} actual={actual_comparable!r}"
            )
            raise
        if equal:
            good_names.append(name)
            continue
        msg = (
            f"name={name} "
            f"max_abs_err={max_abs_err} "
            f"mean_abs_err={mean_abs_err} "
            f"num_exceed={num_exceed} "
            f"expect={expect_comparable!r} actual={actual_comparable!r} "
        )
        if not should_compare:
            info_messages.append(msg)
        elif allow_quant_error and num_exceed == 0:
            info_messages.append(msg + "(within quantization ULP tolerance)")
        else:
            error_messages.append(msg)

    logger.info(f"[check_tensors] equal tensors: {good_names}")
    if len(info_messages) > 0:
        logger.info(f"[check_tensors] info: {info_messages}")
    if len(error_messages) > 0:
        raise Exception(f"check tensor equality failed:\n" + "\n".join(error_messages))


def _random_like(t: torch.Tensor):
    device = t.device
    shape = t.shape
    dtype = t.dtype

    if dtype.is_floating_point:
        out = torch.empty(shape, device=device, dtype=dtype)
        for chunk in out.view(-1).split(CHUNK_NUMEL):
            chunk.copy_(
                torch.rand(chunk.shape, device=device, dtype=torch.float32).to(dtype)
            )
        return out

    if dtype == torch.bool:
        return torch.rand(shape, device=device) > 0.5

    info = torch.iinfo(dtype)
    return torch.randint(
        low=int(info.min), high=int(info.max), size=shape, device=device, dtype=dtype
    )


def _build_quantized_set(model) -> Dict[str, QuantizedWeight]:
    """Run the router over the model: {weight_name: QuantizedWeight} for each
    quantized weight; weights absent from the set compare raw."""
    quantized_set = {}
    for module_name, module in model.named_modules():
        comparable_cls = select_comparable_weight(getattr(module, "quant_method", None))
        if comparable_cls is None:
            continue
        prefix = f"{module_name}." if module_name else ""
        own = {name for name, _ in module.named_parameters(recurse=False)}
        for name in own:
            scale = name.replace("weight", "weight_scale_inv")
            if name.endswith("weight") and scale in own:
                quantized_set[prefix + name] = QuantizedWeight(
                    comparable_cls, prefix + scale
                )
    return quantized_set


def _build_check_entries(
    raw: Dict[str, torch.Tensor],
    skip_compare_names: Set[str],
    quantized_set: Optional[Dict[str, QuantizedWeight]] = None,
) -> Iterable[CheckEntry]:
    """Yields a CheckEntry per weight; quantized weights consume their scale, everything
    else is raw."""
    skip_compare_names = set(skip_compare_names)
    quantized_set = quantized_set or {}
    scale_names = {qw.scale_name for qw in quantized_set.values()}

    for name, tensor in raw.items():
        if name in scale_names:
            continue  # compared via its weight's comparable
        if name in quantized_set:
            qw = quantized_set[name]
            yield CheckEntry(name, True, qw.comparable_cls(tensor, raw[qw.scale_name]))
        else:
            should_compare = name not in skip_compare_names and (
                not _is_non_persistent_buffer_name(name)
            )
            yield CheckEntry(name, should_compare, RawComparable(tensor))