sgl-project--sglang/python/sglang/multimodal_gen/runtime/disaggregation/transport/codec.py

# SPDX-License-Identifier: Apache-2.0
"""Zero-copy tensor codec for ZMQ multipart messages.

Frame 0: JSON metadata (tensor descriptors + scalar fields)
Frame 1-N: Raw tensor data buffers (one per tensor)
"""

import ctypes
import json
import logging
from dataclasses import dataclass

import torch
import zmq

logger = logging.getLogger(__name__)

_DTYPE_TO_STR = {
    torch.float16: "float16",
    torch.float32: "float32",
    torch.float64: "float64",
    torch.bfloat16: "bfloat16",
    torch.int8: "int8",
    torch.int16: "int16",
    torch.int32: "int32",
    torch.int64: "int64",
    torch.uint8: "uint8",
    torch.bool: "bool",
}
_STR_TO_DTYPE = {v: k for k, v in _DTYPE_TO_STR.items()}


def dtype_to_str(dtype: torch.dtype) -> str:
    s = _DTYPE_TO_STR.get(dtype)
    if s is None:
        raise ValueError(f"Unsupported dtype: {dtype}")
    return s


def str_to_dtype(s: str) -> torch.dtype:
    d = _STR_TO_DTYPE.get(s)
    if d is None:
        raise ValueError(f"Unknown dtype string: {s}")
    return d


class TensorWrapper:
    """Expose a CPU-contiguous tensor's data buffer for zero-copy ZMQ send."""

    def __init__(self, tensor: torch.Tensor):
        if tensor.is_cuda or tensor.is_npu:
            tensor = tensor.cpu()
        if not tensor.is_contiguous():
            tensor = tensor.contiguous()
        self.tensor = tensor
        data_ptr = tensor.data_ptr()
        total_bytes = tensor.numel() * tensor.element_size()
        self._c_buf = (ctypes.c_char * total_bytes).from_address(data_ptr)
        self._view = memoryview(self._c_buf)


@dataclass
class TensorDescriptor:
    field_name: str
    shape: list[int]
    dtype: str
    list_index: int = -1  # -1 means not part of a list

    def to_dict(self) -> dict:
        return {
            "field_name": self.field_name,
            "shape": self.shape,
            "dtype": self.dtype,
            "list_index": self.list_index,
        }

    @classmethod
    def from_dict(cls, d: dict) -> "TensorDescriptor":
        return cls(
            field_name=d["field_name"],
            shape=d["shape"],
            dtype=d["dtype"],
            list_index=d.get("list_index", -1),
        )


def pack_tensors(
    tensor_fields: dict[str, torch.Tensor | list[torch.Tensor] | None],
    scalar_fields: dict | None = None,
) -> tuple[bytes, list[TensorWrapper]]:
    """Pack tensor fields into metadata + buffer list for send_multipart."""
    descriptors = []
    buffers = []

    for field_name, value in tensor_fields.items():
        if value is None:
            continue

        if isinstance(value, torch.Tensor):
            wrapper = TensorWrapper(value)
            descriptors.append(
                TensorDescriptor(
                    field_name=field_name,
                    shape=list(value.shape),
                    dtype=dtype_to_str(value.dtype),
                )
            )
            buffers.append(wrapper)

        elif isinstance(value, list):
            for i, t in enumerate(value):
                if t is None:
                    continue
                if not isinstance(t, torch.Tensor):
                    raise TypeError(
                        f"Expected Tensor in list for field '{field_name}', "
                        f"got {type(t)}"
                    )
                wrapper = TensorWrapper(t)
                descriptors.append(
                    TensorDescriptor(
                        field_name=field_name,
                        shape=list(t.shape),
                        dtype=dtype_to_str(t.dtype),
                        list_index=i,
                    )
                )
                buffers.append(wrapper)

    metadata = {
        "tensor_descriptors": [d.to_dict() for d in descriptors],
        "scalar_fields": scalar_fields or {},
    }
    metadata_bytes = json.dumps(metadata, separators=(",", ":")).encode("utf-8")
    return metadata_bytes, buffers


def send_tensors(
    socket: zmq.Socket,
    tensor_fields: dict[str, torch.Tensor | list[torch.Tensor] | None],
    scalar_fields: dict | None = None,
    flags: int = 0,
) -> None:
    """Send tensors over ZMQ using multipart with zero-copy."""
    metadata_bytes, buffers = pack_tensors(tensor_fields, scalar_fields)
    parts: list = [metadata_bytes]
    parts.extend(w._view if isinstance(w, TensorWrapper) else w for w in buffers)
    socket.send_multipart(parts, flags=flags, copy=True)


def unpack_tensors(
    parts: list,
    device: str | torch.device = "cpu",
) -> tuple[dict[str, torch.Tensor | list[torch.Tensor]], dict]:
    """Unpack multipart message frames into tensor fields and scalar fields."""
    metadata_frame = parts[0]
    metadata_bytes = (
        bytes(metadata_frame.buffer)
        if hasattr(metadata_frame, "buffer")
        else bytes(metadata_frame)
    )
    metadata = json.loads(metadata_bytes)

    descriptors = [
        TensorDescriptor.from_dict(d) for d in metadata["tensor_descriptors"]
    ]
    scalar_fields = metadata.get("scalar_fields", {})

    if len(parts) - 1 != len(descriptors):
        raise ValueError(
            f"Expected {len(descriptors)} tensor frames, got {len(parts) - 1}"
        )

    tensor_fields: dict[str, torch.Tensor | list[torch.Tensor]] = {}
    list_sizes: dict[str, int] = {}
    for desc in descriptors:
        if desc.list_index >= 0:
            current_max = list_sizes.get(desc.field_name, 0)
            list_sizes[desc.field_name] = max(current_max, desc.list_index + 1)

    for field_name, size in list_sizes.items():
        tensor_fields[field_name] = [None] * size

    for i, desc in enumerate(descriptors):
        frame = parts[i + 1]
        buf = frame.buffer if hasattr(frame, "buffer") else bytes(frame)
        dtype = str_to_dtype(desc.dtype)
        # clone() to own the memory (decouple from ZMQ buffer lifetime)
        tensor = torch.frombuffer(buf, dtype=dtype).reshape(desc.shape).clone()
        if device != "cpu" and device != torch.device("cpu"):
            tensor = tensor.to(device)

        if desc.list_index >= 0:
            tensor_fields[desc.field_name][desc.list_index] = tensor
        else:
            tensor_fields[desc.field_name] = tensor

    return tensor_fields, scalar_fields