sgl-project--sglang/python/sglang/multimodal_gen/runtime/entrypoints/vla/protocol.py

# SPDX-License-Identifier: Apache-2.0

from __future__ import annotations

import base64
import dataclasses
import io
import time
import uuid
from functools import lru_cache
from typing import Any

import numpy as np
from PIL import Image

from sglang.multimodal_gen.configs.sample.vla import VLASamplingParams
from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
from sglang.multimodal_gen.runtime.server_args import ServerArgs


def pack_numpy_payload(obj):
    if isinstance(obj, (np.ndarray, np.generic)) and obj.dtype.kind in ("V", "O", "c"):
        raise ValueError(f"Unsupported dtype: {obj.dtype}")
    if isinstance(obj, np.ndarray):
        return {
            b"__ndarray__": True,
            b"data": obj.tobytes(),
            b"dtype": obj.dtype.str,
            b"shape": obj.shape,
        }
    if isinstance(obj, np.generic):
        return {
            b"__npgeneric__": True,
            b"data": obj.item(),
            b"dtype": obj.dtype.str,
        }
    return obj


def unpack_numpy_payload(obj):
    ndarray_marker = obj.get("__ndarray__") or obj.get(b"__ndarray__")
    npgeneric_marker = obj.get("__npgeneric__") or obj.get(b"__npgeneric__")
    data = obj.get("data", obj.get(b"data"))
    dtype = obj.get("dtype", obj.get(b"dtype"))
    shape = obj.get("shape", obj.get(b"shape"))
    if ndarray_marker:
        return np.ndarray(
            buffer=data,
            dtype=np.dtype(dtype),
            shape=shape,
        )
    if npgeneric_marker:
        return np.dtype(dtype).type(data)
    return obj


def pack_msgpack(payload: Any) -> bytes:
    import msgpack

    return msgpack.packb(payload, default=pack_numpy_payload, use_bin_type=True)


def unpack_msgpack(payload: bytes) -> Any:
    import msgpack

    return msgpack.unpackb(payload, object_hook=unpack_numpy_payload, raw=False)


def _decode_b64_image(payload: dict[str, Any]) -> Image.Image:
    data = payload.get("b64_json") or payload.get("base64")
    if not data:
        raise ValueError("image payload requires b64_json")
    if isinstance(data, str) and "," in data and data.startswith("data:"):
        data = data.split(",", 1)[1]
    return Image.open(io.BytesIO(base64.b64decode(data))).convert("RGB")


def _decode_tensor_payload(payload: dict[str, Any]) -> Any:
    values = payload.get("values")
    if values is None:
        values = payload.get("data")
    if values is None:
        return payload
    dtype = payload.get("dtype")
    array = np.asarray(values, dtype=np.dtype(dtype) if dtype else None)
    shape = payload.get("shape")
    if shape is not None:
        array = array.reshape(tuple(shape))
    return array


def _normalize_image_value(value: Any) -> Any:
    if not isinstance(value, dict):
        return value
    if "b64_json" in value or "base64" in value:
        return _decode_b64_image(value)
    if "values" in value or "data" in value:
        return _decode_tensor_payload(value)
    return value


def _normalize_observation(observation: dict[str, Any]) -> dict[str, Any]:
    normalized = dict(observation)
    images = normalized.get("images")
    if isinstance(images, dict):
        normalized["images"] = {
            name: _normalize_image_value(value) for name, value in images.items()
        }
    state = normalized.get("state")
    if isinstance(state, dict):
        normalized["state"] = _decode_tensor_payload(state)
    observation_state = normalized.get("observation.state")
    if isinstance(observation_state, dict):
        normalized["observation.state"] = _decode_tensor_payload(observation_state)
    noise = normalized.get("noise")
    if isinstance(noise, dict):
        normalized["noise"] = _decode_tensor_payload(noise)
    observation_noise = normalized.get("observation.noise")
    if isinstance(observation_noise, dict):
        normalized["observation.noise"] = _decode_tensor_payload(observation_noise)
    return normalized


def images_from_observation(
    observation: dict[str, Any],
    pipeline_config: Any,
) -> dict[str, Any]:
    if isinstance(observation.get("images"), dict):
        images = dict(observation["images"])
    else:
        images = {}
    for key in pipeline_config.image_keys:
        if key in observation:
            images[key] = observation[key]
        full_key = f"observation.images.{key}"
        if full_key in observation:
            images[key] = observation[full_key]
    return {name: _normalize_image_value(value) for name, value in images.items()}


def action_metadata(server_args: ServerArgs) -> dict[str, Any]:
    pipeline_config = server_args.pipeline_config
    policy_family = getattr(
        pipeline_config,
        "policy_family",
        type(pipeline_config).__name__.removesuffix("PipelineConfig").lower(),
    )
    return {
        "object": "action.metadata",
        "model": server_args.model_id or server_args.model_path,
        "model_path": server_args.model_path,
        "policy_family": policy_family,
        "input": {
            "image_keys": list(pipeline_config.image_keys),
            "image_size": list(pipeline_config.image_size),
            "state_dim": pipeline_config.state_dim,
        },
        "output": {
            "action_type": "continuous",
            "action_horizon": pipeline_config.action_horizon,
            "action_dim": pipeline_config.output_action_dim,
            "padded_action_dim": pipeline_config.action_dim,
            "dtype": "float32",
        },
        "runtime": {
            "materialize_dtype": pipeline_config.materialize_dtype,
            "enable_autocast": pipeline_config.enable_autocast,
            "parallelism": {
                "num_gpus": server_args.num_gpus,
                "tp_size": server_args.tp_size,
                "sp_degree": server_args.sp_degree,
                "ulysses_degree": server_args.ulysses_degree,
                "ring_degree": server_args.ring_degree,
                "prefix_strategy": pipeline_config.prefix_parallel_strategy,
                "action_strategy": pipeline_config.action_parallel_strategy,
                "layout_version": pipeline_config.parallel_layout_version,
            },
        },
        "defaults": {
            "num_inference_steps": pipeline_config.default_num_inference_steps,
            "prefix_cache": (
                "auto" if pipeline_config.enable_global_prefix_cache else False
            ),
            "cuda_graph": "auto" if pipeline_config.enable_action_cuda_graph else False,
        },
        "capabilities": {
            "exact_prefix_cache": True,
            "cuda_graph": pipeline_config.enable_action_cuda_graph,
            "realtime_websocket": True,
            "openpi_websocket": True,
            "batch_inputs": False,
            "multiple_candidates": False,
        },
    }


def _runtime_bool(value: Any, default: bool) -> bool:
    if value is None:
        return default
    if isinstance(value, str):
        value = value.lower()
        if value == "auto":
            return default
        if value in ("true", "1", "yes"):
            return True
        if value in ("false", "0", "no"):
            return False
    return bool(value)


def _action_request_to_observation(payload: dict[str, Any]) -> dict[str, Any]:
    if "input" not in payload:
        return _normalize_observation(payload)

    input_payload = payload.get("input") or {}
    observation = dict(input_payload.get("observation") or {})
    if "task" in input_payload:
        observation["prompt"] = input_payload["task"]
    elif "prompt" in input_payload:
        observation["prompt"] = input_payload["prompt"]
    if "images" in input_payload:
        observation["images"] = input_payload["images"]
    if "state" in input_payload:
        observation["state"] = input_payload["state"]
    if "noise" in input_payload:
        observation["noise"] = input_payload["noise"]
    return _normalize_observation(observation)


@lru_cache(maxsize=32)
def _resolve_action_sampling_params_cls_cached(
    model_path: str,
    backend: str | None,
    model_id: str | None,
    pipeline_class_name: str | None,
) -> type[VLASamplingParams]:
    if pipeline_class_name:
        from sglang.multimodal_gen.registry import get_pipeline_config_classes

        config_classes = get_pipeline_config_classes(pipeline_class_name)
        if config_classes is not None:
            _, sampling_params_cls = config_classes
            if issubclass(sampling_params_cls, VLASamplingParams):
                return sampling_params_cls

    from sglang.multimodal_gen.registry import get_model_info

    model_info = get_model_info(
        model_path,
        backend=backend,
        model_id=model_id,
    )
    sampling_params_cls = model_info.sampling_param_cls
    if not issubclass(sampling_params_cls, VLASamplingParams):
        raise ValueError(
            f"Action endpoint requires VLASamplingParams, got {sampling_params_cls.__name__}"
        )
    return sampling_params_cls


def _resolve_action_sampling_params_cls(
    server_args: ServerArgs,
) -> type[VLASamplingParams]:
    return _resolve_action_sampling_params_cls_cached(
        server_args.model_path,
        getattr(server_args, "backend", None),
        getattr(server_args, "model_id", None),
        getattr(server_args, "pipeline_class_name", None),
    )


@lru_cache(maxsize=32)
def _sampling_params_field_names(
    sampling_params_cls: type[VLASamplingParams],
) -> frozenset[str]:
    return frozenset(field.name for field in dataclasses.fields(sampling_params_cls))


def build_action_sampling_params(
    payload: dict[str, Any],
    server_args: ServerArgs,
) -> VLASamplingParams:
    pipeline_config = server_args.pipeline_config
    observation = _action_request_to_observation(payload)
    parameters = dict(payload.get("parameters") or {})
    runtime = dict(payload.get("runtime") or {})
    if "return_timing" in payload and "return_timing" not in runtime:
        runtime["return_timing"] = payload["return_timing"]
    images = images_from_observation(observation, pipeline_config)
    state = observation.get("state")
    if state is None:
        state = observation.get("observation.state")
    noise = observation.get("noise")
    if noise is None:
        noise = observation.get("observation.noise")
    prompt = observation.get("prompt") or observation.get("task") or ""
    prefix_cache = runtime.get("prefix_cache")
    if prefix_cache is None:
        prefix_cache = observation.get("enable_prefix_cache")
    if prefix_cache is None:
        prefix_cache = observation.get("enable_pi_prefix_cache")
    cuda_graph = runtime.get("cuda_graph")
    if cuda_graph is None:
        cuda_graph = observation.get("enable_cuda_graph")
    if cuda_graph is None:
        cuda_graph = observation.get("enable_pi_cuda_graph")
    output_format = str(
        runtime.get(
            "output_format",
            parameters.get(
                "output_format",
                observation.get("output_format", "list"),
            ),
        )
    ).lower()
    if output_format not in ("list", "numpy"):
        raise ValueError("output_format must be 'list' or 'numpy'")

    sampling_params_cls = _resolve_action_sampling_params_cls(server_args)
    sampling_kwargs = {
        "request_id": payload.get("request_id") or payload.get("id"),
        "prompt": prompt,
        "images": images,
        "image_masks": observation.get("image_masks"),
        "camera_order": observation.get("camera_order"),
        "state": state,
        "noise": noise,
        "observation": observation,
        "action_horizon": int(
            parameters.get(
                "action_horizon",
                observation.get("action_horizon", pipeline_config.action_horizon),
            )
        ),
        "action_dim": int(
            parameters.get(
                "action_dim",
                observation.get("action_dim", pipeline_config.action_dim),
            )
        ),
        "num_inference_steps": int(
            parameters.get(
                "num_inference_steps",
                observation.get(
                    "num_inference_steps",
                    pipeline_config.default_num_inference_steps,
                ),
            )
        ),
        "output_format": output_format,
        "return_timing": _runtime_bool(runtime.get("return_timing"), True),
        "enable_prefix_cache": _runtime_bool(prefix_cache, True),
        "enable_cuda_graph": _runtime_bool(cuda_graph, True),
    }
    supported_fields = _sampling_params_field_names(sampling_params_cls)
    sp = sampling_params_cls(
        **{
            name: value
            for name, value in sampling_kwargs.items()
            if name in supported_fields
        }
    )
    sp._adjust(server_args)
    return sp


async def infer_action(
    payload: dict[str, Any],
    server_args: ServerArgs,
) -> dict[str, Any]:
    sp = build_action_sampling_params(payload, server_args)
    req = prepare_request(server_args, sp)
    response = await async_scheduler_client.forward(req)
    if getattr(response, "error", None):
        raise RuntimeError(response.error)
    if response.output is None:
        raise RuntimeError("action policy returned no output")
    return response.output[0]


def action_generation_response(
    output: dict[str, Any],
    server_args: ServerArgs,
    *,
    preserve_numpy: bool = False,
) -> dict[str, Any]:
    actions = output["actions"]
    if isinstance(actions, np.ndarray):
        action_shape = list(actions.shape)
        action_values = actions if preserve_numpy else actions.tolist()
    else:
        horizon = len(actions) if isinstance(actions, list) else 0
        action_dim = len(actions[0]) if horizon and isinstance(actions[0], list) else 0
        action_shape = [horizon, action_dim]
        action_values = actions
    response = {
        "id": output.get("request_id") or f"act_{uuid.uuid4().hex}",
        "object": "action.generation",
        "created": int(time.time()),
        "model": server_args.model_id or server_args.model_path,
        "data": [
            {
                "index": 0,
                "input_index": 0,
                "candidate_index": 0,
                "action": {
                    "type": "continuous",
                    "dtype": "float32",
                    "shape": action_shape,
                    "values": action_values,
                },
            }
        ],
        "usage": {
            "action_horizon": action_shape[0] if action_shape else 0,
            "action_dim": action_shape[1] if len(action_shape) > 1 else 0,
            "denoise_steps": output.get("parameters", {}).get(
                "num_inference_steps",
                server_args.pipeline_config.default_num_inference_steps,
            ),
            "prefix_cache_hit": bool(output.get("cache", {}).get("hit", False)),
        },
    }
    if "timings" in output:
        response["timings"] = output["timings"]
    if "cache" in output:
        response["cache"] = output["cache"]
    if "parallel" in output:
        response["parallel"] = output["parallel"]
    return response


def action_raw_response(
    output: dict[str, Any],
    *,
    preserve_numpy: bool = False,
) -> dict[str, Any]:
    response = dict(output)
    actions = response.get("actions")
    if isinstance(actions, np.ndarray) and not preserve_numpy:
        response["actions"] = actions.tolist()
    return response