chore: import upstream snapshot with attribution

python/sglang/multimodal_gen/runtime/entrypoints/__init__.py

@@ -0,0 +1,4 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+from sglang.multimodal_gen.runtime.utils.logging_utils import globally_suppress_loggers
+
+globally_suppress_loggers()

python/sglang/multimodal_gen/runtime/entrypoints/cli/__init__.py

@@ -0,0 +1 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo

python/sglang/multimodal_gen/runtime/entrypoints/cli/cli_types.py

@@ -0,0 +1,30 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+# SPDX-License-Identifier: Apache-2.0
+# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/entrypoints/cli/types.py
+
+import argparse
+
+from sglang.multimodal_gen.utils import FlexibleArgumentParser
+
+
+class CLISubcommand:
+    """Base class for CLI subcommands"""
+
+    name: str
+
+    def cmd(
+        self, args: argparse.Namespace, unknown_args: list[str] | None = None
+    ) -> None:
+        """Execute the command with the given arguments"""
+        raise NotImplementedError
+
+    def validate(self, args: argparse.Namespace) -> None:
+        """Validate the arguments for this command"""
+        pass
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        """Initialize the subparser for this command"""
+        raise NotImplementedError

python/sglang/multimodal_gen/runtime/entrypoints/cli/generate.py

@@ -0,0 +1,248 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+# SPDX-License-Identifier: Apache-2.0
+# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/entrypoints/cli/serve.py
+
+import argparse
+import dataclasses
+import json
+import os
+from typing import cast
+
+from sglang.multimodal_gen import DiffGenerator
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    SamplingParams,
+    generate_request_id,
+)
+from sglang.multimodal_gen.runtime.entrypoints.cli.cli_types import CLISubcommand
+from sglang.multimodal_gen.runtime.entrypoints.cli.utils import (
+    RaiseNotImplementedAction,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import GenerationResult
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.multimodal_gen.runtime.utils.perf_logger import (
+    MemorySnapshot,
+    PerformanceLogger,
+    RequestMetrics,
+)
+from sglang.multimodal_gen.utils import FlexibleArgumentParser
+
+logger = init_logger(__name__)
+
+
+def _resolve_cli_sampling_params_cls(server_args: ServerArgs) -> type[SamplingParams]:
+    pipeline_class_name = getattr(server_args, "pipeline_class_name", None)
+    if pipeline_class_name:
+        from sglang.multimodal_gen.registry import get_pipeline_config_classes
+
+        config_classes = get_pipeline_config_classes(pipeline_class_name)
+        if config_classes is not None:
+            _, sampling_params_cls = config_classes
+            return sampling_params_cls
+
+    try:
+        from sglang.multimodal_gen.registry import get_model_info
+
+        model_info = get_model_info(
+            server_args.model_path,
+            backend=server_args.backend,
+            model_id=server_args.model_id,
+        )
+        if model_info is not None:
+            return model_info.sampling_param_cls
+    except Exception as exc:
+        logger.debug("Falling back to base SamplingParams for CLI parsing: %s", exc)
+
+    return SamplingParams
+
+
+def add_multimodal_gen_generate_args(parser: argparse.ArgumentParser):
+    """Add the arguments for the generate command."""
+    parser.add_argument(
+        "--config",
+        type=str,
+        default="",
+        required=False,
+        help="Read CLI options from a config JSON or YAML file. If provided, --model-path and --prompt are optional.",
+    )
+    parser.add_argument(
+        "--perf-dump-path",
+        type=str,
+        default=None,
+        required=False,
+        help="Path to dump the performance metrics (JSON) for the run.",
+    )
+    parser.add_argument(
+        "--output-file-path",
+        type=str,
+        default=None,
+        required=False,
+        help="Convenience alias that sets both --output-path and --output-file-name.",
+    )
+
+    parser = ServerArgs.add_cli_args(parser)
+    parser = SamplingParams.add_cli_args(parser)
+
+    parser.add_argument(
+        "--text-encoder-configs",
+        action=RaiseNotImplementedAction,
+        help="JSON array of text encoder configurations (NOT YET IMPLEMENTED)",
+    )
+
+    return parser
+
+
+def _apply_output_file_path_override(
+    args: argparse.Namespace, sampling_params_kwargs: dict
+):
+    output_file_path = args.output_file_path
+    if not output_file_path:
+        return
+
+    output_path = os.path.dirname(output_file_path) or "."
+    sampling_params_kwargs["output_path"] = output_path
+    sampling_params_kwargs["output_file_name"] = os.path.basename(output_file_path)
+
+
+def maybe_dump_performance(
+    args: argparse.Namespace,
+    server_args,
+    prompt: str,
+    results: GenerationResult | list[GenerationResult] | None,
+):
+    """dump performance if necessary"""
+    if not (args.perf_dump_path and results):
+        return
+
+    if isinstance(results, list):
+        result = results[0] if results else None
+    else:
+        result = results
+
+    metrics_dict = result.metrics
+    if not (args.perf_dump_path and metrics_dict):
+        return
+
+    metrics = RequestMetrics(request_id=metrics_dict.get("request_id"))
+    metrics.stages = metrics_dict.get("stages", {})
+    metrics.steps = metrics_dict.get("steps", [])
+    metrics.total_duration_ms = metrics_dict.get("total_duration_ms", 0)
+
+    # restore memory snapshots from serialized dict
+    memory_snapshots_dict = metrics_dict.get("memory_snapshots", {})
+    for checkpoint_name, snapshot_dict in memory_snapshots_dict.items():
+        snapshot = MemorySnapshot(
+            allocated_mb=snapshot_dict.get("allocated_mb", 0.0),
+            reserved_mb=snapshot_dict.get("reserved_mb", 0.0),
+            peak_allocated_mb=snapshot_dict.get("peak_allocated_mb", 0.0),
+            peak_reserved_mb=snapshot_dict.get("peak_reserved_mb", 0.0),
+        )
+        metrics.memory_snapshots[checkpoint_name] = snapshot
+
+    PerformanceLogger.dump_benchmark_report(
+        file_path=args.perf_dump_path,
+        metrics=metrics,
+        meta={
+            "prompt": prompt,
+            "model": server_args.model_path,
+        },
+        tag="cli_generate",
+    )
+
+
+def generate_cmd(args: argparse.Namespace, unknown_args: list[str] | None = None):
+    """The entry point for the generate command."""
+    args.request_id = "mocked_fake_id_for_offline_generate"
+
+    server_args = ServerArgs.from_cli_args(args, unknown_args)
+    sampling_params_cls = _resolve_cli_sampling_params_cls(server_args)
+
+    sampling_params_kwargs = {}
+    config_file = getattr(args, "config", None)
+    # respect config file by overriding args with args parsed from it
+    if config_file:
+        config_args = ServerArgs.load_config_file(config_file) or {}
+        sampling_param_fields = {
+            field.name for field in dataclasses.fields(sampling_params_cls)
+        }
+        sampling_params_kwargs.update(
+            {
+                key: value
+                for key, value in config_args.items()
+                if key in sampling_param_fields and value is not None
+            }
+        )
+
+    sampling_params_kwargs.update(sampling_params_cls.get_cli_args(args))
+    _apply_output_file_path_override(args, sampling_params_kwargs)
+    sampling_params_kwargs["request_id"] = generate_request_id()
+
+    # Handle diffusers-specific kwargs passed via CLI
+    if hasattr(args, "diffusers_kwargs") and args.diffusers_kwargs:
+        try:
+            sampling_params_kwargs["diffusers_kwargs"] = json.loads(
+                args.diffusers_kwargs
+            )
+            logger.info(
+                "Parsed diffusers_kwargs: %s",
+                sampling_params_kwargs["diffusers_kwargs"],
+            )
+        except json.JSONDecodeError as e:
+            logger.error("Failed to parse --diffusers-kwargs as JSON: %s", e)
+            raise ValueError(
+                f"--diffusers-kwargs must be valid JSON. Got: {args.diffusers_kwargs}"
+            ) from e
+
+    generator = DiffGenerator.from_pretrained(
+        model_path=server_args.model_path, server_args=server_args, local_mode=True
+    )
+
+    results = generator.generate(sampling_params_kwargs=sampling_params_kwargs)
+
+    prompt = sampling_params_kwargs.get("prompt")
+    maybe_dump_performance(args, server_args, prompt, results)
+
+
+class GenerateSubcommand(CLISubcommand):
+    """The `generate` subcommand for the sglang-diffusion CLI"""
+
+    def __init__(self) -> None:
+        self.name = "generate"
+        super().__init__()
+        self.init_arg_names = self._get_init_arg_names()
+        self.generation_arg_names = self._get_generation_arg_names()
+
+    def _get_init_arg_names(self) -> list[str]:
+        """Get names of arguments for DiffGenerator initialization"""
+        return ["num_gpus", "tp_size", "sp_size", "model_path"]
+
+    def _get_generation_arg_names(self) -> list[str]:
+        """Get names of arguments for generate_video method"""
+        return [field.name for field in dataclasses.fields(SamplingParams)]
+
+    def cmd(
+        self, args: argparse.Namespace, unknown_args: list[str] | None = None
+    ) -> None:
+        generate_cmd(args, unknown_args)
+
+    def validate(self, args: argparse.Namespace) -> None:
+        """Validate the arguments for this command"""
+        if args.num_gpus is not None and args.num_gpus <= 0:
+            raise ValueError("Number of gpus must be positive")
+
+        if args.config and not os.path.exists(args.config):
+            raise ValueError(f"Config file not found: {args.config}")
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        generate_parser = subparsers.add_parser(
+            "generate",
+            help="Run inference on a model",
+            usage="sglang generate (--model-path MODEL_PATH_OR_ID --prompt PROMPT) | --config CONFIG_FILE [OPTIONS]",
+        )
+
+        generate_parser = add_multimodal_gen_generate_args(generate_parser)
+
+        return cast(FlexibleArgumentParser, generate_parser)

python/sglang/multimodal_gen/runtime/entrypoints/cli/main.py

@@ -0,0 +1,44 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+# SPDX-License-Identifier: Apache-2.0
+# adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/entrypoints/cli/main.py
+
+from sglang.multimodal_gen.runtime.entrypoints.cli.cli_types import CLISubcommand
+from sglang.multimodal_gen.runtime.entrypoints.cli.generate import GenerateSubcommand
+from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ServeSubcommand
+from sglang.multimodal_gen.utils import FlexibleArgumentParser
+
+
+def generate_cmd_init() -> list[CLISubcommand]:
+    return [GenerateSubcommand(), ServeSubcommand()]
+
+
+def cmd_init() -> list[CLISubcommand]:
+    """Initialize all commands from separate modules"""
+    commands = []
+    commands.extend(generate_cmd_init())
+    return commands
+
+
+def main() -> None:
+    parser = FlexibleArgumentParser(description="sglang-diffusion CLI")
+    parser.add_argument("-v", "--version", action="version", version="0.1.0")
+
+    subparsers = parser.add_subparsers(required=False, dest="subparser")
+
+    cmds = {}
+    for cmd in cmd_init():
+        cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
+        cmds[cmd.name] = cmd
+    args, unknown_args = parser.parse_known_args()
+    if args.subparser in cmds:
+        cmds[args.subparser].validate(args)
+
+    if hasattr(args, "dispatch_function"):
+        args.dispatch_function(args, unknown_args=unknown_args)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

python/sglang/multimodal_gen/runtime/entrypoints/cli/serve.py

@@ -0,0 +1,75 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+from typing import cast
+
+from sglang.multimodal_gen.apps.webui import run_sgl_diffusion_webui
+from sglang.multimodal_gen.runtime.entrypoints.cli.cli_types import CLISubcommand
+from sglang.multimodal_gen.runtime.launch_server import (
+    dispatch_launch,
+)
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.multimodal_gen.utils import FlexibleArgumentParser
+
+
+def add_multimodal_gen_serve_args(parser: argparse.ArgumentParser):
+    """Add the arguments for the serve command."""
+    parser.add_argument(
+        "--config",
+        type=str,
+        default="",
+        required=False,
+        help="Read CLI options from a config JSON or YAML file.",
+    )
+    return ServerArgs.add_cli_args(parser)
+
+
+def execute_serve_cmd(args: argparse.Namespace, unknown_args: list[str] | None = None):
+    """The entry point for the serve command."""
+    # use server-based warmup for production
+    server_args = ServerArgs.from_cli_args(
+        args, unknown_args, default_args={"warmup_mode": "server"}
+    )
+
+    dispatch_launch(server_args)
+
+    if server_args.webui:
+        run_sgl_diffusion_webui(server_args)
+
+
+class ServeSubcommand(CLISubcommand):
+    """The `serve` subcommand for the sglang-diffusion CLI"""
+
+    def __init__(self) -> None:
+        self.name = "serve"
+        super().__init__()
+
+    def cmd(
+        self, args: argparse.Namespace, unknown_args: list[str] | None = None
+    ) -> None:
+        execute_serve_cmd(args, unknown_args)
+
+    def validate(self, args: argparse.Namespace) -> None:
+        """Validate the arguments for this command"""
+        if args.config and not os.path.exists(args.config):
+            raise ValueError(f"Config file not found: {args.config}")
+
+    def subparser_init(
+        self, subparsers: argparse._SubParsersAction
+    ) -> FlexibleArgumentParser:
+        serve_parser = subparsers.add_parser(
+            "serve",
+            help="Launch the server and start FastAPI listener.",
+            usage="sglang serve --model-path MODEL_PATH_OR_ID [OPTIONS]",
+        )
+
+        serve_parser = add_multimodal_gen_serve_args(serve_parser)
+
+        return cast(FlexibleArgumentParser, serve_parser)
+
+
+def cmd_init() -> list[CLISubcommand]:
+    return [ServeSubcommand()]

python/sglang/multimodal_gen/runtime/entrypoints/cli/utils.py

@@ -0,0 +1,75 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+
+class RaiseNotImplementedAction(argparse.Action):
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        raise NotImplementedError(f"The {option_string} option is not yet implemented")
+
+
+def launch_distributed(
+    num_gpus: int, args: list[str], master_port: int | None = None
+) -> int:
+    """
+    Launch a distributed job with the given arguments
+
+    Args:
+        num_gpus: Number of GPUs to use
+        args: Arguments to pass to v1_sgl_diffusion_inference.py (defaults to sys.argv[1:])
+        master_port: Port for the master process (default: random)
+    """
+
+    current_env = os.environ.copy()
+    python_executable = sys.executable
+    project_root = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "../../../..")
+    )
+    main_script = os.path.join(
+        project_root, "sgl_diffusion/sample/v1_sgl_diffusion_inference.py"
+    )
+
+    cmd = [
+        python_executable,
+        "-m",
+        "torch.distributed.run",
+        f"--nproc_per_node={num_gpus}",
+    ]
+
+    if master_port is not None:
+        cmd.append(f"--master_port={master_port}")
+
+    cmd.append(main_script)
+    cmd.extend(args)
+
+    logger.info("Running inference with %d GPU(s)", num_gpus)
+    logger.info("Launching command: %s", shlex.join(cmd))
+
+    current_env["PYTHONIOENCODING"] = "utf-8"
+    process = subprocess.Popen(
+        cmd,
+        env=current_env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        universal_newlines=True,
+        bufsize=1,
+        encoding="utf-8",
+        errors="replace",
+    )
+
+    if process.stdout:
+        for line in iter(process.stdout.readline, ""):
+            print(line.strip())
+
+    return process.wait()

python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py

@@ -0,0 +1,713 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+# SPDX-License-Identifier: Apache-2.0
+"""
+DiffGenerator module for sglang-diffusion.
+
+This module provides a consolidated interface for generating images/videos using
+diffusion models.
+"""
+
+import dataclasses
+import multiprocessing as mp
+import os
+import time
+from contextlib import ExitStack
+from typing import Any, List, Union
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    DataType,
+    SamplingParams,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    GenerationResult,
+    ListLorasReq,
+    MergeLoraWeightsReq,
+    SetLoraReq,
+    ShutdownReq,
+    UnmergeLoraWeightsReq,
+    expand_request_outputs,
+    format_lora_message,
+    prepare_request,
+    save_outputs,
+)
+from sglang.multimodal_gen.runtime.launch_server import launch_server
+from sglang.multimodal_gen.runtime.pipelines_core import Req
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.scheduler_client import sync_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import PortArgs, ServerArgs
+from sglang.multimodal_gen.runtime.server_warmup import (
+    run_sync_client_warmup,
+    should_run_explicit_client_warmup,
+)
+from sglang.multimodal_gen.runtime.utils.logging_utils import (
+    GREEN,
+    RESET,
+    init_logger,
+    log_batch_completion,
+    log_generation_timer,
+)
+from sglang.multimodal_gen.runtime.utils.trace_wrapper import (
+    init_diffusion_tracing,
+    trace_req,
+)
+
+logger = init_logger(__name__)
+
+try:
+    # Set the start method to 'spawn' to avoid CUDA errors in forked processes.
+    # This must be done at the top level of the module, before any CUDA context
+    # or other processes are initialized.
+    mp.set_start_method("spawn", force=True)
+except RuntimeError:
+    # The start method can only be set once per program execution.
+    pass
+
+
+class DiffGenerator:
+    """
+    A unified class for generating images/videos using diffusion models.
+
+    This class provides a simple interface for image/video generation with rich
+    customization options, similar to popular frameworks like HF Diffusers.
+    """
+
+    def __init__(
+        self,
+        server_args: ServerArgs,
+    ):
+        """
+        Initialize the generator.
+
+        Args:
+            server_args: The inference arguments
+        """
+        self.server_args = server_args
+        self.port_args = PortArgs.from_server_args(server_args)
+
+        # The executor is now a client to the Scheduler service
+        self.local_scheduler_process: list[mp.Process] | None = None
+        self.owns_scheduler_client: bool = False
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        local_mode: bool = True,
+        **kwargs,
+    ) -> "DiffGenerator":
+        """
+        Create a DiffGenerator from a pretrained model.
+
+        Priority level: Default pipeline config < User's pipeline config < User's kwargs
+        """
+        # If users also provide some kwargs, it will override the ServerArgs and PipelineConfig.
+
+        if (server_args := kwargs.get("server_args", None)) is not None:
+            if isinstance(server_args, ServerArgs):
+                pass
+            elif isinstance(server_args, dict):
+                server_args = ServerArgs.from_kwargs(**server_args)
+        else:
+            server_args = ServerArgs.from_kwargs(**kwargs)
+
+        return cls.from_server_args(server_args, local_mode=local_mode)
+
+    @classmethod
+    def from_server_args(
+        cls, server_args: ServerArgs, local_mode: bool = True
+    ) -> "DiffGenerator":
+        """
+        Create a DiffGenerator with the specified arguments.
+
+        Args:
+            server_args: The inference arguments
+
+        Returns:
+            The created DiffGenerator
+        """
+        instance = cls(
+            server_args=server_args,
+        )
+        init_diffusion_tracing(server_args, "DiffGenerator")
+
+        logger.info(f"Local mode: {local_mode}")
+        if local_mode:
+            instance.local_scheduler_process = instance._start_local_server_if_needed()
+            instance.owns_scheduler_client = True
+            instance._run_client_warmup_if_needed()
+        else:
+            # In remote mode, we just need to connect and check.
+            sync_scheduler_client.initialize(server_args)
+            instance._check_remote_scheduler()
+            instance.owns_scheduler_client = True
+        return instance
+
+    def _start_local_server_if_needed(
+        self,
+    ) -> list[mp.Process]:
+        """Check if a local server is running; if not, start it and return the process handles."""
+        # First, we need a client to test the server. Initialize it temporarily.
+        sync_scheduler_client.initialize(self.server_args)
+
+        processes = launch_server(self.server_args, launch_http_server=False)
+
+        return processes
+
+    def _run_client_warmup_if_needed(self) -> None:
+        if not should_run_explicit_client_warmup(self.server_args):
+            return
+
+        run_sync_client_warmup(self.server_args, sync_scheduler_client.forward)
+
+    def _check_remote_scheduler(self):
+        """Check if the remote scheduler is accessible."""
+        if not sync_scheduler_client.ping():
+            raise ConnectionError(
+                f"Could not connect to remote scheduler at "
+                f"{self.server_args.scheduler_endpoint} with `local mode` as False. "
+                "Please ensure the server is running."
+            )
+        logger.info(
+            f"Successfully connected to remote scheduler at "
+            f"{self.server_args.scheduler_endpoint}."
+        )
+
+    @staticmethod
+    def _resolve_image_paths_per_prompt(
+        prompts: list[str], image_paths: str | list[str] | None
+    ) -> list[str | list[str] | None]:
+        if len(prompts) <= 1:
+            return [image_paths]
+
+        if not isinstance(image_paths, list) or len(image_paths) <= 1:
+            return [image_paths for _ in prompts]
+
+        if len(image_paths) != len(prompts):
+            raise ValueError(
+                "When using multiple prompts with multiple input images, "
+                "provide either one shared image or exactly one image per prompt."
+            )
+
+        return [[image_path] for image_path in image_paths]
+
+    def generate(
+        self,
+        sampling_params_kwargs: dict | None = None,
+        external_trace_header: dict[str, str] | None = None,
+    ) -> GenerationResult | list[GenerationResult] | None:
+        """Generate image(s)/video(s) based on the given prompt(s).
+
+        Returns a single GenerationResult for a single prompt, a list for
+        multiple prompts, or None when every request failed.
+        """
+        # 1. prepare requests
+        prompts = self._resolve_prompts(
+            sampling_params_kwargs.get("prompt"),
+            sampling_params_kwargs.get("prompt_path"),
+        )
+        user_output_file_name = sampling_params_kwargs.get("output_file_name")
+
+        if len(prompts) > 1 and user_output_file_name is not None:
+            raise ValueError(
+                "Cannot use multiple prompts with a fixed output_file_name. "
+                "Either remove --output-file-name or use a single prompt."
+            )
+
+        sampling_params_orig = SamplingParams.from_user_sampling_params_args(
+            self.server_args.model_path,
+            server_args=self.server_args,
+            **sampling_params_kwargs,
+        )
+
+        request_groups: list[list[Req]] = []
+        image_paths_per_prompt = self._resolve_image_paths_per_prompt(
+            prompts, sampling_params_orig.image_path
+        )
+
+        for i, p in enumerate(prompts):
+            sampling_params = dataclasses.replace(
+                sampling_params_orig,
+                prompt=p,
+                output_file_name=user_output_file_name,
+                image_path=image_paths_per_prompt[i],
+            )
+            # `dataclasses.replace` drops non-field attrs; restore
+            # `_explicit_fields` so InputValidationStage honors user-supplied
+            # width/height, and mark the keys overridden above as explicit.
+            sampling_params._explicit_fields = getattr(
+                sampling_params_orig, "_explicit_fields", set()
+            ) | {"prompt", "output_file_name", "image_path"}
+            sampling_params._set_output_file_name()
+            req = prepare_request(
+                server_args=self.server_args,
+                sampling_params=sampling_params,
+                external_trace_header=external_trace_header,
+            )
+            request_groups.append(
+                expand_request_outputs(
+                    req,
+                    num_prompts=len(prompts),
+                    prompt_index=i,
+                )
+            )
+
+        results: list[GenerationResult] = []
+        total_start_time = time.perf_counter()
+        global_output_index = 0
+
+        for requests in request_groups:
+            try:
+                timer_prompt = [req.prompt for req in requests]
+                logger.info("Processing %d grouped request(s)", len(requests))
+                with ExitStack() as stack:
+                    for req in requests:
+                        stack.enter_context(trace_req(req.trace_ctx))
+                    timer = stack.enter_context(
+                        log_generation_timer(logger, timer_prompt)
+                    )
+                    output_batch = self._send_to_scheduler_and_wait_for_response(
+                        requests
+                    )
+                    if output_batch.error:
+                        raise Exception(f"{output_batch.error}")
+
+                    if (
+                        output_batch.output is None
+                        and output_batch.output_file_paths is None
+                    ):
+                        logger.error("Received empty output from scheduler")
+                        continue
+
+                    if requests[0].save_output and requests[0].return_file_paths_only:
+                        output_file_paths = output_batch.output_file_paths or []
+                        self._validate_output_count(
+                            len(output_file_paths), len(requests)
+                        )
+                        for idx, path in enumerate(output_file_paths):
+                            req = requests[idx]
+                            results.append(
+                                GenerationResult(
+                                    **self._result_common(
+                                        req, output_batch, timer.duration, idx
+                                    ),
+                                    prompt_index=global_output_index + idx,
+                                    output_file_path=path,
+                                )
+                            )
+                    elif requests[0].data_type == DataType.MESH:
+                        output_file_paths = output_batch.output_file_paths or []
+                        self._validate_output_count(
+                            len(output_file_paths), len(requests)
+                        )
+                        for idx, sample in enumerate(output_file_paths):
+                            req = requests[idx]
+                            results.append(
+                                GenerationResult(
+                                    **self._result_common(
+                                        req, output_batch, timer.duration, idx
+                                    ),
+                                    prompt_index=global_output_index + idx,
+                                    output_file_path=sample,
+                                )
+                            )
+                    else:
+                        self._validate_output_count(
+                            len(output_batch.output), len(requests)
+                        )
+                        samples_out: list[Any] = []
+                        audios_out: list[Any] = []
+                        frames_out: list[Any] = []
+                        save_outputs(
+                            output_batch.output,
+                            requests[0].data_type,
+                            requests[0].fps,
+                            requests[0].save_output,
+                            lambda idx: requests[idx].output_file_path(1, 0),
+                            audio=output_batch.audio,
+                            audio_sample_rate=output_batch.audio_sample_rate,
+                            samples_out=samples_out,
+                            audios_out=audios_out,
+                            frames_out=frames_out,
+                            output_compression=requests[0].output_compression,
+                            enable_frame_interpolation=requests[
+                                0
+                            ].enable_frame_interpolation,
+                            frame_interpolation_exp=requests[0].frame_interpolation_exp,
+                            frame_interpolation_scale=requests[
+                                0
+                            ].frame_interpolation_scale,
+                            frame_interpolation_model_path=requests[
+                                0
+                            ].frame_interpolation_model_path,
+                            enable_upscaling=requests[0].enable_upscaling,
+                            upscaling_model_path=requests[0].upscaling_model_path,
+                            upscaling_scale=requests[0].upscaling_scale,
+                        )
+
+                        for idx in range(len(samples_out)):
+                            req = requests[idx]
+                            results.append(
+                                GenerationResult(
+                                    **self._result_common(
+                                        req, output_batch, timer.duration, idx
+                                    ),
+                                    samples=samples_out[idx],
+                                    frames=frames_out[idx],
+                                    audio=audios_out[idx],
+                                    prompt_index=global_output_index + idx,
+                                    output_file_path=req.output_file_path(1, 0),
+                                )
+                            )
+            except Exception as e:
+                logger.error("Generation failed: %s", e, exc_info=True)
+            finally:
+                global_output_index += len(requests)
+
+        total_gen_time = time.perf_counter() - total_start_time
+        if self.server_args.batching_max_size > 1:
+            log_batch_completion(
+                logger,
+                len(results),
+                total_gen_time,
+            )
+        self._log_summary(results)
+
+        if not results:
+            return None
+        return results[0] if len(results) == 1 else results
+
+    def generate_action(
+        self,
+        sampling_params_kwargs: dict | None = None,
+        external_trace_header: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        sampling_params_kwargs = sampling_params_kwargs or {}
+        sampling_params = SamplingParams.from_user_sampling_params_args(
+            self.server_args.model_path,
+            server_args=self.server_args,
+            **sampling_params_kwargs,
+        )
+        if sampling_params.data_type != DataType.ACTION:
+            raise ValueError(
+                f"generate_action requires an ACTION pipeline, got {sampling_params.data_type}"
+            )
+
+        req = prepare_request(
+            server_args=self.server_args,
+            sampling_params=sampling_params,
+            external_trace_header=external_trace_header,
+        )
+        output_batch = self._send_to_scheduler_and_wait_for_response(req)
+        if output_batch.error:
+            raise RuntimeError(output_batch.error)
+        if output_batch.output is None:
+            raise RuntimeError("action policy returned no output")
+        return output_batch.output[0]
+
+    def _resolve_prompts(
+        self,
+        prompt: str | list[str] | None,
+        prompt_path: str | None = None,
+    ) -> list[str]:
+        """Collect prompts from the argument or from a prompt file."""
+        path = prompt_path or self.server_args.prompt_file_path
+        if path is not None:
+            if not os.path.exists(path):
+                raise FileNotFoundError(f"Prompt text file not found: {path}")
+            with open(path, encoding="utf-8") as f:
+                prompts = [line.strip() for line in f if line.strip()]
+            if not prompts:
+                raise ValueError(f"No prompts found in file: {path}")
+            logger.info("Found %d prompts in %s", len(prompts), path)
+            return prompts
+
+        if prompt is None:
+            return [" "]
+        if isinstance(prompt, str):
+            return [prompt]
+        return list(prompt)
+
+    def _log_summary(self, results: list[GenerationResult]) -> None:
+        if not results:
+            return
+        if self.server_args.warmup:
+            total_duration_ms = results[0].metrics.get("total_duration_ms", 0)
+            logger.info(
+                f"Warmed-up request processed in {GREEN}%.2f{RESET} seconds (with warmup excluded)",
+                total_duration_ms / 1000.0,
+            )
+
+        peak_memories = [r.peak_memory_mb for r in results if r.peak_memory_mb]
+        if peak_memories:
+            logger.info(
+                f"Memory usage - Max peak: {max(peak_memories):.2f} MB, "
+                f"Avg peak: {sum(peak_memories) / len(peak_memories):.2f} MB"
+            )
+
+    @staticmethod
+    def _result_common(
+        req: Req,
+        output_batch: OutputBatch,
+        generation_time: float,
+        output_index: int | None = None,
+    ) -> dict[str, Any]:
+        metrics = output_batch.metrics
+        if (
+            output_index is not None
+            and output_batch.metrics_list is not None
+            and output_index < len(output_batch.metrics_list)
+        ):
+            metrics = output_batch.metrics_list[output_index]
+        if req.data_type == DataType.ACTION:
+            size = ("action",)
+        else:
+            size = (req.height, req.width, req.num_frames)
+        return dict(
+            prompt=req.prompt,
+            size=size,
+            generation_time=generation_time,
+            peak_memory_mb=output_batch.peak_memory_mb,
+            metrics=metrics.to_dict() if metrics else {},
+            action=output_batch.action_pred,
+            trajectory_latents=output_batch.trajectory_latents,
+            trajectory_timesteps=output_batch.trajectory_timesteps,
+            rollout_trajectory_data=output_batch.rollout_trajectory_data,
+            trajectory_decoded=output_batch.trajectory_decoded,
+        )
+
+    @staticmethod
+    def _validate_output_count(output_count: int, request_count: int) -> None:
+        if output_count != request_count:
+            raise RuntimeError(
+                f"Expected {request_count} outputs, got {output_count} from scheduler"
+            )
+
+    def _send_to_scheduler_and_wait_for_response(self, batch: list[Req]) -> OutputBatch:
+        """
+        Sends a request to the scheduler and waits for a response.
+        """
+        return sync_scheduler_client.forward(batch)
+
+    # LoRA
+    def _send_lora_request(self, req: Any, success_msg: str, failure_msg: str):
+        response = sync_scheduler_client.forward(req)
+        if response.error is None:
+            logger.info(success_msg)
+            return response
+        else:
+            error_msg = response.error
+            raise RuntimeError(f"{failure_msg}: {error_msg}")
+
+    def set_lora(
+        self,
+        lora_nickname: Union[str, List[str]],
+        lora_path: Union[str, None, List[Union[str, None]]] = None,
+        target: Union[str, List[str]] = "all",
+        strength: Union[float, List[float]] = 1.0,
+        merge_mode: str | None = None,
+    ) -> None:
+        """
+        Set LoRA adapter(s) for the specified transformer(s).
+        Supports both single LoRA (backward compatible) and multiple LoRA adapters.
+
+        Args:
+            lora_nickname: The nickname(s) of the adapter(s). Can be a string or a list of strings.
+            lora_path: Path(s) to the LoRA adapter(s). Can be a string, None, or a list of strings/None.
+            target: Which transformer(s) to apply the LoRA to. Can be a string or a list of strings.
+                Valid values:
+                - "all": Apply to all transformers (default)
+                - "transformer": Apply only to the primary transformer (high noise for Wan2.2)
+                - "transformer_2": Apply only to transformer_2 (low noise for Wan2.2)
+                - "critic": Apply only to the critic model
+            strength: LoRA strength(s) for merge, default 1.0. Can be a float or a list of floats.
+            merge_mode: Optional LoRA merge mode: "auto", "merge", or "dynamic".
+        """
+        req = SetLoraReq(
+            lora_nickname=lora_nickname,
+            lora_path=lora_path,
+            target=target,
+            strength=strength,
+            merge_mode=merge_mode,
+        )
+        nickname_str, target_str, strength_str = format_lora_message(
+            lora_nickname, target, strength
+        )
+
+        self._send_lora_request(
+            req,
+            f"Successfully set LoRA adapter(s): {nickname_str} (target: {target_str}, strength: {strength_str})",
+            "Failed to set LoRA adapter",
+        )
+
+    def unmerge_lora_weights(self, target: str = "all") -> None:
+        """
+        Unmerge LoRA weights from the base model.
+
+        Args:
+            target: Which transformer(s) to unmerge.
+        """
+        req = UnmergeLoraWeightsReq(target=target)
+        self._send_lora_request(
+            req,
+            f"Successfully unmerged LoRA weights (target: {target})",
+            "Failed to unmerge LoRA weights",
+        )
+
+    def merge_lora_weights(self, target: str = "all", strength: float = 1.0) -> None:
+        """
+        Merge LoRA weights into the base model.
+
+        Args:
+            target: Which transformer(s) to merge.
+            strength: LoRA strength for merge, default 1.0.
+        """
+        req = MergeLoraWeightsReq(target=target, strength=strength)
+        self._send_lora_request(
+            req,
+            f"Successfully merged LoRA weights (target: {target}, strength: {strength})",
+            "Failed to merge LoRA weights",
+        )
+
+    def list_loras(self) -> dict:
+        """List loaded LoRA adapters and current application status per module."""
+        output = self._send_lora_request(
+            req=ListLorasReq(),
+            success_msg="Successfully listed LoRA adapters",
+            failure_msg="Failed to list LoRA adapters",
+        )
+        # _send_lora_request already raises on error, so output.error is always None here
+        return output.output or {}
+
+    def _ensure_lora_state(
+        self,
+        lora_path: str | None,
+        lora_nickname: str | None = None,
+        merge_lora: bool = True,
+    ) -> None:
+        """
+        Ensure the LoRA state matches the desired configuration.
+
+        Note: This method does not cache client-side state. The server handles
+        idempotent operations, so redundant calls are safe but may have minor overhead.
+        """
+        if lora_path is None:
+            # Unmerge all LoRA weights when no lora_path is provided
+            self.unmerge_lora_weights()
+            return
+
+        lora_nickname = lora_nickname or self.server_args.lora_nickname
+
+        # Set the LoRA adapter (server handles idempotent logic)
+        self.set_lora(lora_nickname, lora_path)
+
+        # Merge or unmerge based on the merge_lora flag
+        if merge_lora:
+            self.merge_lora_weights()
+        else:
+            self.unmerge_lora_weights()
+
+    def generate_with_lora(
+        self,
+        prompt: str | list[str] | None = None,
+        sampling_params: SamplingParams | None = None,
+        *,
+        lora_path: str | None = None,
+        lora_nickname: str | None = None,
+        merge_lora: bool = True,
+        **kwargs,
+    ):
+        self._ensure_lora_state(
+            lora_path=lora_path, lora_nickname=lora_nickname, merge_lora=merge_lora
+        )
+        return self.generate(
+            sampling_params_kwargs=dict(
+                prompt=prompt,
+                sampling_params=sampling_params,
+                **kwargs,
+            )
+        )
+
+    def shutdown(self):
+        """
+        Shutdown the generator.
+        If in local mode, it also shuts down the scheduler server.
+        """
+        # sends the shutdown command to the server
+        if self.local_scheduler_process and self.owns_scheduler_client:
+            try:
+                sync_scheduler_client.forward(ShutdownReq(), timeout_ms=5000)
+            except Exception:
+                pass
+
+        if self.local_scheduler_process:
+            for process in self.local_scheduler_process:
+                process.join(timeout=10)
+                if process.is_alive():
+                    logger.warning(
+                        f"Local worker {process.name} did not terminate gracefully, forcing."
+                    )
+                    process.terminate()
+                    process.join(timeout=1)
+                    if process.is_alive():
+                        process.kill()
+                        process.join(timeout=1)
+            self.local_scheduler_process = None
+
+        if self.owns_scheduler_client:
+            sync_scheduler_client.close()
+            self.owns_scheduler_client = False
+
+    def _force_shutdown_local_processes(self) -> None:
+        local_scheduler_process = getattr(self, "local_scheduler_process", None)
+        log = globals().get("logger")
+        if local_scheduler_process:
+            for process in local_scheduler_process:
+                if process.is_alive():
+                    if log is not None:
+                        log.warning(
+                            f"Local worker {process.name} did not terminate gracefully, forcing."
+                        )
+                    process.terminate()
+            for process in local_scheduler_process:
+                process.join(timeout=1)
+                if process.is_alive():
+                    if log is not None:
+                        log.warning(
+                            f"Local worker {process.name} did not terminate after terminate(), killing."
+                        )
+                    process.kill()
+                    process.join(timeout=1)
+            self.local_scheduler_process = None
+
+        if getattr(self, "owns_scheduler_client", False):
+            try:
+                client = globals().get("sync_scheduler_client")
+                if client is not None:
+                    client.close()
+            finally:
+                self.owns_scheduler_client = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.shutdown()
+
+    def __del__(self):
+        owns_scheduler_client = bool(getattr(self, "owns_scheduler_client", False))
+        local_scheduler_process = getattr(self, "local_scheduler_process", None)
+        log = globals().get("logger")
+        if owns_scheduler_client:
+            if log is not None:
+                log.warning(
+                    "Generator was garbage collected without being shut down. "
+                    "Forcing local server and client cleanup."
+                )
+            self._force_shutdown_local_processes()
+        elif local_scheduler_process:
+            if log is not None:
+                log.warning(
+                    "Generator was garbage collected without being shut down. "
+                    "Forcing local server cleanup."
+                )
+            self._force_shutdown_local_processes()

python/sglang/multimodal_gen/runtime/entrypoints/http_server.py

@@ -0,0 +1,413 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+import asyncio
+import base64
+import os
+import signal
+import uuid
+from contextlib import asynccontextmanager, suppress
+from typing import TYPE_CHECKING
+
+import httpx
+import torch
+from fastapi import APIRouter, FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+
+from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    VertexGenerateReqInput,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime import (
+    realtime_video_api,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import build_sampling_params
+from sglang.multimodal_gen.runtime.entrypoints.post_training import (
+    rollout_api,
+    weights_api,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    prepare_request,
+    save_outputs,
+)
+from sglang.multimodal_gen.runtime.entrypoints.vla import api as vla_api
+from sglang.multimodal_gen.runtime.entrypoints.vla import openpi
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import ServerArgs, get_global_server_args
+from sglang.multimodal_gen.runtime.server_warmup import (
+    run_async_client_warmup,
+    should_run_synthetic_server_warmup,
+)
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.srt.utils.json_response import orjson_response
+from sglang.version import __version__
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+
+logger = init_logger(__name__)
+
+VERTEX_ROUTE = os.environ.get("AIP_PREDICT_ROUTE", "/vertex_generate")
+SERVER_WARMUP_BYPASS_PATHS = (
+    "/health",
+    "/health_generate",
+    "/model_info",
+    "/server_info",
+)
+
+
+async def _wait_until_http_ready(server_args: ServerArgs) -> None:
+    """for server warmup"""
+    health_url = f"{server_args.url()}/health"
+    # Probe the local server directly: a loopback readiness check must never be
+    # routed through an HTTP proxy. trust_env=False also avoids crashing startup
+    # on a malformed proxy env var, since httpx parses *_PROXY/NO_PROXY when the
+    # client is constructed (raising httpx.InvalidURL before any request). See #28493.
+    async with httpx.AsyncClient(trust_env=False) as client:
+        for _ in range(120):
+            try:
+                response = await client.get(health_url, timeout=5.0)
+                if response.status_code == 200:
+                    return
+            except httpx.HTTPError:
+                pass
+            await asyncio.sleep(1.0)
+    raise RuntimeError(f"HTTP server did not become ready at {health_url}")
+
+
+async def _run_server_warmup_after_http_ready(
+    server_args: ServerArgs, warmup_done: asyncio.Event
+) -> None:
+    try:
+        if not should_run_synthetic_server_warmup(server_args):
+            warmup_done.set()
+            return
+
+        await _wait_until_http_ready(server_args)
+
+        await run_async_client_warmup(
+            server_args,
+            async_scheduler_client.forward,
+            fail_open=server_args.warmup_resolutions is None,
+        )
+        logger.info("The server is fired up and ready to roll!")
+        warmup_done.set()
+    except asyncio.CancelledError:
+        raise
+    except Exception as e:
+        logger.error("Server warmup failed; aborting startup: %s", e, exc_info=True)
+        os.kill(os.getpid(), signal.SIGTERM)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    from sglang.multimodal_gen.runtime.scheduler_client import (
+        async_scheduler_client,
+        run_zeromq_broker,
+    )
+
+    # 1. Initialize the singleton client that connects to the backend Scheduler
+    server_args = app.state.server_args
+    async_scheduler_client.initialize(server_args)
+    warmup_done = asyncio.Event()
+    app.state.server_warmup_done = warmup_done
+
+    # 2. Start the ZMQ Broker in the background to handle offline requests
+    broker_task = asyncio.create_task(run_zeromq_broker(server_args))
+    warmup_task = None
+    if server_args.server_warmup:
+        warmup_task = asyncio.create_task(
+            _run_server_warmup_after_http_ready(server_args, warmup_done)
+        )
+    else:
+        warmup_done.set()
+
+    try:
+        yield
+    finally:
+        if warmup_task is not None and not warmup_task.done():
+            warmup_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await warmup_task
+
+        # On shutdown
+        logger.info("FastAPI app is shutting down...")
+        broker_task.cancel()
+        async_scheduler_client.close()
+
+
+# Health router
+health_router = APIRouter()
+
+
+@health_router.get("/health")
+async def health():
+    return {"status": "ok"}
+
+
+@health_router.get("/models", deprecated=True)
+async def get_models(request: Request):
+    """
+    Get information about the model served by this server.
+
+    .. deprecated::
+        Use /v1/models instead for OpenAI-compatible model discovery.
+        This endpoint will be removed in a future version.
+    """
+    from sglang.multimodal_gen.registry import get_model_info
+
+    server_args: ServerArgs = request.app.state.server_args
+    model_info = get_model_info(server_args.model_path, model_id=server_args.model_id)
+
+    response = {
+        "model_path": server_args.model_path,
+        "num_gpus": server_args.num_gpus,
+        "task_type": server_args.pipeline_config.task_type.name,
+        "dit_precision": server_args.pipeline_config.dit_precision,
+        "vae_precision": server_args.pipeline_config.vae_precision,
+    }
+
+    if model_info:
+        response["pipeline_name"] = model_info.pipeline_cls.pipeline_name
+        response["pipeline_class"] = model_info.pipeline_cls.__name__
+
+    return response
+
+
+@health_router.get("/server_info")
+async def server_info_endpoint(request: Request):
+    """Get server information.
+
+    Returns fields compatible with the LLM engine's /server_info so that
+    the model gateway can discover diffusion workers.
+    """
+    server_args: ServerArgs = request.app.state.server_args
+
+    return {
+        "model_path": server_args.model_path,
+        "served_model_name": server_args.model_id or server_args.model_path,
+        "tp_size": server_args.tp_size,
+        "dp_size": server_args.dp_size,
+        "version": __version__,
+    }
+
+
+@health_router.get("/model_info")
+async def model_info_endpoint(request: Request):
+    """Get model information.
+
+    Returns fields compatible with the LLM engine's /model_info so that
+    the model gateway can detect capabilities for diffusion workers.
+    """
+    from sglang.multimodal_gen.registry import get_model_info
+
+    server_args: ServerArgs = request.app.state.server_args
+    task_type = server_args.pipeline_config.task_type
+
+    try:
+        registry_info = get_model_info(
+            server_args.model_path,
+            backend=server_args.backend,
+            model_id=server_args.model_id,
+        )
+    except Exception:
+        logger.warning("Failed to resolve model info from registry", exc_info=True)
+        registry_info = None
+
+    return {
+        # Fields consumed by the model gateway for worker discovery
+        "model_path": server_args.model_path,
+        "is_generation": True,
+        "model_type": "diffusion",
+        "architectures": (
+            [registry_info.pipeline_cls.__name__] if registry_info else None
+        ),
+        # Fields matching the LLM engine's /model_info shape
+        "has_image_understanding": task_type.accepts_image_input(),
+        "has_audio_understanding": False,
+        # Diffusion-specific fields
+        "task_type": task_type.name,
+        "is_image_gen": task_type.is_image_gen(),
+    }
+
+
+@health_router.get("/health_generate")
+async def health_generate():
+    # TODO : health generate endpoint
+    return {"status": "ok"}
+
+
+@health_router.get("/stats")
+async def stats_endpoint(request: Request):
+    """Get runtime statistics including disagg pipeline metrics.
+
+    Returns queue depth, request counts, latency, throughput, etc.
+    Sends a GetDisaggStatsReq to the scheduler via ZMQ and returns the result.
+    """
+    from sglang.multimodal_gen.runtime.entrypoints.utils import GetDisaggStatsReq
+
+    server_args: ServerArgs = request.app.state.server_args
+    response: dict = {
+        "status": "ok",
+        "model_path": server_args.model_path,
+    }
+
+    # Query the scheduler for disagg metrics
+    try:
+        stats_response = await async_scheduler_client.forward(GetDisaggStatsReq())
+        if hasattr(stats_response, "output") and stats_response.output is not None:
+            response["disagg"] = stats_response.output
+    except Exception as e:
+        response["disagg"] = {"error": str(e)}
+
+    return response
+
+
+def make_serializable(obj):
+    """Recursively converts Tensors to None for JSON serialization."""
+    if isinstance(obj, torch.Tensor):
+        return None
+    if isinstance(obj, dict):
+        return {k: make_serializable(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [make_serializable(v) for v in obj]
+    return obj
+
+
+def encode_video_to_base64(file_path: str):
+    if not os.path.exists(file_path):
+        return None
+    with open(file_path, "rb") as f:
+        return base64.b64encode(f.read()).decode("utf-8")
+
+
+async def forward_to_scheduler(
+    req_obj: "Req",
+    sp: SamplingParams,
+):
+    """Forwards request to scheduler and processes the result."""
+    try:
+        response = await async_scheduler_client.forward(req_obj)
+        if response.output is None and response.output_file_paths is None:
+            raise RuntimeError("Model generation returned no output.")
+
+        if response.output_file_paths:
+            output_file_path = response.output_file_paths[0]
+        else:
+            output_file_path = sp.output_file_path()
+            save_outputs(
+                [response.output[0]],
+                sp.data_type,
+                sp.fps,
+                True,
+                lambda _idx: output_file_path,
+                audio=response.audio,
+                audio_sample_rate=response.audio_sample_rate,
+                enable_frame_interpolation=sp.enable_frame_interpolation,
+                frame_interpolation_exp=sp.frame_interpolation_exp,
+                frame_interpolation_scale=sp.frame_interpolation_scale,
+                frame_interpolation_model_path=sp.frame_interpolation_model_path,
+                enable_upscaling=sp.enable_upscaling,
+                upscaling_model_path=sp.upscaling_model_path,
+                upscaling_scale=sp.upscaling_scale,
+            )
+
+        if hasattr(response, "model_dump"):
+            data = response.model_dump()
+        else:
+            data = response if isinstance(response, dict) else vars(response)
+
+        if output_file_path:
+            logger.info("Processing output file: %s", output_file_path)
+            b64_video = encode_video_to_base64(output_file_path)
+
+            if b64_video:
+                data["output"] = b64_video
+                data.pop("video_data", None)
+                data.pop("video_tensor", None)
+
+        return make_serializable(data)
+
+    except Exception as e:
+        logger.error("Error during generation: %s", e, exc_info=True)
+        return {"error": str(e)}
+
+
+vertex_router = APIRouter()
+
+
+@vertex_router.post(VERTEX_ROUTE)
+async def vertex_generate(vertex_req: VertexGenerateReqInput):
+    if not vertex_req.instances:
+        return orjson_response({"predictions": []})
+
+    server_args = get_global_server_args()
+    params = vertex_req.parameters or {}
+
+    futures = []
+
+    for inst in vertex_req.instances:
+        rid = f"vertex_{uuid.uuid4()}"
+
+        sp = build_sampling_params(
+            rid,
+            prompt=inst.get("prompt") or inst.get("text"),
+            image_path=inst.get("image") or inst.get("image_url"),
+            num_frames=params.get("num_frames"),
+            fps=params.get("fps"),
+            width=params.get("width"),
+            height=params.get("height"),
+            guidance_scale=params.get("guidance_scale"),
+            save_output=params.get("save_output"),
+        )
+
+        backend_req = prepare_request(server_args, sampling_params=sp)
+        futures.append(forward_to_scheduler(backend_req, sp))
+
+    results = await asyncio.gather(*futures)
+
+    return orjson_response({"predictions": results})
+
+
+def create_app(server_args: ServerArgs):
+    """
+    Create and configure the FastAPI application instance.
+    """
+    app = FastAPI(lifespan=lifespan)
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    @app.middleware("http")
+    async def wait_for_server_warmup(request: Request, call_next):
+        warmup_done = getattr(request.app.state, "server_warmup_done", None)
+        if (
+            warmup_done is not None
+            and not warmup_done.is_set()
+            and request.url.path not in SERVER_WARMUP_BYPASS_PATHS
+        ):
+            await warmup_done.wait()
+        return await call_next(request)
+
+    app.include_router(health_router)
+    app.include_router(vertex_router)
+
+    from sglang.multimodal_gen.runtime.entrypoints.openai import common_api, mesh_api
+
+    app.include_router(common_api.router)
+    app.include_router(image_api.router)
+    app.include_router(video_api.router)
+    app.include_router(realtime_video_api.router)
+    if server_args.pipeline_config.task_type.is_action_gen():
+        app.include_router(vla_api.router)
+        app.include_router(openpi.router)
+    app.include_router(mesh_api.router)
+    app.include_router(weights_api.router)
+    app.include_router(rollout_api.router)
+
+    app.state.server_args = server_args
+    return app

python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py

@@ -0,0 +1,252 @@
+import time
+from typing import Any, List, Optional, Union
+
+from fastapi import APIRouter, Body, HTTPException
+from pydantic import BaseModel, Field
+
+from sglang.multimodal_gen.registry import get_model_info
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    ListLorasReq,
+    MergeLoraWeightsReq,
+    SetLoraReq,
+    UnmergeLoraWeightsReq,
+    format_lora_message,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.srt.utils.json_response import orjson_response
+
+router = APIRouter(prefix="/v1")
+logger = init_logger(__name__)
+
+
+class ModelCard(BaseModel):
+    """Model cards."""
+
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "sglang"
+    root: Optional[str] = None
+    parent: Optional[str] = None
+    max_model_len: Optional[int] = None
+
+
+class DiffusionModelCard(ModelCard):
+    """Extended ModelCard with diffusion-specific fields."""
+
+    num_gpus: Optional[int] = None
+    task_type: Optional[str] = None
+    dit_precision: Optional[str] = None
+    vae_precision: Optional[str] = None
+    pipeline_name: Optional[str] = None
+    pipeline_class: Optional[str] = None
+
+
+async def _handle_lora_request(req: Any, success_msg: str, failure_msg: str):
+    try:
+        output: OutputBatch = await async_scheduler_client.forward(req)
+        if output.error is None:
+            return {"status": "ok", "message": success_msg}
+        else:
+            error_msg = output.error
+            raise HTTPException(status_code=500, detail=f"{failure_msg}: {error_msg}")
+    except Exception as e:
+        if isinstance(e, HTTPException):
+            raise
+        logger.error(f"Error during '{failure_msg}': {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/set_lora")
+async def set_lora(
+    lora_nickname: Union[str, List[str]] = Body(..., embed=True),
+    lora_path: Optional[Union[str, List[Optional[str]]]] = Body(None, embed=True),
+    target: Union[str, List[str]] = Body("all", embed=True),
+    strength: Union[float, List[float]] = Body(1.0, embed=True),
+    merge_mode: Optional[str] = Body(None, embed=True),
+):
+    """
+    Set LoRA adapter(s) for the specified transformer(s).
+    Supports both single LoRA (backward compatible) and multiple LoRA adapters.
+
+    Args:
+        lora_nickname: The nickname(s) of the adapter(s). Can be a string or a list of strings.
+        lora_path: Path(s) to the LoRA adapter(s) (local path or HF repo id).
+            Can be a string, None, or a list of strings/None. Must match the length of lora_nickname.
+        target: Which transformer(s) to apply the LoRA to. Can be a string or a list of strings.
+            If a list, must match the length of lora_nickname. Valid values:
+            - "all": Apply to all transformers (default)
+            - "transformer": Apply only to the primary transformer (high noise for Wan2.2)
+            - "transformer_2": Apply only to transformer_2 (low noise for Wan2.2)
+            - "critic": Apply only to the critic model
+        strength: LoRA strength(s) for merge, default 1.0. Can be a float or a list of floats.
+            If a list, must match the length of lora_nickname. Values < 1.0 reduce the effect,
+            values > 1.0 amplify the effect.
+        merge_mode: Optional LoRA merge mode: "auto", "merge", or "dynamic".
+    """
+    req = SetLoraReq(
+        lora_nickname=lora_nickname,
+        lora_path=lora_path,
+        target=target,
+        strength=strength,
+        merge_mode=merge_mode,
+    )
+    nickname_str, target_str, strength_str = format_lora_message(
+        lora_nickname, target, strength
+    )
+
+    return await _handle_lora_request(
+        req,
+        f"Successfully set LoRA adapter(s): {nickname_str} (target: {target_str}, strength: {strength_str})",
+        "Failed to set LoRA adapter",
+    )
+
+
+@router.post("/merge_lora_weights")
+async def merge_lora_weights(
+    target: str = Body("all", embed=True),
+    strength: float = Body(1.0, embed=True),
+):
+    """
+    Merge LoRA weights into the base model.
+
+    Args:
+        target: Which transformer(s) to merge. One of "all", "transformer",
+                "transformer_2", "critic".
+        strength: LoRA strength for merge, default 1.0. Values < 1.0 reduce the effect,
+            values > 1.0 amplify the effect.
+    """
+    req = MergeLoraWeightsReq(target=target, strength=strength)
+    return await _handle_lora_request(
+        req,
+        f"Successfully merged LoRA weights (target: {target}, strength: {strength})",
+        "Failed to merge LoRA weights",
+    )
+
+
+@router.post("/unmerge_lora_weights")
+async def unmerge_lora_weights(
+    target: str = Body("all", embed=True),
+):
+    """
+    Unmerge LoRA weights from the base model.
+
+    Args:
+        target: Which transformer(s) to unmerge. One of "all", "transformer",
+                "transformer_2", "critic".
+    """
+    req = UnmergeLoraWeightsReq(target=target)
+    return await _handle_lora_request(
+        req,
+        f"Successfully unmerged LoRA weights (target: {target})",
+        "Failed to unmerge LoRA weights",
+    )
+
+
+@router.get("/model_info")
+async def model_info():
+    """Get the model information."""
+    server_args = get_global_server_args()
+    if not server_args:
+        raise HTTPException(status_code=500, detail="Server args not initialized")
+
+    result = {
+        "model_path": server_args.model_path,
+    }
+    return result
+
+
+@router.get("/list_loras")
+async def list_loras():
+    """List loaded LoRA adapters and current application status per module."""
+    try:
+        req = ListLorasReq()
+        output: OutputBatch = await async_scheduler_client.forward(req)
+        if output.error is None:
+            return output.output or {}
+        else:
+            raise HTTPException(status_code=500, detail=output.error)
+    except Exception as e:
+        if isinstance(e, HTTPException):
+            raise
+        logger.error(f"Error during 'list_loras': {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/models")
+async def available_models():
+    """Show available models. OpenAI-compatible endpoint with extended diffusion info."""
+    server_args = get_global_server_args()
+    if not server_args:
+        raise HTTPException(status_code=500, detail="Server args not initialized")
+
+    model_info = get_model_info(
+        server_args.model_path,
+        backend=server_args.backend,
+        model_id=server_args.model_id,
+    )
+
+    card_kwargs = {
+        "id": server_args.model_path,
+        "root": server_args.model_path,
+        # Extended diffusion-specific fields
+        "num_gpus": server_args.num_gpus,
+        "task_type": server_args.pipeline_config.task_type.name,
+        "dit_precision": server_args.pipeline_config.dit_precision,
+        "vae_precision": server_args.pipeline_config.vae_precision,
+    }
+
+    if model_info:
+        card_kwargs["pipeline_name"] = model_info.pipeline_cls.pipeline_name
+        card_kwargs["pipeline_class"] = model_info.pipeline_cls.__name__
+
+    model_card = DiffusionModelCard(**card_kwargs)
+
+    # Return dict directly to preserve extended fields (ModelList strips them)
+    return {"object": "list", "data": [model_card.model_dump()]}
+
+
+@router.get("/models/{model:path}")
+async def retrieve_model(model: str):
+    """Retrieve a model instance. OpenAI-compatible endpoint with extended diffusion info."""
+    server_args = get_global_server_args()
+    if not server_args:
+        raise HTTPException(status_code=500, detail="Server args not initialized")
+
+    if model != server_args.model_path:
+        return orjson_response(
+            {
+                "error": {
+                    "message": f"The model '{model}' does not exist",
+                    "type": "invalid_request_error",
+                    "param": "model",
+                    "code": "model_not_found",
+                }
+            },
+            status_code=404,
+        )
+
+    model_info = get_model_info(
+        server_args.model_path,
+        backend=server_args.backend,
+        model_id=server_args.model_id,
+    )
+
+    card_kwargs = {
+        "id": model,
+        "root": model,
+        "num_gpus": server_args.num_gpus,
+        "task_type": server_args.pipeline_config.task_type.name,
+        "dit_precision": server_args.pipeline_config.dit_precision,
+        "vae_precision": server_args.pipeline_config.vae_precision,
+    }
+
+    if model_info:
+        card_kwargs["pipeline_name"] = model_info.pipeline_cls.pipeline_name
+        card_kwargs["pipeline_class"] = model_info.pipeline_cls.__name__
+
+    # Return dict to preserve extended fields
+    return DiffusionModelCard(**card_kwargs).model_dump()

python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py

@@ -0,0 +1,447 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+import base64
+import contextlib
+import json
+import os
+import time
+from typing import Any, List, Optional
+
+from fastapi import (
+    APIRouter,
+    File,
+    Form,
+    HTTPException,
+    Path,
+    Query,
+    Request,
+    UploadFile,
+)
+from fastapi.responses import FileResponse
+
+from sglang.multimodal_gen.configs.sample.sampling_params import generate_request_id
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    ImageGenerationsRequest,
+    ImageResponse,
+    ImageResponseData,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.storage import cloud_storage
+from sglang.multimodal_gen.runtime.entrypoints.openai.stores import IMAGE_STORE
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    add_common_data_to_response,
+    build_sampling_params,
+    choose_output_image_ext,
+    flatten_extra_params,
+    merge_image_input_list,
+    process_generation_batch,
+    save_image_to_path,
+    temp_dir_if_disabled,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.srt.observability.trace import extract_trace_headers
+
+router = APIRouter(prefix="/v1/images", tags=["images"])
+
+
+def _get_extra_field(request, field_name):
+    """Get a field from model_extra, with fallback to nested extra_body dict."""
+    extra = request.model_extra or {}
+    value = extra.get(field_name)
+    if value is not None:
+        return value
+    if field_name == "use_guardrails" and extra.get("guardrails") is not None:
+        return extra["guardrails"]
+
+    for container_name in ("extra_body", "extra_json", "extra_args", "extra_params"):
+        value = _parse_extra_container(extra.get(container_name)).get(field_name)
+        if value is not None:
+            return value
+
+    return value
+
+
+def _parse_extra_container(value: Any) -> dict[str, Any]:
+    if isinstance(value, str):
+        try:
+            value = json.loads(value)
+        except Exception:
+            return {}
+    if isinstance(value, dict):
+        return flatten_extra_params(dict(value))
+    return {}
+
+
+def _read_b64_for_paths(paths: list[str]) -> list[str]:
+    """Read and base64-encode each file. Must be called before cloud upload deletes them."""
+    result = []
+    for path in paths:
+        with open(path, "rb") as f:
+            result.append(base64.b64encode(f.read()).decode("utf-8"))
+    return result
+
+
+def _build_image_response_kwargs(
+    save_file_path_list: list[str],
+    resp_format: str,
+    prompt: str,
+    request_id: str,
+    result: OutputBatch,
+    *,
+    b64_list: list[str] | None = None,
+    cloud_url: str | None = None,
+    fallback_url: str | None = None,
+    is_persistent: bool = True,
+) -> dict:
+    """Build ImageResponse data list.
+
+    For b64_json: uses pre-read b64_list (call _read_b64_for_paths first).
+    For url: uses cloud_url or fallback_url.
+    file_path is omitted when is_persistent=False to avoid exposing stale temp paths.
+    """
+    ret = None
+    if resp_format == "b64_json":
+        if not b64_list:
+            raise ValueError("b64_list required for b64_json response_format")
+        data = [
+            ImageResponseData(
+                b64_json=b64,
+                revised_prompt=prompt,
+                file_path=os.path.abspath(path) if is_persistent else None,
+            )
+            for b64, path in zip(b64_list, save_file_path_list)
+        ]
+        ret = {"data": data}
+    elif resp_format == "url":
+        url = cloud_url or fallback_url
+        if not url:
+            raise HTTPException(
+                status_code=400,
+                detail="response_format='url' requires cloud storage to be configured.",
+            )
+        ret = {
+            "data": [
+                ImageResponseData(
+                    url=url,
+                    revised_prompt=prompt,
+                    file_path=(
+                        os.path.abspath(save_file_path_list[0])
+                        if is_persistent
+                        else None
+                    ),
+                )
+            ],
+        }
+    else:
+        raise HTTPException(
+            status_code=400, detail=f"response_format={resp_format} is not supported"
+        )
+
+    ret = add_common_data_to_response(ret, request_id=request_id, result=result)
+
+    return ret
+
+
+@router.post("/generations", response_model=ImageResponse)
+async def generations(
+    request: ImageGenerationsRequest,
+    raw_request: Request,
+):
+    request_id = generate_request_id()
+    server_args = get_global_server_args()
+    is_cosmos3 = "cosmos3" in (server_args.model_path or "").lower()
+    ext = (
+        "png"
+        if is_cosmos3 and request.output_format is None
+        else choose_output_image_ext(request.output_format, request.background)
+    )
+
+    with temp_dir_if_disabled(server_args.output_path) as output_dir:
+        sampling = build_sampling_params(
+            request_id,
+            prompt=request.prompt,
+            size=request.size,
+            width=request.width,
+            height=request.height,
+            num_outputs_per_prompt=max(1, min(int(request.n or 1), 10)),
+            output_file_name=f"{request_id}.{ext}",
+            output_path=output_dir,
+            num_frames=1,
+            seed=request.seed,
+            generator_device=request.generator_device,
+            num_inference_steps=request.num_inference_steps,
+            guidance_scale=request.guidance_scale,
+            true_cfg_scale=request.true_cfg_scale,
+            negative_prompt=request.negative_prompt,
+            max_sequence_length=(
+                request.max_sequence_length
+                if request.max_sequence_length is not None
+                else _get_extra_field(request, "max_sequence_length")
+            ),
+            flow_shift=(
+                request.flow_shift
+                if request.flow_shift is not None
+                else _get_extra_field(request, "flow_shift")
+            ),
+            use_duration_template=_get_extra_field(request, "use_duration_template"),
+            use_resolution_template=_get_extra_field(
+                request, "use_resolution_template"
+            ),
+            use_system_prompt=_get_extra_field(request, "use_system_prompt"),
+            use_guardrails=_get_extra_field(request, "use_guardrails"),
+            enable_teacache=request.enable_teacache,
+            output_compression=request.output_compression,
+            output_quality=request.output_quality,
+            diffusers_kwargs=request.diffusers_kwargs,
+            enable_upscaling=request.enable_upscaling,
+            upscaling_model_path=request.upscaling_model_path,
+            upscaling_scale=request.upscaling_scale,
+            perf_dump_path=request.perf_dump_path,
+            use_pe=_get_extra_field(request, "use_pe"),
+            preset=_get_extra_field(request, "preset"),
+            progressive_mode=(
+                request.progressive_mode
+                if request.progressive_mode is not None
+                else _get_extra_field(request, "progressive_mode")
+            ),
+            progressive_levels=(
+                request.progressive_levels
+                if request.progressive_levels is not None
+                else _get_extra_field(request, "progressive_levels")
+            ),
+            progressive_delta=(
+                request.progressive_delta
+                if request.progressive_delta is not None
+                else _get_extra_field(request, "progressive_delta")
+            ),
+        )
+        trace_headers = extract_trace_headers(raw_request.headers)
+        batch = prepare_request(
+            server_args=server_args,
+            sampling_params=sampling,
+            external_trace_header=trace_headers,
+        )
+        # Add diffusers_kwargs if provided
+        if request.diffusers_kwargs:
+            batch.extra["diffusers_kwargs"] = request.diffusers_kwargs
+
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+        resp_format = (request.response_format or "b64_json").lower()
+        if (
+            is_cosmos3
+            and "response_format" not in request.model_fields_set
+            and request.response_format == "url"
+        ):
+            resp_format = "b64_json"
+
+        # read b64 before cloud upload may delete the local file
+        b64_list = (
+            _read_b64_for_paths(save_file_path_list)
+            if resp_format == "b64_json"
+            else None
+        )
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        is_persistent = server_args.output_path is not None
+        await IMAGE_STORE.upsert(
+            request_id,
+            {
+                "id": request_id,
+                "created_at": int(time.time()),
+                "file_path": None if cloud_url or not is_persistent else save_file_path,
+                "url": cloud_url,
+            },
+        )
+
+        response_kwargs = _build_image_response_kwargs(
+            save_file_path_list,
+            resp_format,
+            request.prompt,
+            request_id,
+            result,
+            b64_list=b64_list,
+            cloud_url=cloud_url,
+            fallback_url=f"/v1/images/{request_id}/content" if is_persistent else None,
+            is_persistent=is_persistent,
+        )
+
+    return ImageResponse(**response_kwargs)
+
+
+@router.post("/edits", response_model=ImageResponse)
+async def edits(
+    raw_request: Request,
+    image: Optional[List[UploadFile]] = File(None),
+    image_array: Optional[List[UploadFile]] = File(None, alias="image[]"),
+    url: Optional[List[str]] = Form(None),
+    url_array: Optional[List[str]] = Form(None, alias="url[]"),
+    prompt: str = Form(...),
+    mask: Optional[UploadFile] = File(None),
+    model: Optional[str] = Form(None),
+    n: Optional[int] = Form(1),
+    response_format: Optional[str] = Form(None),
+    size: Optional[str] = Form(None),
+    output_format: Optional[str] = Form(None),
+    background: Optional[str] = Form("auto"),
+    seed: Optional[int] = Form(None),
+    generator_device: Optional[str] = Form("cuda"),
+    user: Optional[str] = Form(None),
+    negative_prompt: Optional[str] = Form(None),
+    guidance_scale: Optional[float] = Form(None),
+    true_cfg_scale: Optional[float] = Form(None),
+    num_inference_steps: Optional[int] = Form(None),
+    output_quality: Optional[str] = Form("default"),
+    output_compression: Optional[int] = Form(None),
+    enable_teacache: Optional[bool] = Form(False),
+    enable_upscaling: Optional[bool] = Form(False),
+    upscaling_model_path: Optional[str] = Form(None),
+    upscaling_scale: Optional[int] = Form(4),
+    num_frames: int = Form(1),
+):
+    request_id = generate_request_id()
+    server_args = get_global_server_args()
+    # Resolve images from either `image` or `image[]` (OpenAI SDK sends `image[]` when list is provided)
+    images = image or image_array
+    urls = url or url_array
+
+    if (not images or len(images) == 0) and (not urls or len(urls) == 0):
+        raise HTTPException(
+            status_code=422, detail="Field 'image' or 'url' is required"
+        )
+
+    image_list = merge_image_input_list(images, urls)
+
+    with contextlib.ExitStack() as stack:
+        uploads_dir = stack.enter_context(
+            temp_dir_if_disabled(server_args.input_save_path)
+        )
+        output_dir = stack.enter_context(temp_dir_if_disabled(server_args.output_path))
+
+        input_paths = []
+        try:
+            for idx, img in enumerate(image_list):
+                filename = img.filename if hasattr(img, "filename") else f"image_{idx}"
+                input_path = await save_image_to_path(
+                    img,
+                    os.path.join(uploads_dir, f"{request_id}_{idx}_{filename}"),
+                    prefer_remote_source=server_args.input_save_path is None,
+                )
+                input_paths.append(input_path)
+        except Exception as e:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Failed to process image source: {str(e)}",
+            )
+
+        ext = choose_output_image_ext(output_format, background)
+        sampling = build_sampling_params(
+            request_id,
+            prompt=prompt,
+            size=size,
+            num_outputs_per_prompt=max(1, min(int(n or 1), 10)),
+            output_file_name=f"{request_id}.{ext}",
+            output_path=output_dir,
+            image_path=input_paths,
+            seed=seed,
+            generator_device=generator_device,
+            negative_prompt=negative_prompt,
+            guidance_scale=guidance_scale,
+            true_cfg_scale=true_cfg_scale,
+            num_inference_steps=num_inference_steps,
+            enable_teacache=enable_teacache,
+            num_frames=num_frames,
+            output_compression=output_compression,
+            output_quality=output_quality,
+            enable_upscaling=enable_upscaling,
+            upscaling_model_path=upscaling_model_path,
+            upscaling_scale=upscaling_scale,
+        )
+        trace_headers = extract_trace_headers(raw_request.headers)
+        batch = prepare_request(
+            server_args=server_args,
+            sampling_params=sampling,
+            external_trace_header=trace_headers,
+        )
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+        resp_format = (response_format or "b64_json").lower()
+
+        # read b64 before cloud upload may delete the local file
+        b64_list = (
+            _read_b64_for_paths(save_file_path_list)
+            if resp_format == "b64_json"
+            else None
+        )
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        is_persistent = server_args.output_path is not None
+        is_input_persistent = server_args.input_save_path is not None
+        await IMAGE_STORE.upsert(
+            request_id,
+            {
+                "id": request_id,
+                "created_at": int(time.time()),
+                "file_path": None if cloud_url or not is_persistent else save_file_path,
+                "url": cloud_url,
+                "input_image_paths": input_paths if is_input_persistent else None,
+                "num_input_images": len(input_paths),
+            },
+        )
+
+        response_kwargs = _build_image_response_kwargs(
+            save_file_path_list,
+            resp_format,
+            prompt,
+            request_id,
+            result,
+            b64_list=b64_list,
+            cloud_url=cloud_url,
+            fallback_url=f"/v1/images/{request_id}/content" if is_persistent else None,
+            is_persistent=is_persistent,
+        )
+
+    return ImageResponse(**response_kwargs)
+
+
+@router.get("/{image_id}/content")
+async def download_image_content(
+    image_id: str = Path(...), variant: Optional[str] = Query(None)
+):
+    item = await IMAGE_STORE.get(image_id)
+    if not item:
+        raise HTTPException(status_code=404, detail="Image not found")
+
+    if item.get("url"):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Image has been uploaded to cloud storage. Please use the cloud URL: {item.get('url')}",
+        )
+
+    file_path = item.get("file_path")
+    if not file_path:
+        raise HTTPException(
+            status_code=404,
+            detail="Image was not persisted on disk (output_path is disabled). Use b64_json response_format or configure cloud storage.",
+        )
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="Image is still being generated")
+
+    ext = os.path.splitext(file_path)[1].lower()
+    media_type = "image/jpeg"
+    if ext == ".png":
+        media_type = "image/png"
+    elif ext == ".webp":
+        media_type = "image/webp"
+
+    return FileResponse(
+        path=file_path, media_type=media_type, filename=os.path.basename(file_path)
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/mesh_api.py

@@ -0,0 +1,296 @@
+import asyncio
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from fastapi import (
+    APIRouter,
+    File,
+    Form,
+    HTTPException,
+    Path,
+    Query,
+    Request,
+    UploadFile,
+)
+from fastapi.responses import FileResponse
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    SamplingParams,
+    generate_request_id,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    MeshGenerationsRequest,
+    MeshListResponse,
+    MeshResponse,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.storage import cloud_storage
+from sglang.multimodal_gen.runtime.entrypoints.openai.stores import MESH_STORE
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    add_common_data_to_response,
+    merge_image_input_list,
+    process_generation_batch,
+    save_image_to_path,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+router = APIRouter(prefix="/v1/meshes", tags=["meshes"])
+
+
+def _normalize_format(fmt: Optional[str]) -> str:
+    fmt = (fmt or "glb").lower()
+    return fmt if fmt in ("glb", "obj") else "glb"
+
+
+def _build_sampling_params_from_request(
+    request_id: str, req: MeshGenerationsRequest, image_path: Optional[str] = None
+) -> SamplingParams:
+    ext = _normalize_format(req.output_format)
+
+    server_args = get_global_server_args()
+    sampling_kwargs: Dict[str, Any] = {
+        "request_id": request_id,
+        "prompt": req.prompt,
+        "num_frames": 1,
+        "image_path": [image_path] if image_path else None,
+        "save_output": True,
+        "output_file_name": f"{request_id}.{ext}",
+        "seed": req.seed,
+        "generator_device": req.generator_device,
+    }
+    if req.num_inference_steps is not None:
+        sampling_kwargs["num_inference_steps"] = req.num_inference_steps
+    if req.guidance_scale is not None:
+        sampling_kwargs["guidance_scale"] = req.guidance_scale
+    if req.negative_prompt is not None:
+        sampling_kwargs["negative_prompt"] = req.negative_prompt
+
+    return SamplingParams.from_user_sampling_params_args(
+        model_path=server_args.model_path,
+        server_args=server_args,
+        **sampling_kwargs,
+    )
+
+
+def _mesh_job_from_sampling(
+    request_id: str, req: MeshGenerationsRequest, sampling: SamplingParams
+) -> Dict[str, Any]:
+    return {
+        "id": request_id,
+        "object": "mesh",
+        "model": req.model or "",
+        "status": "queued",
+        "progress": 0,
+        "created_at": int(time.time()),
+        "format": _normalize_format(req.output_format),
+        "file_path": os.path.abspath(sampling.output_file_path()),
+    }
+
+
+async def _dispatch_job_async(job_id: str, batch: Req) -> None:
+    from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+
+    try:
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+
+        file_size = None
+        if os.path.exists(save_file_path):
+            file_size = os.path.getsize(save_file_path)
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        update_fields: Dict[str, Any] = {
+            "status": "completed",
+            "progress": 100,
+            "completed_at": int(time.time()),
+            "url": cloud_url,
+            "file_path": save_file_path if not cloud_url else None,
+            "file_size_bytes": file_size,
+        }
+        update_fields = add_common_data_to_response(
+            update_fields, request_id=job_id, result=result
+        )
+        await MESH_STORE.update_fields(job_id, update_fields)
+    except Exception as e:
+        logger.error(f"{e}")
+        await MESH_STORE.update_fields(
+            job_id, {"status": "failed", "error": {"message": str(e)}}
+        )
+
+
+@router.post("", response_model=MeshResponse)
+async def create_mesh(
+    request: Request,
+    image: Optional[List[UploadFile]] = File(None),
+    image_array: Optional[List[UploadFile]] = File(None, alias="image[]"),
+    url: Optional[List[str]] = Form(None),
+    url_array: Optional[List[str]] = Form(None, alias="url[]"),
+    prompt: Optional[str] = Form("generate 3d mesh"),
+    model: Optional[str] = Form(None),
+    seed: Optional[int] = Form(None),
+    generator_device: Optional[str] = Form("cuda"),
+    guidance_scale: Optional[float] = Form(None),
+    num_inference_steps: Optional[int] = Form(None),
+    negative_prompt: Optional[str] = Form(None),
+    output_format: Optional[str] = Form("glb"),
+):
+    content_type = request.headers.get("content-type", "").lower()
+    request_id = generate_request_id()
+    server_args = get_global_server_args()
+
+    input_path = None
+
+    if "multipart/form-data" in content_type:
+        images = image or image_array
+        urls = url or url_array
+        image_list = merge_image_input_list(images, urls)
+
+        if not image_list:
+            raise HTTPException(
+                status_code=422,
+                detail="Field 'image' or 'url' is required for mesh generation",
+            )
+
+        uploads_dir = os.path.join("outputs", "uploads")
+        os.makedirs(uploads_dir, exist_ok=True)
+        img = image_list[0]
+        filename = img.filename if hasattr(img, "filename") else "input_image"
+        try:
+            input_path = await save_image_to_path(
+                img, os.path.join(uploads_dir, f"{request_id}_{filename}")
+            )
+        except Exception as e:
+            raise HTTPException(
+                status_code=400, detail=f"Failed to process image source: {str(e)}"
+            )
+
+        req = MeshGenerationsRequest(
+            prompt=prompt or "generate 3d mesh",
+            model=model,
+            seed=seed,
+            generator_device=generator_device,
+            num_inference_steps=num_inference_steps,
+            negative_prompt=negative_prompt,
+            output_format=output_format,
+            **(
+                {"guidance_scale": guidance_scale} if guidance_scale is not None else {}
+            ),
+        )
+    else:
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        try:
+            payload: Dict[str, Any] = dict(body or {})
+
+            if payload.get("input_image"):
+                img_src = payload.pop("input_image")
+                uploads_dir = os.path.join("outputs", "uploads")
+                os.makedirs(uploads_dir, exist_ok=True)
+                input_path = await save_image_to_path(
+                    img_src,
+                    os.path.join(uploads_dir, f"{request_id}_input_image"),
+                )
+
+            req = MeshGenerationsRequest(**payload)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Invalid request body: {e}")
+
+    if not input_path:
+        raise HTTPException(
+            status_code=422,
+            detail="An input image is required for mesh generation",
+        )
+
+    sampling_params = _build_sampling_params_from_request(request_id, req, input_path)
+    job = _mesh_job_from_sampling(request_id, req, sampling_params)
+    await MESH_STORE.upsert(request_id, job)
+
+    batch = prepare_request(
+        server_args=server_args,
+        sampling_params=sampling_params,
+    )
+
+    asyncio.create_task(_dispatch_job_async(request_id, batch))
+    return MeshResponse(**job)
+
+
+@router.get("", response_model=MeshListResponse)
+async def list_meshes(
+    after: Optional[str] = Query(None),
+    limit: Optional[int] = Query(None, ge=1, le=100),
+    order: Optional[str] = Query("desc"),
+):
+    order = (order or "desc").lower()
+    if order not in ("asc", "desc"):
+        order = "desc"
+    jobs = await MESH_STORE.list_values()
+
+    reverse = order != "asc"
+    jobs.sort(key=lambda j: j.get("created_at", 0), reverse=reverse)
+
+    if after is not None:
+        try:
+            idx = next(i for i, j in enumerate(jobs) if j["id"] == after)
+            jobs = jobs[idx + 1 :]
+        except StopIteration:
+            jobs = []
+
+    if limit is not None:
+        jobs = jobs[:limit]
+    items = [MeshResponse(**j) for j in jobs]
+    return MeshListResponse(data=items)
+
+
+@router.get("/{mesh_id}", response_model=MeshResponse)
+async def retrieve_mesh(mesh_id: str = Path(...)):
+    job = await MESH_STORE.get(mesh_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Mesh not found")
+    return MeshResponse(**job)
+
+
+@router.delete("/{mesh_id}", response_model=MeshResponse)
+async def delete_mesh(mesh_id: str = Path(...)):
+    job = await MESH_STORE.pop(mesh_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Mesh not found")
+    job["status"] = "deleted"
+    return MeshResponse(**job)
+
+
+@router.get("/{mesh_id}/content")
+async def download_mesh_content(
+    mesh_id: str = Path(...), variant: Optional[str] = Query(None)
+):
+    job = await MESH_STORE.get(mesh_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Mesh not found")
+
+    if job.get("url"):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Mesh has been uploaded to cloud storage. Please use the cloud URL: {job.get('url')}",
+        )
+
+    file_path = job.get("file_path")
+    if not file_path or not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="Generation is still in-progress")
+
+    ext = os.path.splitext(file_path)[1].lower()
+    media_type = {
+        ".glb": "model/gltf-binary",
+        ".obj": "text/plain",
+    }.get(ext, "application/octet-stream")
+
+    return FileResponse(
+        path=file_path, media_type=media_type, filename=os.path.basename(file_path)
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py

@@ -0,0 +1,227 @@
+import time
+import uuid
+from abc import ABC
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# Image API protocol models
+class ImageResponseData(BaseModel):
+    b64_json: Optional[str] = None
+    url: Optional[str] = None
+    revised_prompt: Optional[str] = None
+    file_path: Optional[str] = None
+
+
+class ImageResponse(BaseModel):
+    id: str
+    created: int = Field(default_factory=lambda: int(time.time()))
+    data: List[ImageResponseData]
+    peak_memory_mb: Optional[float] = None
+    inference_time_s: Optional[float] = None
+
+
+class ImageGenerationsRequest(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    prompt: str
+    model: Optional[str] = None
+    n: Optional[int] = 1
+    quality: Optional[str] = "auto"
+    response_format: Optional[str] = "url"  # url | b64_json
+    size: Optional[str] = "1024x1024"  # e.g., 1024x1024
+    style: Optional[str] = "vivid"
+    background: Optional[str] = "auto"  # transparent | opaque | auto
+    output_format: Optional[str] = None  # png | jpeg | webp
+    user: Optional[str] = None
+    # SGLang extensions
+    width: Optional[int] = None
+    height: Optional[int] = None
+    num_inference_steps: Optional[int] = None
+    guidance_scale: Optional[float] = None
+    true_cfg_scale: Optional[float] = (
+        None  # for CFG vs guidance distillation (e.g., QwenImage)
+    )
+    seed: Optional[Union[int, List[int]]] = None
+    generator_device: Optional[str] = "cuda"
+    negative_prompt: Optional[str] = None
+    output_quality: Optional[str] = "default"
+    output_compression: Optional[int] = None
+    enable_teacache: Optional[bool] = False
+    max_sequence_length: Optional[int] = None
+    flow_shift: Optional[float] = None
+    # Upscaling
+    enable_upscaling: Optional[bool] = False
+    upscaling_model_path: Optional[str] = None
+    upscaling_scale: Optional[int] = 4
+    diffusers_kwargs: Optional[Dict[str, Any]] = None  # kwargs for diffusers backend
+    # Performance profiling
+    perf_dump_path: Optional[str] = None
+    # Progressive resolution generation
+    progressive_mode: Optional[str] = None
+    progressive_levels: Optional[int] = None
+    progressive_delta: Optional[float] = None
+
+
+# Video API protocol models
+class VideoResponse(BaseModel):
+    id: str
+    object: str = "video"
+    model: str = "sora-2"
+    status: str = "queued"
+    progress: int = 0
+    created_at: int = Field(default_factory=lambda: int(time.time()))
+    size: str = ""
+    seconds: str = "4"
+    quality: str = "standard"
+    url: Optional[str] = None
+    remixed_from_video_id: Optional[str] = None
+    completed_at: Optional[int] = None
+    expires_at: Optional[int] = None
+    error: Optional[Dict[str, Any]] = None
+    file_path: Optional[str] = None
+    file_paths: Optional[List[str]] = None
+    num_outputs: Optional[int] = None
+    peak_memory_mb: Optional[float] = None
+    inference_time_s: Optional[float] = None
+    action: Optional[Dict[str, Any]] = None
+
+
+class VideoGenerationsRequest(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    prompt: str
+    input_reference: Optional[str] = None
+    reference_url: Optional[str] = None
+    video_path: Optional[str] = None
+    video_url: Optional[str] = None
+    model: Optional[str] = None
+    n: Optional[int] = 1
+    num_outputs_per_prompt: Optional[int] = None
+    seconds: Optional[int] = 4
+    size: Optional[str] = ""
+    fps: Optional[int] = None
+    num_frames: Optional[int] = None
+    seed: Optional[Union[int, List[int]]] = None
+    generator_device: Optional[str] = "cuda"
+    # SGLang extensions
+    width: Optional[int] = None
+    height: Optional[int] = None
+    num_inference_steps: Optional[int] = None
+    guidance_scale: Optional[float] = None
+    guidance_scale_2: Optional[float] = None
+    true_cfg_scale: Optional[float] = (
+        None  # for CFG vs guidance distillation (e.g., QwenImage)
+    )
+    negative_prompt: Optional[str] = None
+    max_sequence_length: Optional[int] = None
+    flow_shift: Optional[float] = None
+    enable_teacache: Optional[bool] = False
+    # Frame interpolation
+    enable_frame_interpolation: Optional[bool] = False
+    frame_interpolation_exp: Optional[int] = 1  # 1=2×, 2=4×
+    frame_interpolation_scale: Optional[float] = 1.0
+    frame_interpolation_model_path: Optional[str] = None
+    # Upscaling
+    enable_upscaling: Optional[bool] = False
+    upscaling_model_path: Optional[str] = None
+    upscaling_scale: Optional[int] = 4
+    output_quality: Optional[str] = "default"
+    output_compression: Optional[int] = None
+    output_path: Optional[str] = None
+    diffusers_kwargs: Optional[Dict[str, Any]] = None  # kwargs for diffusers backend
+    # Performance profiling
+    perf_dump_path: Optional[str] = None
+
+
+class VideoListResponse(BaseModel):
+    data: List[VideoResponse]
+    object: str = "list"
+
+
+class VideoRemixRequest(BaseModel):
+    prompt: str
+
+
+class RealtimeVideoGenerationsRequest(VideoGenerationsRequest):
+    type: Literal["init"]
+    # WebSocket does not support multipart/form-data image uploads
+    first_frame: Optional[bytes | str] = None
+    condition_inputs: Optional[Dict[str, Any]] = None
+    max_chunks: Optional[int] = Field(default=None, ge=1)
+    seed: Optional[int] = 42
+    guidance_scale: Optional[float] = 1.0
+    size: Optional[str] = "832x480"
+    profile: Optional[bool] = False
+    num_profiled_timesteps: Optional[int] = None
+    profile_all_stages: Optional[bool] = False
+    realtime_output_format: Optional[Literal["raw", "webp", "jpeg"]] = None
+    realtime_preview_max_width: Optional[int] = None
+    realtime_output_pacing: Optional[bool] = False
+    realtime_causal_sink_size: Optional[int] = None
+    realtime_causal_kv_cache_num_frames: Optional[int] = None
+
+
+class RealtimeEvent(BaseModel):
+    type: Literal["event"]
+    kind: str
+    payload: Any = None
+    event_id: Optional[int] = None
+
+
+# Mesh API protocol models
+class MeshResponse(BaseModel):
+    id: str
+    object: str = "mesh"
+    model: str = ""
+    status: str = "queued"
+    progress: int = 0
+    created_at: int = Field(default_factory=lambda: int(time.time()))
+    format: str = "glb"
+    url: Optional[str] = None
+    completed_at: Optional[int] = None
+    expires_at: Optional[int] = None
+    error: Optional[Dict[str, Any]] = None
+    file_path: Optional[str] = None
+    file_size_bytes: Optional[int] = None
+    peak_memory_mb: Optional[float] = None
+    inference_time_s: Optional[float] = None
+
+
+class MeshGenerationsRequest(BaseModel):
+    prompt: str = "generate 3d mesh"
+    input_image: Optional[str] = None
+    model: Optional[str] = None
+    seed: Optional[Union[int, List[int]]] = None
+    generator_device: Optional[str] = "cuda"
+    num_inference_steps: Optional[int] = None
+    guidance_scale: Optional[float] = None
+    negative_prompt: Optional[str] = None
+    output_format: Optional[str] = "glb"
+
+
+class MeshListResponse(BaseModel):
+    data: List[MeshResponse]
+    object: str = "list"
+
+
+@dataclass
+class BaseReq(ABC):
+    rid: Optional[Union[str, List[str]]] = field(default=None, kw_only=True)
+    http_worker_ipc: Optional[str] = field(default=None, kw_only=True)
+
+    def regenerate_rid(self):
+        """Generate a new request ID and return it."""
+        if isinstance(self.rid, list):
+            self.rid = [uuid.uuid4().hex for _ in range(len(self.rid))]
+        else:
+            self.rid = uuid.uuid4().hex
+        return self.rid
+
+
+@dataclass
+class VertexGenerateReqInput(BaseReq):
+    instances: List[dict]
+    parameters: Optional[dict] = None

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/__init__.py

@@ -0,0 +1 @@
+# SPDX-License-Identifier: Apache-2.0

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/lingbot_world_realtime_adapter.py

@@ -0,0 +1,352 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+    BaseRealtimeModelAdapter,
+    RealtimeChunkInputs,
+    build_realtime_sampling_params,
+    save_realtime_first_frame,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.lingbot_world.constants import (
+    LINGBOT_CAMERA_ACTIONS_CONDITION,
+    LINGBOT_PROMPT_UPDATED_CONDITION,
+)
+from sglang.multimodal_gen.runtime.realtime.control_signals import (
+    ControlSignalQueue,
+    ParsedControlEventPayload,
+    parse_control_event_payload,
+)
+from sglang.multimodal_gen.runtime.realtime.states import (
+    RealtimeCameraControlState,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+        RealtimeChunkContext,
+    )
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+LINGBOT_REALTIME_DEFAULT_NUM_INFERENCE_STEPS = 4
+LINGBOT_REALTIME_MIN_CONDITION_CHUNKS = 2
+COMPOSITE_INPUT_EVENT_KIND = "composite_input"
+
+
+class LingBotWorldRealtimeState(RealtimeCameraControlState):
+    def __init__(self):
+        super().__init__(
+            min_pulse_items=1,
+            script_maxlen=512,
+            max_transitions=512,
+        )
+        self.prompt_queue = ControlSignalQueue(max_events={"prompt": 1})
+
+    def clear(self) -> None:
+        super().clear()
+        self.prompt_queue.clear()
+
+    def receive_prompt(self, prompt: str, *, event_id: int | None = None) -> None:
+        self.prompt_queue.push("prompt", prompt, event_id=event_id)
+
+    def parse_camera_control_event_payload(
+        self,
+        payload: Any,
+        *,
+        event_id: int | None,
+    ) -> ParsedControlEventPayload:
+        return parse_control_event_payload(
+            payload,
+            event_id=event_id,
+            kind="camera_actions",
+            normalize_state_payload=self._normalize_state_actions,
+            validate_script_payload=LingBotWorldRealtimeAdapter._validate_camera_actions,
+        )
+
+    def receive_parsed_camera_control_event_payload(
+        self,
+        parsed: ParsedControlEventPayload,
+        *,
+        event_id: int | None,
+    ) -> str:
+        if parsed.mode == "state":
+            transitions = parsed.payload
+            self.receive_camera_state_transitions(transitions)
+            return f"kind=camera_actions, mode=state, transitions={len(transitions)}"
+
+        camera_actions = parsed.payload
+        self.receive_camera_action_script(camera_actions, event_id=event_id)
+        return f"kind=camera_actions, mode=script, frames={len(camera_actions)}"
+
+    def receive_camera_control_event_payload(
+        self,
+        payload: Any,
+        *,
+        event_id: int | None,
+    ) -> str:
+        parsed = self.parse_camera_control_event_payload(payload, event_id=event_id)
+        return self.receive_parsed_camera_control_event_payload(
+            parsed, event_id=event_id
+        )
+
+    def sample_prompt(self) -> str:
+        prompt = self.prompt_queue.pop_latest("prompt")
+        if not isinstance(prompt, str):
+            raise ValueError("prompt event payload must be a string")
+        self.latest_sampled_event_id = self.prompt_queue.last_sampled_seq_id("prompt")
+        return prompt
+
+    def has_prompt(self) -> bool:
+        return self.prompt_queue.has_events("prompt")
+
+
+class LingBotWorldRealtimeAdapter(BaseRealtimeModelAdapter):
+    def create_state(self) -> LingBotWorldRealtimeState:
+        return LingBotWorldRealtimeState()
+
+    def _state(self, session: GenerateSession) -> LingBotWorldRealtimeState:
+        state = session.adapter_state
+        if not isinstance(state, LingBotWorldRealtimeState):
+            raise TypeError("LingBot realtime adapter state is not initialized")
+        return state
+
+    async def on_init(
+        self,
+        session: GenerateSession,
+        request: RealtimeVideoGenerationsRequest,
+    ) -> None:
+        condition_inputs = request.condition_inputs or {}
+        camera_actions = condition_inputs.get(LINGBOT_CAMERA_ACTIONS_CONDITION)
+        if camera_actions is not None:
+            state = self._state(session)
+            state.receive_camera_action_script(
+                self._validate_camera_actions(camera_actions)
+            )
+
+        await save_realtime_first_frame(session, request)
+
+    @staticmethod
+    def _validate_camera_actions(payload: Any) -> list[list[str]]:
+        if not isinstance(payload, list):
+            raise ValueError("camera_actions event payload must be list[list[str]]")
+        normalized = []
+        for frame_actions in payload:
+            if not isinstance(frame_actions, list):
+                raise ValueError("camera_actions event payload must be list[list[str]]")
+            normalized.append(list(frame_actions))
+        return normalized
+
+    def ingest_event(
+        self,
+        session: GenerateSession,
+        event: RealtimeEvent,
+    ) -> str:
+        state = self._state(session)
+        if event.kind == "camera_actions":
+            return self._ingest_camera_actions(state, event.payload, event.event_id)
+        elif event.kind == "prompt":
+            return self._ingest_prompt(state, event.payload, event.event_id)
+        elif event.kind == COMPOSITE_INPUT_EVENT_KIND:
+            return self._ingest_composite_input(state, event.payload, event.event_id)
+        raise ValueError(f"unsupported event kind: {event.kind}")
+
+    def _ingest_camera_actions(
+        self,
+        state: LingBotWorldRealtimeState,
+        payload: Any,
+        event_id: int | None,
+    ) -> str:
+        return state.receive_camera_control_event_payload(
+            payload,
+            event_id=event_id,
+        )
+
+    def _ingest_prompt(
+        self,
+        state: LingBotWorldRealtimeState,
+        payload: Any,
+        event_id: int | None,
+    ) -> str:
+        prompt = self._validate_prompt_payload(payload)
+        state.receive_prompt(prompt, event_id=event_id)
+        return f"kind=prompt, prompt_len={len(prompt)}"
+
+    @staticmethod
+    def _validate_prompt_payload(payload: Any) -> str:
+        if not isinstance(payload, str) or not payload:
+            raise ValueError("prompt event payload must be a non-empty string")
+        return payload
+
+    def _ingest_composite_input(
+        self,
+        state: LingBotWorldRealtimeState,
+        payload: Any,
+        event_id: int | None,
+    ) -> str:
+        if not isinstance(payload, dict):
+            raise ValueError("composite_input event payload must be a map")
+        input_types = payload.get("input_types")
+        if not isinstance(input_types, list) or not input_types:
+            raise ValueError(
+                "composite_input event payload requires non-empty input_types"
+            )
+
+        parsed_inputs = []
+        for input_type in input_types:
+            if not isinstance(input_type, str) or not input_type:
+                raise ValueError(
+                    "composite_input input_types must contain non-empty strings"
+                )
+            if input_type not in payload:
+                raise ValueError(f"composite_input event payload requires {input_type}")
+            parsed_inputs.append(
+                (
+                    input_type,
+                    self._parse_composite_input_item(
+                        state,
+                        input_type,
+                        payload[input_type],
+                        event_id,
+                    ),
+                )
+            )
+
+        input_logs = []
+        for input_type, parsed_payload in parsed_inputs:
+            input_logs.append(
+                self._ingest_parsed_composite_input_item(
+                    state,
+                    input_type,
+                    parsed_payload,
+                    event_id,
+                )
+            )
+        return f"kind=composite_input, inputs={input_logs}"
+
+    def _parse_composite_input_item(
+        self,
+        state: LingBotWorldRealtimeState,
+        input_type: str,
+        payload: Any,
+        event_id: int | None,
+    ) -> Any:
+        if input_type == "camera_actions":
+            return state.parse_camera_control_event_payload(
+                payload,
+                event_id=event_id,
+            )
+        if input_type == "prompt":
+            return self._validate_prompt_payload(payload)
+        raise ValueError(f"unsupported composite_input type: {input_type}")
+
+    def _ingest_parsed_composite_input_item(
+        self,
+        state: LingBotWorldRealtimeState,
+        input_type: str,
+        parsed_payload: Any,
+        event_id: int | None,
+    ) -> str:
+        if input_type == "camera_actions":
+            return state.receive_parsed_camera_control_event_payload(
+                parsed_payload,
+                event_id=event_id,
+            )
+        if input_type == "prompt":
+            state.receive_prompt(parsed_payload, event_id=event_id)
+            return f"kind=prompt, prompt_len={len(parsed_payload)}"
+        raise ValueError(f"unsupported composite_input type: {input_type}")
+
+    def sample_chunk_inputs(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> RealtimeChunkInputs:
+        """Samples user inputs (conditions) for the current RealtimeChunk from RealtimeStates"""
+        state = self._state(session)
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        prompt_updated = False
+        if chunk.index == 0:
+            prompt = request.prompt
+        elif state.has_prompt():
+            prompt = state.sample_prompt()
+            request.prompt = prompt
+            prompt_updated = True
+        else:
+            prompt = request.prompt
+
+        condition_inputs = {}
+        if prompt_updated:
+            condition_inputs[LINGBOT_PROMPT_UPDATED_CONDITION] = True
+        camera_actions = state.sample_camera_actions(chunk_size)
+        if camera_actions is not None:
+            condition_inputs[LINGBOT_CAMERA_ACTIONS_CONDITION] = camera_actions
+        return RealtimeChunkInputs(prompt=prompt, condition_inputs=condition_inputs)
+
+    def build_sampling_params(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_inputs: RealtimeChunkInputs,
+        chunk_size: int,
+    ):
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        num_frames = self._condition_num_frames(
+            request=request,
+            server_args=server_args,
+            chunk_size=chunk_size,
+        )
+
+        return build_realtime_sampling_params(
+            chunk.request_id,
+            request=request,
+            chunk_inputs=chunk_inputs,
+            num_frames=num_frames,
+            num_inference_steps=(
+                request.num_inference_steps
+                or LINGBOT_REALTIME_DEFAULT_NUM_INFERENCE_STEPS
+            ),
+            chunk_size=chunk_size,
+        )
+
+    @staticmethod
+    def _condition_num_frames(
+        *,
+        request: RealtimeVideoGenerationsRequest,
+        server_args: ServerArgs | None,
+        chunk_size: int,
+    ) -> int:
+        if server_args is None:
+            return int(request.num_frames or 0)
+
+        # encode one extra blank condition chunk so repeat-last never reuses
+        # the first-frame image mask on later realtime chunks
+        temporal_ratio = int(
+            server_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio
+        )
+        required_latent_frames = chunk_size * LINGBOT_REALTIME_MIN_CONDITION_CHUNKS
+        required_num_frames = (required_latent_frames - 1) * temporal_ratio + 1
+        return max(int(request.num_frames or 0), required_num_frames)
+
+    def get_realtime_event_id(self, session: GenerateSession) -> int | None:
+        return self._state(session).latest_sampled_event_id
+
+    def clear_state(self, session: GenerateSession) -> None:
+        state = session.adapter_state
+        if isinstance(state, LingBotWorldRealtimeState):
+            state.clear()

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py

@@ -0,0 +1,264 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+    BaseRealtimeModelAdapter,
+    RealtimeChunkInputs,
+    build_realtime_sampling_params,
+    save_realtime_first_frame,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_wm.base import (
+    normalize_sana_wm_camera_actions,
+    parse_sana_wm_action_string,
+    snap_sana_wm_num_frames,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_wm.self_forcing import (
+    SanaWMSelfForcingSampler,
+)
+from sglang.multimodal_gen.runtime.realtime.states import (
+    RealtimeCameraControlState,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+        RealtimeChunkContext,
+    )
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
+        OutputBatch,
+    )
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+SANA_WM_DEFAULT_SIZE = "1280x704"
+SANA_WM_DEFAULT_NUM_FRAMES = 1081
+SANA_WM_DEFAULT_FPS = 16
+SANA_WM_DEFAULT_STEPS = 4
+SANA_WM_DEFAULT_GUIDANCE = 1.0
+SANA_WM_CONTROL_PULSE_FRAMES = 8
+
+
+def _normalize_sana_wm_state_actions(actions: list[Any]) -> list[str]:
+    return [str(action).lower() for action in actions]
+
+
+class SanaWMRealtimeAdapterState(RealtimeCameraControlState):
+    def __init__(self):
+        super().__init__(
+            min_pulse_items=SANA_WM_CONTROL_PULSE_FRAMES,
+            script_maxlen=2048,
+            max_transitions=512,
+            normalize_state_actions=_normalize_sana_wm_state_actions,
+        )
+        self.base_condition_inputs: dict[str, Any] = {}
+
+    def clear(self) -> None:
+        super().clear()
+        self.base_condition_inputs.clear()
+
+    def receive_camera_control_event_payload(
+        self,
+        payload: Any,
+        *,
+        event_id: int | None,
+    ) -> str:
+        return super().receive_camera_control_event_payload(
+            payload,
+            event_id=event_id,
+            validate_camera_actions=SanaWMRealtimeAdapter._validate_camera_actions,
+        )
+
+
+class SanaWMRealtimeAdapter(BaseRealtimeModelAdapter):
+    def create_state(self) -> SanaWMRealtimeAdapterState:
+        return SanaWMRealtimeAdapterState()
+
+    def _state(self, session: GenerateSession) -> SanaWMRealtimeAdapterState:
+        state = session.adapter_state
+        if not isinstance(state, SanaWMRealtimeAdapterState):
+            raise TypeError("SANA-WM realtime adapter state is not initialized")
+        return state
+
+    @staticmethod
+    def _validate_camera_actions(payload: Any) -> list[list[str]]:
+        return normalize_sana_wm_camera_actions(
+            payload, error_label="camera_actions event payload"
+        )
+
+    @staticmethod
+    def _raw_frame_count(result: OutputBatch) -> int | None:
+        if result.raw_frame_batches is None:
+            return None
+        return sum(len(frames) for frames in result.raw_frame_batches)
+
+    async def on_init(
+        self,
+        session: GenerateSession,
+        request: RealtimeVideoGenerationsRequest,
+    ) -> None:
+        request.size = request.size or SANA_WM_DEFAULT_SIZE
+        if request.num_frames is not None:
+            request.num_frames = int(request.num_frames)
+        else:
+            # Open-ended session: keep num_frames unset so prepare_next_request
+            # samples uniform action chunks (no front-loaded segmentation), and
+            # flag the stage explicitly via condition_inputs —
+            # build_sampling_params strips None fields, so the per-chunk batch
+            # would otherwise carry the SamplingParams default num_frames.
+            request.condition_inputs = {
+                **(request.condition_inputs or {}),
+                "sana_wm_open_ended": True,
+            }
+        request.fps = int(request.fps or SANA_WM_DEFAULT_FPS)
+        request.num_inference_steps = int(
+            request.num_inference_steps or SANA_WM_DEFAULT_STEPS
+        )
+        request.guidance_scale = float(
+            request.guidance_scale or SANA_WM_DEFAULT_GUIDANCE
+        )
+        if request.negative_prompt is None:
+            request.negative_prompt = ""
+        if request.generator_device is None:
+            request.generator_device = "cuda"
+
+        state = self._state(session)
+        condition_inputs = dict(request.condition_inputs or {})
+        camera_actions = condition_inputs.pop("camera_actions", None)
+        action = condition_inputs.pop("action", None)
+        if camera_actions is not None and action is not None:
+            raise ValueError("pass only one of camera_actions or action")
+        if camera_actions is not None:
+            state.receive_camera_control_event_payload(camera_actions, event_id=None)
+        if action is not None:
+            if not isinstance(action, str) or not action:
+                raise ValueError("action condition input must be a non-empty string")
+            state.receive_camera_action_script(
+                parse_sana_wm_action_string(action), event_id=None
+            )
+        state.base_condition_inputs = condition_inputs
+
+        await save_realtime_first_frame(
+            session,
+            request,
+            required_error="SANA-WM realtime requires first_frame",
+            cache_remote_urls=True,
+        )
+
+    def ingest_event(
+        self,
+        session: GenerateSession,
+        event: RealtimeEvent,
+    ) -> str:
+        state = self._state(session)
+        if event.kind == "camera_actions":
+            return state.receive_camera_control_event_payload(
+                event.payload,
+                event_id=event.event_id,
+            )
+        if event.kind == "action":
+            if not isinstance(event.payload, str) or not event.payload:
+                raise ValueError("action event payload must be a non-empty string")
+            camera_actions = parse_sana_wm_action_string(event.payload)
+            state.receive_camera_action_script(camera_actions, event_id=event.event_id)
+            return f"kind=action, frames={len(camera_actions)}"
+        raise ValueError(f"unsupported event kind: {event.kind}")
+
+    def sample_chunk_inputs(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> RealtimeChunkInputs:
+        action_chunk_size = self._action_chunk_size(
+            session,
+            server_args,
+            chunk,
+            chunk_size,
+        )
+        state = self._state(session)
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        condition_inputs = dict(state.base_condition_inputs) if chunk.index == 0 else {}
+        camera_actions = state.sample_camera_actions(action_chunk_size)
+        if camera_actions is not None:
+            condition_inputs["camera_actions"] = camera_actions
+        return RealtimeChunkInputs(
+            prompt=request.prompt,
+            condition_inputs=condition_inputs,
+        )
+
+    def build_sampling_params(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_inputs: RealtimeChunkInputs,
+        chunk_size: int,
+    ):
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        return build_realtime_sampling_params(
+            chunk.request_id,
+            request=request,
+            chunk_inputs=chunk_inputs,
+            num_frames=request.num_frames,
+            num_inference_steps=request.num_inference_steps,
+            chunk_size=chunk_size,
+        )
+
+    def _action_chunk_size(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> int:
+        temporal_compression = int(
+            server_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio
+        )
+        # Match action sampling to the latent span used by the batch path. Chunk
+        # 0 may carry a front-loaded remainder, so a fixed nfpb*tc action count
+        # would read static-padded camera poses and drift from batch output.
+        action_chunk_size = chunk_size * temporal_compression
+        req_num_frames = (
+            session.request.num_frames if session.request is not None else None
+        )
+        if req_num_frames is not None:
+            snapped = snap_sana_wm_num_frames(
+                int(req_num_frames), stride=temporal_compression
+            )
+            latent_t = (snapped - 1) // temporal_compression + 1
+            segments = SanaWMSelfForcingSampler.create_autoregressive_segments(
+                latent_t, chunk_size
+            )
+            idx = int(chunk.index)
+            if 0 <= idx and idx + 1 < len(segments):
+                action_chunk_size = (
+                    segments[idx + 1] - segments[idx]
+                ) * temporal_compression
+        return action_chunk_size
+
+    def get_realtime_event_id(self, session: GenerateSession) -> int | None:
+        return self._state(session).latest_sampled_event_id
+
+    def on_chunk_complete(self, session: GenerateSession, result: OutputBatch) -> None:
+        if session.request is not None and self._raw_frame_count(result) == 0:
+            session.request.max_chunks = session.generate_chunk_cnt + 1
+        session.generate_chunk_completed()
+
+    def clear_state(self, session: GenerateSession) -> None:
+        state = session.adapter_state
+        if isinstance(state, SanaWMRealtimeAdapterState):
+            state.clear()

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/generate_session.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+from uuid import uuid4
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.realtime.session import (
+    RealtimeSession,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+        BaseRealtimeModelAdapter,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeChunkContext:
+    session_id: str
+    index: int
+    request_id: str
+
+
+class GenerateSession:
+    """A realtime generation session"""
+
+    def __init__(self):
+        self.id = uuid4().hex
+        self.request: RealtimeVideoGenerationsRequest | None = None
+        self.input_temp_dir: str | None = None
+        self.generate_chunk_cnt = 0
+        self.current_chunk: RealtimeChunkContext | None = None
+        self.realtime_session = RealtimeSession()
+        self.adapter: BaseRealtimeModelAdapter | None = None
+        self.adapter_state: Any = None
+        self.output_pace_next_send_at: float | None = None
+        self.output_pace_last_event_id: int | None = None
+
+    def set_adapter(self, adapter: BaseRealtimeModelAdapter):
+        self.adapter = adapter
+        self.adapter_state = adapter.create_state()
+
+    def set_request(self, request: RealtimeVideoGenerationsRequest):
+        self.request = request
+
+    def dispose(self):
+        if self.adapter is not None:
+            self.adapter.dispose(self)
+        self.request = None
+        self.input_temp_dir = None
+        self.generate_chunk_cnt = 0
+        self.current_chunk = None
+        self.adapter = None
+        self.adapter_state = None
+        self.output_pace_next_send_at = None
+        self.output_pace_last_event_id = None
+        self.realtime_session.dispose()
+
+    def new_chunk(self) -> RealtimeChunkContext:
+        if self.current_chunk is not None:
+            raise RuntimeError("previous realtime chunk is still active")
+        chunk = RealtimeChunkContext(
+            session_id=self.id,
+            index=self.generate_chunk_cnt,
+            request_id=f"{self.id}_{uuid4().hex}",
+        )
+        self.current_chunk = chunk
+        return chunk
+
+    def generate_chunk_completed(self):
+        self.generate_chunk_cnt += 1
+        self.current_chunk = None
+
+    def reached_max_chunks(self) -> bool:
+        return (
+            self.request is not None
+            and self.request.max_chunks is not None
+            and self.generate_chunk_cnt >= self.request.max_chunks
+        )

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/realtime_adapter.py

@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import hashlib
+import os
+import tempfile
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from fastapi import WebSocket
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_output_adapter import (
+    RawRGBRealtimeOutputAdapter,
+    RealtimeFrameSendStats,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    build_sampling_params,
+    save_image_to_path,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    prepare_request,
+)
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+        RealtimeChunkContext,
+    )
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
+        OutputBatch,
+        Req,
+    )
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+@dataclass(slots=True)
+class RealtimeChunkInputs:
+    """Sampled from realtime control state, consumed by the Req"""
+
+    prompt: str
+    condition_inputs: dict[str, Any] = field(default_factory=dict)
+
+
+async def save_realtime_first_frame(
+    session: GenerateSession,
+    request: RealtimeVideoGenerationsRequest,
+    *,
+    required_error: str | None = None,
+    cache_remote_urls: bool = False,
+) -> None:
+    first_frame = request.first_frame
+    if first_frame is None:
+        if required_error is not None:
+            raise ValueError(required_error)
+        return
+
+    server_args = get_global_server_args()
+    if server_args.input_save_path is not None:
+        uploads_dir = server_args.input_save_path
+        os.makedirs(uploads_dir, exist_ok=True)
+    else:
+        if session.input_temp_dir is None:
+            session.input_temp_dir = tempfile.mkdtemp(prefix="sglang_input_")
+        uploads_dir = session.input_temp_dir
+
+    if (
+        cache_remote_urls
+        and isinstance(first_frame, str)
+        and first_frame.lower().startswith(("http://", "https://"))
+    ):
+        suffix = os.path.splitext(first_frame.split("?", 1)[0])[1]
+        digest = hashlib.sha256(first_frame.encode("utf-8")).hexdigest()[:16]
+        target_path = os.path.join(uploads_dir, f"realtime_ref_{digest}{suffix}")
+        if os.path.exists(target_path):
+            request.first_frame = target_path
+            return
+    else:
+        target_path = os.path.join(uploads_dir, f"{session.id}_first_frame")
+
+    request.first_frame = await save_image_to_path(first_frame, target_path)
+
+
+def build_realtime_sampling_params(
+    request_id: str,
+    *,
+    request: RealtimeVideoGenerationsRequest,
+    chunk_inputs: RealtimeChunkInputs,
+    num_frames: int | None,
+    num_inference_steps: int | None,
+    chunk_size: int,
+):
+    return build_sampling_params(
+        request_id,
+        prompt=chunk_inputs.prompt,
+        size=request.size,
+        num_frames=num_frames,
+        fps=request.fps,
+        image_path=request.first_frame,
+        output_file_name=request_id,
+        save_output=False,
+        seed=request.seed,
+        generator_device=request.generator_device,
+        num_inference_steps=num_inference_steps,
+        guidance_scale=request.guidance_scale,
+        guidance_scale_2=request.guidance_scale_2,
+        negative_prompt=request.negative_prompt,
+        enable_teacache=request.enable_teacache,
+        enable_frame_interpolation=request.enable_frame_interpolation,
+        frame_interpolation_exp=request.frame_interpolation_exp,
+        frame_interpolation_scale=request.frame_interpolation_scale,
+        frame_interpolation_model_path=request.frame_interpolation_model_path,
+        enable_upscaling=request.enable_upscaling,
+        upscaling_model_path=request.upscaling_model_path,
+        upscaling_scale=request.upscaling_scale,
+        diffusers_kwargs=request.diffusers_kwargs,
+        profile=request.profile,
+        num_profiled_timesteps=request.num_profiled_timesteps,
+        profile_all_stages=request.profile_all_stages,
+        perf_dump_path=request.perf_dump_path,
+        output_path=request.output_path,
+        output_compression=request.output_compression,
+        output_quality=request.output_quality,
+        condition_inputs=chunk_inputs.condition_inputs,
+        realtime_chunk_size=chunk_size,
+    )
+
+
+class BaseRealtimeModelAdapter:
+    def __init__(self):
+        self.output_adapter = RawRGBRealtimeOutputAdapter()
+
+    async def on_init(
+        self,
+        session: GenerateSession,
+        request: RealtimeVideoGenerationsRequest,
+    ) -> None:
+        raise NotImplementedError
+
+    def create_state(self) -> Any:
+        """create a state for managing runtime states"""
+        raise NotImplementedError
+
+    def ingest_event(
+        self,
+        session: GenerateSession,
+        event: RealtimeEvent,
+    ) -> str:
+        """
+        Ingest a realtime endpoint event and install it into the model's realtime control queues
+        """
+        raise NotImplementedError
+
+    async def wait_for_next_chunk(self, session: GenerateSession) -> None:
+        del session
+
+    def get_chunk_size(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+    ) -> int:
+        del session, chunk
+        arch_config = server_args.pipeline_config.dit_config.arch_config
+        return int(getattr(arch_config, "num_frames_per_block", 3))
+
+    def sample_chunk_inputs(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> RealtimeChunkInputs:
+        raise NotImplementedError
+
+    def build_sampling_params(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_inputs: RealtimeChunkInputs,
+        chunk_size: int,
+    ):
+        raise NotImplementedError
+
+    def get_realtime_event_id(self, session: GenerateSession) -> int | None:
+        del session
+        return None
+
+    def prepare_next_request(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+    ) -> Req:
+        chunk_size = self.get_chunk_size(session, server_args, chunk)
+        chunk_inputs = self.sample_chunk_inputs(
+            session,
+            server_args,
+            chunk,
+            chunk_size,
+        )
+        sampling_params = self.build_sampling_params(
+            session,
+            server_args,
+            chunk,
+            chunk_inputs,
+            chunk_size,
+        )
+        batch = prepare_request(
+            server_args=server_args,
+            sampling_params=sampling_params,
+        )
+        self.apply_realtime_request_fields(
+            batch,
+            session,
+            chunk,
+            event_id=self.get_realtime_event_id(session),
+        )
+        return batch
+
+    def apply_realtime_request_fields(
+        self,
+        batch: Req,
+        session: GenerateSession,
+        chunk: RealtimeChunkContext,
+        *,
+        event_id: int | None,
+    ) -> None:
+        batch.realtime_session_id = session.id
+        batch.return_raw_frames = True
+        batch.block_idx = chunk.index
+        batch.realtime_event_id = event_id
+        if session.request is None:
+            return
+        batch.realtime_output_format = session.request.realtime_output_format
+        batch.realtime_preview_max_width = session.request.realtime_preview_max_width
+        batch.realtime_output_pacing = bool(session.request.realtime_output_pacing)
+        batch.realtime_causal_sink_size = session.request.realtime_causal_sink_size
+        batch.realtime_causal_kv_cache_num_frames = (
+            session.request.realtime_causal_kv_cache_num_frames
+        )
+
+    async def send_output(
+        self,
+        ws: WebSocket,
+        session: GenerateSession,
+        result: OutputBatch,
+        batch: Req,
+    ) -> RealtimeFrameSendStats:
+        """send the generate output (usually frames) back via websocket"""
+        return await self.output_adapter.send(ws, session, result, batch)
+
+    def on_chunk_complete(self, session: GenerateSession, result: OutputBatch) -> None:
+        del result
+        session.generate_chunk_completed()
+
+    def clear_state(self, session: GenerateSession) -> None:
+        del session
+
+    def dispose(self, session: GenerateSession) -> None:
+        self.clear_state(session)
+        self.output_adapter.reset()

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/realtime_output_adapter.py

@@ -0,0 +1,610 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import asyncio
+import io
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+import msgspec.msgpack
+from fastapi import WebSocket
+from PIL import Image
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.timer import (
+    RealtimeStageTimer,
+)
+from sglang.multimodal_gen.runtime.utils.realtime_video import (
+    JPEG_FRAME_CONTENT_TYPE,
+    RAW_RGB_CHANNELS,
+    RAW_RGB_CONTENT_TYPE,
+    WEBP_FRAME_CONTENT_TYPE,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+    )
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
+        OutputBatch,
+        Req,
+    )
+
+
+class RealtimeFrameBatchHeader(TypedDict, total=False):
+    type: str
+    request_id: str
+    chunk_index: int
+    content_type: str
+    num_frames: int
+    total_size: int
+    format: str
+    width: int
+    height: int
+    channels: int
+    bytes_per_frame: int
+    raw_size: int
+    encoding: str
+    delta_reference: str
+    payload_lengths: list[int]
+    event_id: int
+    frame_batch_index: int
+    num_frame_batches: int
+    is_final_frame_batch: bool
+
+
+class RealtimeFrameBatchMessage(RealtimeFrameBatchHeader, total=False):
+    payload: bytes
+
+
+class RealtimeFrameSendStats(TypedDict):
+    header_pack_ms: float
+    header_write_ms: float
+    raw_payload_build_ms: float
+    raw_write_ms: float
+    ws_write_ms: float
+    pace_wait_ms: float
+    raw_bytes: int
+    ws_payload_bytes: int
+    num_frames: int
+    num_batches: int
+    frame_shape: tuple[int, int, int] | None
+    content_type: str
+
+
+def empty_frame_send_stats(content_type: str = "") -> RealtimeFrameSendStats:
+    return {
+        "header_pack_ms": 0.0,
+        "header_write_ms": 0.0,
+        "raw_payload_build_ms": 0.0,
+        "raw_write_ms": 0.0,
+        "ws_write_ms": 0.0,
+        "pace_wait_ms": 0.0,
+        "raw_bytes": 0,
+        "ws_payload_bytes": 0,
+        "num_frames": 0,
+        "num_batches": 0,
+        "frame_shape": None,
+        "content_type": content_type,
+    }
+
+
+def _raw_rgb_frame_metadata(batch: Req) -> dict[str, int | str]:
+    frame_width = batch.width
+    frame_height = batch.height
+    if frame_width is None or frame_height is None:
+        return {}
+
+    frame_width = int(frame_width)
+    frame_height = int(frame_height)
+    if batch.enable_upscaling:
+        upscaling_scale = int(batch.upscaling_scale or 1)
+        frame_width *= upscaling_scale
+        frame_height *= upscaling_scale
+
+    return {
+        "format": "rgb24",
+        "width": frame_width,
+        "height": frame_height,
+        "channels": RAW_RGB_CHANNELS,
+        "bytes_per_frame": frame_width * frame_height * RAW_RGB_CHANNELS,
+    }
+
+
+def _frame_shape_from_metadata(
+    metadata: dict[str, int | str] | None,
+) -> tuple[int, int, int] | None:
+    if not metadata:
+        return None
+    return (
+        int(metadata["height"]),
+        int(metadata["width"]),
+        int(metadata["channels"]),
+    )
+
+
+RAW_RGB_FRAMES_PER_WS_MESSAGE = 16
+ENCODED_PREVIEW_FRAMES_PER_WS_MESSAGE = 6
+FRAME_BATCH_PACK_OFFLOAD_BYTES = 64 * 1024
+WEBP_DEFAULT_QUALITY = 90
+JPEG_DEFAULT_QUALITY = 95
+JPEG_SUBSAMPLING = 0
+RAW_LOSSLESS_OUTPUT_FORMAT = "raw"
+ENCODED_PREVIEW_FORMATS = {"webp", "jpeg"}
+
+
+@dataclass(frozen=True)
+class _TransportPayload:
+    content_type: str
+    payload: bytes
+    metadata: dict[str, int | str | bool | list[int]]
+
+
+def _split_frame_batch(
+    frames: list[bytes],
+    frames_per_message: int = RAW_RGB_FRAMES_PER_WS_MESSAGE,
+) -> list[list[bytes]]:
+    if not frames:
+        return [frames]
+    return [
+        frames[i : i + frames_per_message]
+        for i in range(0, len(frames), frames_per_message)
+    ]
+
+
+def _encode_rgb_frame_to_webp(
+    frame: bytes,
+    *,
+    width: int,
+    height: int,
+    quality: int,
+    preview_max_width: int | None,
+) -> bytes:
+    buffer = io.BytesIO()
+    image = _resize_preview_image(
+        Image.frombytes("RGB", (width, height), frame),
+        preview_max_width=preview_max_width,
+    )
+    image.save(
+        buffer,
+        format="WEBP",
+        quality=quality,
+        method=0,
+    )
+    return buffer.getvalue()
+
+
+def _encode_rgb_frame_to_jpeg(
+    frame: bytes,
+    *,
+    width: int,
+    height: int,
+    quality: int,
+    preview_max_width: int | None,
+) -> bytes:
+    buffer = io.BytesIO()
+    image = _resize_preview_image(
+        Image.frombytes("RGB", (width, height), frame),
+        preview_max_width=preview_max_width,
+    )
+    image.save(
+        buffer,
+        format="JPEG",
+        quality=quality,
+        subsampling=JPEG_SUBSAMPLING,
+    )
+    return buffer.getvalue()
+
+
+def _preview_dimensions(
+    *,
+    width: int,
+    height: int,
+    preview_max_width: int | None,
+) -> tuple[int, int]:
+    if (
+        preview_max_width is None
+        or preview_max_width <= 0
+        or width <= preview_max_width
+    ):
+        return width, height
+    preview_width = int(preview_max_width)
+    preview_height = max(1, round(height * preview_width / width))
+    return preview_width, preview_height
+
+
+def _resize_preview_image(
+    image: Image.Image,
+    *,
+    preview_max_width: int | None,
+) -> Image.Image:
+    width, height = image.size
+    preview_width, preview_height = _preview_dimensions(
+        width=width,
+        height=height,
+        preview_max_width=preview_max_width,
+    )
+    if (preview_width, preview_height) == image.size:
+        return image
+    return image.resize((preview_width, preview_height), Image.Resampling.BICUBIC)
+
+
+def _pack_frame_batch_message(
+    header: RealtimeFrameBatchHeader,
+    payload: bytes,
+) -> bytes:
+    message: RealtimeFrameBatchMessage = {
+        **header,
+        "type": "frame_batch",
+        "payload": payload,
+    }
+    return msgspec.msgpack.encode(message)
+
+
+def _pack_frame_batch_header(header: RealtimeFrameBatchHeader) -> bytes:
+    return msgspec.msgpack.encode(header)
+
+
+def _build_transport_payload(
+    transport_frames: list[bytes],
+    *,
+    content_type: str,
+    metadata: dict[str, int | str],
+    output_format: str | None,
+    transport_quality: int | None,
+    preview_max_width: int | None,
+) -> _TransportPayload:
+    payload_content_type = content_type
+    payload_metadata: dict[str, int | str | bool | list[int]] = {}
+    raw_payload = b""
+
+    if (
+        output_format in ENCODED_PREVIEW_FORMATS
+        and content_type == RAW_RGB_CONTENT_TYPE
+        and transport_frames
+    ):
+        if output_format == "webp":
+            encoded_frames = [
+                _encode_rgb_frame_to_webp(
+                    frame,
+                    width=int(metadata["width"]),
+                    height=int(metadata["height"]),
+                    quality=int(transport_quality or WEBP_DEFAULT_QUALITY),
+                    preview_max_width=preview_max_width,
+                )
+                for frame in transport_frames
+            ]
+            payload_content_type = WEBP_FRAME_CONTENT_TYPE
+        else:
+            encoded_frames = [
+                _encode_rgb_frame_to_jpeg(
+                    frame,
+                    width=int(metadata["width"]),
+                    height=int(metadata["height"]),
+                    quality=int(transport_quality or JPEG_DEFAULT_QUALITY),
+                    preview_max_width=preview_max_width,
+                )
+                for frame in transport_frames
+            ]
+            payload_content_type = JPEG_FRAME_CONTENT_TYPE
+        raw_payload = b"".join(encoded_frames)
+        preview_width, preview_height = _preview_dimensions(
+            width=int(metadata["width"]),
+            height=int(metadata["height"]),
+            preview_max_width=preview_max_width,
+        )
+        payload_metadata = {
+            "format": output_format,
+            "encoding": output_format,
+            "source_width": int(metadata["width"]),
+            "source_height": int(metadata["height"]),
+            "preview_width": preview_width,
+            "preview_height": preview_height,
+            "width": preview_width,
+            "height": preview_height,
+            "payload_lengths": [len(frame) for frame in encoded_frames],
+        }
+    elif content_type == RAW_RGB_CONTENT_TYPE and transport_frames:
+        raw_payload = b"".join(transport_frames)
+        payload_metadata = {
+            "raw_size": len(raw_payload),
+            "encoding": RAW_LOSSLESS_OUTPUT_FORMAT,
+        }
+    else:
+        raw_payload = b"".join(transport_frames)
+
+    return _TransportPayload(
+        content_type=payload_content_type,
+        payload=raw_payload,
+        metadata=payload_metadata,
+    )
+
+
+def _should_build_payload_off_loop(
+    *,
+    content_type: str,
+    output_format: str | None,
+    transport_frames: list[bytes],
+) -> bool:
+    if content_type != RAW_RGB_CONTENT_TYPE or not transport_frames:
+        return False
+    return output_format in ENCODED_PREVIEW_FORMATS or output_format is None
+
+
+def _is_encoded_preview_transport(
+    *,
+    content_type: str,
+    output_format: str | None,
+) -> bool:
+    return (
+        output_format in ENCODED_PREVIEW_FORMATS
+        and content_type == RAW_RGB_CONTENT_TYPE
+    )
+
+
+async def _build_encoded_preview_payloads(
+    split_batches: list[list[bytes]],
+    *,
+    content_type: str,
+    metadata: dict[str, int | str],
+    output_format: str,
+    transport_quality: int | None,
+    preview_max_width: int | None,
+    event_id: int | None,
+) -> list[_TransportPayload]:
+    return list(
+        await asyncio.gather(
+            *(
+                _build_encoded_preview_payload(
+                    transport_frames,
+                    metadata=metadata,
+                    output_format=output_format,
+                    transport_quality=transport_quality,
+                    preview_max_width=preview_max_width,
+                )
+                for transport_frames in split_batches
+            )
+        )
+    )
+
+
+async def _build_encoded_preview_payload(
+    transport_frames: list[bytes],
+    *,
+    metadata: dict[str, int | str],
+    output_format: str,
+    transport_quality: int | None,
+    preview_max_width: int | None,
+) -> _TransportPayload:
+    width = int(metadata["width"])
+    height = int(metadata["height"])
+    if output_format == "webp":
+        encoded_frames = list(
+            await asyncio.gather(
+                *(
+                    asyncio.to_thread(
+                        _encode_rgb_frame_to_webp,
+                        frame,
+                        width=width,
+                        height=height,
+                        quality=int(transport_quality or WEBP_DEFAULT_QUALITY),
+                        preview_max_width=preview_max_width,
+                    )
+                    for frame in transport_frames
+                )
+            )
+        )
+        payload_content_type = WEBP_FRAME_CONTENT_TYPE
+    else:
+        encoded_frames = list(
+            await asyncio.gather(
+                *(
+                    asyncio.to_thread(
+                        _encode_rgb_frame_to_jpeg,
+                        frame,
+                        width=width,
+                        height=height,
+                        quality=int(transport_quality or JPEG_DEFAULT_QUALITY),
+                        preview_max_width=preview_max_width,
+                    )
+                    for frame in transport_frames
+                )
+            )
+        )
+        payload_content_type = JPEG_FRAME_CONTENT_TYPE
+
+    preview_width, preview_height = _preview_dimensions(
+        width=width,
+        height=height,
+        preview_max_width=preview_max_width,
+    )
+    return _TransportPayload(
+        content_type=payload_content_type,
+        payload=b"".join(encoded_frames),
+        metadata={
+            "format": output_format,
+            "encoding": output_format,
+            "source_width": width,
+            "source_height": height,
+            "preview_width": preview_width,
+            "preview_height": preview_height,
+            "width": preview_width,
+            "height": preview_height,
+            "payload_lengths": [len(frame) for frame in encoded_frames],
+        },
+    )
+
+
+class RawRGBRealtimeOutputAdapter:
+    """send raw RGB over WebSocket using lossless transport"""
+
+    def __init__(self) -> None:
+        pass
+
+    def reset(self) -> None:
+        pass
+
+    async def send(
+        self,
+        ws: WebSocket,
+        session: GenerateSession,
+        result: OutputBatch,
+        batch: Req,
+    ) -> RealtimeFrameSendStats:
+        """send frames through ws"""
+        content_type = result.raw_frame_content_type
+        if result.raw_frame_batches is None:
+            return empty_frame_send_stats(content_type)
+        if batch.block_idx == 0:
+            self.reset()
+
+        frame_metadata = (
+            result.raw_frame_metadata or _raw_rgb_frame_metadata(batch)
+            if content_type == RAW_RGB_CONTENT_TYPE
+            else {}
+        )
+        output_format = getattr(batch, "realtime_output_format", None)
+        preview_max_width = getattr(batch, "realtime_preview_max_width", None)
+        stats = await self._send_frame_batches(
+            ws,
+            result.raw_frame_batches,
+            content_type=content_type,
+            chunk_index_start=batch.block_idx,
+            request_id=batch.request_id,
+            event_id=getattr(batch, "realtime_event_id", None),
+            frame_metadata=frame_metadata,
+            output_format=output_format,
+            transport_quality=getattr(batch, "output_compression", None),
+            preview_max_width=preview_max_width,
+        )
+        stats["frame_shape"] = _frame_shape_from_metadata(frame_metadata)
+        return stats
+
+    async def _send_frame_batches(
+        self,
+        ws: WebSocket,
+        frame_batches: list[list[bytes]],
+        *,
+        content_type: str,
+        chunk_index_start: int,
+        request_id: str,
+        event_id: int | None = None,
+        frame_metadata: dict[str, int | str] | None = None,
+        output_format: str | None = None,
+        transport_quality: int | None = None,
+        preview_max_width: int | None = None,
+    ) -> RealtimeFrameSendStats:
+        chunk_index = chunk_index_start
+        metadata = frame_metadata or {}
+        stats = empty_frame_send_stats(content_type)
+        for frames in frame_batches:
+            split_batches = (
+                _split_frame_batch(frames, ENCODED_PREVIEW_FRAMES_PER_WS_MESSAGE)
+                if _is_encoded_preview_transport(
+                    content_type=content_type,
+                    output_format=output_format,
+                )
+                else (
+                    _split_frame_batch(frames)
+                    if content_type == RAW_RGB_CONTENT_TYPE
+                    else [frames]
+                )
+            )
+            num_frame_batches = len(split_batches)
+            encoded_preview_payloads: list[_TransportPayload] | None = None
+            if _is_encoded_preview_transport(
+                content_type=content_type,
+                output_format=output_format,
+            ):
+                timer = RealtimeStageTimer()
+                encoded_preview_payloads = await _build_encoded_preview_payloads(
+                    split_batches,
+                    content_type=content_type,
+                    metadata=metadata,
+                    output_format=output_format,
+                    transport_quality=transport_quality,
+                    preview_max_width=preview_max_width,
+                    event_id=event_id,
+                )
+                stats["raw_payload_build_ms"] += timer.mark_ms()
+            for frame_batch_index, transport_frames in enumerate(split_batches):
+                timer = RealtimeStageTimer()
+                transport_metadata = metadata
+                if encoded_preview_payloads is not None:
+                    transport_payload = encoded_preview_payloads[frame_batch_index]
+                else:
+                    if _should_build_payload_off_loop(
+                        content_type=content_type,
+                        output_format=output_format,
+                        transport_frames=transport_frames,
+                    ):
+                        transport_payload = await asyncio.to_thread(
+                            _build_transport_payload,
+                            transport_frames,
+                            content_type=content_type,
+                            metadata=metadata,
+                            output_format=output_format,
+                            transport_quality=transport_quality,
+                            preview_max_width=preview_max_width,
+                        )
+                    else:
+                        transport_payload = _build_transport_payload(
+                            transport_frames,
+                            content_type=content_type,
+                            metadata=metadata,
+                            output_format=output_format,
+                            transport_quality=transport_quality,
+                            preview_max_width=preview_max_width,
+                        )
+                    stats["raw_payload_build_ms"] += timer.mark_ms()
+
+                header: RealtimeFrameBatchHeader = {
+                    "type": "frame_batch_header",
+                    "request_id": request_id,
+                    "chunk_index": chunk_index,
+                    "content_type": transport_payload.content_type,
+                    "num_frames": len(transport_frames),
+                    "total_size": len(transport_payload.payload),
+                    "frame_batch_index": frame_batch_index,
+                    "num_frame_batches": num_frame_batches,
+                    "is_final_frame_batch": frame_batch_index == num_frame_batches - 1,
+                }
+                if event_id is not None:
+                    header["event_id"] = event_id
+                header.update(transport_metadata)
+                header.update(transport_payload.metadata)
+
+                if len(transport_payload.payload) >= FRAME_BATCH_PACK_OFFLOAD_BYTES:
+                    header_payload = _pack_frame_batch_header(header)
+                    stats["header_pack_ms"] += timer.mark_ms()
+
+                    await ws.send_bytes(header_payload)
+                    stats["header_write_ms"] += timer.mark_ms()
+
+                    await ws.send_bytes(transport_payload.payload)
+                    stats["raw_write_ms"] += timer.mark_ms()
+
+                    stats["ws_payload_bytes"] += len(header_payload) + len(
+                        transport_payload.payload
+                    )
+                else:
+                    message_payload = _pack_frame_batch_message(
+                        header,
+                        transport_payload.payload,
+                    )
+                    stats["header_pack_ms"] += timer.mark_ms()
+
+                    stats["header_write_ms"] += timer.mark_ms()
+                    await ws.send_bytes(message_payload)
+                    stats["raw_write_ms"] += timer.mark_ms()
+
+                    stats["ws_payload_bytes"] += len(message_payload)
+
+                stats["raw_bytes"] += sum(len(frame) for frame in transport_frames)
+                stats["num_frames"] += len(transport_frames)
+                stats["num_batches"] += 1
+                stats["content_type"] = transport_payload.content_type
+            chunk_index += 1
+
+        stats["ws_write_ms"] = stats["header_write_ms"] + stats["raw_write_ms"]
+        return stats

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/realtime_video_api.py

@@ -0,0 +1,500 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import shutil
+import time
+from typing import TYPE_CHECKING
+
+import msgspec.msgpack
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+    GenerateSession,
+    RealtimeChunkContext,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_output_adapter import (
+    RealtimeFrameSendStats,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.registry import (
+    get_realtime_model_adapter,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.timer import (
+    RealtimeStageTimer,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    process_generation_batch,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    ReleaseRealtimeSessionReq,
+)
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+
+logger = init_logger(__name__)
+router = APIRouter(prefix="/v1/realtime_video", tags=["realtime"])
+_ACTIVE_SESSION_IDS: set[str] = set()
+_ACTIVE_SESSION_WAIT_SECONDS = 1.0
+_ACTIVE_SESSION_WAIT_INTERVAL_SECONDS = 0.1
+
+
+def _transport_ms(value: float) -> int:
+    return max(0, int(value + 0.5))
+
+
+async def _wait_for_active_session_slot(
+    *,
+    timeout_s: float = _ACTIVE_SESSION_WAIT_SECONDS,
+    interval_s: float = _ACTIVE_SESSION_WAIT_INTERVAL_SECONDS,
+) -> bool:
+    deadline = time.monotonic() + timeout_s
+    while _ACTIVE_SESSION_IDS and time.monotonic() < deadline:
+        await asyncio.sleep(interval_s)
+    return not _ACTIVE_SESSION_IDS
+
+
+def _log_realtime_chunk_timing(
+    session: GenerateSession,
+    chunk: RealtimeChunkContext,
+    batch: "Req",
+    request_prepare_ms: float,
+    scheduler_forward_ms: float,
+    chunk_total_ms: float,
+    send_stats: RealtimeFrameSendStats,
+) -> None:
+    logger.info(
+        "realtime chunk timing: session_id=%s request_id=%s "
+        "chunk_idx=%s event_id=%s condition_kinds=%s "
+        "request_prepare=%.2fms scheduler_forward=%.2fms "
+        "output_pace=%.2fms "
+        "header_pack=%.2fms "
+        "header_write=%.2fms raw_payload_build=%.2fms raw_write=%.2fms "
+        "ws_write=%.2fms chunk_total=%.2fms batches=%d frames=%d "
+        "frame_shape=%s raw_bytes=%d ws_payload_bytes=%d content_type=%s",
+        session.id,
+        chunk.request_id,
+        batch.block_idx,
+        getattr(batch, "realtime_event_id", None),
+        sorted(batch.condition_inputs) if batch.condition_inputs else [],
+        request_prepare_ms,
+        scheduler_forward_ms,
+        send_stats["pace_wait_ms"],
+        send_stats["header_pack_ms"],
+        send_stats["header_write_ms"],
+        send_stats["raw_payload_build_ms"],
+        send_stats["raw_write_ms"],
+        send_stats["ws_write_ms"],
+        chunk_total_ms,
+        send_stats["num_batches"],
+        send_stats["num_frames"],
+        send_stats["frame_shape"],
+        send_stats["raw_bytes"],
+        send_stats["ws_payload_bytes"],
+        send_stats["content_type"],
+    )
+
+
+async def _send_realtime_chunk_stats(
+    ws: WebSocket,
+    session: GenerateSession,
+    chunk: RealtimeChunkContext,
+    batch: "Req",
+    request_prepare_ms: float,
+    scheduler_forward_ms: float,
+    chunk_total_ms: float,
+    send_stats: RealtimeFrameSendStats,
+) -> None:
+    await ws.send_bytes(
+        msgspec.msgpack.encode(
+            {
+                "type": "chunk_stats",
+                "session_id": session.id,
+                "request_id": chunk.request_id,
+                "chunk_index": batch.block_idx,
+                "event_id": getattr(batch, "realtime_event_id", None),
+                "request_prepare_ms": _transport_ms(request_prepare_ms),
+                "scheduler_forward_ms": _transport_ms(scheduler_forward_ms),
+                "pace_wait_ms": _transport_ms(send_stats["pace_wait_ms"]),
+                "header_write_ms": _transport_ms(send_stats["header_write_ms"]),
+                "raw_payload_build_ms": _transport_ms(
+                    send_stats["raw_payload_build_ms"]
+                ),
+                "raw_write_ms": _transport_ms(send_stats["raw_write_ms"]),
+                "ws_write_ms": _transport_ms(send_stats["ws_write_ms"]),
+                "chunk_total_ms": _transport_ms(chunk_total_ms),
+                "num_batches": send_stats["num_batches"],
+                "num_frames": send_stats["num_frames"],
+                "raw_bytes": send_stats["raw_bytes"],
+                "ws_payload_bytes": send_stats["ws_payload_bytes"],
+                "content_type": send_stats["content_type"],
+            }
+        )
+    )
+
+
+async def _generate_loop(ws: WebSocket, session: GenerateSession):
+    adapter = session.adapter
+    if adapter is None:
+        raise ValueError("realtime adapter is not initialized")
+
+    pending_send_task = None
+    while not session.reached_max_chunks():
+        try:
+            if pending_send_task is not None and pending_send_task.done():
+                await pending_send_task
+                pending_send_task = None
+
+            # send to scheduler and generate video chunk
+            server_args = get_global_server_args()
+
+            await adapter.wait_for_next_chunk(session)
+
+            timer = RealtimeStageTimer()
+            chunk_started = time.perf_counter()
+
+            chunk = session.new_chunk()
+            batch = adapter.prepare_next_request(
+                session,
+                server_args,
+                chunk,
+            )
+            if batch.condition_inputs:
+                logger.debug(
+                    "consume realtime conditions, session_id=%s, block_idx=%s, kinds=%s",
+                    session.id,
+                    batch.block_idx,
+                    sorted(batch.condition_inputs),
+                )
+            request_prepare_ms = timer.mark_ms()
+
+            _, result = await process_generation_batch(async_scheduler_client, batch)
+            scheduler_forward_ms = timer.mark_ms()
+
+            # finish
+            adapter.on_chunk_complete(session, result)
+            if pending_send_task is not None:
+                await pending_send_task
+            if getattr(batch, "realtime_output_pacing", False):
+                await _send_output_and_log(
+                    ws,
+                    session,
+                    chunk,
+                    batch,
+                    result,
+                    request_prepare_ms,
+                    scheduler_forward_ms,
+                    chunk_started,
+                )
+                pending_send_task = None
+            else:
+                pending_send_task = asyncio.create_task(
+                    _send_output_and_log(
+                        ws,
+                        session,
+                        chunk,
+                        batch,
+                        result,
+                        request_prepare_ms,
+                        scheduler_forward_ms,
+                        chunk_started,
+                    )
+                )
+
+        except asyncio.CancelledError:
+            if pending_send_task is not None:
+                pending_send_task.cancel()
+                await _await_realtime_task(pending_send_task)
+            logger.info("generation completed, session_id=%s", session.id)
+            break
+        except WebSocketDisconnect:
+            if pending_send_task is not None:
+                pending_send_task.cancel()
+                await _await_realtime_task(pending_send_task)
+            logger.info(
+                "client disconnected during generation, session_id=%s", session.id
+            )
+            break
+        except Exception as e:
+            if pending_send_task is not None:
+                pending_send_task.cancel()
+                await _await_realtime_task(pending_send_task)
+            err_msg = str(e).splitlines()[0]
+            logger.error("error during generate loop: %s", err_msg)
+            try:
+                await write_error_msg(f"error during generate loop: {err_msg}", ws)
+            except Exception as send_error:
+                logger.error(
+                    "error during sending complete msg: %s",
+                    send_error,
+                )
+            break
+    else:
+        if pending_send_task is not None:
+            await pending_send_task
+        logger.info(
+            "generation reached max chunks, session_id=%s, max_chunks=%s",
+            session.id,
+            session.request.max_chunks if session.request is not None else None,
+        )
+
+
+async def _send_output_and_log(
+    ws: WebSocket,
+    session: GenerateSession,
+    chunk: RealtimeChunkContext,
+    batch: "Req",
+    result,
+    request_prepare_ms: float,
+    scheduler_forward_ms: float,
+    chunk_started: float,
+) -> RealtimeFrameSendStats:
+    if session.adapter is None:
+        raise ValueError("realtime adapter is not initialized")
+    pace_wait_ms = await _wait_for_realtime_output_slot(session, batch, result)
+    send_stats = await session.adapter.send_output(
+        ws,
+        session,
+        result,
+        batch,
+    )
+    send_stats["pace_wait_ms"] = pace_wait_ms
+    chunk_total_ms = (time.perf_counter() - chunk_started) * 1000
+    _log_realtime_chunk_timing(
+        session,
+        chunk,
+        batch,
+        request_prepare_ms,
+        scheduler_forward_ms,
+        chunk_total_ms,
+        send_stats,
+    )
+    await _send_realtime_chunk_stats(
+        ws,
+        session,
+        chunk,
+        batch,
+        request_prepare_ms,
+        scheduler_forward_ms,
+        chunk_total_ms,
+        send_stats,
+    )
+    return send_stats
+
+
+def _result_num_frames(result) -> int:
+    if result.raw_frame_batches is None:
+        return 0
+    return sum(len(frames) for frames in result.raw_frame_batches)
+
+
+def _output_pacing_fps(batch: "Req") -> float:
+    fps = float(batch.fps or 0)
+    if batch.enable_frame_interpolation:
+        fps *= 2 ** int(batch.frame_interpolation_exp or 1)
+    return fps
+
+
+async def _wait_for_realtime_output_slot(
+    session: GenerateSession,
+    batch: "Req",
+    result,
+) -> float:
+    if not getattr(batch, "realtime_output_pacing", False):
+        return 0.0
+
+    frame_count = _result_num_frames(result)
+    output_fps = _output_pacing_fps(batch)
+    if frame_count <= 0 or output_fps <= 0:
+        return 0.0
+
+    now = time.perf_counter()
+    next_send_at = session.output_pace_next_send_at
+    if next_send_at is None:
+        next_send_at = now
+    if (
+        batch.realtime_event_id is not None
+        and batch.realtime_event_id != session.output_pace_last_event_id
+    ):
+        next_send_at = min(next_send_at, now)
+        session.output_pace_last_event_id = batch.realtime_event_id
+
+    wait_s = max(0.0, next_send_at - now)
+    if wait_s > 0:
+        await asyncio.sleep(wait_s)
+
+    send_started_at = time.perf_counter()
+    session.output_pace_next_send_at = (
+        max(next_send_at, send_started_at) + frame_count / output_fps
+    )
+    return wait_s * 1000
+
+
+async def _await_realtime_task(task: asyncio.Task | None) -> None:
+    if task is None:
+        return
+    try:
+        await task
+    except (asyncio.CancelledError, WebSocketDisconnect):
+        pass
+    except Exception as e:
+        logger.debug("realtime task exited with error: %s", e)
+
+
+async def _listen_events(ws: WebSocket, session: GenerateSession):
+    """listen for user events: usually condition inputs"""
+    async for message in ws.iter_bytes():
+        data = None
+        try:
+            data = msgspec.msgpack.decode(message)
+            if not isinstance(data, dict):
+                raise ValueError("realtime event must be a map")
+            realtime_event = RealtimeEvent.model_validate(data)
+            if session.adapter is None:
+                raise ValueError("realtime adapter is not initialized")
+            event_log = session.adapter.ingest_event(session, realtime_event)
+            logger.info(
+                "receive realtime event, session_id=%s, event_id=%s, %s",
+                session.id,
+                realtime_event.event_id,
+                event_log,
+            )
+        except Exception as e:
+            event_kind = data.get("kind") if isinstance(data, dict) else None
+            logger.warning("invalid event, kind=%s, error=%s", event_kind, e)
+            await write_error_msg("invalid event", ws)
+            continue
+
+
+async def _listen_generate_request(ws: WebSocket, session: GenerateSession):
+    while True:
+        try:
+            data = msgspec.msgpack.decode(await ws.receive_bytes())
+            if not isinstance(data, dict):
+                raise ValueError("generate request must be a map")
+
+            realtime_req = RealtimeVideoGenerationsRequest.model_validate(data)
+            adapter = get_realtime_model_adapter(get_global_server_args())
+            session.set_adapter(adapter)
+            await adapter.on_init(session, realtime_req)
+
+            # Keep session state update atomic with validated request.
+            session.set_request(realtime_req)
+            break
+        except WebSocketDisconnect:
+            raise
+        except Exception as e:
+            logger.warning(
+                "invalid generate request, session_id=%s, error=%s",
+                session.id,
+                e,
+            )
+            await write_error_msg("invalid generate request", ws)
+            continue
+
+
+async def _cleanup_realtime_session(
+    session: GenerateSession,
+    generate_task: asyncio.Task | None,
+    listen_task: asyncio.Task | None,
+) -> None:
+    logger.info("terminating session, session_id=%s", session.id)
+    for task in (generate_task, listen_task):
+        if task and not task.done():
+            task.cancel()
+    for task in (generate_task, listen_task):
+        if task is None:
+            continue
+        await _await_realtime_task(task)
+    try:
+        await async_scheduler_client.forward(
+            ReleaseRealtimeSessionReq(session_id=session.id)
+        )
+    except Exception as e:
+        logger.warning(
+            "failed to release realtime session on scheduler, session_id=%s, error=%s",
+            session.id,
+            e,
+        )
+    if session.input_temp_dir is not None:
+        shutil.rmtree(session.input_temp_dir, ignore_errors=True)
+    session.dispose()
+
+
+async def _close_realtime_websocket(
+    websocket: WebSocket,
+    *,
+    code: int,
+    reason: str,
+) -> None:
+    try:
+        await websocket.close(code=code, reason=reason)
+    except (RuntimeError, WebSocketDisconnect):
+        pass
+
+
+async def _wait_for_server_warmup(websocket: WebSocket) -> None:
+    warmup_done = getattr(websocket.app.state, "server_warmup_done", None)
+    if warmup_done is not None and not warmup_done.is_set():
+        await warmup_done.wait()
+
+
+@router.websocket("/generate")
+async def generate(websocket: WebSocket):
+    """endpoint for creating a new realtime session"""
+    await websocket.accept()
+    await _wait_for_server_warmup(websocket)
+    if _ACTIVE_SESSION_IDS and not await _wait_for_active_session_slot():
+        logger.warning(
+            "reject realtime session because another session is active: %s",
+            sorted(_ACTIVE_SESSION_IDS),
+        )
+        try:
+            await write_error_msg(
+                "another realtime session is already active", websocket
+            )
+        finally:
+            await websocket.close(code=1008)
+        return
+
+    session = GenerateSession()
+    _ACTIVE_SESSION_IDS.add(session.id)
+    generate_task = None
+    listen_task = None
+    try:
+        # receive new generate request
+        await _listen_generate_request(websocket, session)
+
+        # continuously generate video chunk
+        generate_task = asyncio.create_task(_generate_loop(websocket, session))
+        # continuously listen for user events
+        listen_task = asyncio.create_task(_listen_events(websocket, session))
+
+        wait_tasks = [generate_task, listen_task]
+        await asyncio.wait(wait_tasks, return_when=asyncio.FIRST_COMPLETED)
+        if generate_task.done() and session.reached_max_chunks():
+            await _close_realtime_websocket(
+                websocket,
+                code=1000,
+                reason="generation complete",
+            )
+
+    except WebSocketDisconnect:
+        logger.info("client disconnected, session_id=%s", session.id)
+    finally:
+        try:
+            await _cleanup_realtime_session(session, generate_task, listen_task)
+        finally:
+            _ACTIVE_SESSION_IDS.discard(session.id)
+
+
+async def write_error_msg(error_msg: str, websocket: WebSocket):
+    await websocket.send_bytes(
+        msgspec.msgpack.encode({"type": "error", "content": error_msg})
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/registry.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+    BaseRealtimeModelAdapter,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+_REALTIME_ADAPTER_REGISTRY: dict[type, type[BaseRealtimeModelAdapter]] = {}
+_BUILTIN_ADAPTERS_REGISTERED = False
+
+
+def register_realtime_model_adapter(
+    pipeline_config_cls: type,
+    adapter_cls: type[BaseRealtimeModelAdapter],
+) -> None:
+    _REALTIME_ADAPTER_REGISTRY[pipeline_config_cls] = adapter_cls
+
+
+def _register_builtin_realtime_model_adapters() -> None:
+    global _BUILTIN_ADAPTERS_REGISTERED
+    if _BUILTIN_ADAPTERS_REGISTERED:
+        return
+
+    from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import (
+        LingBotWorldCausalDMDConfig,
+    )
+    from sglang.multimodal_gen.configs.pipeline_configs.sana_wm import (
+        SanaWMRealtimeConfig,
+    )
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.adapters.lingbot_world_realtime_adapter import (
+        LingBotWorldRealtimeAdapter,
+    )
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.adapters.sana_wm_realtime_adapter import (
+        SanaWMRealtimeAdapter,
+    )
+
+    register_realtime_model_adapter(
+        LingBotWorldCausalDMDConfig,
+        LingBotWorldRealtimeAdapter,
+    )
+    register_realtime_model_adapter(
+        SanaWMRealtimeConfig,
+        SanaWMRealtimeAdapter,
+    )
+    _BUILTIN_ADAPTERS_REGISTERED = True
+
+
+def get_realtime_model_adapter(
+    server_args: ServerArgs,
+) -> BaseRealtimeModelAdapter:
+    _register_builtin_realtime_model_adapters()
+
+    pipeline_config = server_args.pipeline_config
+    for config_cls in type(pipeline_config).__mro__:
+        adapter_cls = _REALTIME_ADAPTER_REGISTRY.get(config_cls)
+        if adapter_cls is not None:
+            return adapter_cls()
+
+    raise ValueError(
+        "Realtime video is not supported for pipeline config "
+        f"{type(pipeline_config).__name__}; no realtime adapter is registered."
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/timer.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import time
+
+
+class RealtimeStageTimer:
+    __slots__ = ("_last", "_start")
+
+    def __init__(self):
+        now = time.perf_counter()
+        self._start = now
+        self._last = now
+
+    def mark_ms(self) -> float:
+        now = time.perf_counter()
+        elapsed_ms = (now - self._last) * 1000.0
+        self._last = now
+        return elapsed_ms
+
+    def total_ms(self) -> float:
+        return (time.perf_counter() - self._start) * 1000.0

python/sglang/multimodal_gen/runtime/entrypoints/openai/storage.py

@@ -0,0 +1,109 @@
+import asyncio
+import os
+from typing import Optional
+
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+
+class CloudStorage:
+    def __init__(self):
+        self.enabled = os.getenv("SGLANG_CLOUD_STORAGE_TYPE", "").lower() == "s3"
+        if not self.enabled:
+            return
+
+        try:
+            import boto3
+        except ImportError:
+            logger.error(
+                "boto3 is not installed. Please install it with `pip install boto3` to use cloud storage."
+            )
+            self.enabled = False
+            return
+
+        self.bucket_name = os.getenv("SGLANG_S3_BUCKET_NAME")
+        if not self.bucket_name:
+            self.enabled = False
+            return
+
+        endpoint_url = os.getenv("SGLANG_S3_ENDPOINT_URL") or None
+        region_name = os.getenv("SGLANG_S3_REGION_NAME") or None
+
+        self.client = boto3.client(
+            "s3",
+            aws_access_key_id=os.getenv("SGLANG_S3_ACCESS_KEY_ID"),
+            aws_secret_access_key=os.getenv("SGLANG_S3_SECRET_ACCESS_KEY"),
+            endpoint_url=endpoint_url,
+            region_name=region_name,
+        )
+        self.endpoint_url = endpoint_url
+        self.region_name = region_name
+
+    def is_enabled(self) -> bool:
+        return self.enabled
+
+    async def upload_file(self, local_path: str, destination_key: str) -> Optional[str]:
+        if not self.is_enabled():
+            return None
+
+        def _sync_upload():
+            """Synchronous part of the upload to run in a thread."""
+            ext = os.path.splitext(local_path)[1].lower()
+            content_type = {
+                ".png": "image/png",
+                ".jpg": "image/jpeg",
+                ".jpeg": "image/jpeg",
+                ".webp": "image/webp",
+                ".mp4": "video/mp4",
+                ".glb": "model/gltf-binary",
+                ".obj": "text/plain",
+            }.get(ext, "application/octet-stream")
+
+            # Use the client created once in __init__
+            self.client.upload_file(
+                local_path,
+                self.bucket_name,
+                destination_key,
+                ExtraArgs={"ContentType": content_type},
+            )
+
+        try:
+            # Offload the blocking I/O call to a thread executor
+            await asyncio.get_running_loop().run_in_executor(None, _sync_upload)
+        except Exception as e:
+            # If upload fails, log the error and return None for fallback
+            logger.error(f"Upload failed for {destination_key}: {e}")
+            return None
+
+        # Simplified URL generation with a default region
+        if self.endpoint_url:
+            url = (
+                f"{self.endpoint_url.rstrip('/')}/{self.bucket_name}/{destination_key}"
+            )
+        else:
+            region = self.region_name or "us-east-1"
+            url = f"https://{self.bucket_name}.s3.{region}.amazonaws.com/{destination_key}"
+
+        logger.info(f"Uploaded {local_path} to {url}")
+        return url
+
+    async def upload_and_cleanup(self, file_path: str) -> Optional[str]:
+        """Helper to upload a file and delete the local copy if successful."""
+        if not self.is_enabled():
+            return None
+
+        key = os.path.basename(file_path)
+        url = await self.upload_file(file_path, key)
+
+        if url:
+            try:
+                # pass if removal fails
+                os.remove(file_path)
+            except OSError as e:
+                logger.warning(f"Failed to remove temporary file {file_path}: {e}")
+        return url
+
+
+# Global instance
+cloud_storage = CloudStorage()

python/sglang/multimodal_gen/runtime/entrypoints/openai/stores.py

@@ -0,0 +1,48 @@
+import asyncio
+from typing import Any, Dict, List, Optional
+
+
+class AsyncDictStore:
+    """A small async-safe in-memory key-value store for dict items.
+
+    This encapsulates the usual pattern of a module-level dict guarded by
+    an asyncio.Lock and provides simple CRUD methods that are safe to call
+    concurrently from FastAPI request handlers and background tasks.
+    """
+
+    def __init__(self) -> None:
+        self._items: Dict[str, Dict[str, Any]] = {}
+        self._lock = asyncio.Lock()
+
+    async def upsert(self, key: str, value: Dict[str, Any]) -> None:
+        async with self._lock:
+            self._items[key] = value
+
+    async def update_fields(
+        self, key: str, updates: Dict[str, Any]
+    ) -> Optional[Dict[str, Any]]:
+        async with self._lock:
+            item = self._items.get(key)
+            if item is None:
+                return None
+            item.update(updates)
+            return item
+
+    async def get(self, key: str) -> Optional[Dict[str, Any]]:
+        async with self._lock:
+            return self._items.get(key)
+
+    async def pop(self, key: str) -> Optional[Dict[str, Any]]:
+        async with self._lock:
+            return self._items.pop(key, None)
+
+    async def list_values(self) -> List[Dict[str, Any]]:
+        async with self._lock:
+            return list(self._items.values())
+
+
+# Global stores shared by OpenAI entrypoints
+# [request_id, dict]
+VIDEO_STORE = AsyncDictStore()
+IMAGE_STORE = AsyncDictStore()
+MESH_STORE = AsyncDictStore()

python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py

@@ -0,0 +1,452 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+import asyncio
+import inspect
+import json
+import os
+import shutil
+import tempfile
+import time
+from contextlib import contextmanager
+from typing import Any, Generator, List, Optional, Union
+
+import httpx
+from fastapi import HTTPException, UploadFile
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    DataType,
+    SamplingParams,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    ListLorasReq,
+    MergeLoraWeightsReq,
+    SetLoraReq,
+    ShutdownReq,
+    UnmergeLoraWeightsReq,
+    format_lora_message,
+    save_outputs,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.scheduler_client import AsyncSchedulerClient
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.common import parse_size
+from sglang.multimodal_gen.runtime.utils.image_io import save_base64_image_to_path
+from sglang.multimodal_gen.runtime.utils.logging_utils import (
+    init_logger,
+    log_batch_completion,
+    log_generation_timer,
+)
+from sglang.multimodal_gen.runtime.utils.trace_wrapper import trace_req
+
+# re-export LoRA protocol types for backward compatibility
+__all__ = [
+    "SetLoraReq",
+    "MergeLoraWeightsReq",
+    "UnmergeLoraWeightsReq",
+    "ListLorasReq",
+    "ShutdownReq",
+    "format_lora_message",
+]
+
+logger = init_logger(__name__)
+
+OUTPUT_QUALITY_MAPPER = {"maximum": 100, "high": 90, "medium": 55, "low": 35}
+DEFAULT_FPS = 24
+DEFAULT_VIDEO_SECONDS = 4
+
+
+def _bad_request(message: str) -> HTTPException:
+    return HTTPException(status_code=400, detail=message)
+
+
+def _parse_size_or_raise(size: str) -> tuple[int, int]:
+    width, height = parse_size(size)
+    if width is None or height is None or width <= 0 or height <= 0:
+        raise _bad_request("size must be formatted as positive WIDTHxHEIGHT")
+    return width, height
+
+
+def _validate_positive_int(kwargs: dict[str, Any], name: str) -> None:
+    value = kwargs.get(name)
+    if value is not None and int(value) <= 0:
+        raise _bad_request(f"{name} must be positive")
+
+
+def flatten_extra_params(payload: Any) -> dict[str, Any]:
+    """Promote vLLM-Omni-style extra_params into regular request fields."""
+    if not isinstance(payload, dict):
+        return {}
+
+    extra_params = payload.pop("extra_params", None)
+    if isinstance(extra_params, str):
+        try:
+            extra_params = json.loads(extra_params)
+        except Exception:
+            extra_params = None
+    if not isinstance(extra_params, dict):
+        if "guardrails" in payload:
+            payload.setdefault("use_guardrails", payload["guardrails"])
+        return payload
+
+    for key, value in extra_params.items():
+        payload.setdefault(key, value)
+    if "guardrails" in extra_params:
+        payload.setdefault("use_guardrails", extra_params["guardrails"])
+
+    return payload
+
+
+@contextmanager
+def temp_dir_if_disabled(
+    configured_path: str | None,
+) -> Generator[str, None, None]:
+    """Yield *configured_path* when it is set, otherwise create a temporary
+    directory that is automatically removed when the context exits."""
+    if configured_path is not None:
+        os.makedirs(configured_path, exist_ok=True)
+        yield configured_path
+    else:
+        tmp = tempfile.mkdtemp(prefix="sglang_")
+        try:
+            yield tmp
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+
+
+def choose_output_image_ext(
+    output_format: Optional[str], background: Optional[str]
+) -> str:
+    fmt = (output_format or "").lower()
+    if fmt in {"png", "webp", "jpeg", "jpg"}:
+        return "jpg" if fmt == "jpeg" else fmt
+    if (background or "auto").lower() == "transparent":
+        return "png"
+    return "jpg"
+
+
+def build_sampling_params(request_id: str, **kwargs) -> SamplingParams:
+    """Build SamplingParams from request parameters.
+
+    Handles size parsing, output_quality resolution, and None filtering before
+    delegating to SamplingParams.from_user_sampling_params_args. Callers pass
+    only the parameters they have; None values are stripped automatically so
+    that SamplingParams defaults apply.
+    """
+    server_args = get_global_server_args()
+
+    # pop HTTP-layer params that aren't SamplingParams fields
+    output_quality = kwargs.pop("output_quality", None)
+
+    has_explicit_compression = kwargs.get("output_compression") is not None
+
+    # parse "WxH" size string if provided
+    size = kwargs.pop("size", None)
+    if size:
+        w, h = _parse_size_or_raise(size)
+        # treat None dimensions as unset so parsed size can fill them
+        if kwargs.get("width") is None:
+            kwargs["width"] = w
+        if kwargs.get("height") is None:
+            kwargs["height"] = h
+
+    for name in (
+        "width",
+        "height",
+        "num_frames",
+        "num_inference_steps",
+        "num_outputs_per_prompt",
+    ):
+        _validate_positive_int(kwargs, name)
+
+    # filter out None values to let SamplingParams defaults apply
+    kwargs = {k: v for k, v in kwargs.items() if v is not None}
+    kwargs.setdefault("save_output", True)
+
+    sampling_params = SamplingParams.from_user_sampling_params_args(
+        model_path=server_args.model_path,
+        server_args=server_args,
+        request_id=request_id,
+        **kwargs,
+    )
+
+    # resolve output_quality → output_compression with the correct data_type.
+    # SamplingParams.__post_init__ may have resolved with the wrong data_type
+    # (default VIDEO) before _adjust() set the correct one.
+    if not has_explicit_compression and output_quality is not None:
+        resolved = adjust_output_quality(output_quality, sampling_params.data_type)
+        if resolved is not None:
+            sampling_params.output_compression = resolved
+
+    return sampling_params
+
+
+async def save_image_to_path(
+    image: Union[UploadFile, bytes, str],
+    target_path: str,
+    *,
+    prefer_remote_source: bool = False,
+) -> str:
+    input_path = await _maybe_url_image(
+        image, target_path, prefer_remote_source=prefer_remote_source
+    )
+    if input_path is None:
+        input_path = await _save_upload_to_path(image, target_path)
+    return input_path
+
+
+# Helpers
+async def _save_upload_to_path(
+    upload: Union[UploadFile, bytes], target_path: str
+) -> str:
+    os.makedirs(os.path.dirname(target_path), exist_ok=True)
+    if isinstance(upload, bytes):
+        content = upload
+    elif isinstance(upload, (bytearray, memoryview)):
+        content = bytes(upload)
+    else:
+        read = getattr(upload, "read", None)
+        if not callable(read):
+            raise TypeError(f"Unsupported image upload type: {type(upload).__name__}")
+        content = read()
+        if inspect.isawaitable(content):
+            content = await content
+        if isinstance(content, (bytearray, memoryview)):
+            content = bytes(content)
+        if not isinstance(content, bytes):
+            raise TypeError(
+                f"Image upload read() returned {type(content).__name__}, expected bytes"
+            )
+    with open(target_path, "wb") as f:
+        f.write(content)
+    return target_path
+
+
+async def _maybe_url_image(
+    img_url: str,
+    target_path: str,
+    *,
+    prefer_remote_source: bool = False,
+) -> str | None:
+    if not isinstance(img_url, str):
+        return None
+
+    if img_url.lower().startswith(("http://", "https://")):
+        # Only bypass persistence when the caller explicitly disables input saves.
+        # Otherwise keep the prefetch outside the measured server stages.
+        if prefer_remote_source:
+            return img_url
+        # download image from URL and persist on disk
+        input_path = await _save_url_image_to_path(img_url, target_path)
+        return input_path
+    elif img_url.startswith("data:image"):
+        if prefer_remote_source:
+            return img_url
+        # encode image base64 url and persist on disk
+        input_path = save_base64_image_to_path(img_url, target_path)
+        return input_path
+    else:
+        raise ValueError("Unsupported image url format")
+
+
+async def _save_url_image_to_path(image_url: str, target_path: str) -> str:
+    """Download image from URL and save to target path."""
+
+    def _is_retryable_download_error(error: Exception) -> bool:
+        if isinstance(error, httpx.HTTPStatusError):
+            status_code = error.response.status_code
+            # Retry on rate limit and transient server-side failures.
+            return status_code == 429 or 500 <= status_code < 600
+        # Retry on transient network/protocol issues.
+        return isinstance(
+            error,
+            (
+                httpx.TimeoutException,
+                httpx.NetworkError,
+                httpx.RemoteProtocolError,
+            ),
+        )
+
+    os.makedirs(os.path.dirname(target_path), exist_ok=True)
+
+    max_attempts = 3
+    backoff_seconds = 0.2
+    last_error: Exception | None = None
+
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            for attempt in range(1, max_attempts + 1):
+                try:
+                    response = await client.get(image_url, timeout=10.0)
+                    response.raise_for_status()
+
+                    # Determine file extension from content type or URL after downloading
+                    if not os.path.splitext(target_path)[1]:
+                        content_type = response.headers.get("content-type", "").lower()
+
+                        url_path = image_url.split("?")[0]
+                        _, url_ext = os.path.splitext(url_path)
+                        url_ext = url_ext.lower()
+
+                        if url_ext in {
+                            ".jpg",
+                            ".jpeg",
+                            ".png",
+                            ".webp",
+                            ".gif",
+                            ".bmp",
+                        }:
+                            ext = ".jpg" if url_ext == ".jpeg" else url_ext
+                        elif content_type.startswith("image/"):
+                            if "jpeg" in content_type or "jpg" in content_type:
+                                ext = ".jpg"
+                            elif "png" in content_type:
+                                ext = ".png"
+                            elif "webp" in content_type:
+                                ext = ".webp"
+                            else:
+                                ext = ".jpg"  # Default to jpg
+                        elif content_type == "application/octet-stream":
+                            # for octet-stream, if we couldn't get it from URL, default to jpg
+                            ext = ".jpg"
+                        else:
+                            raise ValueError(
+                                f"URL does not point to an image. Content-Type: {content_type}"
+                            )
+                        target_path = f"{target_path}{ext}"
+
+                    with open(target_path, "wb") as f:
+                        f.write(response.content)
+
+                    return target_path
+                except Exception as e:
+                    last_error = e
+                    if attempt == max_attempts or not _is_retryable_download_error(e):
+                        raise
+                    wait_s = backoff_seconds * (2 ** (attempt - 1))
+                    logger.warning(
+                        "Retrying image download (%s/%s) for %s after %.1fs due to: %s",
+                        attempt,
+                        max_attempts,
+                        image_url,
+                        wait_s,
+                        e,
+                    )
+                    await asyncio.sleep(wait_s)
+    except Exception as e:
+        final_error = last_error or e
+        raise Exception(
+            f"Failed to download image from URL {image_url}: {str(final_error)}"
+        )
+
+
+async def process_generation_batch(
+    scheduler_client: AsyncSchedulerClient,
+    batch,
+) -> tuple[list[str], OutputBatch]:
+    total_start_time = time.perf_counter()
+    with trace_req(batch.trace_ctx), log_generation_timer(logger, batch.prompt):
+        result = await scheduler_client.forward([batch])
+
+        if (
+            result.output is None
+            and result.output_file_paths is None
+            and result.raw_frame_batches is None
+        ):
+            error_msg = result.error or "Unknown error"
+            raise RuntimeError(
+                f"Model generation returned no output. Error from scheduler: {error_msg}"
+            )
+
+        save_file_path_list = []
+        if result.output_file_paths:
+            save_file_path_list = result.output_file_paths
+        elif result.output is not None:
+            num_outputs = len(result.output)
+            save_file_path_list = save_outputs(
+                result.output,
+                batch.data_type,
+                batch.fps,
+                batch.save_output,
+                lambda idx: str(batch.output_file_path(num_outputs, idx)),
+                audio=result.audio,
+                audio_sample_rate=result.audio_sample_rate,
+                output_compression=batch.output_compression,
+                enable_frame_interpolation=batch.enable_frame_interpolation,
+                frame_interpolation_exp=batch.frame_interpolation_exp,
+                frame_interpolation_scale=batch.frame_interpolation_scale,
+                frame_interpolation_model_path=batch.frame_interpolation_model_path,
+                enable_upscaling=batch.enable_upscaling,
+                upscaling_model_path=batch.upscaling_model_path,
+                upscaling_scale=batch.upscaling_scale,
+            )
+
+    total_time = time.perf_counter() - total_start_time
+    if get_global_server_args().batching_max_size > 1:
+        log_batch_completion(
+            logger,
+            len(save_file_path_list),
+            total_time,
+        )
+
+    if result.peak_memory_mb and result.peak_memory_mb > 0:
+        logger.info(f"Peak memory usage: {result.peak_memory_mb:.2f} MB")
+
+    return save_file_path_list, result
+
+
+def merge_image_input_list(*inputs: Union[List, Any, None]) -> List:
+    """
+    Merge multiple image input sources into a single list.
+
+    This function handles both single items and lists of items, merging them
+    into a single flattened list. Useful for processing images, URLs, or other
+    multimedia inputs that can come as either single items or lists.
+
+    Args:
+        *inputs: Variable number of inputs, each can be None, single item, or list
+
+    Returns:
+        List: Flattened list of all non-None inputs
+
+    Example:
+        >>> merge_image_input_list(["img1", "img2"], "img3", None)
+        ["img1", "img2", "img3"]
+    """
+    result = []
+    for input_item in inputs:
+        if input_item is not None:
+            if isinstance(input_item, list):
+                result.extend(input_item)
+            else:
+                result.append(input_item)
+    return result
+
+
+def add_common_data_to_response(
+    response: dict, request_id: str, result: OutputBatch
+) -> dict:
+    if result.peak_memory_mb and result.peak_memory_mb > 0:
+        response["peak_memory_mb"] = result.peak_memory_mb
+
+    if result.metrics and result.metrics.total_duration_s > 0:
+        response["inference_time_s"] = result.metrics.total_duration_s
+
+    response["id"] = request_id
+
+    if result.action_pred is not None:
+        t = result.action_pred
+        response["action"] = {
+            "data": t.tolist(),
+            "shape": list(t.shape),
+            "dtype": str(t.dtype).replace("torch.", ""),
+            "raw_action_dim": result.action_raw_action_dim,
+            "action_mode": result.action_mode,
+            "domain_id": result.action_domain_id,
+        }
+
+    return response
+
+
+def adjust_output_quality(output_quality: str, data_type: DataType = None) -> int:
+    if output_quality == "default":
+        return 50 if data_type == DataType.VIDEO else 75
+    return OUTPUT_QUALITY_MAPPER.get(output_quality, None)

python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py

@@ -0,0 +1,741 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+import asyncio
+import json
+import os
+import shutil
+import tempfile
+import time
+from typing import Any, Dict, Optional
+
+from fastapi import (
+    APIRouter,
+    File,
+    Form,
+    HTTPException,
+    Path,
+    Query,
+    Request,
+    UploadFile,
+)
+from fastapi.responses import FileResponse
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    SamplingParams,
+    generate_request_id,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    VideoGenerationsRequest,
+    VideoListResponse,
+    VideoResponse,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.storage import cloud_storage
+from sglang.multimodal_gen.runtime.entrypoints.openai.stores import VIDEO_STORE
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    DEFAULT_FPS,
+    DEFAULT_VIDEO_SECONDS,
+    add_common_data_to_response,
+    build_sampling_params,
+    flatten_extra_params,
+    merge_image_input_list,
+    process_generation_batch,
+    save_image_to_path,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.srt.observability.trace import extract_trace_headers
+
+logger = init_logger(__name__)
+router = APIRouter(prefix="/v1/videos", tags=["videos"])
+
+_VIDEO_EXTENSIONS = {
+    ".avi",
+    ".gif",
+    ".m4v",
+    ".mkv",
+    ".mov",
+    ".mp4",
+    ".mpeg",
+    ".mpg",
+    ".webm",
+}
+
+
+def _extra_value(request: VideoGenerationsRequest, name: str) -> Any:
+    return (request.model_extra or {}).get(name)
+
+
+def _request_value(request: VideoGenerationsRequest, name: str) -> Any:
+    value = getattr(request, name, None)
+    if value is not None:
+        return value
+    return _extra_value(request, name)
+
+
+def _parse_form_extra_value(value: Any) -> Any:
+    if not isinstance(value, str):
+        return value
+    try:
+        return json.loads(value)
+    except Exception:
+        return value
+
+
+def _is_probably_video_source(source: Any) -> bool:
+    content_type = (getattr(source, "content_type", "") or "").lower()
+    if content_type.startswith("video/"):
+        return True
+
+    if isinstance(source, str):
+        if source.lower().startswith("data:video"):
+            return True
+        source_name = source
+    else:
+        source_name = getattr(source, "filename", None)
+
+    if not source_name:
+        return False
+    source_name = str(source_name).split("?", 1)[0].split("#", 1)[0]
+    return os.path.splitext(source_name)[1].lower() in _VIDEO_EXTENSIONS
+
+
+def _is_cosmos3_server(server_args) -> bool:
+    from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config
+
+    return isinstance(server_args.pipeline_config, Cosmos3Config)
+
+
+def _normalize_optional_string(value: Any) -> Any:
+    if isinstance(value, str) and not value.strip():
+        return None
+    return value
+
+
+def _coerce_optional_int_list(value: Any) -> list[int] | None:
+    value = _parse_form_extra_value(value)
+    if value is None:
+        return None
+    if isinstance(value, str) and not value.strip():
+        return None
+    if isinstance(value, (list, tuple)):
+        return [int(item) for item in value]
+    return [int(value)]
+
+
+def _resolve_video_path(req: VideoGenerationsRequest) -> str | None:
+    video_path = _request_value(req, "video_path") or _request_value(req, "video_url")
+    if video_path:
+        return str(video_path)
+
+    input_reference = _request_value(req, "input_reference")
+    if _is_probably_video_source(input_reference):
+        return str(input_reference)
+
+    reference_url = _request_value(req, "reference_url")
+    if _is_probably_video_source(reference_url):
+        return str(reference_url)
+
+    return None
+
+
+def _resolve_image_path(
+    req: VideoGenerationsRequest, video_path: str | None
+) -> str | None:
+    image_path = _request_value(req, "input_reference")
+    if video_path and image_path == video_path:
+        return None
+    if _is_probably_video_source(image_path):
+        return None
+    return image_path
+
+
+def _resolve_sound_duration(
+    req: VideoGenerationsRequest, *, num_frames: int, fps: int
+) -> float | None:
+    generate_sound = _request_value(req, "generate_sound")
+    sound_duration = _request_value(req, "sound_duration")
+
+    if generate_sound is False:
+        return 0.0
+    if sound_duration is not None:
+        return float(sound_duration)
+    if generate_sound is True:
+        return float(num_frames) / float(fps)
+    return None
+
+
+def _cosmos3_sampling_param_kwargs(
+    req: VideoGenerationsRequest, *, num_frames: int, fps: int
+) -> Dict[str, Any]:
+    """Map HTTP/API aliases to Cosmos3SamplingParams field names."""
+    kwargs: Dict[str, Any] = {}
+
+    sound_duration = _resolve_sound_duration(req, num_frames=num_frames, fps=fps)
+    if sound_duration is not None:
+        kwargs["sound_duration"] = sound_duration
+
+    condition_frame_indexes = _request_value(req, "condition_frame_indexes")
+    if condition_frame_indexes is None:
+        condition_frame_indexes = _request_value(req, "condition_frame_indexes_vision")
+    condition_frame_indexes = _coerce_optional_int_list(condition_frame_indexes)
+    if condition_frame_indexes is not None:
+        kwargs["condition_frame_indexes"] = condition_frame_indexes
+
+    for name in (
+        "condition_video_keep",
+        "action_mode",
+        "domain_id",
+        "domain_name",
+        "raw_action_dim",
+        "action_fps",
+        "action",
+        "action_view_point",
+        "action_normalization",
+    ):
+        value = _parse_form_extra_value(_request_value(req, name))
+        value = _normalize_optional_string(value)
+        if value is not None:
+            kwargs[name] = value
+
+    return kwargs
+
+
+def _build_video_sampling_params(request_id: str, request: VideoGenerationsRequest):
+    """Resolve video-specific defaults (fps, seconds → num_frames) then
+    delegate to the shared build_sampling_params."""
+    server_args = get_global_server_args()
+    seconds = request.seconds if request.seconds is not None else DEFAULT_VIDEO_SECONDS
+    fps = request.fps if request.fps is not None else DEFAULT_FPS
+    num_frames = request.num_frames if request.num_frames is not None else fps * seconds
+    num_outputs = request.num_outputs_per_prompt
+    if num_outputs is None:
+        num_outputs = request.n or 1
+    video_path = _resolve_video_path(request)
+    image_path = _resolve_image_path(request, video_path)
+    cosmos3_kwargs = {}
+    if _is_cosmos3_server(server_args):
+        cosmos3_kwargs = _cosmos3_sampling_param_kwargs(
+            request, num_frames=num_frames, fps=fps
+        )
+        if server_args.pipeline_config.action_stats_path is not None:
+            cosmos3_kwargs["action_stats_path"] = (
+                server_args.pipeline_config.action_stats_path
+            )
+
+    return build_sampling_params(
+        request_id,
+        prompt=request.prompt,
+        num_outputs_per_prompt=max(1, min(int(num_outputs), 10)),
+        size=request.size,
+        width=request.width,
+        height=request.height,
+        num_frames=num_frames,
+        fps=fps,
+        image_path=image_path,
+        video_path=video_path,
+        output_file_name=request_id,
+        seed=request.seed,
+        generator_device=request.generator_device,
+        num_inference_steps=request.num_inference_steps,
+        guidance_scale=request.guidance_scale,
+        guidance_scale_2=request.guidance_scale_2,
+        negative_prompt=request.negative_prompt,
+        max_sequence_length=request.max_sequence_length,
+        flow_shift=request.flow_shift,
+        use_duration_template=_extra_value(request, "use_duration_template"),
+        use_resolution_template=_extra_value(request, "use_resolution_template"),
+        use_system_prompt=_extra_value(request, "use_system_prompt"),
+        use_guardrails=_extra_value(request, "use_guardrails"),
+        enable_teacache=request.enable_teacache,
+        enable_frame_interpolation=request.enable_frame_interpolation,
+        frame_interpolation_exp=request.frame_interpolation_exp,
+        frame_interpolation_scale=request.frame_interpolation_scale,
+        frame_interpolation_model_path=request.frame_interpolation_model_path,
+        enable_upscaling=request.enable_upscaling,
+        upscaling_model_path=request.upscaling_model_path,
+        upscaling_scale=request.upscaling_scale,
+        output_path=request.output_path,
+        output_compression=request.output_compression,
+        output_quality=request.output_quality,
+        perf_dump_path=request.perf_dump_path,
+        diffusers_kwargs=request.diffusers_kwargs,
+        **cosmos3_kwargs,
+    )
+
+
+# extract metadata which http_server needs to know
+def _video_job_from_sampling(
+    request_id: str, req: VideoGenerationsRequest, sampling: SamplingParams
+) -> Dict[str, Any]:
+    size_str = f"{sampling.width}x{sampling.height}"
+    seconds = int(round((sampling.num_frames or 0) / float(sampling.fps or 24)))
+    return {
+        "id": request_id,
+        "object": "video",
+        "model": req.model or "sora-2",
+        "status": "queued",
+        "progress": 0,
+        "created_at": int(time.time()),
+        "size": size_str,
+        "seconds": str(seconds),
+        "quality": "standard",
+        "file_path": os.path.abspath(sampling.output_file_path()),
+    }
+
+
+async def _save_first_input_image(
+    image_sources,
+    request_id: str,
+    uploads_dir: str,
+    *,
+    prefer_remote_source: bool = False,
+) -> str | None:
+    """Save the first input image from a list of sources and return its path."""
+    image_list = merge_image_input_list(image_sources)
+    if not image_list:
+        return None
+    image = image_list[0]
+
+    os.makedirs(uploads_dir, exist_ok=True)
+
+    filename = image.filename if hasattr(image, "filename") else "url_image"
+    target_path = os.path.join(uploads_dir, f"{request_id}_{filename}")
+    return await save_image_to_path(
+        image, target_path, prefer_remote_source=prefer_remote_source
+    )
+
+
+async def _dispatch_job_async(
+    job_id: str,
+    batch: Req,
+    *,
+    temp_dirs: list[str] | None = None,
+    output_persistent: bool = True,
+) -> None:
+    from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+
+    try:
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        persistent_path = (
+            save_file_path if not cloud_url and output_persistent else None
+        )
+        update_fields = {
+            "status": "completed",
+            "progress": 100,
+            "completed_at": int(time.time()),
+            "url": cloud_url,
+            "file_path": persistent_path,
+            "file_paths": (
+                [os.path.abspath(path) for path in save_file_path_list]
+                if output_persistent
+                else None
+            ),
+            "num_outputs": len(save_file_path_list),
+        }
+        update_fields = add_common_data_to_response(
+            update_fields, request_id=job_id, result=result
+        )
+        await VIDEO_STORE.update_fields(job_id, update_fields)
+    except Exception as e:
+        logger.error(f"{e}")
+        await VIDEO_STORE.update_fields(
+            job_id, {"status": "failed", "error": {"message": str(e)}}
+        )
+    finally:
+        for td in temp_dirs or []:
+            shutil.rmtree(td, ignore_errors=True)
+
+
+# TODO: support image to video generation
+@router.post("", response_model=VideoResponse)
+async def create_video(
+    request: Request,
+    # multipart/form-data fields (optional; used only when content-type is multipart)
+    prompt: Optional[str] = Form(None),
+    input_reference: Optional[UploadFile] = File(None),
+    reference_url: Optional[str] = Form(None),
+    video_reference: Optional[UploadFile] = File(None),
+    video_url: Optional[str] = Form(None),
+    video_path: Optional[str] = Form(None),
+    model: Optional[str] = Form(None),
+    n: Optional[int] = Form(1),
+    num_outputs_per_prompt: Optional[int] = Form(None),
+    seconds: Optional[int] = Form(None),
+    size: Optional[str] = Form(None),
+    fps: Optional[int] = Form(None),
+    num_frames: Optional[int] = Form(None),
+    seed: Optional[int] = Form(None),
+    generator_device: Optional[str] = Form("cuda"),
+    negative_prompt: Optional[str] = Form(None),
+    guidance_scale: Optional[float] = Form(None),
+    num_inference_steps: Optional[int] = Form(None),
+    max_sequence_length: Optional[int] = Form(None),
+    flow_shift: Optional[float] = Form(None),
+    enable_teacache: Optional[bool] = Form(None),
+    enable_frame_interpolation: Optional[bool] = Form(None),
+    frame_interpolation_exp: Optional[int] = Form(None),
+    frame_interpolation_scale: Optional[float] = Form(None),
+    frame_interpolation_model_path: Optional[str] = Form(None),
+    enable_upscaling: Optional[bool] = Form(None),
+    upscaling_model_path: Optional[str] = Form(None),
+    upscaling_scale: Optional[int] = Form(None),
+    output_quality: Optional[str] = Form(None),
+    output_compression: Optional[int] = Form(None),
+    output_path: Optional[str] = Form(None),
+    extra_params: Optional[str] = Form(None),
+    extra_body: Optional[str] = Form(None),
+):
+    content_type = request.headers.get("content-type", "").lower()
+    request_id = generate_request_id()
+
+    server_args = get_global_server_args()
+    task_type = server_args.pipeline_config.task_type
+
+    # Resolve input upload directory (may be a temp dir when saving is disabled)
+    temp_dirs: list[str] = []
+    if server_args.input_save_path is not None:
+        uploads_dir = server_args.input_save_path
+        os.makedirs(uploads_dir, exist_ok=True)
+    else:
+        uploads_dir = tempfile.mkdtemp(prefix="sglang_input_")
+        temp_dirs.append(uploads_dir)
+
+    # Resolve output directory
+    effective_output_path = server_args.output_path
+    output_persistent = True
+    if "multipart/form-data" not in content_type:
+        # JSON body may carry a per-request output_path; checked after parsing below
+        pass
+
+    if "multipart/form-data" in content_type:
+        if not prompt:
+            raise HTTPException(status_code=400, detail="prompt is required")
+
+        video_input_path = None
+        image_sources = merge_image_input_list(input_reference, reference_url)
+        if video_reference is not None:
+            video_input_path = await _save_first_input_image(
+                video_reference,
+                request_id,
+                uploads_dir,
+                prefer_remote_source=server_args.input_save_path is None,
+            )
+        elif video_path or video_url:
+            video_input_path = video_path or video_url
+        elif input_reference is not None and _is_probably_video_source(input_reference):
+            video_input_path = await _save_first_input_image(
+                input_reference,
+                request_id,
+                uploads_dir,
+                prefer_remote_source=server_args.input_save_path is None,
+            )
+            image_sources = merge_image_input_list(reference_url)
+        elif reference_url and _is_probably_video_source(reference_url):
+            video_input_path = reference_url
+            image_sources = merge_image_input_list(input_reference)
+
+        # Validate image input based on model task type
+        if task_type.requires_image_input() and not image_sources:
+            raise HTTPException(
+                status_code=400,
+                detail="input_reference or reference_url is required for image-to-video generation",
+            )
+        input_path = None
+        if image_sources:
+            try:
+                input_path = await _save_first_input_image(
+                    image_sources,
+                    request_id,
+                    uploads_dir,
+                    prefer_remote_source=server_args.input_save_path is None,
+                )
+            except Exception as e:
+                raise HTTPException(
+                    status_code=400, detail=f"Failed to process image source: {str(e)}"
+                )
+
+        # Parse extra_body JSON (if provided in multipart form) to get fps/num_frames overrides
+        extra_from_form: Dict[str, Any] = {}
+        if extra_body:
+            try:
+                extra_from_form = flatten_extra_params(json.loads(extra_body))
+            except Exception:
+                extra_from_form = {}
+        if extra_params:
+            try:
+                extra_from_form.update(
+                    flatten_extra_params({"extra_params": json.loads(extra_params)})
+                )
+            except Exception:
+                pass
+
+        def form_value(name: str, value: Any) -> Any:
+            selected = value if value is not None else extra_from_form.get(name)
+            return _parse_form_extra_value(selected)
+
+        raw_form = await request.form()
+        for key in (
+            "use_duration_template",
+            "use_resolution_template",
+            "use_system_prompt",
+            "use_guardrails",
+            "guardrails",
+            "video_path",
+            "video_url",
+            "generate_sound",
+            "sound_duration",
+            "condition_frame_indexes",
+            "action_mode",
+            "domain_id",
+            "domain_name",
+            "raw_action_dim",
+            "action_fps",
+            "action",
+            "action_view_point",
+            "action_normalization",
+            "condition_frame_indexes_vision",
+            "condition_video_keep",
+        ):
+            if key in raw_form and key not in extra_from_form:
+                extra_from_form[key] = _parse_form_extra_value(raw_form[key])
+        flatten_extra_params(extra_from_form)
+
+        request_field_names = set(VideoGenerationsRequest.model_fields)
+        extra_request_fields = {
+            key: value
+            for key, value in extra_from_form.items()
+            if key not in request_field_names
+        }
+        fps_val = form_value("fps", fps)
+        num_frames_val = form_value("num_frames", num_frames)
+
+        req = VideoGenerationsRequest(
+            prompt=prompt,
+            input_reference=input_path,
+            video_path=form_value("video_path", video_input_path),
+            video_url=form_value("video_url", video_url),
+            model=form_value("model", model),
+            n=form_value("n", n),
+            num_outputs_per_prompt=form_value(
+                "num_outputs_per_prompt", num_outputs_per_prompt
+            ),
+            seconds=form_value("seconds", seconds) or 4,
+            size=form_value("size", size),
+            fps=fps_val,
+            num_frames=num_frames_val,
+            seed=form_value("seed", seed),
+            generator_device=form_value("generator_device", generator_device),
+            negative_prompt=form_value("negative_prompt", negative_prompt),
+            num_inference_steps=form_value("num_inference_steps", num_inference_steps),
+            guidance_scale=form_value("guidance_scale", guidance_scale),
+            max_sequence_length=form_value("max_sequence_length", max_sequence_length),
+            flow_shift=form_value("flow_shift", flow_shift),
+            enable_teacache=form_value("enable_teacache", enable_teacache),
+            enable_frame_interpolation=form_value(
+                "enable_frame_interpolation", enable_frame_interpolation
+            ),
+            frame_interpolation_exp=form_value(
+                "frame_interpolation_exp", frame_interpolation_exp
+            ),
+            frame_interpolation_scale=form_value(
+                "frame_interpolation_scale", frame_interpolation_scale
+            ),
+            frame_interpolation_model_path=form_value(
+                "frame_interpolation_model_path", frame_interpolation_model_path
+            ),
+            enable_upscaling=form_value("enable_upscaling", enable_upscaling),
+            upscaling_model_path=form_value(
+                "upscaling_model_path", upscaling_model_path
+            ),
+            upscaling_scale=form_value("upscaling_scale", upscaling_scale),
+            output_compression=form_value("output_compression", output_compression),
+            output_quality=form_value("output_quality", output_quality),
+            output_path=form_value("output_path", output_path),
+            diffusers_kwargs=form_value("diffusers_kwargs", None),
+            **extra_request_fields,
+        )
+    else:
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        try:
+            # If client uses extra_body, merge it into the top-level payload
+            payload: Dict[str, Any] = dict(body or {})
+            extra = payload.pop("extra_body", None)
+            if isinstance(extra, str):
+                extra = json.loads(extra)
+            if isinstance(extra, dict):
+                payload.update(flatten_extra_params(extra))
+            # openai may turn extra_body to extra_json
+            extra_json = payload.pop("extra_json", None)
+            if isinstance(extra_json, str):
+                extra_json = json.loads(extra_json)
+            if isinstance(extra_json, dict):
+                payload.update(flatten_extra_params(extra_json))
+            flatten_extra_params(payload)
+            # Validate image input based on model task type
+            if payload.get("video_url") and not payload.get("video_path"):
+                payload["video_path"] = payload["video_url"]
+            if _is_probably_video_source(payload.get("reference_url")):
+                payload.setdefault("video_path", payload.get("reference_url"))
+            if _is_probably_video_source(payload.get("input_reference")):
+                payload.setdefault("video_path", payload.get("input_reference"))
+
+            has_image_input = (
+                payload.get("reference_url")
+                and not _is_probably_video_source(payload.get("reference_url"))
+            ) or (
+                payload.get("input_reference")
+                and not _is_probably_video_source(payload.get("input_reference"))
+            )
+            if task_type.requires_image_input() and not has_image_input:
+                raise HTTPException(
+                    status_code=400,
+                    detail="input_reference or reference_url is required for image-to-video generation",
+                )
+            # for non-multipart/form-data type
+            if payload.get("reference_url") and not _is_probably_video_source(
+                payload.get("reference_url")
+            ):
+                try:
+                    input_path = await _save_first_input_image(
+                        payload.get("reference_url"),
+                        request_id,
+                        uploads_dir,
+                        prefer_remote_source=server_args.input_save_path is None,
+                    )
+                except Exception as e:
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Failed to process image source: {str(e)}",
+                    )
+                payload["input_reference"] = input_path
+            req = VideoGenerationsRequest(**payload)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Invalid request body: {e}")
+
+    # Resolve per-request output_path override
+    effective_output_path = req.output_path or server_args.output_path
+    if effective_output_path is None:
+        output_tmp = tempfile.mkdtemp(prefix="sglang_output_")
+        temp_dirs.append(output_tmp)
+        effective_output_path = output_tmp
+        output_persistent = False
+
+    # Inject resolved output_path so _build_video_sampling_params picks it up
+    req.output_path = effective_output_path
+
+    logger.debug(f"Server received from create_video endpoint: req={req}")
+
+    try:
+        sampling_params = _build_video_sampling_params(request_id, req)
+    except (ValueError, TypeError) as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    job = _video_job_from_sampling(request_id, req, sampling_params)
+    await VIDEO_STORE.upsert(request_id, job)
+
+    # Build Req for scheduler
+    trace_headers = extract_trace_headers(request.headers)
+    batch = prepare_request(
+        server_args=server_args,
+        sampling_params=sampling_params,
+        external_trace_header=trace_headers,
+    )
+    # Add diffusers_kwargs if provided
+    if req.diffusers_kwargs:
+        batch.extra["diffusers_kwargs"] = req.diffusers_kwargs
+        if "max_sequence_length" in req.diffusers_kwargs:
+            batch.max_sequence_length = req.diffusers_kwargs["max_sequence_length"]
+        if "flow_shift" in req.diffusers_kwargs:
+            batch.flow_shift = req.diffusers_kwargs["flow_shift"]
+    # Enqueue the job asynchronously and return immediately
+    asyncio.create_task(
+        _dispatch_job_async(
+            request_id,
+            batch,
+            temp_dirs=temp_dirs or None,
+            output_persistent=output_persistent,
+        )
+    )
+    return VideoResponse(**job)
+
+
+@router.get("", response_model=VideoListResponse)
+async def list_videos(
+    after: Optional[str] = Query(None),
+    limit: Optional[int] = Query(None, ge=1, le=100),
+    order: Optional[str] = Query("desc"),
+):
+    # Normalize order
+    order = (order or "desc").lower()
+    if order not in ("asc", "desc"):
+        order = "desc"
+    jobs = await VIDEO_STORE.list_values()
+
+    reverse = order != "asc"
+    jobs.sort(key=lambda j: j.get("created_at", 0), reverse=reverse)
+
+    if after is not None:
+        try:
+            idx = next(i for i, j in enumerate(jobs) if j["id"] == after)
+            jobs = jobs[idx + 1 :]
+        except StopIteration:
+            jobs = []
+
+    if limit is not None:
+        jobs = jobs[:limit]
+    items = [VideoResponse(**j) for j in jobs]
+    return VideoListResponse(data=items)
+
+
+@router.get("/{video_id}", response_model=VideoResponse)
+async def retrieve_video(video_id: str = Path(...)):
+    job = await VIDEO_STORE.get(video_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Video not found")
+    return VideoResponse(**job)
+
+
+# TODO: support aborting a job.
+@router.delete("/{video_id}", response_model=VideoResponse)
+async def delete_video(video_id: str = Path(...)):
+    job = await VIDEO_STORE.pop(video_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Video not found")
+    # Mark as deleted in response semantics
+    job["status"] = "deleted"
+    return VideoResponse(**job)
+
+
+@router.get("/{video_id}/content")
+async def download_video_content(
+    video_id: str = Path(...), variant: Optional[str] = Query(None)
+):
+    job = await VIDEO_STORE.get(video_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Video not found")
+
+    if job.get("url"):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Video has been uploaded to cloud storage. Please use the cloud URL: {job.get('url')}",
+        )
+
+    file_path = job.get("file_path")
+    if not file_path or not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="Generation is still in-progress")
+
+    media_type = "video/mp4"  # default variant
+    return FileResponse(
+        path=file_path, media_type=media_type, filename=os.path.basename(file_path)
+    )

python/sglang/multimodal_gen/runtime/entrypoints/post_training/io_struct.py

@@ -0,0 +1,110 @@
+"""Request/response data structures for post-training APIs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+
+@dataclass
+class UpdateWeightFromDiskReqInput:
+    """Request to update model weights from disk for diffusion models."""
+
+    model_path: str
+    flush_cache: bool = True
+    target_modules: list[str] | None = None
+
+
+@dataclass
+class UpdateWeightFromTensorReqInput:
+    """Request to update model weights from tensor payloads for diffusion models."""
+
+    serialized_named_tensors: list[str | bytes]
+    load_format: str | None = None
+    target_modules: list[str] | None = None
+
+
+@dataclass
+class UpdateWeightFromTensorCheckerReqInput:
+    """Request to verify live module weights against expected SHA-256 values."""
+
+    target_module: str
+    expected_named_tensors_sha256: dict[str, str]
+
+
+@dataclass
+class GetWeightsChecksumReqInput:
+    """Compute SHA-256 checksum of loaded module weights for verification."""
+
+    module_names: list[str] | None = None
+
+
+@dataclass
+class ReleaseMemoryOccupationReqInput:
+    """Request to release (sleep) GPU memory occupation for the diffusion engine."""
+
+    pass
+
+
+@dataclass
+class ResumeMemoryOccupationReqInput:
+    """Request to resume (wake) GPU memory occupation for the diffusion engine."""
+
+    pass
+
+
+class RolloutRequest(BaseModel):
+    prompt: str
+    negative_prompt: Optional[str] = None
+    seed: Optional[int] = None
+    generator_device: str = "cuda"
+
+    width: Optional[int] = None
+    height: Optional[int] = None
+    num_inference_steps: Optional[int] = None
+    num_outputs_per_prompt: Optional[int] = None
+
+    guidance_scale: Optional[float] = None
+    true_cfg_scale: Optional[float] = None
+
+    # video-specific (ignored by image pipelines)
+    num_frames: Optional[int] = None
+    fps: Optional[int] = None
+
+    rollout: bool = True
+    rollout_sde_type: str = "sde"
+    rollout_noise_level: float = 0.7
+    rollout_log_prob_no_const: bool = False
+    rollout_debug_mode: bool = True
+
+    rollout_return_denoising_env: bool = False
+    rollout_return_dit_trajectory: bool = False
+
+    # 0-indexed denoising-loop step filters. None = all steps.
+    rollout_sde_step_indices: Optional[list[int]] = None
+    rollout_return_step_indices: Optional[list[int]] = None
+
+    image_path: Optional[list[str]] = None
+
+    # suppress verbose per-request logging (also gates peak_memory_mb collection)
+    suppress_logs: bool = False
+
+    extra_sampling_params: Optional[dict[str, Any]] = None
+
+
+class RolloutResponse(BaseModel):
+    request_id: str
+    prompt: str
+    seed: int
+
+    generated_output: Any = None
+
+    rollout_log_probs: Optional[dict[str, Any]] = None
+    rollout_debug_tensors: Optional[dict[str, Any]] = None
+    denoising_env: Optional[dict[str, Any]] = None
+    dit_trajectory: Optional[dict[str, Any]] = None
+
+    inference_time_s: Optional[float] = None
+    peak_memory_mb: Optional[float] = None

python/sglang/multimodal_gen/runtime/entrypoints/post_training/rollout_api.py

@@ -0,0 +1,329 @@
+"""Rollout HTTP API (``POST /rollout/generate``)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import ORJSONResponse
+
+from sglang.multimodal_gen.configs.sample.sampling_params import generate_request_id
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import build_sampling_params
+from sglang.multimodal_gen.runtime.entrypoints.post_training.io_struct import (
+    RolloutRequest,
+    RolloutResponse,
+)
+from sglang.multimodal_gen.runtime.entrypoints.post_training.utils import (
+    _maybe_serialize,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
+    RolloutDebugTensors,
+    RolloutDenoisingEnv,
+    RolloutDitTrajectory,
+    RolloutTrajectoryData,
+)
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+router = APIRouter(prefix="/rollout", tags=["rollout"])
+
+
+def _extract_single_sample_tensor(
+    obj: Any, sample_idx: int, batch_size: int, *, current_key: str | None = None
+) -> Any:
+    if isinstance(obj, torch.Tensor):
+        if obj.dim() >= 1 and obj.shape[0] == batch_size:
+            return obj[sample_idx].contiguous()
+        return obj
+    if isinstance(obj, dict):
+        return {
+            k: _extract_single_sample_tensor(v, sample_idx, batch_size, current_key=k)
+            for k, v in obj.items()
+        }
+    if isinstance(obj, list):
+        if current_key == "img_shapes" and len(obj) == batch_size:
+            return [obj[sample_idx]]
+        return [
+            _extract_single_sample_tensor(
+                v, sample_idx, batch_size, current_key=current_key
+            )
+            for v in obj
+        ]
+    if isinstance(obj, tuple):
+        return tuple(
+            _extract_single_sample_tensor(
+                v, sample_idx, batch_size, current_key=current_key
+            )
+            for v in obj
+        )
+    return obj
+
+
+def _slice_rollout_trajectory_for_sample(
+    rtd: RolloutTrajectoryData | None,
+    sample_idx: int,
+    batch_size: int,
+) -> RolloutTrajectoryData | None:
+    if rtd is None:
+        return None
+    log_probs = rtd.rollout_log_probs
+    if (
+        isinstance(log_probs, torch.Tensor)
+        and log_probs.dim() >= 1
+        and log_probs.shape[0] == batch_size
+    ):
+        log_probs = log_probs[sample_idx].contiguous()
+    debug_tensors = None
+    if rtd.rollout_debug_tensors:
+        rd = rtd.rollout_debug_tensors
+        debug_tensors = RolloutDebugTensors(
+            rollout_variance_noises=_extract_single_sample_tensor(
+                rd.rollout_variance_noises, sample_idx, batch_size
+            ),
+            rollout_prev_sample_means=_extract_single_sample_tensor(
+                rd.rollout_prev_sample_means, sample_idx, batch_size
+            ),
+            rollout_noise_std_devs=_extract_single_sample_tensor(
+                rd.rollout_noise_std_devs, sample_idx, batch_size
+            ),
+            rollout_model_outputs=_extract_single_sample_tensor(
+                rd.rollout_model_outputs, sample_idx, batch_size
+            ),
+        )
+    denoising_env = None
+    if rtd.denoising_env:
+        env = rtd.denoising_env
+        denoising_env = RolloutDenoisingEnv(
+            image_kwargs=(
+                _extract_single_sample_tensor(env.image_kwargs, sample_idx, batch_size)
+                if env.image_kwargs
+                else None
+            ),
+            pos_cond_kwargs=(
+                _extract_single_sample_tensor(
+                    env.pos_cond_kwargs, sample_idx, batch_size
+                )
+                if env.pos_cond_kwargs
+                else None
+            ),
+            neg_cond_kwargs=(
+                _extract_single_sample_tensor(
+                    env.neg_cond_kwargs, sample_idx, batch_size
+                )
+                if env.neg_cond_kwargs
+                else None
+            ),
+            guidance=(
+                _extract_single_sample_tensor(env.guidance, sample_idx, batch_size)
+                if env.guidance is not None
+                else None
+            ),
+        )
+    dit_trajectory = None
+    if rtd.dit_trajectory:
+        dit = rtd.dit_trajectory
+        dit_trajectory = RolloutDitTrajectory(
+            latents=_extract_single_sample_tensor(dit.latents, sample_idx, batch_size),
+            timesteps=dit.timesteps,
+        )
+    return RolloutTrajectoryData(
+        rollout_log_probs=log_probs,
+        rollout_debug_tensors=debug_tensors,
+        denoising_env=denoising_env,
+        dit_trajectory=dit_trajectory,
+    )
+
+
+def _serialize_rollout_trajectory(
+    rtd: RolloutTrajectoryData | None,
+    *,
+    serialized_dit_timesteps: dict | None = None,
+) -> tuple[dict | None, dict | None, dict | None, dict | None]:
+    """Return order: rollout_log_probs, rollout_debug_tensors, denoising_env, dit_trajectory."""
+    if rtd is None:
+        return None, None, None, None
+    serialized_log_probs = _maybe_serialize(rtd.rollout_log_probs)
+    serialized_debug_tensors = None
+    if rtd.rollout_debug_tensors:
+        rd = rtd.rollout_debug_tensors
+        serialized_debug_tensors = {
+            "rollout_variance_noises": _maybe_serialize(rd.rollout_variance_noises),
+            "rollout_prev_sample_means": _maybe_serialize(rd.rollout_prev_sample_means),
+            "rollout_noise_std_devs": _maybe_serialize(rd.rollout_noise_std_devs),
+            "rollout_model_outputs": _maybe_serialize(rd.rollout_model_outputs),
+        }
+    serialized_denoising_env = None
+    if rtd.denoising_env:
+        env = rtd.denoising_env
+        serialized_denoising_env = {
+            "image_kwargs": (
+                _maybe_serialize(env.image_kwargs) if env.image_kwargs else None
+            ),
+            "pos_cond_kwargs": (
+                _maybe_serialize(env.pos_cond_kwargs) if env.pos_cond_kwargs else None
+            ),
+            "neg_cond_kwargs": (
+                _maybe_serialize(env.neg_cond_kwargs) if env.neg_cond_kwargs else None
+            ),
+            "guidance": (
+                _maybe_serialize(env.guidance) if env.guidance is not None else None
+            ),
+        }
+    serialized_dit_trajectory = None
+    if rtd.dit_trajectory:
+        dit = rtd.dit_trajectory
+        serialized_dit_trajectory = {
+            "latents": (
+                _maybe_serialize(dit.latents) if dit.latents is not None else None
+            ),
+            "timesteps": serialized_dit_timesteps,
+        }
+    return (
+        serialized_log_probs,
+        serialized_debug_tensors,
+        serialized_denoising_env,
+        serialized_dit_trajectory,
+    )
+
+
+def _build_response(
+    request_id: str, prompt: str, seed: int, rollout: bool, result: OutputBatch
+) -> list[RolloutResponse]:
+    """
+    rollout: bool - set to False when evaluating the model
+    """
+    batch_size = result.output.shape[0]
+    inference_time_s = (
+        result.metrics.total_duration_s
+        if result.metrics and result.metrics.total_duration_s > 0
+        else None
+    )
+    peak_memory_mb = result.peak_memory_mb if result.peak_memory_mb > 0 else None
+    rollout_trajectory_data = result.rollout_trajectory_data
+    if rollout:
+        assert (
+            rollout_trajectory_data is not None
+        ), "rollout_trajectory_data must be present when rollout=True"
+
+    serialized_dit_timesteps = None
+    if rollout and rollout_trajectory_data and rollout_trajectory_data.dit_trajectory:
+        serialized_dit_timesteps = _maybe_serialize(
+            rollout_trajectory_data.dit_trajectory.timesteps
+        )
+
+    responses: list[RolloutResponse] = []
+    for sample_idx in range(batch_size):
+        out_i = result.output[sample_idx]
+        if isinstance(out_i, torch.Tensor):
+            out_i = out_i.contiguous()
+        serialized_generated_output = _maybe_serialize(out_i)
+        if not rollout:
+            responses.append(
+                RolloutResponse(
+                    request_id=request_id,
+                    prompt=prompt,
+                    seed=seed,
+                    generated_output=serialized_generated_output,
+                    inference_time_s=inference_time_s,
+                    peak_memory_mb=peak_memory_mb,
+                )
+            )
+            continue
+        per_sample_trajectory = _slice_rollout_trajectory_for_sample(
+            result.rollout_trajectory_data, sample_idx, batch_size
+        )
+        (
+            serialized_log_probs,
+            serialized_debug_tensors,
+            serialized_denoising_env,
+            serialized_dit_trajectory,
+        ) = _serialize_rollout_trajectory(
+            per_sample_trajectory,
+            serialized_dit_timesteps=serialized_dit_timesteps,
+        )
+        responses.append(
+            RolloutResponse(
+                request_id=request_id,
+                prompt=prompt,
+                seed=seed,
+                generated_output=serialized_generated_output,
+                rollout_log_probs=serialized_log_probs,
+                rollout_debug_tensors=serialized_debug_tensors,
+                denoising_env=serialized_denoising_env,
+                dit_trajectory=serialized_dit_trajectory,
+                inference_time_s=inference_time_s,
+                peak_memory_mb=peak_memory_mb,
+            )
+        )
+    return responses
+
+
+def _build_sampling_kwargs(request: RolloutRequest) -> dict:
+    sampling_kwargs: dict = dict(
+        prompt=request.prompt,
+        negative_prompt=request.negative_prompt,
+        seed=request.seed,
+        generator_device=request.generator_device,
+        width=request.width,
+        height=request.height,
+        num_inference_steps=request.num_inference_steps,
+        num_outputs_per_prompt=request.num_outputs_per_prompt,
+        guidance_scale=request.guidance_scale,
+        true_cfg_scale=request.true_cfg_scale,
+        num_frames=request.num_frames,
+        fps=request.fps,
+        image_path=request.image_path,
+        rollout=request.rollout,
+        rollout_sde_type=request.rollout_sde_type,
+        rollout_noise_level=request.rollout_noise_level,
+        rollout_log_prob_no_const=request.rollout_log_prob_no_const,
+        rollout_debug_mode=request.rollout_debug_mode,
+        rollout_return_denoising_env=request.rollout_return_denoising_env,
+        rollout_return_dit_trajectory=request.rollout_return_dit_trajectory,
+        rollout_sde_step_indices=request.rollout_sde_step_indices,
+        rollout_return_step_indices=request.rollout_return_step_indices,
+        suppress_logs=request.suppress_logs,
+        save_output=False,
+        return_trajectory_latents=False,
+        return_trajectory_decoded=False,
+    )
+    if request.extra_sampling_params:
+        sampling_kwargs.update(request.extra_sampling_params)
+        sampling_kwargs["rollout"] = request.rollout
+    return {k: v for k, v in sampling_kwargs.items() if v is not None}
+
+
+@router.post("/generate", response_model=list[RolloutResponse])
+async def rollout_generate(request: RolloutRequest):
+    request_id = generate_request_id()
+    server_args = get_global_server_args()
+    sampling_kwargs = _build_sampling_kwargs(request)
+    try:
+        sampling_params = build_sampling_params(request_id, **sampling_kwargs)
+    except Exception as exc:
+        raise HTTPException(
+            status_code=400, detail=f"Invalid sampling params: {exc}"
+        ) from exc
+    pipeline_request = prepare_request(
+        server_args=server_args, sampling_params=sampling_params
+    )
+    try:
+        output_batch: OutputBatch = await async_scheduler_client.forward(
+            pipeline_request
+        )
+    except Exception as exc:
+        logger.error("Rollout generation failed: %s", exc, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail=f"Generation failed: {exc}"
+        ) from exc
+    if output_batch.error:
+        raise HTTPException(status_code=500, detail=output_batch.error)
+    rollout_responses = _build_response(
+        request_id, request.prompt, request.seed, request.rollout, output_batch
+    )
+    return ORJSONResponse(content=[r.model_dump() for r in rollout_responses])

python/sglang/multimodal_gen/runtime/entrypoints/post_training/utils.py

@@ -0,0 +1,48 @@
+"""Tensor serialization for post-training / rollout HTTP responses."""
+
+from __future__ import annotations
+
+import base64
+from typing import Any
+
+import numpy as np
+import torch
+from safetensors.torch import load, save
+
+
+def tensor_to_base64(t: torch.Tensor) -> str:
+    t = t.detach().contiguous().cpu()
+    raw = save({"t": t})
+    return base64.b64encode(raw).decode("ascii")
+
+
+def base64_to_tensor(s: str) -> torch.Tensor:
+    raw = base64.b64decode(s)
+    return load(raw)["t"]
+
+
+def _maybe_serialize(obj: Any) -> Any:
+    if isinstance(obj, torch.Tensor):
+        return {
+            "__tensor__": True,
+            "data": tensor_to_base64(obj),
+            "shape": list(obj.shape),
+            "dtype": str(obj.dtype),
+        }
+    if isinstance(obj, np.ndarray):
+        return _maybe_serialize(torch.from_numpy(obj))
+    if isinstance(obj, dict):
+        return {k: _maybe_serialize(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_maybe_serialize(v) for v in obj]
+    return obj
+
+
+def _maybe_deserialize(obj: Any) -> Any:
+    if isinstance(obj, dict):
+        if obj.get("__tensor__"):
+            return base64_to_tensor(obj["data"])
+        return {k: _maybe_deserialize(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [_maybe_deserialize(v) for v in obj]
+    return obj

python/sglang/multimodal_gen/runtime/entrypoints/post_training/weights_api.py

@@ -0,0 +1,199 @@
+"""Weight update API for the diffusion engine."""
+
+from fastapi import APIRouter, Request
+
+from sglang.multimodal_gen.runtime.entrypoints.post_training.io_struct import (
+    GetWeightsChecksumReqInput,
+    ReleaseMemoryOccupationReqInput,
+    ResumeMemoryOccupationReqInput,
+    UpdateWeightFromDiskReqInput,
+    UpdateWeightFromTensorCheckerReqInput,
+    UpdateWeightFromTensorReqInput,
+)
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.srt.utils.json_response import orjson_response
+
+router = APIRouter()
+
+
+@router.post("/update_weights_from_disk")
+async def update_weights_from_disk(request: Request):
+    """Update model weights from disk inplace without restarting the server."""
+    body = await request.json()
+    model_path = body.get("model_path")
+    if not model_path:
+        return orjson_response(
+            {"success": False, "message": "model_path is required"},
+            status_code=400,
+        )
+
+    req = UpdateWeightFromDiskReqInput(
+        model_path=model_path,
+        flush_cache=body.get("flush_cache", True),
+        target_modules=body.get("target_modules"),
+    )
+
+    try:
+        response = await async_scheduler_client.forward(req)
+    except Exception as e:
+        return orjson_response(
+            {"success": False, "message": str(e)},
+            status_code=500,
+        )
+
+    if response.output is None:
+        return orjson_response(
+            {
+                "success": False,
+                "message": response.error or "Unknown status",
+            },
+            status_code=500,
+        )
+
+    result = response.output
+    return orjson_response(
+        result,
+        status_code=200 if result["success"] else 400,
+    )
+
+
+@router.post("/update_weights_from_tensor")
+async def update_weights_from_tensor(request: Request):
+    """Update model weights from serialized tensor payloads."""
+    body = await request.json()
+    serialized_named_tensors = body.get("serialized_named_tensors")
+    if not serialized_named_tensors:
+        return orjson_response(
+            {"success": False, "message": "serialized_named_tensors is required"},
+            status_code=400,
+        )
+
+    req = UpdateWeightFromTensorReqInput(
+        serialized_named_tensors=serialized_named_tensors,
+        load_format=body.get("load_format"),
+        target_modules=body.get("target_modules"),
+    )
+
+    try:
+        response = await async_scheduler_client.forward(req)
+    except Exception as e:
+        return orjson_response(
+            {"success": False, "message": str(e)},
+            status_code=500,
+        )
+
+    result = response.output
+    return orjson_response(
+        result,
+        status_code=200 if result["success"] else 400,
+    )
+
+
+@router.post("/update_weights_from_tensor_checker")
+async def update_weights_from_tensor_checker(request: Request):
+    """Verify live module weights against expected SHA-256 values."""
+    body = await request.json()
+    target_module = body.get("target_module")
+    if not target_module:
+        return orjson_response(
+            {"success": False, "message": "target_module is required"},
+            status_code=400,
+        )
+
+    expected_named_tensors_sha256 = body.get("expected_named_tensors_sha256")
+    if (
+        not isinstance(expected_named_tensors_sha256, dict)
+        or not expected_named_tensors_sha256
+    ):
+        return orjson_response(
+            {
+                "success": False,
+                "message": "expected_named_tensors_sha256 is required",
+            },
+            status_code=400,
+        )
+
+    req = UpdateWeightFromTensorCheckerReqInput(
+        target_module=target_module,
+        expected_named_tensors_sha256=expected_named_tensors_sha256,
+    )
+
+    try:
+        response = await async_scheduler_client.forward(req)
+    except Exception as e:
+        return orjson_response(
+            {"success": False, "message": str(e)},
+            status_code=500,
+        )
+
+    result = response.output
+    success = result.get("success", False)
+    message = result.get("message", "Unknown status")
+    return orjson_response(
+        {"success": success, "message": message},
+        status_code=200 if success else 400,
+    )
+
+
+@router.post("/get_weights_checksum")
+async def get_weights_checksum(request: Request):
+    """Return SHA-256 checksum of each requested module's weights."""
+    body = await request.json()
+    req = GetWeightsChecksumReqInput(
+        module_names=body.get("module_names"),
+    )
+
+    try:
+        response = await async_scheduler_client.forward(req)
+    except Exception as e:
+        return orjson_response({"error": str(e)}, status_code=500)
+
+    return orjson_response(response.output, status_code=200)
+
+
+@router.post("/release_memory_occupation")
+async def release_memory_occupation():
+    """Release GPU memory occupation (sleep the engine)."""
+    try:
+        response = await async_scheduler_client.forward(
+            ReleaseMemoryOccupationReqInput()
+        )
+    except Exception as e:
+        return orjson_response({"success": False, "message": str(e)}, status_code=500)
+
+    if response.output is None:
+        return orjson_response(
+            {
+                "success": False,
+                "message": response.error or "Unknown status",
+            },
+            status_code=500,
+        )
+
+    payload = response.output
+    success = bool(payload["success"])
+    return orjson_response(payload, status_code=200 if success else 400)
+
+
+@router.post("/resume_memory_occupation")
+async def resume_memory_occupation():
+    """Resume GPU memory occupation (wake the engine)."""
+    try:
+        response = await async_scheduler_client.forward(
+            ResumeMemoryOccupationReqInput()
+        )
+    except Exception as e:
+        return orjson_response({"success": False, "message": str(e)}, status_code=500)
+
+    if response.output is None:
+        return orjson_response(
+            {
+                "success": False,
+                "message": response.error or "Unknown status",
+            },
+            status_code=500,
+        )
+
+    payload = response.output
+    success = bool(payload["success"])
+    return orjson_response(payload, status_code=200 if success else 400)

python/sglang/multimodal_gen/runtime/entrypoints/utils.py

@@ -0,0 +1,755 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+# SPDX-License-Identifier: Apache-2.0
+"""
+DiffGenerator module for sglang-diffusion.
+
+This module provides a consolidated interface for generating videos using
+diffusion models.
+"""
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+from copy import copy
+from dataclasses import dataclass, field
+from typing import Any, Callable, List, Optional, Sequence, Union
+
+import imageio
+import numpy as np
+import torch
+from PIL import Image
+
+try:
+    import scipy.io.wavfile as scipy_wavfile
+except ImportError:  # pragma: no cover
+    scipy_wavfile = None
+
+try:
+    import imageio_ffmpeg as _imageio_ffmpeg
+except ImportError:  # pragma: no cover
+    _imageio_ffmpeg = None
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    DataType,
+    SamplingParams,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.multimodal_gen.runtime.utils.logging_utils import CYAN, RESET, init_logger
+from sglang.srt.observability.trace import TraceReqContext
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class SetLoraReq:
+    lora_nickname: Union[str, List[str]]
+    lora_path: Optional[Union[str, List[Optional[str]]]] = None
+    target: Union[str, List[str]] = "all"
+    strength: Union[float, List[float]] = 1.0
+    merge_mode: Optional[str] = None
+
+
+@dataclass
+class MergeLoraWeightsReq:
+    target: str = "all"
+    strength: float = 1.0
+
+
+@dataclass
+class UnmergeLoraWeightsReq:
+    target: str = "all"
+
+
+@dataclass
+class ListLorasReq:
+    pass
+
+
+@dataclass
+class ShutdownReq:
+    pass
+
+
+@dataclass
+class ReleaseRealtimeSessionReq:
+    session_id: str
+
+
+@dataclass
+class GetDisaggStatsReq:
+    """Request to get disagg pipeline metrics from the scheduler."""
+
+    pass
+
+
+def format_lora_message(
+    lora_nickname: Union[str, List[str]],
+    target: Union[str, List[str]],
+    strength: Union[float, List[float]],
+) -> tuple[str, str, str]:
+    """Format success message for single or multiple LoRAs."""
+    if isinstance(lora_nickname, list):
+        nickname_str = ", ".join(lora_nickname)
+        target_str = ", ".join(target) if isinstance(target, list) else target
+        strength_str = (
+            ", ".join(f"{s:.2f}" for s in strength)
+            if isinstance(strength, list)
+            else f"{strength:.2f}"
+        )
+    else:
+        nickname_str = lora_nickname
+        target_str = target if isinstance(target, str) else ", ".join(target)
+        strength_str = (
+            f"{strength:.2f}"
+            if isinstance(strength, (int, float))
+            else ", ".join(f"{s:.2f}" for s in strength)
+        )
+    return nickname_str, target_str, strength_str
+
+
+@dataclass
+class GenerationResult:
+    """Result of a single generation request from DiffGenerator."""
+
+    samples: Any = None
+    frames: Any = None
+    audio: Any = None
+    action: Any = None  # [T, raw_action_dim] predicted action (policy/inverse_dynamics)
+    prompt: str | None = None
+    size: tuple | None = None  # (height, width, num_frames)
+    generation_time: float = 0.0
+    peak_memory_mb: float = 0.0
+    metrics: dict = field(default_factory=dict)
+    trajectory_latents: Any = None
+    trajectory_timesteps: Any = None
+    rollout_trajectory_data: Any = None
+    trajectory_decoded: Any = None
+    prompt_index: int = 0
+    output_file_path: str | None = None
+
+
+@dataclass
+class MaterializedOutput:
+    sample: Any
+    frames: list[Any]
+    audio: Any = None
+    fps: int = 0
+
+
+def normalize_output_seeds(
+    seed: int | list[int],
+    *,
+    num_outputs_per_prompt: int,
+    num_prompts: int = 1,
+    prompt_index: int = 0,
+) -> list[int]:
+    """
+    return a list of seed with size equal to `num_outputs_per_prompt`
+    """
+    if num_outputs_per_prompt <= 0:
+        raise ValueError(
+            f"num_outputs_per_prompt must be positive, got {num_outputs_per_prompt}"
+        )
+
+    if isinstance(seed, list):
+        seeds = [int(item) for item in seed]
+        total_outputs = num_outputs_per_prompt * num_prompts
+        if len(seeds) == num_outputs_per_prompt:
+            return seeds
+        if len(seeds) == total_outputs:
+            start = prompt_index * num_outputs_per_prompt
+            return seeds[start : start + num_outputs_per_prompt]
+        raise ValueError(
+            "seed list length must match num_outputs_per_prompt "
+            f"({num_outputs_per_prompt}) or total outputs ({total_outputs}), "
+            f"got {len(seeds)}"
+        )
+
+    base_seed = int(seed)
+    return [base_seed + i for i in range(num_outputs_per_prompt)]
+
+
+def _with_output_index_suffix(output_file_name: str, output_index: int) -> str:
+    base, ext = os.path.splitext(output_file_name)
+    return f"{base}_{output_index}{ext}"
+
+
+def _copy_trace_ctx_for_output(req: Req, request_id: str | None, output_index: int):
+    trace_ctx = req.trace_ctx
+    if output_index == 0 or not trace_ctx.tracing_enable:
+        return trace_ctx
+
+    output_trace_ctx = TraceReqContext(
+        rid=request_id,
+        module_name=trace_ctx.module_name,
+        external_trace_header=trace_ctx.external_trace_header,
+    )
+    output_trace_ctx.trace_req_start()
+    return output_trace_ctx
+
+
+def _copy_req_for_output(
+    req: Req,
+    *,
+    request_id: str | None,
+    output_index: int,
+) -> Req:
+    """Create a lightweight per-output ``Req`` without deep-copying tensors."""
+    output_req = copy(req)
+    output_req.sampling_params = copy(req.sampling_params)
+    output_req.extra = dict(req.extra)
+    output_req.condition_inputs = dict(req.condition_inputs)
+    output_req.trace_ctx = _copy_trace_ctx_for_output(req, request_id, output_index)
+    return output_req
+
+
+def expand_request_outputs(
+    req: Req,
+    *,
+    num_prompts: int = 1,
+    prompt_index: int = 0,
+) -> list[Req]:
+    """
+    Expand a req to a list with size equal to `num_prompts`
+    """
+    num_outputs = int(req.num_outputs_per_prompt)
+    # each req must has different seed
+    seeds = normalize_output_seeds(
+        req.seed,
+        num_outputs_per_prompt=num_outputs,
+        num_prompts=num_prompts,
+        prompt_index=prompt_index,
+    )
+
+    if num_outputs == 1:
+        req.seed = seeds[0]
+        req.seeds = None
+        req.generator = None
+        return [req]
+
+    expanded: list[Req] = []
+    for output_index, seed in enumerate(seeds):
+        output_request_id = (
+            f"{req.request_id}:{output_index}" if req.request_id is not None else None
+        )
+        output_req = _copy_req_for_output(
+            req, request_id=output_request_id, output_index=output_index
+        )
+        output_req.seed = seed
+        output_req.num_outputs_per_prompt = 1
+        output_req.seeds = None
+        output_req.generator = None
+        output_req.extra["parent_request_id"] = req.request_id
+        output_req.extra["output_index"] = output_index
+
+        if output_request_id is not None:
+            output_req.request_id = output_request_id
+
+        if req.output_file_name:
+            output_req.output_file_name = _with_output_index_suffix(
+                req.output_file_name, output_index
+            )
+        output_req.validate()
+        expanded.append(output_req)
+
+    return expanded
+
+
+def _normalize_audio_to_numpy(audio: Any) -> np.ndarray | None:
+    """Convert audio (torch / numpy) into a float32 numpy array in [-1, 1], best-effort."""
+    if audio is None:
+        return None
+    if isinstance(audio, torch.Tensor):
+        audio_np = audio.detach().float().clamp(-1.0, 1.0).cpu().numpy()
+    elif isinstance(audio, np.ndarray):
+        audio_np = audio.astype(np.float32, copy=False)
+        audio_np = np.clip(audio_np, -1.0, 1.0)
+    else:
+        return None
+
+    # 1. Squeeze leading singleton dimensions (Batch, etc.)
+    while audio_np.ndim > 1 and audio_np.shape[0] == 1:
+        audio_np = audio_np.squeeze(0)
+
+    # 2. Handle (C, L) -> (L, C)
+    if audio_np.ndim == 2 and audio_np.shape[0] < audio_np.shape[1]:
+        audio_np = audio_np.transpose(1, 0)
+
+    # 3. Final safety check: if still 2D and channels (dim 1) is huge, something is wrong
+    if audio_np.ndim == 2 and audio_np.shape[1] > 256 and audio_np.shape[0] == 1:
+        audio_np = audio_np.flatten()
+
+    return audio_np
+
+
+def _pick_audio_sample_rate(
+    *,
+    audio_np: np.ndarray,
+    audio_sample_rate: Optional[int],
+    fps: int,
+    num_frames: int,
+) -> int:
+    """Pick a plausible sample rate, falling back to inferring from video duration."""
+    selected_sr = int(audio_sample_rate) if audio_sample_rate is not None else None
+    if selected_sr is None or not (8000 <= selected_sr <= 192000):
+        selected_sr = 24000
+        try:
+            duration_s = float(num_frames) / float(fps) if fps else 0.0
+            if duration_s > 0:
+                audio_len = (
+                    int(audio_np.shape[0])
+                    if audio_np.ndim == 2
+                    else int(audio_np.shape[-1])
+                )
+                inferred_sr = int(round(float(audio_len) / duration_s))
+                if 8000 <= inferred_sr <= 192000:
+                    selected_sr = inferred_sr
+        except Exception:
+            pass
+    return selected_sr
+
+
+def _resolve_ffmpeg_exe() -> str:
+    ffmpeg_exe = "ffmpeg"
+    ffmpeg_on_path = shutil.which("ffmpeg")
+    if ffmpeg_on_path:
+        ffmpeg_exe = ffmpeg_on_path
+    try:
+        if _imageio_ffmpeg is not None:
+            ffmpeg_exe = _imageio_ffmpeg.get_ffmpeg_exe()
+    except Exception:
+        pass
+
+    ffmpeg_ok = False
+    if ffmpeg_exe:
+        if os.path.isabs(ffmpeg_exe):
+            ffmpeg_ok = os.path.exists(ffmpeg_exe)
+        else:
+            ffmpeg_ok = shutil.which(ffmpeg_exe) is not None
+    if not ffmpeg_ok:
+        raise RuntimeError("ffmpeg not found")
+    return ffmpeg_exe
+
+
+def _mux_audio_np_into_mp4(
+    *,
+    save_file_path: str,
+    audio_np: np.ndarray,
+    sample_rate: int,
+    ffmpeg_exe: str,
+) -> None:
+    merged_path = save_file_path.rsplit(".", 1)[0] + ".tmp_mux.mp4"
+    tmp_wav_path = None
+    try:
+        if scipy_wavfile is None:
+            raise RuntimeError(
+                "scipy is required to mux audio into mp4 (pip install scipy)"
+            )
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+            tmp_wav_path = f.name
+        scipy_wavfile.write(tmp_wav_path, sample_rate, audio_np)
+        subprocess.run(
+            [
+                ffmpeg_exe,
+                "-y",
+                "-i",
+                save_file_path,
+                "-i",
+                tmp_wav_path,
+                "-c:v",
+                "copy",
+                "-c:a",
+                "aac",
+                "-strict",
+                "experimental",
+                merged_path,
+            ],
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        os.replace(merged_path, save_file_path)
+    finally:
+        if tmp_wav_path:
+            try:
+                os.remove(tmp_wav_path)
+            except OSError:
+                pass
+        if os.path.exists(merged_path):
+            try:
+                os.remove(merged_path)
+            except OSError:
+                pass
+
+
+def _maybe_mux_audio_into_mp4(
+    *,
+    save_file_path: str,
+    audio: Any,
+    frames: list,
+    fps: int,
+    audio_sample_rate: Optional[int],
+) -> None:
+    """Best-effort mux audio into an already-written mp4 at save_file_path.
+
+    Any failure should keep the silent video and only log a warning.
+    """
+    audio_np = _normalize_audio_to_numpy(audio)
+    if audio_np is None:
+        return
+    selected_sr = _pick_audio_sample_rate(
+        audio_np=audio_np,
+        audio_sample_rate=audio_sample_rate,
+        fps=fps,
+        num_frames=len(frames),
+    )
+
+    try:
+        ffmpeg_exe = _resolve_ffmpeg_exe()
+        _mux_audio_np_into_mp4(
+            save_file_path=save_file_path,
+            audio_np=audio_np,
+            sample_rate=selected_sr,
+            ffmpeg_exe=ffmpeg_exe,
+        )
+    except Exception as e:
+        logger.warning(
+            "Failed to mux audio into mp4 (saved silent video): %s",
+            str(e),
+        )
+
+
+def prepare_request(
+    server_args: ServerArgs,
+    sampling_params: SamplingParams,
+    external_trace_header: dict[str, str] | None = None,
+) -> Req:
+    """
+    Create a Req object with sampling_params as a parameter.
+    """
+    req = Req(
+        sampling_params=sampling_params,
+        VSA_sparsity=server_args.attention_backend_config.VSA_sparsity,
+    )
+    sampling_params.apply_request_extra(req)
+    if getattr(sampling_params, "max_sequence_length", None) is not None:
+        req.max_sequence_length = sampling_params.max_sequence_length
+
+    diffusers_kwargs = getattr(sampling_params, "diffusers_kwargs", None)
+    if diffusers_kwargs and "max_sequence_length" in diffusers_kwargs:
+        req.max_sequence_length = diffusers_kwargs["max_sequence_length"]
+
+    if not isinstance(req.prompt, str):
+        raise TypeError(f"`prompt` must be a string, but got {type(req.prompt)}")
+
+    req_width = getattr(req, "width", None)
+    req_height = getattr(req, "height", None)
+    if (req_width is not None and req_width <= 0) or (
+        req_height is not None and req_height <= 0
+    ):
+        raise ValueError(
+            f"Height and width must be positive, got height={req_height}, width={req_width}"
+        )
+
+    if server_args.enable_trace:
+        trace_ctx = TraceReqContext(
+            rid=sampling_params.request_id,
+            module_name="diffusion",
+            external_trace_header=external_trace_header,
+        )
+        trace_ctx.trace_req_start()
+        req.trace_ctx = trace_ctx
+
+    return req
+
+
+def attach_audio_to_video_sample(
+    sample: Any,
+    audio: Any,
+    output_idx: int,
+) -> Any:
+    """Attach per-sample audio for video outputs when available."""
+    audio = select_output_audio(audio, output_idx)
+    if audio is None:
+        return sample
+    if not (isinstance(sample, (tuple, list)) and len(sample) == 2):
+        return (sample, audio)
+    return sample
+
+
+def select_output_audio(audio: Any, output_idx: int) -> Any:
+    if isinstance(audio, torch.Tensor) and audio.ndim >= 2:
+        return audio[output_idx] if audio.shape[0] > output_idx else None
+    if isinstance(audio, np.ndarray) and audio.ndim >= 2:
+        return audio[output_idx] if audio.shape[0] > output_idx else None
+    return audio
+
+
+def _split_sample_audio(sample: Any) -> tuple[Any, Any]:
+    if isinstance(sample, (tuple, list)) and len(sample) == 2:
+        return sample[0], sample[1]
+    return sample, None
+
+
+def _sample_to_uint8_frames(sample: Any) -> list[Any]:
+    """return numpy frames in THCW format"""
+    if isinstance(sample, torch.Tensor):
+        # sample is raw tensor
+        if sample.dim() == 3:
+            sample = sample.unsqueeze(1)
+        sample = (sample * 255).clamp(0, 255).to(torch.uint8)
+        videos = sample.permute(1, 2, 3, 0).contiguous().cpu().numpy()
+        return list(videos)
+
+    if not isinstance(sample, np.ndarray):
+        raise TypeError(f"Unsupported sample type: {type(sample)}")
+
+    # sample is numpy frames
+    arr = sample
+    if arr.ndim == 3:
+        if arr.shape[-1] in (1, 3, 4):
+            arr = arr[None, ...]
+        else:
+            arr = arr[..., None]
+    if arr.ndim != 4:
+        raise ValueError(f"Unexpected numpy sample shape: {tuple(arr.shape)}")
+
+    if arr.shape[-1] not in (1, 3, 4) and arr.shape[0] in (1, 3, 4):
+        t = torch.from_numpy(arr)
+        if t.dim() == 3:
+            t = t.unsqueeze(1)
+        t = (t * 255).clamp(0, 255).to(torch.uint8)
+        videos = t.permute(1, 2, 3, 0).contiguous().cpu().numpy()
+        return list(videos)
+
+    if arr.dtype != np.uint8:
+        arr = (np.clip(arr, 0.0, 1.0) * 255.0).astype(np.uint8)
+    return list(arr)
+
+
+def materialize_output_sample(
+    sample: Any,
+    data_type: DataType,
+    fps: int,
+    *,
+    enable_frame_interpolation: bool = False,
+    frame_interpolation_exp: int = 1,
+    frame_interpolation_scale: float = 1.0,
+    frame_interpolation_model_path: Optional[str] = None,
+    enable_upscaling: bool = False,
+    upscaling_model_path: Optional[str] = None,
+    upscaling_scale: int = 4,
+) -> MaterializedOutput:
+    """materialize samples, apply postprocessing if applicable"""
+    sample_without_audio, audio = _split_sample_audio(sample)
+    frames = _sample_to_uint8_frames(sample_without_audio)
+
+    # frames are uint8 numpy arrays in THWC format at this point
+    if enable_frame_interpolation and data_type == DataType.VIDEO and len(frames) > 1:
+        from sglang.multimodal_gen.runtime.postprocess import (
+            interpolate_video_frames,
+        )
+
+        frames, multiplier = interpolate_video_frames(
+            frames,
+            exp=frame_interpolation_exp,
+            scale=frame_interpolation_scale,
+            model_path=frame_interpolation_model_path,
+        )
+        fps = fps * multiplier
+
+    if enable_upscaling and frames:
+        from sglang.multimodal_gen.runtime.postprocess import upscale_frames
+
+        frames = upscale_frames(
+            frames,
+            model_path=upscaling_model_path,
+            scale=upscaling_scale,
+        )
+
+    return MaterializedOutput(sample=sample, frames=frames, audio=audio, fps=fps)
+
+
+def save_materialized_output(
+    materialized: MaterializedOutput,
+    data_type: DataType,
+    save_file_path: Optional[str],
+    *,
+    save_output: bool = True,
+    audio_sample_rate: Optional[int] = None,
+    output_compression: Optional[int] = None,
+) -> None:
+    if not save_output:
+        return
+    if not save_file_path:
+        logger.info("No output path provided, output not saved")
+        return
+
+    os.makedirs(os.path.dirname(save_file_path), exist_ok=True)
+    if data_type == DataType.VIDEO:
+        quality = output_compression / 10 if output_compression is not None else 5
+        imageio.mimsave(
+            save_file_path,
+            materialized.frames,
+            fps=materialized.fps,
+            format=data_type.get_default_extension(),
+            codec="libx264",
+            quality=quality,
+        )
+
+        _maybe_mux_audio_into_mp4(
+            save_file_path=save_file_path,
+            audio=materialized.audio,
+            frames=materialized.frames,
+            fps=materialized.fps,
+            audio_sample_rate=audio_sample_rate,
+        )
+    else:
+        quality = output_compression if output_compression is not None else 75
+        if len(materialized.frames) > 1:
+            for i, image in enumerate(materialized.frames):
+                parts = save_file_path.rsplit(".", 1)
+                if len(parts) == 2:
+                    indexed_path = f"{parts[0]}_{i}.{parts[1]}"
+                else:
+                    indexed_path = f"{save_file_path}_{i}"
+                _save_image_frame(indexed_path, image, quality, output_compression)
+        else:
+            _save_image_frame(
+                save_file_path, materialized.frames[0], quality, output_compression
+            )
+    logger.info(f"Output saved to {CYAN}{save_file_path}{RESET}")
+
+
+def _save_image_frame(
+    path: str, frame: np.ndarray, quality: int | None, output_compression: int | None
+) -> None:
+    ext = os.path.splitext(path)[1].lower()
+    if ext == ".png":
+        compress_level = 1
+        if output_compression is not None and output_compression != 75:
+            compress_level = max(0, min(9, round(output_compression / 100 * 9)))
+        if frame.ndim == 3 and frame.shape[-1] == 1:
+            frame = frame[..., 0]
+        Image.fromarray(frame).save(path, format="PNG", compress_level=compress_level)
+    else:
+        imageio.imwrite(path, frame, quality=quality)
+
+
+def save_outputs(
+    outputs: Sequence[Any],
+    data_type: DataType,
+    fps: int,
+    save_output: bool,
+    build_output_path: Callable[[int], str],
+    *,
+    audio: Any = None,
+    audio_sample_rate: Optional[int] = None,
+    samples_out: Optional[list[Any]] = None,
+    audios_out: Optional[list[Any]] = None,
+    frames_out: Optional[list[Any]] = None,
+    output_compression: Optional[int] = None,
+    enable_frame_interpolation: bool = False,
+    frame_interpolation_exp: int = 1,
+    frame_interpolation_scale: float = 1.0,
+    frame_interpolation_model_path: Optional[str] = None,
+    enable_upscaling: bool = False,
+    upscaling_model_path: Optional[str] = None,
+    upscaling_scale: int = 4,
+) -> list[str]:
+    output_paths: list[str] = []
+    for idx, sample in enumerate(outputs):
+        save_file_path = build_output_path(idx)
+        if data_type == DataType.ACTION:
+            if samples_out is not None:
+                samples_out.append(sample)
+            if audios_out is not None:
+                audios_out.append(None)
+            if frames_out is not None:
+                frames_out.append([])
+            if save_output and save_file_path:
+                os.makedirs(os.path.dirname(save_file_path) or ".", exist_ok=True)
+                with open(save_file_path, "w", encoding="utf-8") as f:
+                    json.dump(sample, f, ensure_ascii=False)
+                logger.info(f"Output saved to {CYAN}{save_file_path}{RESET}")
+            output_paths.append(save_file_path)
+            continue
+
+        if data_type == DataType.VIDEO:
+            sample = attach_audio_to_video_sample(sample, audio, idx)
+
+        frames = post_process_sample(
+            sample,
+            data_type,
+            fps,
+            save_output,
+            save_file_path,
+            audio_sample_rate=audio_sample_rate,
+            output_compression=output_compression,
+            enable_frame_interpolation=enable_frame_interpolation,
+            frame_interpolation_exp=frame_interpolation_exp,
+            frame_interpolation_scale=frame_interpolation_scale,
+            frame_interpolation_model_path=frame_interpolation_model_path,
+            enable_upscaling=enable_upscaling,
+            upscaling_model_path=upscaling_model_path,
+            upscaling_scale=upscaling_scale,
+        )
+
+        if samples_out is not None:
+            samples_out.append(sample)
+        if audios_out is not None:
+            if data_type == DataType.VIDEO:
+                audios_out.append(select_output_audio(audio, idx))
+            else:
+                audios_out.append(audio)
+        if frames_out is not None:
+            frames_out.append(frames)
+        output_paths.append(save_file_path)
+    return output_paths
+
+
+def post_process_sample(
+    sample: Any,
+    data_type: DataType,
+    fps: int,
+    save_output: bool = True,
+    save_file_path: Optional[str] = None,
+    audio_sample_rate: Optional[int] = None,
+    output_compression: Optional[int] = None,
+    enable_frame_interpolation: bool = False,
+    frame_interpolation_exp: int = 1,
+    frame_interpolation_scale: float = 1.0,
+    frame_interpolation_model_path: Optional[str] = None,
+    enable_upscaling: bool = False,
+    upscaling_model_path: Optional[str] = None,
+    upscaling_scale: int = 4,
+) -> list[Any]:
+    """materialize frames and save outputs (optional)"""
+    if data_type == DataType.ACTION:
+        return []
+
+    materialized = materialize_output_sample(
+        sample,
+        data_type,
+        fps,
+        enable_frame_interpolation=enable_frame_interpolation,
+        frame_interpolation_exp=frame_interpolation_exp,
+        frame_interpolation_scale=frame_interpolation_scale,
+        frame_interpolation_model_path=frame_interpolation_model_path,
+        enable_upscaling=enable_upscaling,
+        upscaling_model_path=upscaling_model_path,
+        upscaling_scale=upscaling_scale,
+    )
+    save_materialized_output(
+        materialized,
+        data_type,
+        save_file_path,
+        save_output=save_output,
+        audio_sample_rate=audio_sample_rate,
+        output_compression=output_compression,
+    )
+    return materialized.frames

python/sglang/multimodal_gen/runtime/entrypoints/vla/__init__.py

@@ -0,0 +1 @@
+# SPDX-License-Identifier: Apache-2.0

python/sglang/multimodal_gen/runtime/entrypoints/vla/api.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Request, Response, WebSocket
+
+from sglang.multimodal_gen.runtime.entrypoints.vla.protocol import (
+    action_generation_response,
+    action_metadata,
+    action_raw_response,
+    infer_action,
+    pack_msgpack,
+    unpack_msgpack,
+)
+from sglang.multimodal_gen.runtime.entrypoints.vla.ws_utils import (
+    run_action_msgpack_ws,
+)
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.srt.utils.json_response import orjson_response
+
+router = APIRouter(prefix="/v1/actions", tags=["actions"])
+
+
+def _wants_msgpack(request: Request) -> bool:
+    content_type = request.headers.get("content-type", "").lower()
+    accept = request.headers.get("accept", "").lower()
+    return "msgpack" in content_type or "msgpack" in accept
+
+
+def _response_format(payload: dict) -> str:
+    runtime = payload.get("runtime") or {}
+    response_format = str(runtime.get("response_format", "envelope")).lower()
+    if response_format not in ("envelope", "raw"):
+        raise ValueError("runtime.response_format must be 'envelope' or 'raw'")
+    return response_format
+
+
+def _prefer_numpy_output(payload: dict) -> None:
+    runtime = payload.setdefault("runtime", {})
+    runtime.setdefault("output_format", "numpy")
+
+
+@router.post("/generations")
+async def create_action_generation(request: Request):
+    server_args: ServerArgs = request.app.state.server_args
+    try:
+        if "msgpack" in request.headers.get("content-type", "").lower():
+            payload = unpack_msgpack(await request.body())
+        else:
+            payload = await request.json()
+        wants_msgpack = _wants_msgpack(request)
+        if wants_msgpack:
+            _prefer_numpy_output(payload)
+        output = await infer_action(payload, server_args)
+        if _response_format(payload) == "raw":
+            response = action_raw_response(output, preserve_numpy=wants_msgpack)
+        else:
+            response = action_generation_response(
+                output,
+                server_args,
+                preserve_numpy=wants_msgpack,
+            )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    if wants_msgpack:
+        return Response(
+            content=pack_msgpack(response), media_type="application/msgpack"
+        )
+    return orjson_response(response)
+
+
+@router.get("/metadata")
+async def action_metadata_endpoint(request: Request):
+    return orjson_response(action_metadata(request.app.state.server_args))
+
+
+@router.websocket("/realtime")
+async def action_realtime_ws(websocket: WebSocket):
+    server_args: ServerArgs = websocket.app.state.server_args
+    await run_action_msgpack_ws(
+        websocket,
+        server_args,
+        prepare_payload=_prefer_numpy_output,
+        build_response=lambda output: action_generation_response(
+            output,
+            server_args,
+            preserve_numpy=True,
+        ),
+    )

python/sglang/multimodal_gen/runtime/entrypoints/vla/openpi.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, WebSocket
+
+from sglang.multimodal_gen.runtime.entrypoints.vla.ws_utils import (
+    run_action_msgpack_ws,
+)
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+router = APIRouter()
+
+
+def _prefer_numpy_output(observation: dict[str, Any]) -> None:
+    observation.setdefault("output_format", "numpy")
+
+
+@router.websocket("/openpi/policy")
+async def openpi_policy_ws(websocket: WebSocket):
+    server_args: ServerArgs = websocket.app.state.server_args
+    await run_action_msgpack_ws(
+        websocket,
+        server_args,
+        prepare_payload=_prefer_numpy_output,
+        build_response=lambda output: output,
+    )

python/sglang/multimodal_gen/runtime/entrypoints/vla/protocol.py

@@ -0,0 +1,443 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import base64
+import dataclasses
+import io
+import time
+import uuid
+from functools import lru_cache
+from typing import Any
+
+import numpy as np
+from PIL import Image
+
+from sglang.multimodal_gen.configs.sample.vla import VLASamplingParams
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+def pack_numpy_payload(obj):
+    if isinstance(obj, (np.ndarray, np.generic)) and obj.dtype.kind in ("V", "O", "c"):
+        raise ValueError(f"Unsupported dtype: {obj.dtype}")
+    if isinstance(obj, np.ndarray):
+        return {
+            b"__ndarray__": True,
+            b"data": obj.tobytes(),
+            b"dtype": obj.dtype.str,
+            b"shape": obj.shape,
+        }
+    if isinstance(obj, np.generic):
+        return {
+            b"__npgeneric__": True,
+            b"data": obj.item(),
+            b"dtype": obj.dtype.str,
+        }
+    return obj
+
+
+def unpack_numpy_payload(obj):
+    ndarray_marker = obj.get("__ndarray__") or obj.get(b"__ndarray__")
+    npgeneric_marker = obj.get("__npgeneric__") or obj.get(b"__npgeneric__")
+    data = obj.get("data", obj.get(b"data"))
+    dtype = obj.get("dtype", obj.get(b"dtype"))
+    shape = obj.get("shape", obj.get(b"shape"))
+    if ndarray_marker:
+        return np.ndarray(
+            buffer=data,
+            dtype=np.dtype(dtype),
+            shape=shape,
+        )
+    if npgeneric_marker:
+        return np.dtype(dtype).type(data)
+    return obj
+
+
+def pack_msgpack(payload: Any) -> bytes:
+    import msgpack
+
+    return msgpack.packb(payload, default=pack_numpy_payload, use_bin_type=True)
+
+
+def unpack_msgpack(payload: bytes) -> Any:
+    import msgpack
+
+    return msgpack.unpackb(payload, object_hook=unpack_numpy_payload, raw=False)
+
+
+def _decode_b64_image(payload: dict[str, Any]) -> Image.Image:
+    data = payload.get("b64_json") or payload.get("base64")
+    if not data:
+        raise ValueError("image payload requires b64_json")
+    if isinstance(data, str) and "," in data and data.startswith("data:"):
+        data = data.split(",", 1)[1]
+    return Image.open(io.BytesIO(base64.b64decode(data))).convert("RGB")
+
+
+def _decode_tensor_payload(payload: dict[str, Any]) -> Any:
+    values = payload.get("values")
+    if values is None:
+        values = payload.get("data")
+    if values is None:
+        return payload
+    dtype = payload.get("dtype")
+    array = np.asarray(values, dtype=np.dtype(dtype) if dtype else None)
+    shape = payload.get("shape")
+    if shape is not None:
+        array = array.reshape(tuple(shape))
+    return array
+
+
+def _normalize_image_value(value: Any) -> Any:
+    if not isinstance(value, dict):
+        return value
+    if "b64_json" in value or "base64" in value:
+        return _decode_b64_image(value)
+    if "values" in value or "data" in value:
+        return _decode_tensor_payload(value)
+    return value
+
+
+def _normalize_observation(observation: dict[str, Any]) -> dict[str, Any]:
+    normalized = dict(observation)
+    images = normalized.get("images")
+    if isinstance(images, dict):
+        normalized["images"] = {
+            name: _normalize_image_value(value) for name, value in images.items()
+        }
+    state = normalized.get("state")
+    if isinstance(state, dict):
+        normalized["state"] = _decode_tensor_payload(state)
+    observation_state = normalized.get("observation.state")
+    if isinstance(observation_state, dict):
+        normalized["observation.state"] = _decode_tensor_payload(observation_state)
+    noise = normalized.get("noise")
+    if isinstance(noise, dict):
+        normalized["noise"] = _decode_tensor_payload(noise)
+    observation_noise = normalized.get("observation.noise")
+    if isinstance(observation_noise, dict):
+        normalized["observation.noise"] = _decode_tensor_payload(observation_noise)
+    return normalized
+
+
+def images_from_observation(
+    observation: dict[str, Any],
+    pipeline_config: Any,
+) -> dict[str, Any]:
+    if isinstance(observation.get("images"), dict):
+        images = dict(observation["images"])
+    else:
+        images = {}
+    for key in pipeline_config.image_keys:
+        if key in observation:
+            images[key] = observation[key]
+        full_key = f"observation.images.{key}"
+        if full_key in observation:
+            images[key] = observation[full_key]
+    return {name: _normalize_image_value(value) for name, value in images.items()}
+
+
+def action_metadata(server_args: ServerArgs) -> dict[str, Any]:
+    pipeline_config = server_args.pipeline_config
+    policy_family = getattr(
+        pipeline_config,
+        "policy_family",
+        type(pipeline_config).__name__.removesuffix("PipelineConfig").lower(),
+    )
+    return {
+        "object": "action.metadata",
+        "model": server_args.model_id or server_args.model_path,
+        "model_path": server_args.model_path,
+        "policy_family": policy_family,
+        "input": {
+            "image_keys": list(pipeline_config.image_keys),
+            "image_size": list(pipeline_config.image_size),
+            "state_dim": pipeline_config.state_dim,
+        },
+        "output": {
+            "action_type": "continuous",
+            "action_horizon": pipeline_config.action_horizon,
+            "action_dim": pipeline_config.output_action_dim,
+            "padded_action_dim": pipeline_config.action_dim,
+            "dtype": "float32",
+        },
+        "runtime": {
+            "materialize_dtype": pipeline_config.materialize_dtype,
+            "enable_autocast": pipeline_config.enable_autocast,
+            "parallelism": {
+                "num_gpus": server_args.num_gpus,
+                "tp_size": server_args.tp_size,
+                "sp_degree": server_args.sp_degree,
+                "ulysses_degree": server_args.ulysses_degree,
+                "ring_degree": server_args.ring_degree,
+                "prefix_strategy": pipeline_config.prefix_parallel_strategy,
+                "action_strategy": pipeline_config.action_parallel_strategy,
+                "layout_version": pipeline_config.parallel_layout_version,
+            },
+        },
+        "defaults": {
+            "num_inference_steps": pipeline_config.default_num_inference_steps,
+            "prefix_cache": (
+                "auto" if pipeline_config.enable_global_prefix_cache else False
+            ),
+            "cuda_graph": "auto" if pipeline_config.enable_action_cuda_graph else False,
+        },
+        "capabilities": {
+            "exact_prefix_cache": True,
+            "cuda_graph": pipeline_config.enable_action_cuda_graph,
+            "realtime_websocket": True,
+            "openpi_websocket": True,
+            "batch_inputs": False,
+            "multiple_candidates": False,
+        },
+    }
+
+
+def _runtime_bool(value: Any, default: bool) -> bool:
+    if value is None:
+        return default
+    if isinstance(value, str):
+        value = value.lower()
+        if value == "auto":
+            return default
+        if value in ("true", "1", "yes"):
+            return True
+        if value in ("false", "0", "no"):
+            return False
+    return bool(value)
+
+
+def _action_request_to_observation(payload: dict[str, Any]) -> dict[str, Any]:
+    if "input" not in payload:
+        return _normalize_observation(payload)
+
+    input_payload = payload.get("input") or {}
+    observation = dict(input_payload.get("observation") or {})
+    if "task" in input_payload:
+        observation["prompt"] = input_payload["task"]
+    elif "prompt" in input_payload:
+        observation["prompt"] = input_payload["prompt"]
+    if "images" in input_payload:
+        observation["images"] = input_payload["images"]
+    if "state" in input_payload:
+        observation["state"] = input_payload["state"]
+    if "noise" in input_payload:
+        observation["noise"] = input_payload["noise"]
+    return _normalize_observation(observation)
+
+
+@lru_cache(maxsize=32)
+def _resolve_action_sampling_params_cls_cached(
+    model_path: str,
+    backend: str | None,
+    model_id: str | None,
+    pipeline_class_name: str | None,
+) -> type[VLASamplingParams]:
+    if pipeline_class_name:
+        from sglang.multimodal_gen.registry import get_pipeline_config_classes
+
+        config_classes = get_pipeline_config_classes(pipeline_class_name)
+        if config_classes is not None:
+            _, sampling_params_cls = config_classes
+            if issubclass(sampling_params_cls, VLASamplingParams):
+                return sampling_params_cls
+
+    from sglang.multimodal_gen.registry import get_model_info
+
+    model_info = get_model_info(
+        model_path,
+        backend=backend,
+        model_id=model_id,
+    )
+    sampling_params_cls = model_info.sampling_param_cls
+    if not issubclass(sampling_params_cls, VLASamplingParams):
+        raise ValueError(
+            f"Action endpoint requires VLASamplingParams, got {sampling_params_cls.__name__}"
+        )
+    return sampling_params_cls
+
+
+def _resolve_action_sampling_params_cls(
+    server_args: ServerArgs,
+) -> type[VLASamplingParams]:
+    return _resolve_action_sampling_params_cls_cached(
+        server_args.model_path,
+        getattr(server_args, "backend", None),
+        getattr(server_args, "model_id", None),
+        getattr(server_args, "pipeline_class_name", None),
+    )
+
+
+@lru_cache(maxsize=32)
+def _sampling_params_field_names(
+    sampling_params_cls: type[VLASamplingParams],
+) -> frozenset[str]:
+    return frozenset(field.name for field in dataclasses.fields(sampling_params_cls))
+
+
+def build_action_sampling_params(
+    payload: dict[str, Any],
+    server_args: ServerArgs,
+) -> VLASamplingParams:
+    pipeline_config = server_args.pipeline_config
+    observation = _action_request_to_observation(payload)
+    parameters = dict(payload.get("parameters") or {})
+    runtime = dict(payload.get("runtime") or {})
+    if "return_timing" in payload and "return_timing" not in runtime:
+        runtime["return_timing"] = payload["return_timing"]
+    images = images_from_observation(observation, pipeline_config)
+    state = observation.get("state")
+    if state is None:
+        state = observation.get("observation.state")
+    noise = observation.get("noise")
+    if noise is None:
+        noise = observation.get("observation.noise")
+    prompt = observation.get("prompt") or observation.get("task") or ""
+    prefix_cache = runtime.get("prefix_cache")
+    if prefix_cache is None:
+        prefix_cache = observation.get("enable_prefix_cache")
+    if prefix_cache is None:
+        prefix_cache = observation.get("enable_pi_prefix_cache")
+    cuda_graph = runtime.get("cuda_graph")
+    if cuda_graph is None:
+        cuda_graph = observation.get("enable_cuda_graph")
+    if cuda_graph is None:
+        cuda_graph = observation.get("enable_pi_cuda_graph")
+    output_format = str(
+        runtime.get(
+            "output_format",
+            parameters.get(
+                "output_format",
+                observation.get("output_format", "list"),
+            ),
+        )
+    ).lower()
+    if output_format not in ("list", "numpy"):
+        raise ValueError("output_format must be 'list' or 'numpy'")
+
+    sampling_params_cls = _resolve_action_sampling_params_cls(server_args)
+    sampling_kwargs = {
+        "request_id": payload.get("request_id") or payload.get("id"),
+        "prompt": prompt,
+        "images": images,
+        "image_masks": observation.get("image_masks"),
+        "camera_order": observation.get("camera_order"),
+        "state": state,
+        "noise": noise,
+        "observation": observation,
+        "action_horizon": int(
+            parameters.get(
+                "action_horizon",
+                observation.get("action_horizon", pipeline_config.action_horizon),
+            )
+        ),
+        "action_dim": int(
+            parameters.get(
+                "action_dim",
+                observation.get("action_dim", pipeline_config.action_dim),
+            )
+        ),
+        "num_inference_steps": int(
+            parameters.get(
+                "num_inference_steps",
+                observation.get(
+                    "num_inference_steps",
+                    pipeline_config.default_num_inference_steps,
+                ),
+            )
+        ),
+        "output_format": output_format,
+        "return_timing": _runtime_bool(runtime.get("return_timing"), True),
+        "enable_prefix_cache": _runtime_bool(prefix_cache, True),
+        "enable_cuda_graph": _runtime_bool(cuda_graph, True),
+    }
+    supported_fields = _sampling_params_field_names(sampling_params_cls)
+    sp = sampling_params_cls(
+        **{
+            name: value
+            for name, value in sampling_kwargs.items()
+            if name in supported_fields
+        }
+    )
+    sp._adjust(server_args)
+    return sp
+
+
+async def infer_action(
+    payload: dict[str, Any],
+    server_args: ServerArgs,
+) -> dict[str, Any]:
+    sp = build_action_sampling_params(payload, server_args)
+    req = prepare_request(server_args, sp)
+    response = await async_scheduler_client.forward(req)
+    if getattr(response, "error", None):
+        raise RuntimeError(response.error)
+    if response.output is None:
+        raise RuntimeError("action policy returned no output")
+    return response.output[0]
+
+
+def action_generation_response(
+    output: dict[str, Any],
+    server_args: ServerArgs,
+    *,
+    preserve_numpy: bool = False,
+) -> dict[str, Any]:
+    actions = output["actions"]
+    if isinstance(actions, np.ndarray):
+        action_shape = list(actions.shape)
+        action_values = actions if preserve_numpy else actions.tolist()
+    else:
+        horizon = len(actions) if isinstance(actions, list) else 0
+        action_dim = len(actions[0]) if horizon and isinstance(actions[0], list) else 0
+        action_shape = [horizon, action_dim]
+        action_values = actions
+    response = {
+        "id": output.get("request_id") or f"act_{uuid.uuid4().hex}",
+        "object": "action.generation",
+        "created": int(time.time()),
+        "model": server_args.model_id or server_args.model_path,
+        "data": [
+            {
+                "index": 0,
+                "input_index": 0,
+                "candidate_index": 0,
+                "action": {
+                    "type": "continuous",
+                    "dtype": "float32",
+                    "shape": action_shape,
+                    "values": action_values,
+                },
+            }
+        ],
+        "usage": {
+            "action_horizon": action_shape[0] if action_shape else 0,
+            "action_dim": action_shape[1] if len(action_shape) > 1 else 0,
+            "denoise_steps": output.get("parameters", {}).get(
+                "num_inference_steps",
+                server_args.pipeline_config.default_num_inference_steps,
+            ),
+            "prefix_cache_hit": bool(output.get("cache", {}).get("hit", False)),
+        },
+    }
+    if "timings" in output:
+        response["timings"] = output["timings"]
+    if "cache" in output:
+        response["cache"] = output["cache"]
+    if "parallel" in output:
+        response["parallel"] = output["parallel"]
+    return response
+
+
+def action_raw_response(
+    output: dict[str, Any],
+    *,
+    preserve_numpy: bool = False,
+) -> dict[str, Any]:
+    response = dict(output)
+    actions = response.get("actions")
+    if isinstance(actions, np.ndarray) and not preserve_numpy:
+        response["actions"] = actions.tolist()
+    return response

python/sglang/multimodal_gen/runtime/entrypoints/vla/ws_utils.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import time
+import traceback
+from collections.abc import Callable
+from typing import Any
+
+from fastapi import WebSocket, WebSocketDisconnect
+
+from sglang.multimodal_gen.runtime.entrypoints.vla.protocol import (
+    action_metadata,
+    infer_action,
+    pack_msgpack,
+    unpack_msgpack,
+)
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+async def run_action_msgpack_ws(
+    websocket: WebSocket,
+    server_args: ServerArgs,
+    *,
+    prepare_payload: Callable[[dict[str, Any]], None],
+    build_response: Callable[[dict[str, Any]], dict[str, Any]],
+) -> None:
+    await websocket.accept()
+    await websocket.send_bytes(pack_msgpack(action_metadata(server_args)))
+
+    prev_total_time = None
+    while True:
+        try:
+            start_time = time.monotonic()
+            payload = unpack_msgpack(await websocket.receive_bytes())
+            prepare_payload(payload)
+            infer_start = time.monotonic()
+            output = await infer_action(payload, server_args)
+            response = build_response(output)
+            response.setdefault("server_timing", {})["infer_ms"] = (
+                time.monotonic() - infer_start
+            ) * 1000
+            if prev_total_time is not None:
+                response["server_timing"]["prev_total_ms"] = prev_total_time * 1000
+            await websocket.send_bytes(pack_msgpack(response))
+            prev_total_time = time.monotonic() - start_time
+        except WebSocketDisconnect:
+            break
+        except Exception:
+            try:
+                await websocket.send_bytes(
+                    pack_msgpack({"error": traceback.format_exc()})
+                )
+            except Exception:
+                pass
+            await websocket.close(code=1011, reason="Internal server error")
+            raise