chore: import upstream snapshot with attribution

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md

@@ -0,0 +1,58 @@
+# ComfyUI SGLDiffusion Plugin
+
+A ComfyUI plugin for integrating with SGLang Diffusion server, supporting image and video generation capabilities.
+
+## Installation
+
+1. **Install SGLang**: Follow the [Installation Guide](../../../../../docs/diffusion/installation.md) to install `sglang[diffusion]`.
+2. **Install Plugin**: Copy this entire directory (`ComfyUI_SGLDiffusion`) to your ComfyUI `custom_nodes/` folder.
+3. **Restart ComfyUI**: Restart ComfyUI to load the plugin.
+
+## Usage
+
+The plugin supports two modes of operation: **Server Mode** (via HTTP API) and **Integrated Mode** (tight integration with ComfyUI).
+
+### Supported Models
+- **Z-Image**: High-speed image generation models (e.g., `Z-Image-Turbo`)
+- **FLUX**: State-of-the-art text-to-image models (e.g., `FLUX.1-dev`)
+- **Qwen-Image**: Multi-modal image generation models (e.g., `Qwen-Image`,`Qwen-Image-2512`). *Note: Image editing support is currently experimental and may have some issues.*
+
+### Mode 1: Server Mode (HTTP API)
+Connect to a standalone SGLang Diffusion server.
+
+1. **Start SGLang Diffusion Server**: Ensure the server is running and accessible.
+2. **Connect to Server**: Use the `SGLDiffusion Server Model` node to connect (default: `http://localhost:3000/v1`).
+3. **Generate Content**:
+   - `SGLDiffusion Generate Image`: For text-to-image and image editing.
+   - `SGLDiffusion Generate Video`: For text-to-video and image-to-video.
+4. **LoRA Support**: Use `SGLDiffusion Server Set LoRA` and `SGLDiffusion Server Unset LoRA`.
+
+### Mode 2: Integrated Mode (Tight Integration)
+Leverage SGLang's high-performance sampling directly within ComfyUI while using ComfyUI's front-end nodes (CLIP, VAE, etc.).
+
+1. **Load Model**: Use the `SGLDiffusion UNET Loader` node to load your diffusion model.
+2. **Configure Options**: Use the `SGLDiffusion Options` node to set runtime parameters like `num_gpus`, `tp_size`, `model_type`, or `enable_torch_compile`.
+3. **Sample**: Connect the loaded model to standard ComfyUI samplers. SGLang will handle the sampling process efficiently.
+4. **LoRA Support**: Use the `SGLDiffusion LoRA Loader` for native LoRA integration.
+
+## Example Workflows
+
+Reference workflow files are provided in the `workflows/` directory:
+
+- **`flux_sgld_sp.json`**: Multi-GPU (Sequence Parallelism) workflow for FLUX models. High-performance inference across multiple cards.
+- **`qwen_image_sgld.json`**: Qwen-Image generation with LoRA support. Optimized for multi-modal image tasks.
+- **`z-image_sgld.json`**: High-speed image generation using Z-Image.
+- **`sgld_text2img.json`**: Server-mode text-to-image generation with LoRA support.
+- **`sgld_image2video.json`**: Server-mode image-to-video generation.
+
+For other workflows supporting the models, you can easily use SGLang by replacing the official `UNET Loader` node with the `SGLDUNETLoader` node. Similarly, for LoRA support, replace the official LoRA loader with the `SGLDiffusion LoRA Loader`.
+
+To use these workflows:
+1. Open ComfyUI.
+2. Load the workflow JSON file from the `workflows/` directory.
+3. Adjust the parameters and model paths as needed.
+4. Run the workflow.
+
+## Current Implementation
+
+This plugin provides a high-performance backend for diffusion models in ComfyUI. By leveraging SGLang's optimized kernels and parallelization techniques (Tensor Parallelism, TeaCache, etc.), it significantly accelerates the sampling process, especially for large models like FLUX.

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/__init__.py

@@ -0,0 +1,13 @@
+"""
+ComfyUI SGLang Diffusion nodes package.
+"""
+
+try:
+    from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
+
+    __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+except ImportError:
+    # ComfyUI dependencies not available (e.g., in test environment)
+    NODE_CLASS_MAPPINGS = {}
+    NODE_DISPLAY_NAME_MAPPINGS = {}
+    __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/__init__.py

@@ -0,0 +1,14 @@
+"""
+Core components for SGLang Diffusion ComfyUI integration.
+Provides generator, model patcher, and server API client.
+"""
+
+from .generator import SGLDiffusionGenerator
+from .model_patcher import SGLDModelPatcher
+from .server_api import SGLDiffusionServerAPI
+
+__all__ = [
+    "SGLDiffusionGenerator",
+    "SGLDModelPatcher",
+    "SGLDiffusionServerAPI",
+]

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py

@@ -0,0 +1,231 @@
+"""
+Generator for SGLang Diffusion ComfyUI integration.
+"""
+
+import logging
+import os
+
+import psutil
+from comfy import model_detection, model_management
+from comfy.utils import (
+    calculate_parameters,
+    load_torch_file,
+    state_dict_prefix_replace,
+    unet_to_diffusers,
+)
+
+logger = logging.getLogger(__name__)
+
+try:
+    from sglang.multimodal_gen import DiffGenerator
+except ImportError:
+    logger.error(
+        "Error: sglang.multimodal_gen is not installed. Please install it using 'pip install sglang[diffusion]'"
+    )
+
+from ..executors import (
+    FluxExecutor,
+    QwenImageEditExecutor,
+    QwenImageExecutor,
+    ZImageExecutor,
+)
+from .model_patcher import SGLDModelPatcher
+
+
+class SGLDiffusionGenerator:
+    """Generator for SGLang Diffusion models in ComfyUI."""
+
+    def __init__(self):
+        self.model_path = None
+        self.generator = None
+        self.executor = None
+        self.last_options = None
+
+        self.pipeline_class_dict = {
+            "flux": "ComfyUIFluxPipeline",
+            "lumina2": "ComfyUIZImagePipeline",  # zimage
+            "qwen_image": "ComfyUIQwenImagePipeline",
+            "qwen_image_edit": "ComfyUIQwenImageEditPipeline",
+        }
+        self.executor_class_dict = {
+            "flux": FluxExecutor,
+            "lumina2": ZImageExecutor,
+            "qwen_image": QwenImageExecutor,
+            "qwen_image_edit": QwenImageEditExecutor,
+        }
+
+    def __del__(self):
+        self.close_generator()
+
+    def init_generator(
+        self, model_path: str, pipeline_class_name: str, kwargs: dict = None
+    ):
+        """Initialize the diffusion generator."""
+        if self.generator is not None:
+            return self.generator
+        if kwargs is None:
+            kwargs = {}
+        # Set comfyui_mode for ComfyUI integration
+        kwargs["comfyui_mode"] = True
+        self.generator = DiffGenerator.from_pretrained(
+            model_path=model_path,
+            pipeline_class_name=pipeline_class_name,
+            **kwargs,
+        )
+        return self.generator
+
+    def kill_generator(self):
+        """Kill worker processes manually because generator shutdown cannot terminate them."""
+        current_pid = os.getpid()
+        worker_processes = []
+        for proc in psutil.process_iter(["pid", "name", "cmdline"]):
+            try:
+                # Look for sglang-diffusionWorker processes
+                if proc.info["cmdline"]:
+                    cmdline = " ".join(proc.info["cmdline"])
+                    if "sgl_diffusion::" in cmdline:
+                        if proc.info["pid"] != current_pid:
+                            worker_processes.append(proc)
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+
+        if worker_processes:
+            logger.info(
+                f"Found {len(worker_processes)} worker processes to terminate..."
+            )
+            for proc in worker_processes:
+                try:
+                    logger.info(
+                        f"Terminating worker process {proc.info['pid']}: {proc.info['name']}"
+                    )
+                    proc.terminate()
+                    proc.wait(timeout=5)
+                except psutil.TimeoutExpired:
+                    logger.warning(
+                        f"Process {proc.info['pid']} did not terminate, forcing kill..."
+                    )
+                    try:
+                        proc.kill()
+                        proc.wait(timeout=2)
+                    except (psutil.NoSuchProcess, psutil.TimeoutExpired):
+                        pass
+                except (psutil.NoSuchProcess, psutil.AccessDenied):
+                    pass
+
+    def close_generator(self):
+        """Close and cleanup the generator and all associated resources."""
+        if self.generator is not None:
+            self.generator.shutdown()
+            self.kill_generator()
+            # Clear other references
+            self.last_options = None
+            self.model_path = None
+            self.generator = None
+            self.executor = None
+
+    def get_comfyui_model(self, model_path: str, model_options: dict = None):
+        """Get ComfyUI model from model path."""
+        if model_options is None:
+            model_options = {}
+        dtype = model_options.get("dtype", None)
+        # Allow loading unets from checkpoint files
+        sd = load_torch_file(model_path)
+        diffusion_model_prefix = model_detection.unet_prefix_from_state_dict(sd)
+        temp_sd = state_dict_prefix_replace(
+            sd, {diffusion_model_prefix: ""}, filter_keys=True
+        )
+        if len(temp_sd) > 0:
+            sd = temp_sd
+
+        parameters = calculate_parameters(sd)
+        load_device = model_management.get_torch_device()
+
+        model_detect_config = model_detection.detect_unet_config(sd, "")
+        model_type = model_detect_config.get("image_model", None)
+        if model_type is None or model_type not in self.pipeline_class_dict:
+            raise ValueError(f"Unsupported model type: {model_type}")
+        model_config = model_detection.model_config_from_unet(sd, "")
+
+        if model_config is not None:
+            new_sd = sd
+        else:
+            new_sd = model_detection.convert_diffusers_mmdit(sd, "")
+            if new_sd is not None:  # diffusers mmdit
+                model_config = model_detection.model_config_from_unet(new_sd, "")
+                if model_config is None:
+                    return None
+            else:  # diffusers unet
+                model_config = model_detection.model_config_from_diffusers_unet(sd)
+                if model_config is None:
+                    return None
+
+                diffusers_keys = unet_to_diffusers(model_config.unet_config)
+                new_sd = {}
+                for k in diffusers_keys:
+                    if k in sd:
+                        new_sd[diffusers_keys[k]] = sd.pop(k)
+        if dtype is None:
+            unet_dtype = model_management.unet_dtype(
+                model_params=parameters,
+                supported_dtypes=model_config.supported_inference_dtypes,
+            )
+        else:
+            unet_dtype = dtype
+
+        manual_cast_dtype = model_management.unet_manual_cast(
+            unet_dtype, load_device, model_config.supported_inference_dtypes
+        )
+        model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)
+        model_config.custom_operations = model_options.get("custom_operations", None)
+        model_config.unet_config["disable_unet_model_creation"] = True
+        comfyui_model = model_config.get_model({})
+        return comfyui_model, model_config, model_type
+
+    def load_model(
+        self, model_path: str, model_options: dict = None, sgld_options: dict = None
+    ):
+        """Load model and return model patcher."""
+        gather_options = {
+            "model_path": model_path,
+            "model_options": model_options,
+            "sgld_options": sgld_options,
+        }
+        if (
+            self.last_options is not None
+            and self.last_options == gather_options
+            and self.generator is not None
+        ):
+            return self.generator
+        else:
+            self.close_generator()
+
+        self.last_options = gather_options
+        self.model_path = model_path
+
+        comfyui_model, model_config, model_type = self.get_comfyui_model(
+            model_path, model_options
+        )
+        if model_type is None or model_type not in self.pipeline_class_dict:
+            raise ValueError(f"Unsupported model type: {model_type}")
+
+        set_model_type = sgld_options.pop("model_type", None) if sgld_options else None
+        if set_model_type is not None and set_model_type in self.pipeline_class_dict:
+            model_type = set_model_type
+
+        pipeline_class_name = self.pipeline_class_dict[model_type]
+        self.generator = self.init_generator(
+            model_path, pipeline_class_name, sgld_options
+        )
+
+        executor_class = self.executor_class_dict[model_type]
+        self.executor = executor_class(
+            self.generator, model_path, comfyui_model, model_config
+        )
+        comfyui_model.diffusion_model = self.executor
+
+        load_device = model_management.get_torch_device()
+        offload_device = model_management.unet_offload_device()
+
+        return SGLDModelPatcher(
+            comfyui_model, load_device, offload_device, model_type=model_type
+        )

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/model_patcher.py

@@ -0,0 +1,82 @@
+"""
+Model patcher for SGLang Diffusion ComfyUI integration.
+"""
+
+import copy
+
+from comfy.model_patcher import ModelPatcher
+
+
+class SGLDModelPatcher(ModelPatcher):
+    """Model patcher for SGLang Diffusion models in ComfyUI."""
+
+    def __init__(
+        self,
+        model,
+        load_device,
+        offload_device,
+        size=0,
+        weight_inplace_update=False,
+        model_type=None,
+    ):
+        super().__init__(
+            model, load_device, offload_device, size, weight_inplace_update
+        )
+        self.lora_cache = {}
+        self.model_type = model_type
+        self.model_size_dict = {
+            "flux": 27 * 1024 * 1024 * 1024,
+            "lumina2": 8 * 1024 * 1024 * 1024,
+        }
+
+    def clone(self):
+        """Clone the model patcher."""
+        n = SGLDModelPatcher(
+            self.model,
+            self.load_device,
+            self.offload_device,
+            self.size,
+            weight_inplace_update=self.weight_inplace_update,
+        )
+        n.patches = {}
+        for k in self.patches:
+            n.patches[k] = self.patches[k][:]
+        n.patches_uuid = self.patches_uuid
+
+        n.object_patches = self.object_patches.copy()
+        n.model_options = copy.deepcopy(self.model_options)
+        n.backup = self.backup
+        n.object_patches_backup = self.object_patches_backup
+        n.lora_cache = copy.copy(self.lora_cache)
+        return n
+
+    def model_size(self):
+        """Get the model size in bytes."""
+        if self.model_type in self.model_size_dict:
+            return self.model_size_dict[self.model_type]
+        else:
+            return 0
+
+    def load(
+        self,
+        device_to=None,
+        lowvram_model_memory=0,
+        force_patch_weights=False,
+        full_load=False,
+    ):
+        """Load model (no-op for SGLang Diffusion)."""
+        pass
+
+    def patch_model(
+        self,
+        device_to=None,
+        lowvram_model_memory=0,
+        load_weights=True,
+        force_patch_weights=False,
+    ):
+        """Patch model (no-op for SGLang Diffusion)."""
+        pass
+
+    def unpatch_model(self, device_to=None, unpatch_weights=True):
+        """Unpatch model (no-op for SGLang Diffusion)."""
+        pass

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/server_api.py

@@ -0,0 +1,539 @@
+"""
+SGLang Diffusion Server API client.
+Provides a low-level interface for interacting with SGLang Diffusion HTTP server.
+"""
+
+import base64
+import io
+import os
+import time
+from typing import Any, Dict, Optional
+
+import requests
+from PIL import Image
+
+
+class SGLDiffusionServerAPI:
+    """Client for SGLang Diffusion HTTP server API."""
+
+    def __init__(self, base_url: str, api_key: str = "sk-proj-1234567890"):
+        """
+        Initialize the API client.
+
+        Args:
+            base_url: Base URL of the SGLang Diffusion server (e.g., "http://localhost:30010/v1")
+            api_key: API key for authentication (default: "sk-proj-1234567890")
+        """
+        # Ensure base_url doesn't end with /v1 if it's already there
+        if base_url.endswith("/v1"):
+            self.base_url = base_url
+        elif base_url.endswith("/v1/"):
+            self.base_url = base_url.rstrip("/")
+        else:
+            self.base_url = f"{base_url.rstrip('/')}/v1"
+
+        self.api_key = api_key
+        self.headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+
+    def get_model_info(self) -> Dict[str, Any]:
+        """
+        Get information about the model served by this server.
+
+        Returns:
+            Dictionary containing model information including:
+            - model_path: Path to the model
+            - task_type: Type of task (e.g., "T2V", "I2I")
+            - pipeline_name: Name of the pipeline
+            - num_gpus: Number of GPUs
+            - dit_precision: DiT model precision
+            - vae_precision: VAE model precision
+        """
+        try:
+            # Remove /v1 from base_url for /models endpoint
+            models_url = self.base_url.removesuffix("/v1") + "/models"
+            response = requests.get(models_url, headers=self.headers, timeout=30)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise RuntimeError(f"Failed to get model info: {str(e)}")
+
+    def generate_image(
+        self,
+        prompt: str,
+        image_path: Optional[str] = None,
+        mask_path: Optional[str] = None,
+        size: Optional[str] = None,
+        width: Optional[int] = None,
+        height: Optional[int] = None,
+        n: int = 1,
+        negative_prompt: Optional[str] = None,
+        guidance_scale: Optional[float] = None,
+        num_inference_steps: Optional[int] = None,
+        seed: Optional[int] = None,
+        enable_teacache: bool = False,
+        response_format: str = "b64_json",
+        quality: Optional[str] = "auto",
+        style: Optional[str] = "vivid",
+        background: Optional[str] = "auto",
+        output_format: Optional[str] = None,
+        generator_device: Optional[str] = "cuda",
+    ) -> Dict[str, Any]:
+        """
+        Generate or edit an image using SGLang Diffusion API.
+        If image_path is provided, calls the edit endpoint; otherwise calls the generation endpoint.
+
+        Args:
+            prompt: Text prompt for image generation/editing
+            image_path: Optional path to input image file for editing. If provided, uses edit API.
+            mask_path: Optional path to mask image file (only used when image_path is provided)
+            size: Image size in format "WIDTHxHEIGHT" (e.g., "1024x1024")
+            width: Image width (used if size is not provided)
+            height: Image height (used if size is not provided)
+            n: Number of images to generate (1-10)
+            negative_prompt: Negative prompt to avoid certain elements
+            guidance_scale: Classifier-free guidance scale
+            num_inference_steps: Number of denoising steps
+            seed: Random seed for reproducible generation
+            enable_teacache: Enable TEA cache acceleration
+            response_format: Response format ("b64_json" or "url")
+            quality: Image quality ("auto", "standard", "hd") - only for generation
+            style: Image style ("vivid" or "natural") - only for generation
+            background: Background type ("auto", "transparent", "opaque")
+            output_format: Output format ("png", "jpeg", "webp")
+            generator_device: Device for random generator ("cuda" or "cpu")
+
+        Returns:
+            Dictionary containing the API response with generated/edited image data
+        """
+        if not prompt:
+            raise ValueError("Prompt cannot be empty")
+
+        # Determine size
+        if size is None:
+            if width is not None and height is not None:
+                size = f"{width}x{height}"
+            else:
+                size = "1024x1024"
+
+        # Build common parameters
+        common_params = self._build_image_common_params(
+            prompt=prompt,
+            size=size,
+            n=n,
+            response_format=response_format,
+            negative_prompt=negative_prompt,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            seed=seed,
+            enable_teacache=enable_teacache,
+            background=background,
+            output_format=output_format,
+            generator_device=generator_device,
+        )
+
+        # If image_path is provided, use edit endpoint
+        if image_path:
+            if not os.path.exists(image_path):
+                raise FileNotFoundError(f"Image file not found: {image_path}")
+
+            # Prepare multipart form data for edit
+            files: Dict[str, Any] = {}
+            data = common_params.copy()
+
+            # Add image file
+            files["image"] = (
+                os.path.basename(image_path),
+                open(image_path, "rb"),
+                self._get_content_type(image_path),
+            )
+
+            # Add mask file if provided
+            if mask_path:
+                if not os.path.exists(mask_path):
+                    raise FileNotFoundError(f"Mask file not found: {mask_path}")
+                files["mask"] = (
+                    os.path.basename(mask_path),
+                    open(mask_path, "rb"),
+                    self._get_content_type(mask_path),
+                )
+
+            # Prepare headers for multipart form data
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+            }
+
+            try:
+                response = requests.post(
+                    f"{self.base_url}/images/edits",
+                    files=files,
+                    data=data,
+                    headers=headers,
+                    timeout=300,  # 5 minutes timeout for generation
+                )
+                response.raise_for_status()
+                return response.json()
+            except requests.exceptions.RequestException as e:
+                raise RuntimeError(f"Failed to edit image: {str(e)}")
+            finally:
+                # Close file handles
+                for file_tuple in files.values():
+                    if isinstance(file_tuple, tuple) and len(file_tuple) > 1:
+                        file_tuple[1].close()
+        else:
+            # Use generation endpoint - add generation-specific parameters
+            payload = common_params.copy()
+            if quality:
+                payload["quality"] = quality
+            if style:
+                payload["style"] = style
+
+            try:
+                response = requests.post(
+                    f"{self.base_url}/images/generations",
+                    json=payload,
+                    headers=self.headers,
+                    timeout=300,  # 5 minutes timeout for generation
+                )
+                response.raise_for_status()
+                return response.json()
+            except requests.exceptions.RequestException as e:
+                raise RuntimeError(f"Failed to generate image: {str(e)}")
+
+    def generate_video(
+        self,
+        prompt: str,
+        size: Optional[str] = None,
+        width: Optional[int] = None,
+        height: Optional[int] = None,
+        seconds: Optional[int] = 4,
+        fps: Optional[int] = None,
+        num_frames: Optional[int] = None,
+        negative_prompt: Optional[str] = None,
+        guidance_scale: Optional[float] = None,
+        num_inference_steps: Optional[int] = None,
+        seed: Optional[int] = None,
+        enable_teacache: bool = False,
+        generator_device: Optional[str] = "cuda",
+        input_reference: Optional[str] = None,
+        output_path: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Generate a video using SGLang Diffusion API and wait for completion.
+
+        Args:
+            prompt: Text prompt for video generation
+            size: Video size in format "WIDTHxHEIGHT" (e.g., "1280x720")
+            width: Video width (used if size is not provided)
+            height: Video height (used if size is not provided)
+            seconds: Duration of the video in seconds
+            fps: Frames per second
+            num_frames: Number of frames (overrides seconds * fps if provided)
+            negative_prompt: Negative prompt to avoid certain elements
+            guidance_scale: Classifier-free guidance scale
+            num_inference_steps: Number of denoising steps
+            seed: Random seed for reproducible generation
+            enable_teacache: Enable TEA cache acceleration
+            generator_device: Device for random generator ("cuda" or "cpu")
+            input_reference: Path to input reference image for image-to-video
+
+        Returns:
+            Dictionary containing completed video job information with file_path
+        """
+        if not prompt:
+            raise ValueError("Prompt cannot be empty")
+
+        # Determine size
+        if size is None:
+            if width is not None and height is not None:
+                size = f"{width}x{height}"
+            else:
+                size = "720x1280"
+
+        # Prepare request payload
+        payload: Dict[str, Any] = {
+            "prompt": prompt,
+            "size": size,
+        }
+
+        # Add optional parameters
+        if seconds is not None:
+            payload["seconds"] = seconds
+        if fps is not None:
+            payload["fps"] = fps
+        if num_frames is not None:
+            payload["num_frames"] = num_frames
+        if negative_prompt:
+            payload["negative_prompt"] = negative_prompt
+        if guidance_scale is not None:
+            payload["guidance_scale"] = guidance_scale
+        if num_inference_steps is not None:
+            payload["num_inference_steps"] = num_inference_steps
+        if seed is not None and seed >= 0:
+            payload["seed"] = seed
+        if enable_teacache:
+            payload["enable_teacache"] = True
+        if generator_device:
+            payload["generator_device"] = generator_device
+        if input_reference:
+            payload["input_reference"] = input_reference
+        if output_path:
+            payload["output_path"] = output_path
+
+        try:
+            # Create video generation job
+            response = requests.post(
+                f"{self.base_url}/videos",
+                json=payload,
+                headers=self.headers,
+                timeout=30,
+            )
+            response.raise_for_status()
+            video_job = response.json()
+            video_id = video_job.get("id")
+
+            # Wait for completion with fixed polling
+            poll_interval = 5  # 5 seconds
+            max_wait_time = 3600  # 1 hour
+            max_consecutive_errors = 5
+            consecutive_errors = 0
+            start_time = time.time()
+
+            while time.time() - start_time < max_wait_time:
+                try:
+                    status_response = requests.get(
+                        f"{self.base_url}/videos/{video_id}",
+                        headers=self.headers,
+                        timeout=30,
+                    )
+                    status_response.raise_for_status()
+                    status = status_response.json()
+
+                    # Reset error counter on successful request
+                    consecutive_errors = 0
+
+                    if status.get("status") == "completed":
+                        return status
+                    elif status.get("status") == "failed":
+                        error = status.get("error", {})
+                        error_msg = (
+                            error.get("message", "Unknown error")
+                            if error
+                            else "Unknown error"
+                        )
+                        raise RuntimeError(f"Video generation failed: {error_msg}")
+                except requests.exceptions.ConnectionError as e:
+                    # Connection errors - likely server is down
+                    consecutive_errors += 1
+                    if consecutive_errors >= max_consecutive_errors:
+                        raise RuntimeError(
+                            f"Lost connection to server after {consecutive_errors} consecutive errors. "
+                            f"Server may be unavailable: {str(e)}"
+                        )
+                except requests.exceptions.RequestException as e:
+                    # Other network errors - continue polling but track errors
+                    consecutive_errors += 1
+                    if consecutive_errors >= max_consecutive_errors:
+                        raise RuntimeError(
+                            f"Network error after {consecutive_errors} consecutive failures: {str(e)}"
+                        )
+
+                time.sleep(poll_interval)
+
+            raise TimeoutError(
+                f"Video generation timed out after {max_wait_time} seconds"
+            )
+        except requests.exceptions.RequestException as e:
+            raise RuntimeError(f"Failed to generate video: {str(e)}")
+
+    def _build_image_common_params(
+        self,
+        prompt: str,
+        size: str,
+        n: int,
+        response_format: str,
+        negative_prompt: Optional[str] = None,
+        guidance_scale: Optional[float] = None,
+        num_inference_steps: Optional[int] = None,
+        seed: Optional[int] = None,
+        enable_teacache: bool = False,
+        background: Optional[str] = None,
+        output_format: Optional[str] = None,
+        generator_device: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Build common parameters for both image generation and editing.
+
+        Returns:
+            Dictionary containing common parameters
+        """
+        params: Dict[str, Any] = {
+            "prompt": prompt,
+            "size": size,
+            "n": max(1, min(n, 10)),
+            "response_format": response_format,
+        }
+
+        # Add optional parameters
+        if negative_prompt:
+            params["negative_prompt"] = negative_prompt
+        if guidance_scale is not None:
+            params["guidance_scale"] = guidance_scale
+        if num_inference_steps is not None:
+            params["num_inference_steps"] = num_inference_steps
+        if seed is not None and seed >= 0:
+            params["seed"] = seed
+        if enable_teacache:
+            params["enable_teacache"] = True
+        if background:
+            params["background"] = background
+        if output_format:
+            params["output_format"] = output_format
+        if generator_device:
+            params["generator_device"] = generator_device
+
+        return params
+
+    def _get_content_type(self, file_path: str) -> str:
+        """Get content type based on file extension."""
+        ext = os.path.splitext(file_path)[1].lower()
+        content_types = {
+            ".png": "image/png",
+            ".jpg": "image/jpeg",
+            ".jpeg": "image/jpeg",
+            ".webp": "image/webp",
+        }
+        return content_types.get(ext, "image/png")
+
+    def decode_image_from_response(
+        self, response_data: Dict[str, Any], index: int = 0
+    ) -> Image.Image:
+        """
+        Decode base64 image from API response.
+
+        Args:
+            response_data: API response dictionary
+            index: Index of the image in the response (default: 0)
+
+        Returns:
+            PIL Image object
+        """
+        if "data" not in response_data or not response_data["data"]:
+            raise ValueError("No image data in response")
+
+        if index >= len(response_data["data"]):
+            raise IndexError(f"Image index {index} out of range")
+
+        image_data = response_data["data"][index]
+        if "b64_json" not in image_data or not image_data["b64_json"]:
+            raise ValueError("No base64 image data found")
+
+        image_bytes = base64.b64decode(image_data["b64_json"])
+        image = Image.open(io.BytesIO(image_bytes))
+
+        # Convert to RGB if needed
+        if image.mode != "RGB":
+            image = image.convert("RGB")
+
+        return image
+
+    def set_lora(
+        self,
+        lora_nickname: str,
+        lora_path: Optional[str] = None,
+        target: str = "all",
+    ) -> Dict[str, Any]:
+        """
+        Set a LoRA adapter for the specified transformer(s).
+
+        Args:
+            lora_nickname: The nickname of the adapter (required).
+            lora_path: Path to the LoRA adapter (local path or HF repo id).
+                      Required for the first load; optional if re-activating a cached nickname.
+            target: Which transformer(s) to apply the LoRA to. One of:
+                - "all": Apply to all transformers (default)
+                - "transformer": Apply only to the primary transformer (high noise for Wan2.2)
+                - "transformer_2": Apply only to transformer_2 (low noise for Wan2.2)
+                - "critic": Apply only to the critic model
+
+        Returns:
+            Dictionary containing the API response with status and message
+        """
+        if not lora_nickname:
+            raise ValueError("lora_nickname cannot be empty")
+
+        # Prepare request payload
+        payload: Dict[str, Any] = {
+            "lora_nickname": lora_nickname,
+            "target": target,
+        }
+
+        # Add optional lora_path if provided
+        if lora_path:
+            payload["lora_path"] = lora_path
+
+        try:
+            response = requests.post(
+                f"{self.base_url}/set_lora",
+                json=payload,
+                headers=self.headers,
+                timeout=30,
+            )
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise RuntimeError(f"Failed to set LoRA adapter: {str(e)}")
+
+    def unset_lora(
+        self,
+        target: str = "all",
+    ) -> Dict[str, Any]:
+        """
+        Unset (unmerge) LoRA weights from the base model.
+
+        Args:
+            target: same as set_lora
+
+        Returns:
+            Dictionary containing the API response with status and message
+        """
+        # Prepare request payload
+        payload: Dict[str, Any] = {
+            "target": target,
+        }
+
+        try:
+            response = requests.post(
+                f"{self.base_url}/unmerge_lora_weights",
+                json=payload,
+                headers=self.headers,
+                timeout=30,
+            )
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            raise RuntimeError(f"Failed to unset LoRA adapter: {str(e)}")
+
+
+if __name__ == "__main__":
+    api = SGLDiffusionServerAPI(
+        base_url="http://localhost:30010/v1", api_key="sk-proj-1234567890"
+    )
+    model_info = api.get_model_info()
+    print(api.get_model_info())
+    if model_info.get("task_type") == "T2V" or model_info.get("task_type") == "I2V":
+        print(
+            api.generate_video(
+                prompt="A calico cat playing a piano on stage",
+                num_inference_steps=1,
+                size="480x480",
+            )
+        )
+    else:
+        print(
+            api.generate_image(
+                prompt="A calico cat playing a piano on stage", size="1024x1024"
+            )
+        )

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/__init__.py

@@ -0,0 +1,17 @@
+"""
+ComfyUI SGLang Diffusion executors package.
+Provides executor classes for different model types.
+"""
+
+from .base import SGLDiffusionExecutor
+from .flux import FluxExecutor
+from .qwen_image import QwenImageEditExecutor, QwenImageExecutor
+from .zimage import ZImageExecutor
+
+__all__ = [
+    "SGLDiffusionExecutor",
+    "FluxExecutor",
+    "ZImageExecutor",
+    "QwenImageExecutor",
+    "QwenImageEditExecutor",
+]

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py

@@ -0,0 +1,56 @@
+"""
+Base executor class for SGLang Diffusion ComfyUI integration.
+"""
+
+import torch
+
+
+class SGLDiffusionExecutor(torch.nn.Module):
+    """Base executor class for SGLang Diffusion models in ComfyUI."""
+
+    def __init__(self, generator, model_path, model, config):
+        super(SGLDiffusionExecutor, self).__init__()
+        self.generator = generator
+        self.model_path = model_path
+        self.model = model
+        self.dtype = config.unet_config["dtype"]
+        self.config = config
+        self.loras = []
+
+    @staticmethod
+    def should_suppress_logs(timestep):
+        """Determine if logs should be suppressed based on timestep value."""
+        if torch.is_tensor(timestep):
+            return bool((timestep < 1.0).item())
+        return bool(timestep < 1.0)
+
+    def set_lora(self, lora_nickname=None, lora_path=None, strength=None, target=None):
+        """Set LoRA adapter using SGLang Diffusion API."""
+        if len(lora_nickname) > 0:
+            self.generator.set_lora(
+                lora_nickname=lora_nickname,
+                lora_path=lora_path,
+                strength=strength,
+                target=target,
+            )
+
+    def _unpack_latents(self, latents, height, width, channels):
+        """Unpack latents from packed format to standard format."""
+        batch_size = latents.shape[0]
+        latents = latents.view(batch_size, height // 2, width // 2, channels, 2, 2)
+        latents = latents.permute(0, 3, 1, 4, 2, 5)
+        latents = latents.reshape(batch_size, channels, height, width)
+
+        return latents
+
+    def _pack_latents(self, latents):
+        """Pack latents from standard format to packed format."""
+        batch_size, num_channels_latents, height, width = latents.shape
+        latents = latents.view(
+            batch_size, num_channels_latents, height // 2, 2, width // 2, 2
+        )
+        latents = latents.permute(0, 2, 4, 1, 3, 5)
+        latents = latents.reshape(
+            batch_size, (height // 2) * (width // 2), num_channels_latents * 4
+        )
+        return latents

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/flux.py

@@ -0,0 +1,69 @@
+"""
+Flux executor for SGLang Diffusion ComfyUI integration.
+"""
+
+import torch
+
+try:
+    from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+    from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+except ImportError:
+    print(
+        "Error: sglang.multimodal_gen is not installed. Please install it using 'pip install sglang[diffusion]'"
+    )
+
+from .base import SGLDiffusionExecutor
+
+
+class FluxExecutor(SGLDiffusionExecutor):
+    """Executor for Flux models in ComfyUI."""
+
+    def __init__(self, generator, model_path, model, config):
+        super().__init__(generator, model_path, model, config)
+
+    def forward(self, x, timestep, context, y=None, guidance=None, **kwargs):
+        """Forward pass for Flux model."""
+        hidden_states = self._pack_latents(x)
+        timesteps = timestep * 1000.0
+        encoder_hidden_states = context
+        pooled_projections = y
+        guidance = guidance * 1000.0
+
+        B, C, H, W = x.shape
+        height = H * 8
+        width = W * 8
+        # Create SamplingParams
+        sampling_params = SamplingParams.from_user_sampling_params_args(
+            self.model_path,
+            server_args=self.generator.server_args,
+            prompt=" ",
+            guidance_scale=3.5,  # Flux typically uses embedded_cfg_scale=3.5
+            height=height,
+            width=width,
+            num_frames=1,
+            num_inference_steps=1,
+            save_output=False,
+            suppress_logs=self.should_suppress_logs(timestep),
+        )
+
+        # Prepare request (converts SamplingParams to Req)
+        req = prepare_request(
+            server_args=self.generator.server_args,
+            sampling_params=sampling_params,
+        )
+        req.latents = hidden_states  # Set as [B, S, D] format directly
+        req.timesteps = timesteps  # ComfyUI's timesteps parameter
+        req.prompt_embeds = [pooled_projections, encoder_hidden_states]  # [CLIP, T5]
+        req.raw_latent_shape = torch.tensor(hidden_states.shape, dtype=torch.long)
+
+        # Set pooled_projections (required by Flux)
+        req.pooled_embeds = [pooled_projections]  # List format as per Req definition
+        req.do_classifier_free_guidance = False
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+        # Send request to scheduler
+        output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
+        noise_pred = output_batch.noise_pred
+        return self._unpack_latents(noise_pred, H, W, C).to(x.device)

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py

@@ -0,0 +1,172 @@
+"""
+QwenImage executor for SGLang Diffusion ComfyUI integration.
+"""
+
+import torch
+
+try:
+    from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+    from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+except ImportError:
+    print(
+        "Error: sglang.multimodal_gen is not installed. Please install it using 'pip install sglang[diffusion]'"
+    )
+
+import comfy.ldm.common_dit
+
+from .base import SGLDiffusionExecutor
+
+
+class QwenImageExecutor(SGLDiffusionExecutor):
+    """Executor for QwenImage models in ComfyUI."""
+
+    def __init__(self, generator, model_path, model, config):
+        super().__init__(generator, model_path, model, config)
+        self.patch_size = 2
+
+    def _pack_latents(self, x):
+        """Process hidden states for QwenImage model."""
+        latents = comfy.ldm.common_dit.pad_to_patch_size(
+            x, (1, self.patch_size, self.patch_size)
+        )
+        orig_shape = latents.shape
+        latents = latents.view(
+            orig_shape[0],
+            orig_shape[1],
+            orig_shape[-3],
+            orig_shape[-2] // 2,
+            2,
+            orig_shape[-1] // 2,
+            2,
+        )
+        latents = latents.permute(0, 2, 3, 5, 1, 4, 6)
+        latents = latents.reshape(
+            orig_shape[0],
+            orig_shape[-3] * (orig_shape[-2] // 2) * (orig_shape[-1] // 2),
+            orig_shape[1] * 4,
+        )
+        return latents, orig_shape
+
+    def _unpack_latents(self, latents, num_embeds, orig_shape, x):
+        """Unpack hidden states from packed format to standard format."""
+        latents = latents[:, :num_embeds].view(
+            orig_shape[0],
+            orig_shape[-3],
+            orig_shape[-2] // 2,
+            orig_shape[-1] // 2,
+            orig_shape[1],
+            2,
+            2,
+        )
+        latents = latents.permute(0, 4, 1, 2, 5, 3, 6)
+        latents = latents.reshape(orig_shape)[:, :, :, : x.shape[-2], : x.shape[-1]]
+        return latents
+
+    def forward(self, x, timestep, context, **kwargs):
+        """Forward pass for QwenImage model."""
+        latents, orig_shape = self._pack_latents(x)
+        num_embeds = latents.shape[1]
+        height = orig_shape[-2] * 8
+        width = orig_shape[-1] * 8
+
+        sampling_params = SamplingParams.from_user_sampling_params_args(
+            self.model_path,
+            server_args=self.generator.server_args,
+            prompt=" ",
+            guidance_scale=1.0,
+            height=height,
+            width=width,
+            num_frames=1,
+            num_inference_steps=1,
+            save_output=False,
+            suppress_logs=self.should_suppress_logs(timestep),
+        )
+
+        # Prepare request (converts SamplingParams to Req)
+        req = prepare_request(
+            server_args=self.generator.server_args,
+            sampling_params=sampling_params,
+        )
+        # Set ComfyUI-specific inputs directly on the Req object
+        req.latents = latents
+        req.timesteps = timestep * 1000.0
+        req.prompt_embeds = [context]
+        req.raw_latent_shape = torch.tensor(latents.shape, dtype=torch.long)
+        req.do_classifier_free_guidance = False
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+        output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
+        noise_pred = output_batch.noise_pred
+
+        return self._unpack_latents(noise_pred, num_embeds, orig_shape, x)
+
+
+class QwenImageEditExecutor(QwenImageExecutor):
+    """Executor for QwenImageEdit models in ComfyUI."""
+
+    def __init__(self, generator, model_path, model, config):
+        super().__init__(generator, model_path, model, config)
+
+    def forward(
+        self,
+        x,
+        timestep,
+        context,
+        attention_mask=None,
+        ref_latents=None,
+        additional_t_cond=None,
+        transformer_options={},
+        **kwargs,
+    ):
+        """Forward pass for QwenImageEdit model."""
+        latents, orig_shape = self._pack_latents(x)
+        num_embeds = latents.shape[1]
+        height = orig_shape[-2] * 8
+        width = orig_shape[-1] * 8
+
+        # Prepare vae_image_sizes for the condition image (ref_latents)
+        vae_image_sizes = []
+        pack_ref_latents = None
+
+        # TODO: sgld now don't support multiple condition images, so we only support one condition image for now.
+        if ref_latents is not None and len(ref_latents) > 0:
+            pack_ref_latents, orig_ref_shape = self._pack_latents(ref_latents[0])
+            vae_image_sizes = [(orig_ref_shape[-1], orig_ref_shape[-2])]
+
+        sampling_params = SamplingParams.from_user_sampling_params_args(
+            self.model_path,
+            server_args=self.generator.server_args,
+            prompt=" ",
+            guidance_scale=1.0,
+            image_path="",
+            height=height,
+            width=width,
+            num_frames=1,
+            num_inference_steps=1,
+            save_output=False,
+            suppress_logs=self.should_suppress_logs(timestep),
+        )
+
+        # Prepare request (converts SamplingParams to Req)
+        req = prepare_request(
+            server_args=self.generator.server_args,
+            sampling_params=sampling_params,
+        )
+        # Set ComfyUI-specific inputs directly on the Req object
+        req.latents = latents
+        req.image_latent = pack_ref_latents
+        req.timesteps = timestep * 1000.0
+        req.vae_image_sizes = vae_image_sizes
+        req.prompt_embeds = [context]
+        req.raw_latent_shape = torch.tensor(latents.shape, dtype=torch.long)
+        req.do_classifier_free_guidance = False
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+        output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
+        noise_pred = output_batch.noise_pred
+
+        return self._unpack_latents(noise_pred, num_embeds, orig_shape, x)

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/zimage.py

@@ -0,0 +1,64 @@
+"""
+ZImage executor for SGLang Diffusion ComfyUI integration.
+"""
+
+import torch
+
+try:
+    from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+    from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+except ImportError:
+    print(
+        "Error: sglang.multimodal_gen is not installed. Please install it using 'pip install sglang[diffusion]'"
+    )
+
+from .base import SGLDiffusionExecutor
+
+
+class ZImageExecutor(SGLDiffusionExecutor):
+    """Executor for ZImage models in ComfyUI."""
+
+    def __init__(self, generator, model_path, model, config):
+        super().__init__(generator, model_path, model, config)
+
+    def forward(self, x, timesteps, context, **kwargs):
+        """Forward pass for ZImage model."""
+        B, C, H, W = x.shape
+        height = H * 8
+        width = W * 8
+        sampling_params = SamplingParams.from_user_sampling_params_args(
+            self.model_path,
+            server_args=self.generator.server_args,
+            prompt=" ",
+            guidance_scale=1.0,
+            height=height,
+            width=width,
+            num_frames=1,  # For images
+            num_inference_steps=1,  # Single step for ComfyUI
+            save_output=False,
+            suppress_logs=self.should_suppress_logs(timesteps),
+        )
+
+        # Prepare request (converts SamplingParams to Req)
+        req = prepare_request(
+            server_args=self.generator.server_args,
+            sampling_params=sampling_params,
+        )
+        latents = x.unsqueeze(2)
+        context = context.squeeze(0)
+        # Set ComfyUI-specific inputs directly on the Req object
+        req.latents = latents  # ComfyUI's x parameter
+        req.timesteps = timesteps * 1000.0  # ComfyUI's timesteps parameter
+        req.prompt_embeds = [
+            context
+        ]  # ComfyUI's context parameter (must be List[Tensor])
+        req.raw_latent_shape = torch.tensor(latents.shape, dtype=torch.long)
+        req.do_classifier_free_guidance = False
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+        output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
+        noise_pred = output_batch.noise_pred
+
+        return noise_pred.permute(1, 0, 2, 3).to(x.device)

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py

@@ -0,0 +1,715 @@
+"""
+ComfyUI nodes for SGLang Diffusion integration.
+Provides nodes for connecting to SGLang Diffusion server and generating images/videos.
+"""
+
+import os
+import uuid
+
+import folder_paths
+import torch
+
+from .core import SGLDiffusionGenerator, SGLDiffusionServerAPI
+from .utils import (
+    convert_b64_to_tensor_image,
+    convert_video_to_comfy_video,
+    get_image_path,
+    is_empty_image,
+)
+
+
+class SGLDOptions:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {},
+            "optional": {
+                "model_type": (
+                    ["auto-detect", "qwen_image", "qwen_image_edit", "flux", "lumina2"],
+                    {"default": "auto-detect"},
+                ),
+                "enable_torch_compile": (
+                    "BOOLEAN",
+                    {"default": False},
+                ),
+                "num_gpus": ("INT", {"default": 1, "min": 1, "step": 1}),
+                "tp_size": ("INT", {"default": -1, "min": -1, "step": 1}),
+                "sp_degree": ("INT", {"default": -1, "min": -1, "step": 1}),
+                "ulysses_degree": (
+                    "INT",
+                    {
+                        "default": -1,
+                        "min": -1,
+                        "step": 1,
+                    },
+                ),
+                "ring_degree": (
+                    "INT",
+                    {
+                        "default": -1,
+                        "min": -1,
+                        "step": 1,
+                    },
+                ),
+                "dp_size": ("INT", {"default": 1, "min": 1, "step": 1}),
+                "dp_degree": ("INT", {"default": 1, "min": 1, "step": 1}),
+                "enable_cfg_parallel": (
+                    "BOOLEAN",
+                    {"default": False},
+                ),
+                "attention_backend": (
+                    "STRING",
+                    {"default": ""},
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("SGLD_OPTIONS",)
+    RETURN_NAMES = ("sgld_options",)
+    FUNCTION = "create_options"
+    CATEGORY = "SGLDiffusion"
+
+    def create_options(
+        self,
+        model_type: str = "auto-detect",
+        enable_torch_compile: bool = False,
+        num_gpus: int = 1,
+        tp_size: int = -1,
+        sp_degree: int = -1,
+        ulysses_degree: int = -1,
+        ring_degree: int = -1,
+        dp_size: int = 1,
+        dp_degree: int = 1,
+        enable_cfg_parallel: bool = False,
+        attention_backend: str = "",
+    ):
+        """
+        Build a dictionary of SGLang Diffusion runtime options.
+        """
+        # Convert -1 to None for optional parameters (matching ServerArgs defaults)
+        ulysses_degree = None if ulysses_degree == -1 else ulysses_degree
+        ring_degree = None if ring_degree == -1 else ring_degree
+        attention_backend = None if attention_backend == "" else attention_backend
+
+        options = {
+            "model_type": model_type,
+            "enable_torch_compile": enable_torch_compile,
+            "num_gpus": num_gpus,
+            "tp_size": tp_size,
+            "sp_degree": sp_degree,
+            "ulysses_degree": ulysses_degree,
+            "ring_degree": ring_degree,
+            "dp_size": dp_size,
+            "dp_degree": dp_degree,
+            "enable_cfg_parallel": enable_cfg_parallel,
+            "attention_backend": attention_backend,
+        }
+
+        # Strip None to keep payload clean
+        options = {k: v for k, v in options.items() if v is not None}
+        return (options,)
+
+
+class SGLDLoraLoader:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "model": ("MODEL",),
+                "lora_name": (folder_paths.get_filename_list("loras"),),
+                "strength_model": (
+                    "FLOAT",
+                    {"default": 1.0, "min": 0, "max": 10, "step": 0.01},
+                ),
+                "nickname": ("STRING", {"default": ""}),
+                "target": (
+                    ["all", "transformer", "transformer_2", "critic"],
+                    {"default": "all"},
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("MODEL",)
+    FUNCTION = "load_lora"
+
+    CATEGORY = "SGLDiffusion"
+
+    def load_lora(
+        self, model, lora_name, strength_model=1.0, nickname="", target="all"
+    ):
+        """Load LoRA adapter using SGLang Diffusion API."""
+        lora_path = folder_paths.get_full_path("loras", lora_name)
+        assert model is not None
+        bi = model.clone()
+        nickname = nickname if nickname != "" else str("lora" + str(uuid.uuid4()))
+        # set lora in the model
+        bi.patches[nickname] = (lora_path, strength_model, target)
+
+        # prepare input for the SGLang Diffusion API
+        lora_input = {
+            "lora_nickname": [],
+            "lora_path": [],
+            "strength": [],
+            "target": [],
+        }
+        for nickname, lora_info in bi.patches.items():
+            lora_input["lora_nickname"].append(nickname)
+            lora_input["lora_path"].append(lora_info[0])
+            lora_input["strength"].append(lora_info[1])
+            lora_input["target"].append(lora_info[2])
+
+        # call the SGLang Diffusion API
+        model.model.diffusion_model.set_lora(**lora_input)
+        return (model,)
+
+
+class SGLDUNETLoader:
+    def __init__(self):
+        self.generator = SGLDiffusionGenerator()
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "unet_name": (folder_paths.get_filename_list("diffusion_models"),),
+                "weight_dtype": (["default", "fp8_e4m3fn", "fp8_e5m2"],),
+            },
+            "optional": {
+                "sgld_options": ("SGLD_OPTIONS",),
+            },
+        }
+
+    RETURN_TYPES = ("MODEL",)
+    FUNCTION = "load_unet"
+
+    CATEGORY = "SGLDiffusion"
+
+    def load_unet(self, unet_name, weight_dtype, sgld_options: dict = None):
+        model_options = {}
+        if weight_dtype == "fp8_e4m3fn":
+            model_options["dtype"] = torch.float8_e4m3fn
+        elif weight_dtype == "fp8_e5m2":
+            model_options["dtype"] = torch.float8_e5m2
+
+        unet_path = folder_paths.get_full_path("diffusion_models", unet_name)
+
+        model = self.generator.load_model(
+            unet_path, model_options=model_options, sgld_options=sgld_options
+        )
+        return (model,)
+
+
+class SGLDiffusionServerModel:
+    """Node to load and manage SGLang Diffusion server connection."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "base_url": (
+                    "STRING",
+                    {
+                        "default": "http://localhost:3000/v1",
+                        "multiline": False,
+                    },
+                ),
+                "api_key": (
+                    "STRING",
+                    {
+                        "default": "sk-proj-1234567890",
+                        "multiline": False,
+                    },
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("SGLD_CLIENT", "STRING")
+    RETURN_NAMES = ("sgld_client", "model_info")
+    FUNCTION = "load_server"
+    CATEGORY = "SGLDiffusion"
+
+    def load_server(self, base_url: str, api_key: str):
+        """Initialize OpenAI client for SGLang Diffusion server."""
+        client = SGLDiffusionServerAPI(base_url=base_url, api_key=api_key)
+        try:
+            model_info = client.get_model_info()
+            # Format model_info as a readable string
+            info_lines = ["=== SGLDiffusion Model Info ==="]
+            for key, value in model_info.items():
+                info_lines.append(f"{key}: {value}")
+            model_info_str = "\n".join(info_lines)
+        except Exception as e:
+            model_info_str = f"Failed to get model info: {str(e)}"
+        return (client, model_info_str)
+
+
+class SGLDiffusionGenerateImage:
+    """Node to generate images using SGLang Diffusion."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "sgld_client": ("SGLD_CLIENT",),
+                "positive_prompt": (
+                    "STRING",
+                    {
+                        "default": "",
+                        "tooltip": "Text prompt for image generation",
+                    },
+                ),
+            },
+            "optional": {
+                "negative_prompt": (
+                    "STRING",
+                    {
+                        "default": "",
+                        "tooltip": "Negative prompt to avoid certain elements",
+                    },
+                ),
+                "image": (
+                    "IMAGE",
+                    {
+                        "default": None,
+                        "tooltip": "input image to use for editing",
+                    },
+                ),
+                "seed": (
+                    "INT",
+                    {
+                        "default": 1024,
+                        "min": -1,
+                        "max": 2**32 - 1,
+                    },
+                ),
+                "steps": (
+                    "INT",
+                    {
+                        "default": 6,
+                        "min": 1,
+                        "max": 100,
+                        "step": 1,
+                    },
+                ),
+                "cfg": (
+                    "FLOAT",
+                    {
+                        "default": 7.0,
+                        "min": 1.0,
+                        "max": 20.0,
+                        "step": 0.1,
+                    },
+                ),
+                "width": (
+                    "INT",
+                    {
+                        "default": 1024,
+                        "min": 256,
+                        "max": 4096,
+                        "step": 64,
+                    },
+                ),
+                "height": (
+                    "INT",
+                    {
+                        "default": 1024,
+                        "min": 256,
+                        "max": 4096,
+                        "step": 64,
+                    },
+                ),
+                "enable_teacache": (
+                    "BOOLEAN",
+                    {
+                        "default": False,
+                    },
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("image",)
+    FUNCTION = "generate_image"
+    CATEGORY = "SGLDiffusion"
+    OUTPUT_NODE = False
+
+    def generate_image(
+        self,
+        sgld_client: SGLDiffusionServerAPI,
+        positive_prompt: str,
+        negative_prompt: str = "",
+        image: torch.Tensor = None,
+        seed: int = 1024,
+        steps: int = 6,
+        cfg: float = 7.0,
+        width: int = 1024,
+        height: int = 1024,
+        enable_teacache: bool = False,
+    ):
+        """Generate image using SGLang Diffusion API."""
+        if not positive_prompt:
+            raise ValueError("Prompt cannot be empty")
+
+        size = f"{width}x{height}"
+
+        # Prepare request parameters
+        request_params = {
+            "prompt": positive_prompt,
+            "size": size,
+            "response_format": "b64_json",
+        }
+
+        # Add optional parameters if provided
+        if negative_prompt:
+            request_params["negative_prompt"] = negative_prompt
+        if cfg is not None:
+            request_params["guidance_scale"] = cfg
+        if steps is not None:
+            request_params["num_inference_steps"] = steps
+        if seed is not None and seed >= 0:
+            request_params["seed"] = seed
+        if enable_teacache:
+            request_params["enable_teacache"] = True
+        if image is not None:
+            # If the image is empty, use the size of the image to generate the image
+            if is_empty_image(image):
+                width, height = image.shape[2], image.shape[1]
+                size = f"{width}x{height}"
+                request_params["size"] = size
+            else:
+                request_params["image_path"] = get_image_path(image)
+
+        # Call API
+        try:
+            response = sgld_client.generate_image(**request_params)
+        except Exception as e:
+            raise RuntimeError(f"Failed to generate image: {str(e)}")
+
+        # Decode base64 image
+        if not response["data"] or not response["data"][0]["b64_json"]:
+            raise RuntimeError("No image data in response")
+        image_data = response["data"][0]["b64_json"]
+        image = convert_b64_to_tensor_image(image_data)
+
+        return (image,)
+
+
+class SGLDiffusionGenerateVideo:
+    """Node to generate videos using SGLang Diffusion."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "sgld_client": ("SGLD_CLIENT",),
+                "positive_prompt": (
+                    "STRING",
+                    {
+                        "default": "",
+                        "tooltip": "Text prompt for video generation",
+                    },
+                ),
+            },
+            "optional": {
+                "negative_prompt": (
+                    "STRING",
+                    {
+                        "default": "",
+                        "tooltip": "Negative prompt to avoid certain elements",
+                    },
+                ),
+                "image": (
+                    "IMAGE",
+                    {
+                        "default": None,
+                        "tooltip": "input image to use for image-to-video",
+                    },
+                ),
+                "seed": (
+                    "INT",
+                    {
+                        "default": 1024,
+                        "min": -1,
+                        "max": 2**32 - 1,
+                    },
+                ),
+                "steps": (
+                    "INT",
+                    {
+                        "default": 6,
+                        "min": 1,
+                        "max": 100,
+                        "step": 1,
+                    },
+                ),
+                "cfg": (
+                    "FLOAT",
+                    {
+                        "default": 7.0,
+                        "min": 1.0,
+                        "max": 20.0,
+                        "step": 0.1,
+                    },
+                ),
+                "width": (
+                    "INT",
+                    {
+                        "default": 1280,
+                        "min": 256,
+                        "max": 4096,
+                        "step": 1,
+                    },
+                ),
+                "height": (
+                    "INT",
+                    {
+                        "default": 720,
+                        "min": 256,
+                        "max": 4096,
+                        "step": 1,
+                    },
+                ),
+                "num_frames": (
+                    "INT",
+                    {
+                        "default": 120,
+                        "min": 1,
+                        "max": 1000,
+                        "step": 1,
+                    },
+                ),
+                "fps": (
+                    "INT",
+                    {
+                        "default": 24,
+                        "min": 1,
+                        "max": 60,
+                        "step": 1,
+                    },
+                ),
+                "seconds": (
+                    "INT",
+                    {
+                        "default": 5,
+                        "min": 1,
+                        "max": 60,
+                        "step": 1,
+                    },
+                ),
+                "enable_teacache": (
+                    "BOOLEAN",
+                    {
+                        "default": False,
+                    },
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("VIDEO", "STRING")
+    RETURN_NAMES = ("video", "video_path")
+    FUNCTION = "generate_video"
+    CATEGORY = "SGLDiffusion"
+    OUTPUT_NODE = False
+
+    def generate_video(
+        self,
+        sgld_client: SGLDiffusionServerAPI,
+        positive_prompt: str,
+        negative_prompt: str = "",
+        image: torch.Tensor = None,
+        seed: int = 1024,
+        steps: int = 6,
+        cfg: float = 7.0,
+        width: int = 1280,
+        height: int = 720,
+        num_frames: int = 120,
+        fps: int = 24,
+        seconds: int = 5,
+        enable_teacache: bool = False,
+    ):
+        """Generate video using SGLang Diffusion API."""
+        if not positive_prompt:
+            raise ValueError("Prompt cannot be empty")
+
+        size = f"{width}x{height}"
+        output_dir = folder_paths.get_temp_directory()
+
+        # Prepare request parameters
+        request_params = {
+            "prompt": positive_prompt,
+            "size": size,
+            "seconds": seconds,
+            "fps": fps,
+            "output_path": output_dir,
+        }
+
+        # Add optional parameters if provided
+        if negative_prompt:
+            request_params["negative_prompt"] = negative_prompt
+        if cfg is not None:
+            request_params["guidance_scale"] = cfg
+        if steps is not None:
+            request_params["num_inference_steps"] = steps
+        if seed is not None and seed >= 0:
+            request_params["seed"] = seed
+        if enable_teacache:
+            request_params["enable_teacache"] = True
+        if num_frames is not None:
+            request_params["num_frames"] = num_frames
+        if image is not None:
+            # If the image is empty, use the size of the image to generate the video
+            if is_empty_image(image):
+                width, height = image.shape[2], image.shape[1]
+                size = f"{width}x{height}"
+                request_params["size"] = size
+            else:
+                request_params["input_reference"] = get_image_path(image)
+
+        # Call API
+        try:
+            response = sgld_client.generate_video(**request_params)
+            video_path = response.get("file_path", "")
+            video = convert_video_to_comfy_video(video_path, height, width)
+        except Exception as e:
+            raise RuntimeError(f"Failed to generate video: {str(e)}")
+
+        return (video, video_path)
+
+
+class SGLDiffusionServerSetLora:
+    """Node to set LoRA adapter for SGLang Diffusion server."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "sgld_client": ("SGLD_CLIENT",),
+                "lora_name": (
+                    "STRING",
+                    {
+                        "default": "",
+                        "tooltip": "The name of the LoRA adapter",
+                    },
+                ),
+            },
+            "optional": {
+                "lora_nickname": (
+                    "STRING",
+                    {
+                        "default": "",
+                        "tooltip": "The nickname of the LoRA adapter",
+                    },
+                ),
+                "target": (
+                    [
+                        "all",
+                        "transformer",
+                        "transformer_2",
+                        "critic",
+                    ],
+                    {
+                        "default": "all",
+                        "tooltip": "Which transformer(s) to apply the LoRA to",
+                    },
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("SGLD_CLIENT",)
+    RETURN_NAMES = ("sgld_client",)
+    FUNCTION = "set_lora"
+    CATEGORY = "SGLDiffusion"
+    OUTPUT_NODE = False
+
+    def set_lora(
+        self,
+        sgld_client: SGLDiffusionServerAPI,
+        lora_name: str = "",
+        lora_nickname: str = "",
+        target: str = "all",
+    ):
+        """Set LoRA adapter using SGLang Diffusion API."""
+        if lora_nickname == "":
+            lora_nickname = os.path.splitext(lora_name)[0]
+
+        # Prepare request parameters
+        request_params = {
+            "lora_nickname": lora_nickname,
+            "lora_path": lora_name,
+            "target": target,
+        }
+
+        # Call API
+        try:
+            sgld_client.set_lora(**request_params)
+            return (sgld_client,)
+        except Exception as e:
+            raise RuntimeError(f"Failed to set LoRA adapter: {str(e)}")
+
+
+class SGLDiffusionServerUnsetLora:
+    """Node to unset LoRA adapter for SGLang Diffusion server."""
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "sgld_client": ("SGLD_CLIENT",),
+            },
+            "optional": {
+                "target": (
+                    [
+                        "all",
+                        "transformer",
+                        "transformer_2",
+                        "critic",
+                    ],
+                    {
+                        "default": "all",
+                        "tooltip": "Which transformer(s) to unset the LoRA from",
+                    },
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("SGLD_CLIENT",)
+    RETURN_NAMES = ("sgld_client",)
+    FUNCTION = "unset_lora"
+    CATEGORY = "SGLDiffusion"
+    OUTPUT_NODE = False
+
+    def unset_lora(
+        self,
+        sgld_client: SGLDiffusionServerAPI,
+        target: str = "all",
+    ):
+        """Unset LoRA adapter using SGLang Diffusion API."""
+        try:
+            sgld_client.unset_lora(target=target)
+            return (sgld_client,)
+        except Exception as e:
+            raise RuntimeError(f"Failed to unset LoRA adapter: {str(e)}")
+
+
+# Register nodes
+NODE_CLASS_MAPPINGS = {
+    "SGLDiffusionServerModel": SGLDiffusionServerModel,
+    "SGLDiffusionGenerateImage": SGLDiffusionGenerateImage,
+    "SGLDiffusionGenerateVideo": SGLDiffusionGenerateVideo,
+    "SGLDiffusionServerSetLora": SGLDiffusionServerSetLora,
+    "SGLDiffusionServerUnsetLora": SGLDiffusionServerUnsetLora,
+    "SGLDUNETLoader": SGLDUNETLoader,
+    "SGLDOptions": SGLDOptions,
+    "SGLDLoraLoader": SGLDLoraLoader,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "SGLDiffusionServerModel": "SGLDiffusion Server Model",
+    "SGLDiffusionGenerateImage": "SGLDiffusion Generate Image",
+    "SGLDiffusionGenerateVideo": "SGLDiffusion Generate Video",
+    "SGLDiffusionServerSetLora": "SGLDiffusion Server Set LoRA",
+    "SGLDiffusionServerUnsetLora": "SGLDiffusion Server Unset LoRA",
+    "SGLDUNETLoader": "SGLDiffusion UNET Loader",
+    "SGLDOptions": "SGLDiffusion Options",
+    "SGLDLoraLoader": "SGLDiffusion LoRA Loader",
+}

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/README.md

@@ -0,0 +1,66 @@
+# ComfyUI SGLDiffusion Pipeline Tests
+
+This directory contains tests for each ComfyUI pipeline integration.
+
+## Test Files
+
+- `test_zimage_pipeline.py` - Tests for ComfyUIZImagePipeline
+- `test_flux_pipeline.py` - Tests for ComfyUIFluxPipeline
+- `test_qwen_image_pipeline.py` - Tests for ComfyUIQwenImagePipeline
+- `test_qwen_image_edit_pipeline.py` - Tests for ComfyUIQwenImageEditPipeline (I2I/edit mode)
+
+## Running Tests
+
+### Run all tests
+
+```bash
+pytest python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/ -v -s
+```
+
+### Run a specific test file
+
+```bash
+pytest python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py -v -s
+```
+
+## Environment Variables
+
+You can configure model paths via environment variables. Model paths support two formats:
+- **Safetensors file**: Path to a single `.safetensors` file (e.g., `/path/to/model.safetensors`)
+- **Diffusers format**: HuggingFace model ID or local diffusers directory (e.g., `Tongyi-MAI/Z-Image-Turbo`)
+
+Environment variables:
+- `SGLANG_TEST_ZIMAGE_MODEL_PATH` - Path to ZImage model (default: `Tongyi-MAI/Z-Image-Turbo`)
+- `SGLANG_TEST_FLUX_MODEL_PATH` - Path to Flux model (default: `black-forest-labs/FLUX.1-dev`)
+- `SGLANG_TEST_QWEN_IMAGE_MODEL_PATH` - Path to QwenImage model (default: `Qwen/Qwen-Image`)
+- `SGLANG_TEST_QWEN_IMAGE_EDIT_MODEL_PATH` - Path to QwenImageEdit model (default: `Qwen/Qwen-Image-Edit-2511`)
+
+Examples:
+
+```bash
+# Using HuggingFace model ID (diffusers format)
+export SGLANG_TEST_ZIMAGE_MODEL_PATH="Tongyi-MAI/Z-Image-Turbo"
+pytest python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py -v -s
+
+# Using safetensors file
+export SGLANG_TEST_ZIMAGE_MODEL_PATH="/path/to/z_image_turbo_bf16.safetensors"
+pytest python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py -v -s
+```
+
+## Test Structure
+
+Each test file follows a similar structure:
+
+1. **Setup**: Creates a `DiffGenerator` with the appropriate pipeline class
+2. **Input Preparation**: Creates dummy tensors for latents, timesteps, and embeddings
+3. **Request Preparation**: Uses `prepare_request` to convert `SamplingParams` to `Req`
+4. **ComfyUI Inputs**: Sets ComfyUI-specific inputs directly on the `Req` object
+5. **Execution**: Sends request to scheduler and waits for response
+6. **Validation**: Checks that `noise_pred` is retrieved from `OutputBatch`
+
+## Notes
+
+- These tests use `comfyui_mode=True` to enable ComfyUI-specific behavior
+- Tests use pre-processed inputs (latents, timesteps, embeddings) as ComfyUI would provide
+- The tests verify that `noise_pred` can be retrieved from the `OutputBatch` after processing
+- All tests use dummy/ones tensors for simplicity - in production, these would be actual model outputs

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/__init__.py

@@ -0,0 +1,9 @@
+"""
+Test suite for ComfyUI SGLDiffusion pipelines.
+
+This package contains tests for each ComfyUI pipeline integration:
+- ZImagePipeline
+- FluxPipeline
+- QwenImagePipeline
+- QwenImageEditPipeline
+"""

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_flux_pipeline.py

@@ -0,0 +1,156 @@
+"""Test for ComfyUIFluxPipeline with pass-through scheduler."""
+
+import os
+import sys
+
+import pytest
+import torch
+
+from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.runtime.entrypoints.diffusion_generator import DiffGenerator
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+
+
+def test_comfyui_flux_pipeline_direct() -> None:
+    """Test ComfyUIFluxPipeline with custom inputs."""
+    model_path = os.environ.get(
+        "SGLANG_TEST_FLUX_MODEL_PATH",
+        "black-forest-labs/FLUX.1-dev",  # Supports both safetensors file and diffusers format
+    )
+
+    generator = DiffGenerator.from_pretrained(
+        model_path=model_path,
+        pipeline_class_name="ComfyUIFluxPipeline",
+        num_gpus=2,
+        comfyui_mode=True,
+    )
+
+    batch_size = 1
+    hidden_states_seq_len = 3600
+    hidden_states_dim = 64
+    height = 1280
+    width = 720
+
+    encoder_seq_len = 512
+    encoder_dim = 4096
+    pooled_dim = 768
+
+    hidden_states = torch.ones(
+        batch_size,
+        hidden_states_seq_len,
+        hidden_states_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    encoder_hidden_states = torch.ones(
+        batch_size,
+        encoder_seq_len,
+        encoder_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    pooled_projections = torch.ones(
+        batch_size,
+        pooled_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    timesteps = torch.tensor([1000], dtype=torch.long, device="cuda")
+
+    sampling_params = SamplingParams.from_user_sampling_params_args(
+        generator.server_args.model_path,
+        server_args=generator.server_args,
+        prompt="a beautiful girl",
+        height=height,
+        width=width,
+        num_frames=1,
+        num_inference_steps=1,
+        save_output=True,
+        return_trajectory_latents=True,
+    )
+
+    req = prepare_request(
+        server_args=generator.server_args,
+        sampling_params=sampling_params,
+    )
+
+    req.latents = hidden_states
+    req.timesteps = timesteps
+    req.raw_latent_shape = torch.tensor(hidden_states.shape, dtype=torch.long)
+
+    clip_dim = 768
+    req.prompt_embeds = [pooled_projections, encoder_hidden_states]
+
+    if req.guidance_scale > 1.0:
+        dummy_neg_clip_embedding = torch.zeros(
+            batch_size,
+            77,
+            clip_dim,
+            device="cuda",
+            dtype=torch.bfloat16,
+        )
+        negative_encoder_hidden_states = torch.ones(
+            batch_size,
+            encoder_seq_len,
+            encoder_dim,
+            device="cuda",
+            dtype=torch.bfloat16,
+        )
+        req.negative_prompt_embeds = [
+            dummy_neg_clip_embedding,
+            negative_encoder_hidden_states,
+        ]
+    else:
+        req.negative_prompt_embeds = None
+
+    req.pooled_embeds = [pooled_projections]
+    req.neg_pooled_embeds = []
+
+    if (
+        req.guidance_scale > 1.0
+        and req.negative_prompt_embeds is not None
+        and len(req.negative_prompt_embeds) > 0
+    ):
+        req.do_classifier_free_guidance = True
+    else:
+        req.do_classifier_free_guidance = False
+
+    if req.seed is not None:
+        generator_device = req.generator_device
+        device_str = "cuda" if generator_device == "cuda" else "cpu"
+        req.generator = [
+            torch.Generator(device_str).manual_seed(req.seed + i)
+            for i in range(req.num_outputs_per_prompt)
+        ]
+    else:
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+    output_batch = generator._send_to_scheduler_and_wait_for_response([req])
+    noise_pred = output_batch.noise_pred
+
+    assert noise_pred is not None, "noise_pred should not be None in OutputBatch"
+    assert isinstance(noise_pred, torch.Tensor), "noise_pred should be a torch.Tensor"
+    assert (
+        noise_pred.device.type == "cuda"
+    ), f"noise_pred should be on cuda, got {noise_pred.device}"
+    assert (
+        noise_pred.dtype == torch.bfloat16
+    ), f"noise_pred should be bfloat16, got {noise_pred.dtype}"
+
+    print("✓ Successfully retrieved noise_pred from OutputBatch!")
+    print(f"  noise_pred shape: {noise_pred.shape}")
+    print(f"  noise_pred dtype: {noise_pred.dtype}")
+    print(f"  noise_pred device: {noise_pred.device}")
+
+    latents = output_batch.output if output_batch.output is not None else req.latents
+    assert latents is not None, "latents should not be None"
+    print(f"latents.shape: {latents.shape}")
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_edit_pipeline.py

@@ -0,0 +1,136 @@
+"""Test for ComfyUIQwenImageEditPipeline with pass-through scheduler (I2I/edit mode)."""
+
+import os
+import sys
+
+import pytest
+import torch
+
+from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.runtime.entrypoints.diffusion_generator import DiffGenerator
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+
+
+def test_comfyui_qwen_image_edit_pipeline_direct() -> None:
+    """Test ComfyUIQwenImageEditPipeline with edit mode (I2I) and custom inputs."""
+    model_path = os.environ.get(
+        "SGLANG_TEST_QWEN_IMAGE_EDIT_MODEL_PATH",
+        "Qwen/Qwen-Image-Edit-2511",  # Supports both safetensors file and diffusers format
+    )
+
+    generator = DiffGenerator.from_pretrained(
+        model_path=model_path,
+        pipeline_class_name="ComfyUIQwenImageEditPipeline",
+        num_gpus=1,
+        comfyui_mode=True,
+        dit_layerwise_offload=False,
+    )
+
+    batch_size = 1
+    noisy_image_seq_len = 3600
+    hidden_states_dim = 64
+    condition_image_seq_len = 6889
+    condition_image_dim = 64
+    encoder_seq_len = 45
+    encoder_dim = 3584
+    height = 720
+    width = 1280
+
+    vae_scale_factor = 8
+    condition_height_latent = 1328 // vae_scale_factor
+    condition_width_latent = 1328 // vae_scale_factor
+
+    noisy_image_latents = torch.ones(
+        batch_size,
+        noisy_image_seq_len,
+        hidden_states_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    condition_image_latents = torch.ones(
+        batch_size,
+        condition_image_seq_len,
+        condition_image_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    encoder_hidden_states = torch.ones(
+        batch_size,
+        encoder_seq_len,
+        encoder_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    timesteps = torch.tensor([1000], dtype=torch.long, device="cuda")
+
+    sampling_params = SamplingParams.from_user_sampling_params_args(
+        generator.server_args.model_path,
+        server_args=generator.server_args,
+        prompt=" ",
+        guidance_scale=1.0,
+        height=height,
+        width=width,
+        image_path="",
+        num_frames=1,
+        num_inference_steps=1,
+        seed=42,
+        save_output=False,
+        return_frames=False,
+    )
+
+    req = prepare_request(
+        server_args=generator.server_args,
+        sampling_params=sampling_params,
+    )
+
+    req.latents = noisy_image_latents
+    req.image_latent = condition_image_latents
+    req.timesteps = timesteps
+    req.prompt_embeds = [encoder_hidden_states]
+    req.negative_prompt_embeds = None
+    req.vae_image_sizes = [(condition_width_latent, condition_height_latent)]
+    req.raw_latent_shape = torch.tensor(noisy_image_latents.shape, dtype=torch.long)
+
+    if req.guidance_scale > 1.0 and req.negative_prompt_embeds is not None:
+        req.do_classifier_free_guidance = True
+    else:
+        req.do_classifier_free_guidance = False
+
+    if req.seed is not None:
+        generator_device = req.generator_device
+        device_str = "cpu" if generator_device == "cpu" else "cuda"
+        req.generator = [
+            torch.Generator(device_str).manual_seed(req.seed + i)
+            for i in range(req.num_outputs_per_prompt)
+        ]
+    else:
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+    output_batch = generator._send_to_scheduler_and_wait_for_response([req])
+    noise_pred = output_batch.noise_pred
+
+    assert noise_pred is not None, "noise_pred should not be None in OutputBatch"
+    assert isinstance(noise_pred, torch.Tensor), "noise_pred should be a torch.Tensor"
+    assert (
+        noise_pred.device.type == "cuda"
+    ), f"noise_pred should be on cuda, got {noise_pred.device}"
+    assert (
+        noise_pred.dtype == torch.bfloat16
+    ), f"noise_pred should be bfloat16, got {noise_pred.dtype}"
+
+    print("✓ Successfully retrieved noise_pred from OutputBatch (Edit Mode)!")
+    print(f"  noise_pred shape: {noise_pred.shape}")
+    print(f"  noise_pred dtype: {noise_pred.dtype}")
+    print(f"  noise_pred device: {noise_pred.device}")
+
+    latents = output_batch.output if output_batch.output is not None else req.latents
+    assert latents is not None, "latents should not be None"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_pipeline.py

@@ -0,0 +1,120 @@
+"""Test for ComfyUIQwenImagePipeline with pass-through scheduler."""
+
+import os
+import sys
+
+import pytest
+import torch
+
+from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.runtime.entrypoints.diffusion_generator import DiffGenerator
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+
+
+def test_comfyui_qwen_image_pipeline_direct() -> None:
+    """Test ComfyUIQwenImagePipeline with custom inputs."""
+    model_path = os.environ.get(
+        "SGLANG_TEST_QWEN_IMAGE_MODEL_PATH",
+        "Qwen/Qwen-Image",  # Supports both safetensors file and diffusers format
+    )
+
+    generator = DiffGenerator.from_pretrained(
+        model_path=model_path,
+        pipeline_class_name="ComfyUIQwenImagePipeline",
+        num_gpus=2,
+        comfyui_mode=True,
+        dit_layerwise_offload=False,
+    )
+
+    batch_size = 1
+    hidden_states_seq_len = 6889
+    hidden_states_dim = 64
+    encoder_seq_len = 45
+    encoder_dim = 3584
+    height = 1328
+    width = 1328
+    dtype = torch.bfloat16
+
+    hidden_states = torch.ones(
+        batch_size,
+        hidden_states_seq_len,
+        hidden_states_dim,
+        device="cuda",
+        dtype=dtype,
+    )
+
+    encoder_hidden_states = torch.ones(
+        batch_size,
+        encoder_seq_len,
+        encoder_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    timesteps = torch.tensor([1000], dtype=torch.long, device="cuda")
+
+    sampling_params = SamplingParams.from_user_sampling_params_args(
+        generator.server_args.model_path,
+        server_args=generator.server_args,
+        prompt=" ",
+        guidance_scale=3.0,
+        height=height,
+        width=width,
+        num_frames=1,
+        num_inference_steps=1,
+        seed=42,
+        save_output=False,
+        return_frames=False,
+    )
+
+    req = prepare_request(
+        server_args=generator.server_args,
+        sampling_params=sampling_params,
+    )
+
+    req.latents = hidden_states
+    req.timesteps = timesteps
+    req.prompt_embeds = [encoder_hidden_states]
+    req.negative_prompt_embeds = [encoder_hidden_states]
+    req.raw_latent_shape = torch.tensor(hidden_states.shape, dtype=torch.long)
+
+    if req.guidance_scale > 1.0 and req.negative_prompt_embeds is not None:
+        req.do_classifier_free_guidance = True
+    else:
+        req.do_classifier_free_guidance = False
+
+    if req.seed is not None:
+        generator_device = req.generator_device
+        device_str = "cpu" if generator_device == "cpu" else "cuda"
+        req.generator = [
+            torch.Generator(device_str).manual_seed(req.seed + i)
+            for i in range(req.num_outputs_per_prompt)
+        ]
+    else:
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+    output_batch = generator._send_to_scheduler_and_wait_for_response([req])
+    noise_pred = output_batch.noise_pred
+
+    assert noise_pred is not None, "noise_pred should not be None in OutputBatch"
+    assert isinstance(noise_pred, torch.Tensor), "noise_pred should be a torch.Tensor"
+    assert (
+        noise_pred.device.type == "cuda"
+    ), f"noise_pred should be on cuda, got {noise_pred.device}"
+    assert (
+        noise_pred.dtype == torch.bfloat16
+    ), f"noise_pred should be bfloat16, got {noise_pred.dtype}"
+
+    print("✓ Successfully retrieved noise_pred from OutputBatch!")
+    print(f"  noise_pred shape: {noise_pred.shape}")
+    print(f"  noise_pred dtype: {noise_pred.dtype}")
+    print(f"  noise_pred device: {noise_pred.device}")
+
+    latents = output_batch.output if output_batch.output is not None else req.latents
+    assert latents is not None, "latents should not be None"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py

@@ -0,0 +1,122 @@
+"""Test for ComfyUIZImagePipeline with pass-through scheduler."""
+
+import os
+import sys
+
+import pytest
+import torch
+
+from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
+from sglang.multimodal_gen.runtime.entrypoints.diffusion_generator import DiffGenerator
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+
+
+def test_comfyui_zimage_pipeline_direct() -> None:
+    """Test ComfyUIZImagePipeline with custom inputs."""
+    model_path = os.environ.get(
+        "SGLANG_TEST_ZIMAGE_MODEL_PATH",
+        "Tongyi-MAI/Z-Image-Turbo",  # Supports both safetensors file and diffusers format
+    )
+
+    generator = DiffGenerator.from_pretrained(
+        model_path=model_path,
+        pipeline_class_name="ComfyUIZImagePipeline",
+        num_gpus=1,
+        sp_degree=1,
+        comfyui_mode=True,
+    )
+
+    batch_size = 1
+    num_channels = 16
+    num_frames = 1
+    height = 720
+    width = 1280
+    latent_height = height // 8
+    latent_width = width // 8
+
+    latents = torch.ones(
+        batch_size,
+        num_channels,
+        num_frames,
+        latent_height,
+        latent_width,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    timesteps = torch.tensor([1000], dtype=torch.long, device="cuda")
+
+    context_seq_len = 19
+    context_dim = 2560
+    context = torch.ones(
+        context_seq_len,
+        context_dim,
+        device="cuda",
+        dtype=torch.bfloat16,
+    )
+
+    sampling_params = SamplingParams.from_user_sampling_params_args(
+        generator.server_args.model_path,
+        server_args=generator.server_args,
+        prompt="a beautiful girl",
+        guidance_scale=1.0,
+        height=height,
+        width=width,
+        num_frames=1,
+        num_inference_steps=1,
+        seed=42,
+        save_output=False,
+        return_frames=False,
+    )
+
+    req = prepare_request(
+        server_args=generator.server_args,
+        sampling_params=sampling_params,
+    )
+
+    req.latents = latents
+    req.timesteps = timesteps
+    req.prompt_embeds = [context]
+    req.negative_prompt_embeds = None
+    req.raw_latent_shape = torch.tensor(latents.shape, dtype=torch.long)
+
+    if req.guidance_scale > 1.0 and req.negative_prompt_embeds is not None:
+        req.do_classifier_free_guidance = True
+    else:
+        req.do_classifier_free_guidance = False
+
+    if req.seed is not None:
+        generator_device = req.generator_device
+        device_str = "cpu" if generator_device == "cpu" else "cuda"
+        req.generator = [
+            torch.Generator(device_str).manual_seed(req.seed + i)
+            for i in range(req.num_outputs_per_prompt)
+        ]
+    else:
+        req.generator = [
+            torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
+        ]
+
+    output_batch = generator._send_to_scheduler_and_wait_for_response([req])
+    noise_pred = output_batch.noise_pred
+
+    assert noise_pred is not None, "noise_pred should not be None in OutputBatch"
+    assert isinstance(noise_pred, torch.Tensor), "noise_pred should be a torch.Tensor"
+    assert (
+        noise_pred.device.type == "cuda"
+    ), f"noise_pred should be on cuda, got {noise_pred.device}"
+    assert (
+        noise_pred.dtype == torch.bfloat16
+    ), f"noise_pred should be bfloat16, got {noise_pred.dtype}"
+
+    print("✓ Successfully retrieved noise_pred from OutputBatch!")
+    print(f"  noise_pred shape: {noise_pred.shape}")
+    print(f"  noise_pred dtype: {noise_pred.dtype}")
+    print(f"  noise_pred device: {noise_pred.device}")
+
+    latents = output_batch.output if output_batch.output is not None else req.latents
+    assert latents is not None, "latents should not be None"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/utils.py

@@ -0,0 +1,176 @@
+import base64
+import io
+import os
+import shutil
+import time
+import uuid
+
+import folder_paths
+import numpy as np
+import torch
+from comfy_api.input import VideoInput
+from PIL import Image
+
+
+def _ensure_dir(path: str) -> None:
+    os.makedirs(path, exist_ok=True)
+
+
+def _to_numpy_image(image: torch.Tensor) -> np.ndarray:
+    """Convert ComfyUI image tensor to uint8 numpy array (H, W, C)."""
+    if image.dim() == 4:
+        image = image[0]
+    if image.dim() == 3 and image.shape[0] in (1, 3, 4):
+        image = image.permute(1, 2, 0)
+    elif image.dim() == 2:
+        image = image.unsqueeze(-1)
+    np_img = image.detach().cpu().numpy()
+    np_img = np.clip(np_img, 0.0, 1.0)
+    np_img = (np_img * 255).astype(np.uint8)
+    if np_img.shape[-1] == 1:
+        np_img = np.repeat(np_img, 3, axis=-1)
+    return np_img
+
+
+def _to_hwc_tensor(image: torch.Tensor) -> torch.Tensor:
+    """Convert ComfyUI image tensor to HWC format (normalized [0, 1])."""
+    img = image.clone()
+    if img.dim() == 4:
+        img = img[0]
+    if img.dim() == 3 and img.shape[0] in (1, 3, 4):
+        img = img.permute(1, 2, 0)
+    elif img.dim() == 2:
+        img = img.unsqueeze(-1)
+
+    img = torch.clamp(img, 0.0, 1.0)
+    if img.shape[-1] == 1:
+        img = img.repeat(1, 1, 3)
+
+    return img
+
+
+def is_empty_image(image: torch.Tensor, tolerance: float = 1e-6) -> bool:
+    """
+    Check if the input image is an empty/solid color image (like ComfyUI's empty image).
+    Args:
+        image: Input tensor image in ComfyUI format (BCHW, CHW, HWC, etc.)
+        tolerance: Tolerance for floating point comparison (default: 1e-6)
+
+    Returns:
+        True if the image is empty (all pixels have same color), False otherwise
+    """
+    if image is None:
+        return True
+
+    # Convert to HWC format
+    img_hwc = _to_hwc_tensor(image)
+
+    # Get the first pixel's RGB values
+    first_pixel = img_hwc[0, 0, :]
+
+    h, w, c = img_hwc.shape
+    pixels = img_hwc.reshape(-1, c)
+
+    diff = torch.abs(pixels - first_pixel)
+    max_diff = torch.max(diff)
+
+    return max_diff.item() <= tolerance
+
+
+def get_image_path(image: torch.Tensor) -> str:
+    """
+    Save tensor image to ComfyUI temp directory as PNG and return the path.
+    """
+    temp_dir = folder_paths.get_temp_directory()
+
+    # Build file name
+    ts = time.strftime("%Y%m%d-%H%M%S")
+    unique = uuid.uuid4().hex[:8]
+    file_name = f"sgl_output_{ts}_{unique}.png"
+    file_path = os.path.join(temp_dir, file_name)
+
+    # Save image
+    np_img = _to_numpy_image(image)
+    img = Image.fromarray(np_img)
+    img.save(file_path, format="PNG")
+
+    return file_path
+
+
+def convert_b64_to_tensor_image(b64_image: str) -> torch.Tensor:
+    """
+    Convert base64 encoded image to ComfyUI IMAGE format (torch.Tensor).
+
+    Args:
+        b64_image: Base64 encoded image string
+
+    Returns:
+        torch.Tensor with shape [batch_size, height, width, channels] (BHWC format),
+        values normalized to [0, 1] range, RGB format (3 channels)
+    """
+    # Decode base64
+    image_bytes = base64.b64decode(b64_image)
+
+    # Open image and convert to RGB
+    pil_image = Image.open(io.BytesIO(image_bytes))
+    if pil_image.mode != "RGB":
+        pil_image = pil_image.convert("RGB")
+
+    # Convert to numpy array and normalize to [0, 1]
+    image_array = np.array(pil_image).astype(np.float32) / 255.0
+
+    # Add batch dimension: [height, width, channels] -> [1, height, width, channels]
+    image_array = image_array[np.newaxis, ...]
+
+    # Convert to torch.Tensor
+    tensor_image = torch.from_numpy(image_array)
+
+    return tensor_image
+
+
+class SGLDVideoInput(VideoInput):
+    def __init__(self, video_path: str, height: int, width: int):
+        super().__init__()
+
+        self.video_path = video_path
+        self.height = height
+        self.width = width
+
+    def get_dimensions(self) -> tuple[int, int]:
+        """
+        Returns the dimensions of the video input.
+
+        Returns:
+            Tuple of (width, height)
+        """
+        return self.width, self.height
+
+    def get_components(self):
+        """
+        Returns the components of the video input.
+        This is required by the VideoInput abstract base class.
+        """
+        return [self.video_path]
+
+    def save_to(self, path: str, format=None, codec=None, metadata=None):
+        """
+        Abstract method to save the video input to a file.
+        """
+        save_path = path
+        # Copy video file from video_path to save_path
+        if os.path.exists(self.video_path):
+            # Ensure destination directory exists
+            save_dir = os.path.dirname(save_path)
+            if save_dir:
+                os.makedirs(save_dir, exist_ok=True)
+            shutil.copy2(self.video_path, save_path)
+
+
+def convert_video_to_comfy_video(
+    video_path: str, height: int, width: int
+) -> VideoInput:
+    """
+    Convert video to ComfyUI VIDEO format (VideoInput).
+    """
+    video_input = SGLDVideoInput(video_path, height, width)
+    return video_input

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/flux_sgld_sp.json

@@ -0,0 +1,222 @@
+{
+    "8": {
+      "inputs": {
+        "samples": [
+          "40",
+          0
+        ],
+        "vae": [
+          "10",
+          0
+        ]
+      },
+      "class_type": "VAEDecode",
+      "_meta": {
+        "title": "VAE Decode"
+      }
+    },
+    "10": {
+      "inputs": {
+        "vae_name": "ae.safetensors"
+      },
+      "class_type": "VAELoader",
+      "_meta": {
+        "title": "Load VAE"
+      }
+    },
+    "11": {
+      "inputs": {
+        "clip_name1": "t5xxl_fp16.safetensors",
+        "clip_name2": "clip_l.safetensors",
+        "type": "flux",
+        "device": "default"
+      },
+      "class_type": "DualCLIPLoader",
+      "_meta": {
+        "title": "DualCLIPLoader"
+      }
+    },
+    "17": {
+      "inputs": {
+        "scheduler": "normal",
+        "steps": 25,
+        "denoise": 1,
+        "model": [
+          "46",
+          0
+        ]
+      },
+      "class_type": "BasicScheduler",
+      "_meta": {
+        "title": "BasicScheduler"
+      }
+    },
+    "38": {
+      "inputs": {
+        "model": [
+          "46",
+          0
+        ],
+        "conditioning": [
+          "42",
+          0
+        ]
+      },
+      "class_type": "BasicGuider",
+      "_meta": {
+        "title": "BasicGuider"
+      }
+    },
+    "39": {
+      "inputs": {
+        "filename_prefix": "ComfyUI",
+        "images": [
+          "8",
+          0
+        ]
+      },
+      "class_type": "SaveImage",
+      "_meta": {
+        "title": "Save Image"
+      }
+    },
+    "40": {
+      "inputs": {
+        "noise": [
+          "45",
+          0
+        ],
+        "guider": [
+          "38",
+          0
+        ],
+        "sampler": [
+          "47",
+          0
+        ],
+        "sigmas": [
+          "17",
+          0
+        ],
+        "latent_image": [
+          "44",
+          0
+        ]
+      },
+      "class_type": "SamplerCustomAdvanced",
+      "_meta": {
+        "title": "SamplerCustomAdvanced"
+      }
+    },
+    "42": {
+      "inputs": {
+        "guidance": 3.5,
+        "conditioning": [
+          "43",
+          0
+        ]
+      },
+      "class_type": "FluxGuidance",
+      "_meta": {
+        "title": "FluxGuidance"
+      }
+    },
+    "43": {
+      "inputs": {
+        "text": "beautiful photography of a gonger haired artist with Lots of Colorful coloursplashes in face and pn her hands, she is natural, having her hair in a casual bun, looking happily into camera, cinematic,",
+        "speak_and_recognation": {
+          "__value__": [
+            false,
+            true
+          ]
+        },
+        "clip": [
+          "11",
+          0
+        ]
+      },
+      "class_type": "CLIPTextEncode",
+      "_meta": {
+        "title": "CLIP Text Encode (Prompt)"
+      }
+    },
+    "44": {
+      "inputs": {
+        "width": 1024,
+        "height": 1024,
+        "batch_size": 1
+      },
+      "class_type": "EmptySD3LatentImage",
+      "_meta": {
+        "title": "EmptySD3LatentImage"
+      }
+    },
+    "45": {
+      "inputs": {
+        "noise_seed": 747172083610812
+      },
+      "class_type": "RandomNoise",
+      "_meta": {
+        "title": "RandomNoise"
+      }
+    },
+    "46": {
+      "inputs": {
+        "max_shift": 1.15,
+        "base_shift": 0.5,
+        "width": 1024,
+        "height": 1024,
+        "model": [
+          "51",
+          0
+        ]
+      },
+      "class_type": "ModelSamplingFlux",
+      "_meta": {
+        "title": "ModelSamplingFlux"
+      }
+    },
+    "47": {
+      "inputs": {
+        "sampler_name": "euler"
+      },
+      "class_type": "KSamplerSelect",
+      "_meta": {
+        "title": "KSamplerSelect"
+      }
+    },
+    "51": {
+      "inputs": {
+        "unet_name": "flux1-dev.safetensors",
+        "weight_dtype": "default",
+        "sgld_options": [
+          "52",
+          0
+        ]
+      },
+      "class_type": "SGLDUNETLoader",
+      "_meta": {
+        "title": "SGLDiffusion UNET Loader"
+      }
+    },
+    "52": {
+      "inputs": {
+        "model_type": "auto-detect",
+        "enable_torch_compile": false,
+        "num_gpus": 2,
+        "tp_size": -1,
+        "sp_degree": -1,
+        "ulysses_degree": -1,
+        "ring_degree": -1,
+        "dp_size": 1,
+        "dp_degree": 1,
+        "enable_cfg_parallel": false,
+        "attention_backend": "",
+        "cache_strategy": "none"
+      },
+      "class_type": "SGLDOptions",
+      "_meta": {
+        "title": "SGLDiffusion Options"
+      }
+    }
+  }

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/qwen_image_sgld.json

@@ -0,0 +1,165 @@
+{
+    "3": {
+      "inputs": {
+        "seed": 808633539418610,
+        "steps": 4,
+        "cfg": 1,
+        "sampler_name": "euler",
+        "scheduler": "simple",
+        "denoise": 1,
+        "model": [
+          "66",
+          0
+        ],
+        "positive": [
+          "6",
+          0
+        ],
+        "negative": [
+          "7",
+          0
+        ],
+        "latent_image": [
+          "58",
+          0
+        ]
+      },
+      "class_type": "KSampler",
+      "_meta": {
+        "title": "KSampler"
+      }
+    },
+    "6": {
+      "inputs": {
+        "text": "\"A vibrant, warm neon-lit street scene in Hong Kong at the afternoon, with a mix of colorful Chinese and English signs glowing brightly. The atmosphere is lively, cinematic, and rain-washed with reflections on the pavement. The colors are vivid, full of pink, blue, red, and green hues. Crowded buildings with overlapping neon signs. 1980s Hong Kong style. Signs include:\n\"龍鳳冰室\" \"金華燒臘\" \"HAPPY HAIR\" \"鴻運茶餐廳\" \"EASY BAR\" \"永發魚蛋粉\" \"添記粥麵\" \"SUNSHINE MOTEL\" \"美都餐室\" \"富記糖水\" \"太平館\" \"雅芳髮型屋\" \"STAR KTV\" \"銀河娛樂城\" \"百樂門舞廳\" \"BUBBLE CAFE\" \"萬豪麻雀館\" \"CITY LIGHTS BAR\" \"瑞祥香燭莊\" \"文記文具\" \"GOLDEN JADE HOTEL\" \"LOVELY BEAUTY\" \"合興百貨\" \"興旺電器\" And the background is warm yellow street and with all stores' lights on.",
+        "speak_and_recognation": {
+          "__value__": [
+            false,
+            true
+          ]
+        },
+        "clip": [
+          "38",
+          0
+        ]
+      },
+      "class_type": "CLIPTextEncode",
+      "_meta": {
+        "title": "CLIP Text Encode (Positive Prompt)"
+      }
+    },
+    "7": {
+      "inputs": {
+        "text": "",
+        "speak_and_recognation": {
+          "__value__": [
+            false,
+            true
+          ]
+        },
+        "clip": [
+          "38",
+          0
+        ]
+      },
+      "class_type": "CLIPTextEncode",
+      "_meta": {
+        "title": "CLIP Text Encode (Negative Prompt)"
+      }
+    },
+    "8": {
+      "inputs": {
+        "samples": [
+          "3",
+          0
+        ],
+        "vae": [
+          "39",
+          0
+        ]
+      },
+      "class_type": "VAEDecode",
+      "_meta": {
+        "title": "VAE Decode"
+      }
+    },
+    "38": {
+      "inputs": {
+        "clip_name": "qwen_2.5_vl_7b_fp8_scaled.safetensors",
+        "type": "qwen_image",
+        "device": "default"
+      },
+      "class_type": "CLIPLoader",
+      "_meta": {
+        "title": "Load CLIP"
+      }
+    },
+    "39": {
+      "inputs": {
+        "vae_name": "qwen_image_vae.safetensors"
+      },
+      "class_type": "VAELoader",
+      "_meta": {
+        "title": "Load VAE"
+      }
+    },
+    "58": {
+      "inputs": {
+        "width": 1328,
+        "height": 1328,
+        "batch_size": 1
+      },
+      "class_type": "EmptySD3LatentImage",
+      "_meta": {
+        "title": "EmptySD3LatentImage"
+      }
+    },
+    "60": {
+      "inputs": {
+        "filename_prefix": "ComfyUI"
+      },
+      "class_type": "SaveImage",
+      "_meta": {
+        "title": "Save Image"
+      }
+    },
+    "66": {
+      "inputs": {
+        "shift": 3.1000000000000005,
+        "model": [
+          "78",
+          0
+        ]
+      },
+      "class_type": "ModelSamplingAuraFlow",
+      "_meta": {
+        "title": "ModelSamplingAuraFlow"
+      }
+    },
+    "77": {
+      "inputs": {
+        "unet_name": "qwen_image_2512_bf16.safetensors",
+        "weight_dtype": "default"
+      },
+      "class_type": "SGLDUNETLoader",
+      "_meta": {
+        "title": "SGLDiffusion UNET Loader"
+      }
+    },
+    "78": {
+      "inputs": {
+        "lora_name": "Qwen-Image-2512-Lightning-4steps-V1.0-bf16.safetensors",
+        "strength_model": 1,
+        "nickname": "",
+        "target": "all",
+        "model": [
+          "77",
+          0
+        ]
+      },
+      "class_type": "SGLDLoraLoader",
+      "_meta": {
+        "title": "SGLDiffusion LoRA Loader"
+      }
+    }
+  }

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/sgld_image2video.json

@@ -0,0 +1,97 @@
+{
+    "1": {
+      "inputs": {
+        "base_url": "http://localhost:3000/v1",
+        "api_key": "sk-proj-1234567890"
+      },
+      "class_type": "SGLDiffusionServerModel",
+      "_meta": {
+        "title": "SGLDiffusion Server Model"
+      }
+    },
+    "3": {
+      "inputs": {
+        "prompt": "The girl turn the body and spin around in place.",
+        "main": "none",
+        "lighting": "none",
+        "speak_and_recognation": {
+          "__value__": [
+            false,
+            true
+          ]
+        }
+      },
+      "class_type": "easy prompt",
+      "_meta": {
+        "title": "Prompt"
+      }
+    },
+    "4": {
+      "inputs": {
+        "text": "",
+        "anything": [
+          "1",
+          1
+        ]
+      },
+      "class_type": "easy showAnything",
+      "_meta": {
+        "title": "Show Any"
+      }
+    },
+    "15": {
+      "inputs": {
+        "positive_prompt": [
+          "3",
+          0
+        ],
+        "negative_prompt": "",
+        "seed": 2435791308,
+        "steps": 50,
+        "cfg": 4,
+        "width": 704,
+        "height": 1280,
+        "num_frames": 16,
+        "fps": 16,
+        "seconds": 1,
+        "enable_teacache": false,
+        "sgld_client": [
+          "1",
+          0
+        ],
+        "image": [
+          "17",
+          0
+        ]
+      },
+      "class_type": "SGLDiffusionGenerateVideo",
+      "_meta": {
+        "title": "SGLDiffusion Generate Video"
+      }
+    },
+    "16": {
+      "inputs": {
+        "filename_prefix": "video/ComfyUI",
+        "format": "auto",
+        "codec": "auto",
+        "video-preview": "",
+        "video": [
+          "15",
+          0
+        ]
+      },
+      "class_type": "SaveVideo",
+      "_meta": {
+        "title": "save video"
+      }
+    },
+    "17": {
+      "inputs": {
+        "image": "tmpe_w0bd_0.jpg"
+      },
+      "class_type": "LoadImage",
+      "_meta": {
+        "title": "load image"
+      }
+    }
+  }

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/sgld_text2img.json

@@ -0,0 +1,109 @@
+{
+    "1": {
+      "inputs": {
+        "base_url": "http://localhost:3000/v1",
+        "api_key": "sk-proj-1234567890"
+      },
+      "class_type": "SGLDiffusionServerModel",
+      "_meta": {
+        "title": "SGLDiffusion Server Model"
+      }
+    },
+    "3": {
+      "inputs": {
+        "prompt": "a bicycle, illustration in the style of SMPL, thick black lines on a white background",
+        "main": "none",
+        "lighting": "none",
+        "speak_and_recognation": {
+          "__value__": [
+            false,
+            true
+          ]
+        }
+      },
+      "class_type": "easy prompt",
+      "_meta": {
+        "title": "Prompt"
+      }
+    },
+    "4": {
+      "inputs": {
+        "text": "",
+        "anything": [
+          "1",
+          1
+        ]
+      },
+      "class_type": "easy showAnything",
+      "_meta": {
+        "title": "Show Any"
+      }
+    },
+    "5": {
+      "inputs": {
+        "filename_prefix": "ComfyUI",
+        "images": [
+          "6",
+          0
+        ]
+      },
+      "class_type": "SaveImage",
+      "_meta": {
+        "title": "save image"
+      }
+    },
+    "6": {
+      "inputs": {
+        "positive_prompt": [
+          "3",
+          0
+        ],
+        "negative_prompt": "",
+        "seed": 4215918563,
+        "steps": 50,
+        "cfg": 4,
+        "width": 512,
+        "height": 512,
+        "enable_teacache": false,
+        "sgld_client": [
+          "11",
+          0
+        ],
+        "image": [
+          "14",
+          0
+        ]
+      },
+      "class_type": "SGLDiffusionGenerateImage",
+      "_meta": {
+        "title": "SGLDiffusion Generate Image"
+      }
+    },
+    "11": {
+      "inputs": {
+        "lora_name": "dvyio/flux-lora-simple-illustration",
+        "lora_nickname": "",
+        "target": "all",
+        "sgld_client": [
+          "1",
+          0
+        ]
+      },
+      "class_type": "SGLDiffusionSetLora",
+      "_meta": {
+        "title": "SGLDiffusion Set LoRA"
+      }
+    },
+    "14": {
+      "inputs": {
+        "width": 512,
+        "height": 512,
+        "batch_size": 1,
+        "color": 0
+      },
+      "class_type": "EmptyImage",
+      "_meta": {
+        "title": "empty image"
+      }
+    }
+  }

python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/z-image_sgld.json

@@ -0,0 +1,140 @@
+{
+    "3": {
+      "inputs": {
+        "seed": 3338398,
+        "steps": 9,
+        "cfg": 1,
+        "sampler_name": "euler",
+        "scheduler": "simple",
+        "denoise": 1,
+        "model": [
+          "28",
+          0
+        ],
+        "positive": [
+          "6",
+          0
+        ],
+        "negative": [
+          "7",
+          0
+        ],
+        "latent_image": [
+          "13",
+          0
+        ]
+      },
+      "class_type": "KSampler",
+      "_meta": {
+        "title": "KSampler"
+      }
+    },
+    "6": {
+      "inputs": {
+        "text": "cute anime style girl with massive fluffy fennec ears and a big fluffy tail blonde messy long hair blue eyes wearing a maid outfit with a long black gold leaf pattern dress and a white apron, it is a postcard held by a hand in front of a beautiful realistic city at sunset and there is cursive writing that says \"ZImage, Now in ComfyUI\"",
+        "speak_and_recognation": {
+          "__value__": [
+            false,
+            true
+          ]
+        },
+        "clip": [
+          "18",
+          0
+        ]
+      },
+      "class_type": "CLIPTextEncode",
+      "_meta": {
+        "title": "CLIP Text Encode (Positive Prompt)"
+      }
+    },
+    "7": {
+      "inputs": {
+        "text": "blurry ugly bad",
+        "speak_and_recognation": {
+          "__value__": [
+            false,
+            true
+          ]
+        },
+        "clip": [
+          "18",
+          0
+        ]
+      },
+      "class_type": "CLIPTextEncode",
+      "_meta": {
+        "title": "CLIP Text Encode (Negative Prompt)"
+      }
+    },
+    "8": {
+      "inputs": {
+        "samples": [
+          "3",
+          0
+        ],
+        "vae": [
+          "17",
+          0
+        ]
+      },
+      "class_type": "VAEDecode",
+      "_meta": {
+        "title": "VAE Decode"
+      }
+    },
+    "9": {
+      "inputs": {
+        "filename_prefix": "ComfyUI",
+        "images": [
+          "8",
+          0
+        ]
+      },
+      "class_type": "SaveImage",
+      "_meta": {
+        "title": "Save Image"
+      }
+    },
+    "13": {
+      "inputs": {
+        "width": 1024,
+        "height": 1024,
+        "batch_size": 1
+      },
+      "class_type": "EmptySD3LatentImage",
+      "_meta": {
+        "title": "EmptySD3LatentImage"
+      }
+    },
+    "17": {
+      "inputs": {
+        "vae_name": "ae.safetensors"
+      },
+      "class_type": "VAELoader",
+      "_meta": {
+        "title": "VAE Loader"
+      }
+    },
+    "18": {
+      "inputs": {
+        "clip_name": "qwen_3_4b.safetensors",
+        "type": "lumina2",
+        "device": "default"
+      },
+      "class_type": "CLIPLoader",
+      "_meta": {
+        "title": "CLIP Loader"
+      }
+    },
+    "28": {
+      "inputs": {
+        "unet_name": "z_image_turbo_bf16.safetensors",
+        "weight_dtype": "default"
+      },
+      "class_type": "SGLDUNETLoader",
+      "_meta": {
+        "title": "SGLDiffusion UNET Loader"
+      }
+    }
+}

python/sglang/multimodal_gen/apps/realtime_webui/README.md

@@ -0,0 +1,24 @@
+# SGLang Diffusion Realtime WebUI
+
+Standalone browser demo for `/v1/realtime_video/generate`.
+
+Open `index.html` directly in a browser, point it at an SGLang Diffusion server,
+and generate. The app sends msgpack init / event messages and renders lossless
+raw RGB frame batches on a canvas.
+
+The first version is intentionally static: no npm install, no build step, and no
+server-side dependencies. Presets are UI-side templates for prompt, LingBot
+example images, album artwork references, and session parameters. The default
+preset preloads a reference image so the demo can be tested without a file
+upload.
+
+By default, `Continuous session` is enabled for long-running camera control.
+Keyboard and pointer controls send state transitions instead of scripted preset
+actions. The telemetry `Chunk wait` measures request-to-chunk arrival time, not
+client-side RGB decode time. Continuous playback adapts to the measured chunk
+production rate so the canvas does not play a chunk at target FPS and then sit
+on the last frame while waiting for the next chunk.
+
+The interface shape follows camera-control-first video playgrounds such as
+Reactor LingBot: reference image, scene prompt, enhancement, clip controls,
+move/look camera controls, recordings history, and model telemetry.

python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js

@@ -0,0 +1,172 @@
+const RAW_RGB_CONTENT_TYPE = "application/x-raw-rgb";
+const RAW_RGB_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgb-delta-gzip";
+const RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip";
+const WEBP_FRAME_CONTENT_TYPE = "image/webp";
+const JPEG_FRAME_CONTENT_TYPE = "image/jpeg";
+
+let lastFrame = null;
+
+function reset() {
+  lastFrame = null;
+}
+
+async function gunzipBytes(payload) {
+  if (typeof DecompressionStream === "undefined") {
+    throw new Error("This browser does not support gzip stream decoding");
+  }
+  const stream = new Blob([payload]).stream().pipeThrough(new DecompressionStream("gzip"));
+  return new Uint8Array(await new Response(stream).arrayBuffer());
+}
+
+async function restoreDeltaGzipFrames(header, payload) {
+  const frameBytes = Number(header.bytes_per_frame);
+  const count = Number(header.num_frames);
+  const expectedSize = frameBytes * count;
+  const restored = await gunzipBytes(payload);
+  if (restored.length !== expectedSize) {
+    throw new Error(`delta payload size mismatch: expected ${expectedSize}, got ${restored.length}`);
+  }
+
+  let previous = header.delta_reference === "previous-frame" ? lastFrame : null;
+  if (header.delta_reference === "previous-frame") {
+    if (!previous) throw new Error("Missing previous frame for delta payload");
+    if (previous.byteLength !== frameBytes) {
+      throw new Error("Previous frame size does not match current delta payload");
+    }
+  }
+
+  for (let f = 0; f < count; f++) {
+    const offset = f * frameBytes;
+    if (previous) {
+      for (let i = 0; i < frameBytes; i++) restored[offset + i] ^= previous[i];
+    }
+    previous = restored.slice(offset, offset + frameBytes);
+  }
+  lastFrame = previous;
+  return restored;
+}
+
+function rawFramesToRgbaBuffers(header, payload) {
+  const width = Number(header.width);
+  const height = Number(header.height);
+  const channels = Number(header.channels);
+  const count = Number(header.num_frames);
+  const frameBytes = Number(header.bytes_per_frame);
+  const pixels = width * height;
+  const buffers = [];
+
+  for (let f = 0; f < count; f++) {
+    const offset = f * frameBytes;
+    if (channels === 4) {
+      buffers.push(payload.buffer.slice(
+        payload.byteOffset + offset,
+        payload.byteOffset + offset + frameBytes,
+      ));
+      continue;
+    }
+
+    const rgba = new Uint8ClampedArray(pixels * 4);
+    let src = offset;
+    let dst = 0;
+    for (let p = 0; p < pixels; p++) {
+      rgba[dst++] = payload[src++];
+      rgba[dst++] = payload[src++];
+      rgba[dst++] = payload[src++];
+      src += channels - 3;
+      rgba[dst++] = 255;
+    }
+    buffers.push(rgba.buffer);
+  }
+  return buffers;
+}
+
+function splitEncodedPayload(header, payload) {
+  const bytes = payload instanceof Uint8Array ? payload : new Uint8Array(payload);
+  const lengths = Array.isArray(header.payload_lengths) && header.payload_lengths.length
+    ? header.payload_lengths.map(Number)
+    : [bytes.byteLength];
+  const payloads = [];
+  let offset = 0;
+  for (const length of lengths) {
+    payloads.push(bytes.buffer.slice(
+      bytes.byteOffset + offset,
+      bytes.byteOffset + offset + length,
+    ));
+    offset += length;
+  }
+  return payloads;
+}
+
+async function encodedFramesToImageBitmaps(header, payload) {
+  if (typeof createImageBitmap === "undefined") {
+    throw new Error("This browser does not support worker image decoding");
+  }
+
+  const frames = await Promise.all(splitEncodedPayload(header, payload).map((framePayload) => (
+    createImageBitmap(new Blob([framePayload], { type: header.content_type }))
+  )));
+  return {
+    width: frames[0]?.width || 0,
+    height: frames[0]?.height || 0,
+    frame_type: "bitmap",
+    frames,
+  };
+}
+
+async function decode(header, payload) {
+  let rawPayload;
+  if (
+    header.content_type === WEBP_FRAME_CONTENT_TYPE ||
+    header.content_type === JPEG_FRAME_CONTENT_TYPE
+  ) {
+    const decoded = await encodedFramesToImageBitmaps(header, payload);
+    return {
+      id: header.__decode_id,
+      width: decoded.width,
+      height: decoded.height,
+      chunk: Number(header.chunk_index),
+      frame_type: decoded.frame_type,
+      frames: decoded.frames,
+    };
+  } else if (header.content_type === RAW_RGB_CONTENT_TYPE) {
+    rawPayload = new Uint8Array(payload);
+    const frameBytes = Number(header.bytes_per_frame);
+    const count = Number(header.num_frames);
+    lastFrame = count > 0
+      ? rawPayload.slice((count - 1) * frameBytes, count * frameBytes)
+      : null;
+  } else if (
+    header.content_type === RAW_RGB_DELTA_GZIP_CONTENT_TYPE ||
+    header.content_type === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE
+  ) {
+    rawPayload = await restoreDeltaGzipFrames(header, payload);
+  } else {
+    throw new Error(`Unsupported content type ${header.content_type}`);
+  }
+
+  return {
+    id: header.__decode_id,
+    width: Number(header.width),
+    height: Number(header.height),
+    chunk: Number(header.chunk_index),
+    frames: rawFramesToRgbaBuffers(header, rawPayload),
+  };
+}
+
+self.onmessage = async (event) => {
+  const message = event.data;
+  try {
+    if (message.type === "reset") {
+      reset();
+      return;
+    }
+    const result = await decode(message.header, message.payload);
+    self.postMessage({ type: "decoded", ...result }, result.frames);
+  } catch (error) {
+    self.postMessage({
+      type: "error",
+      id: message.header?.__decode_id,
+      message: error.message || "decode failed",
+    });
+  }
+};

python/sglang/multimodal_gen/apps/realtime_webui/index.html

@@ -0,0 +1,168 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>sglang-diffusion Realtime Studio</title>
+    <link rel="stylesheet" href="./styles.css?v=realtime-record-v49" />
+  </head>
+  <body>
+    <main class="shell">
+      <section class="panel controls" aria-label="Session controls">
+        <div class="brand">
+          <span>sglang-diffusion</span>
+          <strong>Realtime Studio</strong>
+        </div>
+        <label>Server<input id="serverUrl" value="ws://127.0.0.1:30000/v1/realtime_video/generate" /></label>
+        <label>Model<input id="model" value="" placeholder="auto from /v1/models" /></label>
+        <div class="section-title">Reference</div>
+        <label class="reference-upload">
+          <input id="firstFrame" type="file" accept="image/*" />
+          <canvas id="referencePreview" width="320" height="180"></canvas>
+          <span id="referenceName">Preset reference</span>
+        </label>
+        <div class="section-title">Generate the scene</div>
+        <label>Prompt<textarea id="prompt" rows="4">A cinematic handheld shot of a quiet city street at dusk, soft reflections, natural motion.</textarea></label>
+        <button id="enhanceBtn" class="wide">Enhance</button>
+        <div class="split">
+          <label>Size<input id="size" value="832x480" /></label>
+          <label>FPS<input id="fps" type="number" value="25" min="1" max="60" /></label>
+        </div>
+        <div class="split">
+          <label>Frames<input id="numFrames" type="number" value="9" min="5" step="4" /></label>
+          <label>Seed<input id="seed" type="number" value="42" /></label>
+        </div>
+        <div class="split">
+          <label>Steps<input id="steps" type="number" value="4" min="1" /></label>
+          <label>Guidance<input id="guidance" type="number" value="1" step="0.1" /></label>
+        </div>
+        <div class="split">
+          <label>Sink<input id="sinkSize" type="number" value="9" min="0" /></label>
+          <label>Window<input id="windowFrames" type="number" value="18" min="1" /></label>
+        </div>
+        <div class="split">
+          <label>Transport
+            <select id="transportFormat">
+              <option value="webp" selected>WebP preview</option>
+              <option value="jpeg">JPEG preview</option>
+              <option value="">Lossless delta</option>
+              <option value="raw">Raw RGB</option>
+            </select>
+          </label>
+          <label>Quality<input id="transportQuality" type="number" value="95" min="1" max="100" /></label>
+        </div>
+        <div class="split output-options">
+          <label class="toggle-row"><input id="superResolution" type="checkbox" />Super resolution</label>
+          <label>Scale
+            <select id="upscalingScale">
+              <option value="2" selected>2x</option>
+              <option value="4">4x</option>
+            </select>
+          </label>
+        </div>
+        <label>SR model
+          <select id="upscalingModel">
+            <option value="">Quality x2</option>
+            <option
+              value="https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth"
+              selected
+            >
+              Fast general
+            </option>
+            <option value="/scratch/realesr-animevideov3.pth">Fast anime</option>
+          </select>
+        </label>
+        <label class="toggle-row"><input id="frameInterpolation" type="checkbox" />Smooth 2x frames</label>
+        <label class="toggle-row"><input id="continuous" type="checkbox" checked />Continuous session</label>
+        <div class="actions">
+          <button id="connectBtn" class="primary">Generate</button>
+          <button id="stopBtn">Close session</button>
+        </div>
+        <button id="sendPromptBtn" class="wide">Send prompt update</button>
+      </section>
+
+      <section class="workspace" aria-label="Realtime workspace">
+        <section class="stage" aria-label="Realtime preview">
+          <div class="topbar">
+            <span id="statusDot" class="dot"></span>
+            <span id="statusText">Idle</span>
+            <span id="chunkText">chunk -</span>
+            <button id="recordBtn" class="record-button" type="button" aria-pressed="false" title="Record preview">
+              <span class="record-button-icon" aria-hidden="true"></span>
+              <span id="recordLabel">Record</span>
+              <span id="recordDuration" class="record-button-duration">00:00</span>
+            </button>
+            <span class="topbar-spacer"></span>
+            <label class="preview-scale-control">Preview
+              <input id="previewScale" type="range" min="80" max="170" value="120" />
+              <b id="previewScaleText">120%</b>
+            </label>
+            <span class="stage-stat">output <b id="outputSizeText">832x480</b></span>
+            <span class="stage-stat">render <b id="renderFps">0</b> fps</span>
+            <span class="stage-stat">source <b id="theoreticalFpsText">-</b></span>
+            <span class="stage-stat">buffer <b id="stageLatencyText">-</b></span>
+          </div>
+          <div class="preview-frame">
+            <canvas id="viewport" width="1280" height="720"></canvas>
+            <div id="previewOverlay" class="preview-overlay" aria-hidden="true">
+              <span class="preview-loader"></span>
+            </div>
+          </div>
+          <div class="stage-controls" aria-label="Camera controls">
+            <div class="control-cluster" aria-label="Move camera">
+              <span class="control-title">Move</span>
+              <div class="camera-pad move-pad">
+                <span></span>
+                <button data-action="w" data-key="W">Forward</button>
+                <span></span>
+                <button data-action="a" data-key="A">Left</button>
+                <button data-action="s" data-key="S">Back</button>
+                <button data-action="d" data-key="D">Right</button>
+              </div>
+            </div>
+            <div class="control-cluster" aria-label="Look around">
+              <span class="control-title">Look</span>
+              <div class="camera-pad look-pad">
+                <span></span>
+                <button data-action="i" data-key="↑">Pitch +</button>
+                <span></span>
+                <button data-action="j" data-key="←">Yaw -</button>
+                <button data-action="k" data-key="↓">Pitch -</button>
+                <button data-action="l" data-key="→">Yaw +</button>
+              </div>
+            </div>
+          </div>
+          <div class="timeline">
+            <span id="queueText">queue 0</span>
+            <span id="frameText">frames 0</span>
+            <span id="byteText">0 MB</span>
+          </div>
+          <div class="telemetry stage-telemetry">
+            <span>Payload<b id="payloadMode">webp</b></span>
+            <span>Server send<b id="serverSendText">-</b></span>
+            <span>Chunk bytes<b id="chunkPayloadText">-</b></span>
+            <span>Chunk wait<b id="latencyText">-</b></span>
+            <span>Decode<b id="decodeText">-</b></span>
+            <span>Display lag<b id="displayLagText">-</b></span>
+          </div>
+        </section>
+
+        <section class="panel presets" aria-label="Presets and camera">
+          <div class="section-title">LingBot</div>
+          <div class="spec-grid">
+            <span><b>25 fps</b> target</span>
+            <span><b>chunked</b> stream</span>
+            <span><b>480p/720p</b></span>
+            <span><b>Cam + Act</b></span>
+          </div>
+          <div class="section-title">Presets</div>
+          <div id="presetList" class="preset-list"></div>
+          <div class="section-title">History</div>
+          <div id="historyList" class="history-list"></div>
+        </section>
+      </section>
+    </main>
+  <script src="./playback_controller.js?v=realtime-playback-v13"></script>
+  <script src="./app.js?v=realtime-record-v75"></script>
+</body>
+</html>

python/sglang/multimodal_gen/apps/realtime_webui/playback_controller.js

@@ -0,0 +1,517 @@
+(function attachRealtimePlaybackController(global) {
+  const DEFAULT_CONFIG = {
+    targetFps: 25,
+    minSourceFps: 1,
+    serverFpsAlphaUp: 0.28,
+    serverFpsAlphaDown: 0.2,
+    deliveryFpsAlphaUp: 0.08,
+    deliveryFpsAlphaDown: 0.55,
+    targetLeadChunkRatio: 1.5,
+    minTargetLeadMs: 1500,
+    maxTargetLeadMs: 2600,
+    maxLeadExtraChunkRatio: 8.0,
+    startLeadChunkRatio: 1.85,
+    minStartLeadMs: 1700,
+    resumeLeadChunkRatio: 2.5,
+    minResumeLeadMs: 1000,
+    maxResumeLeadMs: 1800,
+    rebufferLeadBoostMs: 250,
+    rebufferLeadBoostDecayMsPerSecond: 120,
+    deliveryLeadBoostDecayMsPerSecond: 80,
+    maxDeliveryLeadBoostMs: 2000,
+    deliveryStallExpectedMultiplier: 1.25,
+    receiveStallPlaybackRateMin: 0.65,
+    receiveStallPlaybackRateSlewPerSecond: 0.5,
+    lowWaterRatio: 0.4,
+    playbackRateGain: 0.14,
+    playbackRateMin: 0.92,
+    playbackRateMax: 1.08,
+    emergencyPlaybackRateMin: 0.9,
+    emergencyPlaybackRateMax: 1.12,
+    playbackRateSlewPerSecond: 0.08,
+    eventCutoverMaxMs: 420,
+    eventCutoverMaxFrames: 10,
+    settleEventCutoverMaxMs: 720,
+    settleEventCutoverMaxFrames: 18,
+    startupWarmupMinMs: 1500,
+    startupWarmupExpectedMultiplier: 3,
+  };
+
+  function clamp(value, min, max) {
+    return Math.min(max, Math.max(min, value));
+  }
+
+  function finitePositive(value) {
+    return Number.isFinite(value) && value > 0;
+  }
+
+  class RealtimePlaybackController {
+    constructor(config = {}) {
+      this.config = { ...DEFAULT_CONFIG, ...config };
+      this.reset({ targetFps: this.config.targetFps });
+    }
+
+    reset({ targetFps } = {}) {
+      this.targetFps = Math.max(1, Number(targetFps || this.config.targetFps));
+      this.sourceFps = this.targetFps;
+      this.serverFps = this.targetFps;
+      this.deliveryFps = this.targetFps;
+      this.hasServerSample = false;
+      this.hasDeliverySample = false;
+      this.latestChunkDurationMs = 1000 / this.targetFps;
+      this.latestChunkFrames = 1;
+      this.playbackRate = 1;
+      this.renderFps = this.targetFps;
+      this.queue = [];
+      this.lastDrawAt = 0;
+      this.lastRateUpdateAt = 0;
+      this.renderedFrames = 0;
+      this.droppedFrames = 0;
+      this.buffering = true;
+      this.pendingEventId = 0;
+      this.pendingEventSentAt = 0;
+      this.pendingEventCutoverMode = "motion";
+      this.lastDropReason = "";
+      this.lastDropAt = 0;
+      this.lastDropCount = 0;
+      this.rebufferLeadBoostMs = 0;
+      this.deliveryLeadBoostMs = 0;
+      this.chunkReceives = new Map();
+      this.serverStatChunks = new Set();
+      this.lastFinalReceiveAt = 0;
+      this.receiveStalled = false;
+    }
+
+    setTargetFps(targetFps) {
+      const nextTargetFps = Math.max(1, Number(targetFps || this.config.targetFps));
+      this.targetFps = nextTargetFps;
+      if (!this.hasServerSample && !this.hasDeliverySample) {
+        this.serverFps = nextTargetFps;
+        this.deliveryFps = nextTargetFps;
+        this.sourceFps = nextTargetFps;
+        this.renderFps = nextTargetFps;
+      } else {
+        this.serverFps = clamp(this.serverFps, this.config.minSourceFps, nextTargetFps);
+        this.deliveryFps = clamp(this.deliveryFps, this.config.minSourceFps, nextTargetFps);
+        this.sourceFps = clamp(this.sourceFps, this.config.minSourceFps, nextTargetFps);
+        this.renderFps = this.sourceFps * this.playbackRate;
+      }
+      this.latestChunkDurationMs = Math.max(this.latestChunkDurationMs, 1000 / this.targetFps);
+    }
+
+    clear() {
+      const frames = this.queue.splice(0);
+      this.lastDrawAt = 0;
+      this.buffering = true;
+      return frames;
+    }
+
+    noteInputEvent(eventId, now, { cutoverMode = "motion" } = {}) {
+      this.pendingEventId = Number(eventId || 0);
+      this.pendingEventSentAt = Number(now || 0);
+      this.pendingEventCutoverMode = cutoverMode;
+    }
+
+    observeServerStats(stats, now) {
+      const chunkIndex = Number(stats.chunk_index || 0);
+      const numFrames = Number(stats.num_frames || 0);
+      const chunkTotalMs = Number(stats.chunk_total_ms || 0);
+      if (numFrames > 0 && chunkTotalMs > 0) {
+        this.serverStatChunks.add(chunkIndex);
+        if (this.serverStatChunks.size > 128) {
+          this.serverStatChunks.delete(this.serverStatChunks.values().next().value);
+        }
+        const expectedMs = numFrames / Math.max(1, this.targetFps) * 1000;
+        const isStartupWarmup =
+          chunkIndex === 0 &&
+          chunkTotalMs > Math.max(
+            this.config.startupWarmupMinMs,
+            expectedMs * this.config.startupWarmupExpectedMultiplier,
+          );
+        if (isStartupWarmup) return this.snapshot();
+        this.#observeFpsSample("server", {
+          fps: numFrames / (chunkTotalMs / 1000),
+          frameCount: numFrames,
+          durationMs: chunkTotalMs,
+          now,
+        });
+      }
+      return this.snapshot();
+    }
+
+    enqueueDecodedFrames(header, frames, now) {
+      const chunkIndex = Number(header.chunk_index || 0);
+      const eventId = Number(header.event_id || 0);
+      const receivedAt = Number(header.__received_at || now);
+      const preparedFrames = frames.map((frame) => ({
+        ...frame,
+        chunk: Number(frame.chunk ?? chunkIndex),
+        chunkIndex,
+        eventId,
+      }));
+      const droppedFrames = [];
+      let cutover = null;
+
+      if (this.pendingEventId && eventId >= this.pendingEventId) {
+        const oldEventFrameCount = this.#oldEventFrameCount(eventId);
+        const graceFrames = this.#eventGraceFrames();
+        const dropCount = Math.max(0, oldEventFrameCount - graceFrames);
+        if (dropCount > 0) {
+          droppedFrames.push(...this.queue.splice(graceFrames, dropCount));
+          this.#recordDrop(dropCount, "event cutover", now);
+        }
+        cutover = {
+          eventId,
+          latencyMs: this.pendingEventSentAt ? now - this.pendingEventSentAt : 0,
+        };
+        this.pendingEventId = 0;
+        this.pendingEventSentAt = 0;
+        this.pendingEventCutoverMode = "motion";
+      }
+
+      this.queue.push(...preparedFrames);
+      this.#observeChunkArrival(header, preparedFrames.length, receivedAt, now);
+      droppedFrames.push(...this.#trimBacklog(now));
+      return { droppedFrames, cutover, snapshot: this.snapshot() };
+    }
+
+    render(now, { hasPendingInput = true } = {}) {
+      this.#decayRebufferBoost(now);
+      this.#updateReceiveStallGuard(now);
+      const droppedFrames = this.#trimBacklog(now);
+      if (!this.queue.length) {
+        if (this.renderedFrames && hasPendingInput && !this.buffering) {
+          this.buffering = true;
+          this.rebufferLeadBoostMs = Math.max(
+            this.rebufferLeadBoostMs,
+            this.config.rebufferLeadBoostMs,
+          );
+        }
+        return { action: "hold", droppedFrames, snapshot: this.snapshot() };
+      }
+
+      const bufferMs = this.bufferDurationMs;
+      if (
+        hasPendingInput &&
+        this.receiveStalled &&
+        this.renderedFrames &&
+        bufferMs < this.targetLeadMs
+      ) {
+        this.buffering = true;
+        this.lastDrawAt = 0;
+        return { action: "hold", droppedFrames, snapshot: this.snapshot() };
+      }
+
+      if (
+        hasPendingInput &&
+        this.buffering &&
+        bufferMs < (this.renderedFrames ? this.#resumeLeadMs() : this.#startLeadMs())
+      ) {
+        this.buffering = true;
+        this.lastDrawAt = 0;
+        return { action: "hold", droppedFrames, snapshot: this.snapshot() };
+      }
+
+      if (this.buffering) {
+        this.buffering = false;
+        this.lastDrawAt = 0;
+      }
+
+      this.#updatePlaybackRate(now);
+      const targetMs = 1000 / Math.max(1, this.renderFps);
+      const elapsedMs = this.lastDrawAt ? now - this.lastDrawAt : targetMs;
+      if (elapsedMs < targetMs) {
+        return { action: "wait", droppedFrames, snapshot: this.snapshot() };
+      }
+
+      const frame = this.queue.shift();
+      this.renderedFrames += 1;
+      this.lastDrawAt = !this.lastDrawAt || elapsedMs > targetMs * 4
+        ? now
+        : now - (elapsedMs % targetMs);
+      return { action: "draw", frame, droppedFrames, snapshot: this.snapshot() };
+    }
+
+    get queuedFrames() {
+      return this.queue.length;
+    }
+
+    get bufferDurationMs() {
+      return this.queue.length / Math.max(1, this.sourceFps) * 1000;
+    }
+
+    get targetLeadMs() {
+      const base = clamp(
+        this.latestChunkDurationMs * this.config.targetLeadChunkRatio,
+        this.config.minTargetLeadMs,
+        this.config.maxTargetLeadMs,
+      );
+      return clamp(
+        base + this.rebufferLeadBoostMs + this.deliveryLeadBoostMs,
+        this.config.minTargetLeadMs,
+        this.config.maxTargetLeadMs +
+          this.config.rebufferLeadBoostMs +
+          this.config.maxDeliveryLeadBoostMs,
+      );
+    }
+
+    get maxLeadMs() {
+      return this.targetLeadMs + this.latestChunkDurationMs * this.config.maxLeadExtraChunkRatio;
+    }
+
+    snapshot() {
+      return {
+        queueFrames: this.queue.length,
+        bufferMs: this.bufferDurationMs,
+        targetLeadMs: this.targetLeadMs,
+        maxLeadMs: this.maxLeadMs,
+        sourceFps: this.sourceFps,
+        serverFps: this.serverFps,
+        deliveryFps: this.deliveryFps,
+        targetFps: this.targetFps,
+        renderFps: this.renderFps,
+        playbackRate: this.playbackRate,
+        droppedFrames: this.droppedFrames,
+        lastDropAt: this.lastDropAt,
+        lastDropCount: this.lastDropCount,
+        buffering: this.buffering,
+        lastDropReason: this.lastDropReason,
+      };
+    }
+
+    #observeFpsSample(kind, { fps, frameCount, durationMs, now }) {
+      if (!finitePositive(fps)) return;
+      const cappedFps = clamp(fps, this.config.minSourceFps, this.targetFps);
+      const isDelivery = kind === "delivery";
+      const currentFps = isDelivery ? this.deliveryFps : this.serverFps;
+      const hasSample = isDelivery ? this.hasDeliverySample : this.hasServerSample;
+      let nextFps;
+      if (!hasSample) {
+        nextFps = cappedFps;
+      } else {
+        const alpha = cappedFps > currentFps
+          ? (isDelivery ? this.config.deliveryFpsAlphaUp : this.config.serverFpsAlphaUp)
+          : (isDelivery ? this.config.deliveryFpsAlphaDown : this.config.serverFpsAlphaDown);
+        nextFps = currentFps * (1 - alpha) + cappedFps * alpha;
+      }
+      if (isDelivery) {
+        this.deliveryFps = nextFps;
+        this.hasDeliverySample = true;
+        this.#observeDeliveryJitter(frameCount, durationMs);
+      } else {
+        this.serverFps = nextFps;
+        this.hasServerSample = true;
+      }
+      const effectiveFps = this.hasServerSample
+        ? this.serverFps
+        : (this.hasDeliverySample ? this.deliveryFps : this.targetFps);
+      this.sourceFps = clamp(effectiveFps, this.config.minSourceFps, this.targetFps);
+      if (!isDelivery || !this.hasServerSample) {
+        this.latestChunkFrames = Math.max(1, Number(frameCount || this.latestChunkFrames));
+        this.latestChunkDurationMs = clamp(
+          Number(durationMs || (this.latestChunkFrames / Math.max(1, this.sourceFps) * 1000)),
+          1000 / Math.max(1, this.targetFps),
+          2500,
+        );
+      }
+      this.#updatePlaybackRate(now);
+    }
+
+    #observeDeliveryJitter(frameCount, durationMs) {
+      if (!this.hasServerSample || !finitePositive(durationMs)) return;
+      const expectedMs = Number(frameCount || 0) / Math.max(1, this.serverFps) * 1000;
+      if (expectedMs <= 0) return;
+      if (durationMs <= expectedMs * this.config.deliveryStallExpectedMultiplier) return;
+      const boostMs = clamp(
+        durationMs - expectedMs,
+        0,
+        this.config.maxDeliveryLeadBoostMs,
+      );
+      this.deliveryLeadBoostMs = Math.max(this.deliveryLeadBoostMs, boostMs);
+    }
+
+    #updateReceiveStallGuard(now) {
+      this.receiveStalled = false;
+      if (!this.lastFinalReceiveAt || !this.hasServerSample) return;
+      const elapsedMs = now - this.lastFinalReceiveAt;
+      const expectedMs = Math.max(
+        this.latestChunkDurationMs,
+        this.latestChunkFrames / Math.max(1, this.serverFps) * 1000,
+      );
+      if (elapsedMs <= expectedMs * this.config.deliveryStallExpectedMultiplier) return;
+      this.receiveStalled = true;
+      this.deliveryLeadBoostMs = Math.max(
+        this.deliveryLeadBoostMs,
+        clamp(elapsedMs - expectedMs, 0, this.config.maxDeliveryLeadBoostMs),
+      );
+    }
+
+    #observeChunkArrival(header, frameCount, receivedAt, now) {
+      const chunkIndex = Number(header.chunk_index || 0);
+      const state = this.chunkReceives.get(chunkIndex) || {
+        firstAt: receivedAt,
+        frames: 0,
+      };
+      state.frames += Number(frameCount || 0);
+      state.lastAt = receivedAt;
+      this.chunkReceives.set(chunkIndex, state);
+
+      const frameBatchIndex = Number(header.frame_batch_index || 0);
+      const numFrameBatches = Number(header.num_frame_batches || 1);
+      const isFinalFrameBatch =
+        Boolean(header.is_final_frame_batch) ||
+        frameBatchIndex + 1 >= numFrameBatches;
+      if (!isFinalFrameBatch) return;
+      const durationMs = this.lastFinalReceiveAt
+        ? receivedAt - this.lastFinalReceiveAt
+        : 0;
+      this.lastFinalReceiveAt = receivedAt;
+      if (state.frames > 0 && durationMs > 0) {
+        this.#observeFpsSample("delivery", {
+          fps: state.frames / (durationMs / 1000),
+          frameCount: state.frames,
+          durationMs,
+          now,
+        });
+      }
+      this.chunkReceives.delete(chunkIndex);
+    }
+
+    #updatePlaybackRate(now) {
+      const bufferMs = this.bufferDurationMs;
+      const targetLeadMs = Math.max(1, this.targetLeadMs);
+      const error = (bufferMs - targetLeadMs) / targetLeadMs;
+      const emergency =
+        bufferMs > this.maxLeadMs ||
+        bufferMs < targetLeadMs * this.config.lowWaterRatio ||
+        (this.receiveStalled && bufferMs < targetLeadMs);
+      const minRate = emergency
+        ? (
+            this.receiveStalled
+              ? this.config.receiveStallPlaybackRateMin
+              : this.config.emergencyPlaybackRateMin
+          )
+        : this.config.playbackRateMin;
+      const maxRate = this.receiveStalled && bufferMs < targetLeadMs
+        ? 1
+        : emergency
+        ? this.config.emergencyPlaybackRateMax
+        : this.config.playbackRateMax;
+      const desiredRate = clamp(
+        1 + error * this.config.playbackRateGain,
+        minRate,
+        maxRate,
+      );
+
+      if (!this.lastRateUpdateAt) {
+        this.playbackRate = desiredRate;
+      } else {
+        const dtSeconds = Math.max(0.001, (now - this.lastRateUpdateAt) / 1000);
+        const slewPerSecond = this.receiveStalled
+          ? this.config.receiveStallPlaybackRateSlewPerSecond
+          : this.config.playbackRateSlewPerSecond;
+        const maxDelta = slewPerSecond * dtSeconds;
+        this.playbackRate = clamp(
+          desiredRate,
+          this.playbackRate - maxDelta,
+          this.playbackRate + maxDelta,
+        );
+      }
+      this.lastRateUpdateAt = now;
+      this.renderFps = clamp(
+        this.sourceFps * this.playbackRate,
+        this.config.minSourceFps,
+        this.targetFps * this.config.emergencyPlaybackRateMax,
+      );
+    }
+
+    #trimBacklog(now) {
+      const droppedFrames = [];
+      while (this.queue.length && this.bufferDurationMs > this.maxLeadMs) {
+        const firstChunk = this.queue[0].chunkIndex;
+        let dropCount = 0;
+        while (
+          dropCount < this.queue.length &&
+          this.queue[dropCount].chunkIndex === firstChunk
+        ) {
+          dropCount += 1;
+        }
+        if (!dropCount || dropCount >= this.queue.length) break;
+        droppedFrames.push(...this.queue.splice(0, dropCount));
+        this.#recordDrop(dropCount, "backlog", now);
+      }
+      return droppedFrames;
+    }
+
+    #oldEventFrameCount(nextEventId) {
+      let count = 0;
+      while (count < this.queue.length && this.queue[count].eventId < nextEventId) {
+        count += 1;
+      }
+      return count;
+    }
+
+    #eventGraceFrames() {
+      const byTime = Math.max(
+        1,
+        Math.round(this.sourceFps * this.#eventCutoverMaxMs() / 1000),
+      );
+      const byChunkRatio = this.pendingEventCutoverMode === "settle" ? 1.5 : 0.85;
+      const byChunk = Math.max(1, Math.round(this.latestChunkFrames * byChunkRatio));
+      return Math.min(this.#eventCutoverMaxFrames(), byTime, byChunk);
+    }
+
+    #eventCutoverMaxMs() {
+      return this.pendingEventCutoverMode === "settle"
+        ? this.config.settleEventCutoverMaxMs
+        : this.config.eventCutoverMaxMs;
+    }
+
+    #eventCutoverMaxFrames() {
+      return this.pendingEventCutoverMode === "settle"
+        ? this.config.settleEventCutoverMaxFrames
+        : this.config.eventCutoverMaxFrames;
+    }
+
+    #startLeadMs() {
+      return Math.max(
+        this.config.minStartLeadMs,
+        this.latestChunkDurationMs * this.config.startLeadChunkRatio,
+        this.targetLeadMs,
+      );
+    }
+
+    #resumeLeadMs() {
+      return clamp(
+        this.latestChunkDurationMs * this.config.resumeLeadChunkRatio,
+        this.config.minResumeLeadMs,
+        this.config.maxResumeLeadMs,
+      );
+    }
+
+    #decayRebufferBoost(now) {
+      if ((!this.rebufferLeadBoostMs && !this.deliveryLeadBoostMs) || !this.lastRateUpdateAt) return;
+      const dtSeconds = Math.max(0, (now - this.lastRateUpdateAt) / 1000);
+      this.rebufferLeadBoostMs = Math.max(
+        0,
+        this.rebufferLeadBoostMs - dtSeconds * this.config.rebufferLeadBoostDecayMsPerSecond,
+      );
+      this.deliveryLeadBoostMs = Math.max(
+        0,
+        this.deliveryLeadBoostMs - dtSeconds * this.config.deliveryLeadBoostDecayMsPerSecond,
+      );
+    }
+
+    #recordDrop(count, reason, now) {
+      this.droppedFrames += count;
+      this.lastDropAt = Number(now || 0);
+      this.lastDropCount = count;
+      this.lastDropReason = reason;
+    }
+  }
+
+  global.RealtimePlaybackController = RealtimePlaybackController;
+  if (typeof module !== "undefined" && module.exports) {
+    module.exports = { RealtimePlaybackController };
+  }
+})(typeof globalThis !== "undefined" ? globalThis : window);

python/sglang/multimodal_gen/apps/realtime_webui/playback_controller_test.js

@@ -0,0 +1,116 @@
+const assert = require("node:assert/strict");
+const { RealtimePlaybackController } = require("./playback_controller.js");
+
+function frames(count, chunk) {
+  return Array.from({ length: count }, (_, index) => ({
+    image: { close() {} },
+    chunk,
+    index,
+  }));
+}
+
+function enqueueChunk(controller, {
+  chunk,
+  eventId = 0,
+  frameCount = 12,
+  durationMs = 480,
+  now,
+}) {
+  controller.observeServerStats({
+    chunk_index: chunk,
+    num_frames: frameCount,
+    chunk_total_ms: durationMs,
+  }, now);
+  return controller.enqueueDecodedFrames({
+    chunk_index: chunk,
+    event_id: eventId,
+    num_frames: frameCount,
+    __received_at: now,
+    is_final_frame_batch: true,
+  }, frames(frameCount, chunk), now);
+}
+
+function renderFor(controller, startMs, durationMs) {
+  let rendered = 0;
+  for (let now = startMs; now <= startMs + durationMs; now += 16.67) {
+    const decision = controller.render(now, { hasPendingInput: true });
+    if (decision.action === "draw") rendered += 1;
+  }
+  return rendered;
+}
+
+function stableSourceDoesNotDrop() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  let now = 0;
+  for (let chunk = 0; chunk < 8; chunk += 1) {
+    now += 480;
+    enqueueChunk(controller, { chunk, now });
+    renderFor(controller, now, 480);
+  }
+  const snapshot = controller.snapshot();
+  assert.equal(snapshot.droppedFrames, 0);
+  assert.ok(snapshot.sourceFps > 24 && snapshot.sourceFps <= 25);
+}
+
+function slowServerCapsRenderFps() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  let now = 0;
+  for (let chunk = 0; chunk < 8; chunk += 1) {
+    now += 1360;
+    enqueueChunk(controller, { chunk, durationMs: 1360, now });
+    renderFor(controller, now, 1360);
+  }
+  const snapshot = controller.snapshot();
+  assert.ok(snapshot.sourceFps > 8 && snapshot.sourceFps < 10);
+  assert.ok(snapshot.renderFps <= 10);
+}
+
+function backlogDropsContiguousOldFrames() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  let now = 100;
+  for (let chunk = 0; chunk < 13; chunk += 1) {
+    enqueueChunk(controller, { chunk, now, durationMs: 480 });
+    now += 20;
+  }
+  const snapshot = controller.snapshot();
+  assert.ok(snapshot.droppedFrames > 0);
+  assert.equal(snapshot.lastDropReason, "backlog");
+}
+
+function eventCutoverKeepsShortGrace() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  enqueueChunk(controller, { chunk: 1, frameCount: 24, durationMs: 960, now: 1000 });
+  controller.noteInputEvent(5, 1050);
+  const result = enqueueChunk(controller, {
+    chunk: 2,
+    eventId: 5,
+    frameCount: 12,
+    durationMs: 480,
+    now: 1150,
+  });
+  assert.ok(result.cutover);
+  assert.ok(result.droppedFrames.length >= 14);
+  assert.equal(controller.queue[0].chunk, 1);
+  assert.equal(controller.queue[0].index, 0);
+}
+
+function settleEventCutoverKeepsWiderGrace() {
+  const controller = new RealtimePlaybackController({ targetFps: 25 });
+  enqueueChunk(controller, { chunk: 1, frameCount: 24, durationMs: 960, now: 1000 });
+  controller.noteInputEvent(5, 1050, { cutoverMode: "settle" });
+  const result = enqueueChunk(controller, {
+    chunk: 2,
+    eventId: 5,
+    frameCount: 12,
+    durationMs: 480,
+    now: 1150,
+  });
+  assert.ok(result.cutover);
+  assert.ok(result.droppedFrames.length <= 12);
+}
+
+stableSourceDoesNotDrop();
+slowServerCapsRenderFps();
+backlogDropsContiguousOldFrames();
+eventCutoverKeepsShortGrace();
+settleEventCutoverKeepsWiderGrace();

python/sglang/multimodal_gen/apps/realtime_webui/styles.css

@@ -0,0 +1,714 @@
+:root {
+  --paper: #eef1ec;
+  --panel: #fbfaf5;
+  --ink: #171a16;
+  --muted: #687164;
+  --line: #cbd2c4;
+  --accent: #b9543c;
+  --green: #4d765f;
+  --blue: #3f607c;
+  --pressed: #8c9288;
+  --pressed-border: #aeb4aa;
+  --pressed-ring: rgba(238, 241, 236, 0.2);
+  --shadow: 0 18px 60px rgba(23, 26, 22, 0.12);
+}
+
+* { box-sizing: border-box; }
+
+body {
+  margin: 0;
+  overflow-x: hidden;
+  min-height: 100vh;
+  background:
+    linear-gradient(90deg, rgba(23, 26, 22, 0.035) 1px, transparent 1px),
+    linear-gradient(180deg, rgba(23, 26, 22, 0.035) 1px, transparent 1px),
+    var(--paper);
+  background-size: 28px 28px;
+  color: var(--ink);
+  font-family: ui-sans-serif, "Avenir Next", "Helvetica Neue", sans-serif;
+}
+
+button, input, textarea, select { font: inherit; }
+button:disabled { cursor: wait; opacity: 0.64; transform: none; }
+
+.shell {
+  display: grid;
+  grid-template-columns: minmax(260px, 320px) minmax(0, 1fr);
+  gap: 18px;
+  width: 100%;
+  max-width: 100vw;
+  min-height: 100vh;
+  padding: 18px;
+}
+
+.panel {
+  background: color-mix(in oklch, var(--panel), white 20%);
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  box-shadow: var(--shadow);
+  padding: 18px;
+}
+
+.brand {
+  display: flex;
+  align-items: baseline;
+  gap: 10px;
+  margin-bottom: 22px;
+}
+
+.brand span {
+  color: var(--panel);
+  background: var(--ink);
+  border-radius: 4px;
+  padding: 3px 7px;
+  font-size: 12px;
+  letter-spacing: 0;
+}
+
+.brand strong { font-size: 18px; font-weight: 650; }
+
+label {
+  display: grid;
+  gap: 7px;
+  margin: 12px 0;
+  color: var(--muted);
+  font-size: 12px;
+}
+
+.label-row {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 8px;
+}
+
+.help-tooltip {
+  position: relative;
+  display: inline-grid;
+  place-items: center;
+  width: 18px;
+  height: 18px;
+  border: 1px solid var(--line);
+  border-radius: 50%;
+  color: var(--muted);
+  background: #fffdf7;
+  cursor: help;
+  font-size: 11px;
+  line-height: 1;
+}
+
+.help-tooltip::after {
+  content: attr(aria-label);
+  position: absolute;
+  right: 0;
+  bottom: calc(100% + 8px);
+  z-index: 20;
+  width: 280px;
+  max-width: min(280px, calc(100vw - 48px));
+  padding: 9px 10px;
+  border-radius: 6px;
+  background: var(--ink);
+  box-shadow: 0 12px 36px rgba(23, 26, 22, 0.24);
+  color: var(--panel);
+  font-size: 11px;
+  font-weight: 400;
+  line-height: 1.4;
+  opacity: 0;
+  pointer-events: none;
+  transform: translateY(4px);
+  transition: opacity 120ms ease, transform 120ms ease;
+}
+
+.help-tooltip:hover::after,
+.help-tooltip:focus-visible::after {
+  opacity: 1;
+  transform: translateY(0);
+}
+
+input, textarea, select {
+  width: 100%;
+  border: 1px solid var(--line);
+  border-radius: 6px;
+  background: #fffdf7;
+  color: var(--ink);
+  padding: 10px 11px;
+  outline: none;
+}
+
+textarea { resize: vertical; line-height: 1.45; }
+input:focus, textarea:focus, select:focus { border-color: var(--accent); box-shadow: 0 0 0 3px rgba(185, 84, 60, 0.12); }
+
+.split { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
+.output-options {
+  align-items: end;
+}
+.output-options .toggle-row {
+  min-height: 40px;
+  margin: 12px 0;
+}
+.actions { display: grid; grid-template-columns: 1fr 0.7fr; gap: 10px; margin-top: 16px; }
+.toggle-row {
+  display: flex;
+  align-items: center;
+  gap: 9px;
+  margin-top: 14px;
+}
+.toggle-row input {
+  width: 16px;
+  height: 16px;
+  accent-color: var(--ink);
+}
+
+button {
+  border: 1px solid var(--line);
+  border-radius: 6px;
+  color: var(--ink);
+  background: #fffdf7;
+  min-height: 38px;
+  padding: 0 12px;
+  cursor: pointer;
+  transition:
+    background-color 120ms ease,
+    border-color 120ms ease,
+    box-shadow 120ms ease,
+    color 120ms ease,
+    transform 120ms ease;
+}
+
+button:hover:not(:disabled) {
+  border-color: var(--ink);
+  background: color-mix(in oklch, #fffdf7, var(--green) 10%);
+  box-shadow: 0 8px 18px rgba(23, 26, 22, 0.08);
+  transform: translateY(-1px);
+}
+button:active:not(:disabled),
+button.is-pressed:not(:disabled) {
+  border-color: var(--pressed-border);
+  background: var(--pressed);
+  color: #fffdf7;
+  box-shadow:
+    inset 0 0 0 1px rgba(255, 253, 247, 0.18),
+    inset 0 2px 7px rgba(23, 26, 22, 0.16);
+  transform: translateY(0);
+}
+button.is-key-active:not(:disabled) {
+  border-color: var(--pressed-border);
+  background: var(--pressed);
+  color: #fffdf7;
+  box-shadow:
+    inset 0 0 0 1px rgba(255, 253, 247, 0.22),
+    0 0 0 3px var(--pressed-ring),
+    0 10px 22px rgba(23, 26, 22, 0.18);
+  transform: none;
+}
+button:focus-visible {
+  outline: none;
+  box-shadow: 0 0 0 3px rgba(185, 84, 60, 0.18);
+}
+.primary { background: var(--ink); color: var(--panel); border-color: var(--ink); }
+.primary:hover:not(:disabled) {
+  background: color-mix(in oklch, var(--ink), var(--green) 18%);
+  color: var(--panel);
+}
+.primary:active:not(:disabled),
+.primary.is-pressed:not(:disabled) {
+  background: var(--pressed);
+  border-color: var(--pressed-border);
+  color: var(--panel);
+}
+.wide { width: 100%; margin-top: 10px; }
+
+.workspace {
+  display: grid;
+  gap: 18px;
+  min-width: 0;
+  max-width: 100%;
+}
+
+.stage {
+  position: relative;
+  display: grid;
+  grid-template-rows: auto auto auto auto auto;
+  align-self: start;
+  justify-self: center;
+  min-width: 0;
+  width: 100%;
+  max-width: min(1500px, 100%);
+  overflow: hidden;
+  border: 1px solid #11140f;
+  border-radius: 8px;
+  background: #11140f;
+  box-shadow: var(--shadow);
+}
+
+.preview-frame {
+  position: relative;
+  display: grid;
+  place-items: center;
+  justify-self: center;
+  width: min(calc(1040px * var(--preview-scale, 1.2)), 100%);
+  overflow: hidden;
+  background: #11140f;
+  contain: paint;
+  isolation: isolate;
+}
+
+.preview-frame::before {
+  content: "";
+  position: absolute;
+  inset: 0;
+  z-index: 0;
+  pointer-events: none;
+  background: linear-gradient(
+    180deg,
+    rgba(238, 241, 236, 0.045),
+    transparent 34%,
+    rgba(0, 0, 0, 0.18)
+  );
+}
+
+.preview-frame::after {
+  content: none;
+}
+
+.stage[data-preview-state="waiting"] .preview-frame::after {
+  animation: none;
+}
+
+.topbar, .timeline {
+  display: flex;
+  align-items: center;
+  gap: 10px;
+  min-width: 0;
+  height: 44px;
+  padding: 0 14px;
+  color: #e8eadf;
+  background: rgba(17, 20, 15, 0.88);
+  font-size: 12px;
+  font-variant-numeric: tabular-nums;
+  white-space: nowrap;
+}
+
+.topbar > * {
+  flex: 0 0 auto;
+  align-self: center;
+}
+
+.topbar-spacer { flex: 1; }
+.record-button {
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  gap: 6px;
+  flex: 0 0 118px;
+  width: 118px;
+  min-height: 28px;
+  height: 28px;
+  padding: 0 9px;
+  border-color: rgba(232, 234, 223, 0.22);
+  background: rgba(238, 241, 236, 0.08);
+  color: #e8eadf;
+  font-size: 11px;
+  font-variant-numeric: tabular-nums;
+}
+.record-button:hover:not(:disabled) {
+  border-color: rgba(232, 234, 223, 0.44);
+  background: rgba(238, 241, 236, 0.14);
+  box-shadow: none;
+  transform: none;
+}
+.record-button:active:not(:disabled),
+.record-button.is-pressed:not(:disabled),
+.record-button:focus-visible {
+  transform: none;
+}
+.record-button.is-recording {
+  border-color: color-mix(in oklch, var(--accent), white 18%);
+  background: var(--accent);
+  color: #fffdf7;
+}
+.record-button.is-saving {
+  cursor: wait;
+  opacity: 0.76;
+}
+#recordLabel {
+  flex: 0 0 36px;
+  text-align: left;
+}
+.record-button-icon {
+  flex: 0 0 9px;
+  width: 9px;
+  height: 9px;
+  min-width: 9px;
+  border-radius: 50%;
+  background: var(--accent);
+  box-shadow: 0 0 0 3px rgba(185, 84, 60, 0.16);
+}
+.record-button.is-recording .record-button-icon {
+  border-radius: 2px;
+  background: #fffdf7;
+  box-shadow: none;
+}
+.record-button-duration {
+  display: inline-block;
+  flex: 0 0 34px;
+  min-width: 34px;
+  text-align: right;
+  color: rgba(232, 234, 223, 0.7);
+}
+.record-button.is-recording .record-button-duration {
+  color: rgba(255, 253, 247, 0.86);
+}
+.preview-scale-control {
+  display: inline-flex;
+  align-items: center;
+  gap: 8px;
+  flex: 0 0 170px;
+  min-width: 170px;
+  margin: 0;
+  color: rgba(232, 234, 223, 0.72);
+  font-size: 11px;
+  line-height: 1;
+}
+.preview-scale-control input {
+  width: 92px;
+  min-width: 72px;
+  padding: 0;
+  border: 0;
+  background: transparent;
+  accent-color: #eef1ec;
+}
+.preview-scale-control b {
+  min-width: 36px;
+  color: #fffdf7;
+  font-weight: 650;
+}
+#statusText {
+  display: inline-block;
+  min-width: 92px;
+  line-height: 1;
+}
+#chunkText {
+  display: inline-block;
+  min-width: 70px;
+  line-height: 1;
+}
+.stage-stat {
+  display: inline-flex;
+  align-items: center;
+  gap: 5px;
+  flex: 0 1 auto;
+  min-width: 0;
+  color: rgba(232, 234, 223, 0.72);
+  line-height: 1;
+}
+.stage-stat b {
+  display: inline-block;
+  color: #fffdf7;
+  font-weight: 650;
+  font-variant-numeric: tabular-nums;
+}
+
+#outputSizeText { min-width: 206px; }
+#renderFps { min-width: 2ch; text-align: right; }
+#theoreticalFpsText { min-width: 116px; }
+#stageLatencyText { min-width: 120px; }
+
+@media (max-width: 1180px) {
+  .topbar {
+    flex-wrap: wrap;
+    height: auto;
+    min-height: 44px;
+    padding: 8px 14px;
+    row-gap: 7px;
+  }
+  .topbar-spacer { display: none; }
+  .preview-scale-control { flex-basis: 170px; min-width: 170px; }
+  #outputSizeText { min-width: 156px; }
+  #theoreticalFpsText { min-width: 100px; }
+  #stageLatencyText { min-width: 108px; }
+}
+
+.timeline { justify-content: flex-end; border-top: 1px solid rgba(232, 234, 223, 0.12); }
+.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--muted); }
+.dot.live { background: #8ecf9d; box-shadow: 0 0 0 4px rgba(142, 207, 157, 0.14); }
+.dot.error { background: var(--accent); }
+
+#viewport {
+  position: relative;
+  z-index: 1;
+  display: block;
+  width: 100%;
+  height: auto;
+  max-height: min(calc(56vh * var(--preview-scale, 1.2)), 82vh);
+  min-height: 0;
+  object-fit: contain;
+  image-rendering: auto;
+  transform: translateZ(0);
+}
+
+.preview-overlay {
+  position: absolute;
+  inset: 0;
+  z-index: 3;
+  display: none;
+  place-items: center;
+  pointer-events: none;
+  background: transparent;
+}
+
+.stage[data-preview-state="waiting"] .preview-overlay {
+  display: grid;
+}
+
+.preview-loader {
+  width: 18px;
+  height: 18px;
+  border-radius: 50%;
+  border: 2px solid rgba(238, 241, 236, 0.22);
+  border-top-color: rgba(238, 241, 236, 0.82);
+  animation: previewProgressSpin 0.8s linear infinite;
+}
+
+.stage-controls {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(180px, 1fr));
+  gap: 12px;
+  padding: 12px 14px 13px;
+  border-top: 1px solid rgba(232, 234, 223, 0.12);
+  background: #151912;
+}
+
+.control-cluster {
+  display: grid;
+  grid-template-columns: 46px 1fr;
+  gap: 10px;
+  align-items: center;
+}
+
+.control-title {
+  color: rgba(232, 234, 223, 0.62);
+  font-size: 11px;
+  text-transform: uppercase;
+  letter-spacing: 0.08em;
+}
+
+.stage-controls .camera-pad {
+  margin: 0;
+}
+
+.stage-controls .camera-pad button {
+  position: relative;
+  border-color: rgba(232, 234, 223, 0.18);
+  background: #eef1ec;
+  color: #11140f;
+}
+
+.stage-controls .camera-pad button:active:not(:disabled),
+.stage-controls .camera-pad button.is-pressed:not(:disabled),
+.stage-controls .camera-pad button.is-key-active:not(:disabled) {
+  border-color: var(--pressed-border);
+  background: var(--pressed);
+  color: #fffdf7;
+  box-shadow:
+    inset 0 0 0 1px rgba(255, 253, 247, 0.22),
+    0 0 0 3px var(--pressed-ring),
+    0 10px 22px rgba(23, 26, 22, 0.18);
+}
+
+.stage-controls .camera-pad button::after {
+  content: attr(data-key);
+  position: absolute;
+  right: 7px;
+  top: 5px;
+  color: color-mix(in oklch, var(--muted), var(--ink) 18%);
+  font-size: 10px;
+  font-weight: 650;
+}
+
+.stage-controls .camera-pad button:active::after,
+.stage-controls .camera-pad button.is-pressed::after,
+.stage-controls .camera-pad button.is-key-active::after {
+  color: rgba(255, 253, 247, 0.78);
+}
+
+.section-title {
+  margin: 16px 0 10px;
+  color: var(--muted);
+  font-size: 12px;
+  text-transform: uppercase;
+  letter-spacing: 0.08em;
+}
+
+.reference-upload {
+  margin-top: 0;
+}
+
+.reference-upload input {
+  border: 1px dashed var(--line);
+}
+
+#referencePreview {
+  display: block;
+  width: 100%;
+  aspect-ratio: 16 / 9;
+  min-height: 0;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  background: #e5e7df;
+}
+
+#referenceName {
+  min-height: 18px;
+  color: var(--muted);
+  font-size: 11px;
+}
+
+.spec-grid {
+  display: grid;
+  grid-template-columns: repeat(4, minmax(120px, 1fr));
+  gap: 8px;
+  margin-bottom: 12px;
+}
+
+.spec-grid span {
+  display: grid;
+  gap: 2px;
+  min-height: 46px;
+  align-content: center;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  background: #fffdf7;
+  padding: 9px;
+  color: var(--muted);
+  font-size: 11px;
+}
+
+.spec-grid b {
+  color: var(--ink);
+  font-size: 14px;
+}
+
+.presets {
+  position: static;
+  max-height: none;
+  overflow: visible;
+  scrollbar-gutter: stable;
+}
+
+.preset-list {
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
+  gap: 7px;
+  min-height: 0;
+  max-height: 230px;
+  margin-bottom: 12px;
+  overflow: auto;
+  padding-right: 3px;
+}
+.preset {
+  display: grid;
+  grid-template-columns: 72px minmax(0, 1fr);
+  gap: 4px 9px;
+  align-items: center;
+  padding: 8px;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  background: #fffdf7;
+  text-align: left;
+}
+.preset-thumb {
+  display: block;
+  grid-row: span 2;
+  width: 72px;
+  height: 46px;
+  min-height: 0;
+  object-fit: cover;
+  border-radius: 5px;
+  border: 1px solid color-mix(in oklch, var(--line), var(--ink) 8%);
+}
+.preset b { min-width: 0; font-size: 13px; }
+.preset span { min-width: 0; color: var(--muted); font-size: 11px; line-height: 1.25; }
+.preset[data-tone="green"] { border-left: 4px solid var(--green); }
+.preset[data-tone="blue"] { border-left: 4px solid var(--blue); }
+.preset[data-tone="accent"] { border-left: 4px solid var(--accent); }
+.preset:hover:not(:disabled) {
+  background: color-mix(in oklch, #fffdf7, var(--blue) 9%);
+}
+
+.camera-pad {
+  display: grid;
+  grid-template-columns: repeat(3, 1fr);
+  gap: 6px;
+  margin-bottom: 8px;
+}
+
+.camera-pad span { min-height: 36px; }
+.camera-pad button { min-height: 36px; font-size: 12px; }
+.telemetry { display: grid; gap: 7px; margin-top: 10px; }
+.telemetry span {
+  display: flex;
+  justify-content: space-between;
+  border-bottom: 1px solid var(--line);
+  padding-bottom: 8px;
+  color: var(--muted);
+  font-size: 12px;
+}
+.telemetry b { color: var(--ink); font-weight: 650; }
+
+.stage-telemetry {
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  gap: 0;
+  margin-top: 0;
+  border-top: 1px solid rgba(232, 234, 223, 0.12);
+  background: #11140f;
+}
+
+.stage-telemetry span {
+  min-height: 36px;
+  align-items: center;
+  gap: 8px;
+  border-right: 1px solid rgba(232, 234, 223, 0.1);
+  border-bottom: 1px solid rgba(232, 234, 223, 0.1);
+  padding: 0 14px;
+  color: rgba(232, 234, 223, 0.62);
+  font-size: 11px;
+}
+
+.stage-telemetry b {
+  color: #fffdf7;
+  font-size: 12px;
+}
+
+.history-list {
+  display: grid;
+  gap: 7px;
+  max-height: 92px;
+  overflow: auto;
+}
+
+.history-list span {
+  display: block;
+  border-left: 3px solid var(--blue);
+  background: #fffdf7;
+  padding: 8px 9px;
+  color: var(--muted);
+  font-size: 12px;
+}
+
+@media (max-width: 980px) {
+  .shell { grid-template-columns: 1fr; }
+  .presets { position: static; max-height: none; overflow: visible; }
+  .spec-grid { grid-template-columns: repeat(2, 1fr); }
+  .preset-list { min-height: 260px; max-height: 420px; }
+  #viewport { max-height: 420px; }
+  .stage-controls { grid-template-columns: 1fr; }
+  .stage-telemetry { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+  .topbar { flex-wrap: wrap; height: auto; min-height: 44px; padding: 10px 14px; }
+  .topbar-spacer { display: none; }
+  .preview-scale-control { min-width: 160px; }
+}
+
+@keyframes previewProgressSpin {
+  to { transform: rotate(360deg); }
+}

python/sglang/multimodal_gen/apps/webui/README.md

@@ -0,0 +1,58 @@
+# SGLang Diffusion WebUI User Guide
+
+SGLang Diffusion WebUI provides an intuitive Gradio-based interface for image and video generation, supporting parameter
+tuning and real-time previews.
+
+## Prerequisites
+
+The WebUI runs on Gradio. To get started, install Gradio first:
+
+```bash
+pip install gradio==6.1.0
+```
+
+## Launch WebUI Service
+
+SGLang Diffusion now includes an integrated WebUI. Simply add the `--webui` parameter when starting the service.
+
+### Launch Text-to-Image Service
+
+```bash
+sglang serve --model-path black-forest-labs/FLUX.1-dev --num-gpus 1 --webui --webui-port 2333
+```
+
+### Launch Text-to-Video Service
+
+```bash
+sglang serve --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers --num-gpus 1 --webui --webui-port 2333
+```
+
+### Launch Image-to-Image Service
+```bash
+sglang serve --model-path Qwen/Qwen-Image-Edit-2511 --num-gpus 1 --webui --webui-port 2333
+```
+
+### Launch Image-to-Video Service
+```bash
+sglang serve --model-path Wan-AI/Wan2.2-TI2V-5B-Diffusers --num-gpus 1 --webui --webui-port 2333
+```
+
+## Port Forwarding
+
+Once the WebUI service is running, you need to use **SSH port forwarding** to securely access the remote service from
+your local machine.
+
+In most cases: Your IDE (like VS Code, Cursor, etc.) can handle this automatically. Check your IDE's remote development
+or port forwarding features. Otherwise, execute this command manually.
+
+```bash
+ssh -L ${WEBUI_PORT}:localhost:${WEBUI_PORT} user_name@machine_name
+```
+
+Learn more about port forwarding: [Port Forwarding](https://en.wikipedia.org/wiki/Port_forwarding).
+
+## Interface Instructions
+
+You can view your model path and task name directly in the UI. We'd appreciate any feedback you'd like to share.
+
+Once launched, access the interface at `http://localhost:${WEBUI_PORT}` in your browser.

python/sglang/multimodal_gen/apps/webui/__init__.py

@@ -0,0 +1,3 @@
+from .main import run_sgl_diffusion_webui
+
+__all__ = ["run_sgl_diffusion_webui"]

python/sglang/multimodal_gen/apps/webui/main.py

@@ -0,0 +1,265 @@
+import argparse
+import os
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    DataType,
+    SamplingParams,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    post_process_sample,
+    prepare_request,
+)
+from sglang.multimodal_gen.runtime.scheduler_client import sync_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.srt.environ import envs
+
+logger = init_logger(__name__)
+
+
+def add_webui_args(parser: argparse.ArgumentParser):
+    """Add the arguments for the generate command."""
+    parser = ServerArgs.add_cli_args(parser)
+    parser = SamplingParams.add_cli_args(parser)
+    return parser
+
+
+def run_sgl_diffusion_webui(server_args: ServerArgs):
+    # import gradio in function to avoid CI crash
+
+    import gradio as gr
+
+    def resolve_model_repo_id(model_path: str) -> str:
+        from pathlib import Path
+
+        from huggingface_hub.utils import HFValidationError, validate_repo_id
+
+        try:
+            validate_repo_id(model_path)
+            return model_path
+        except HFValidationError:
+            pass
+
+        p = Path(model_path).expanduser()
+        parts = p.parts
+
+        if len(parts) < 2:
+            raise ValueError(f"Invalid model_path: {model_path}")
+
+        candidate = f"{parts[-2]}/{parts[-1]}"
+        validate_repo_id(candidate)  # let it raise if invalid
+        return candidate
+
+    # Prefer the hub pipeline tag for Hub models; fall back to the loaded pipeline's
+    # own task_type for local checkpoints (e.g. a single .safetensors path), which
+    # have no hub repo to query.
+    task_name = None
+    try:
+        repo_id = resolve_model_repo_id(server_args.model_path)
+        if envs.SGLANG_USE_MODELSCOPE.get():
+            from modelscope.hub.api import HubApi
+
+            api = HubApi()
+            model_info_obj = api.model_info(repo_id)
+            task_name = model_info_obj.tasks[0]["Name"].replace("-synthesis", "")
+        else:
+            from huggingface_hub import model_info
+
+            task_name = model_info(repo_id).pipeline_tag
+    except Exception as e:
+        logger.info(
+            "Could not resolve task from the model hub (%s); using the loaded "
+            "pipeline's task_type.",
+            e,
+        )
+
+    # init client
+    sync_scheduler_client.initialize(server_args)
+
+    if task_name in ("text-to-video", "image-to-video", "video-to-video"):
+        task_type = "video"
+    elif task_name in ("text-to-image", "image-to-image"):
+        task_type = "image"
+    else:
+        task_type = (
+            "image" if server_args.pipeline_config.task_type.is_image_gen() else "video"
+        )
+        task_name = task_name or server_args.pipeline_config.task_type.name
+    video_visible_only = task_type == "video"
+    image_visible_only = task_type == "image"
+
+    # server_args will be reused in gradio_generate function
+    def gradio_generate(
+        prompt,
+        negative_prompt,
+        reference_image_paths_str,
+        seed,
+        num_frames,
+        frames_per_second,
+        width,
+        height,
+        num_inference_steps,
+        guidance_scale,
+        enable_teacache,
+    ):
+        """
+        NOTE: The input and output of function which is called by gradio button must be gradio components
+        So we use global variable sampling_params_kwargs to avoid pass this param, because gradio does not support this.
+        return [ np.ndarray, None ] | [None, np.ndarray]
+        """
+        if reference_image_paths_str:
+            if "，" in reference_image_paths_str:
+                logger.warning(
+                    f"Warning: please use English comma to separate the reference image paths, and the reference image paths is: {reference_image_paths_str}"
+                )
+                reference_image_paths_str = reference_image_paths_str.replace("，", ",")
+            image_path = [path.strip() for path in reference_image_paths_str.split(",")]
+        else:
+            image_path = None
+
+        sampling_params_kwargs = dict(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            image_path=image_path,
+            seed=seed,
+            num_frames=num_frames,
+            fps=frames_per_second,
+            width=width,
+            height=height,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            enable_teacache=enable_teacache,
+            return_file_paths_only=False,
+        )
+        sampling_params = SamplingParams.from_user_sampling_params_args(
+            server_args.model_path,
+            server_args=server_args,
+            **sampling_params_kwargs,
+        )
+        batch = prepare_request(
+            server_args=server_args,
+            sampling_params=sampling_params,
+        )
+        result = sync_scheduler_client.forward([batch])
+        save_file_path = str(os.path.join(batch.output_path, batch.output_file_name))
+        if result.output is None:
+            sampling_params_str = "\n".join(
+                [f"{key}: {value}" for key, value in sampling_params_kwargs.items()]
+            )
+            no_output_msg = f"No output is generated by client, and their sampling params is: {sampling_params_str}"
+
+            if batch.data_type == DataType.VIDEO:
+                if os.path.exists(save_file_path):
+                    logger.warning(no_output_msg)
+                    return None, save_file_path
+                else:
+                    no_output_msg += f"\nAnd the expected output file was not found at: {save_file_path}"
+                    raise ValueError(no_output_msg)
+            else:
+                raise ValueError(no_output_msg)
+
+        frames = post_process_sample(
+            result.output[0],
+            batch.data_type,
+            batch.fps,
+            batch.save_output,
+            save_file_path,
+        )
+        if batch.data_type == DataType.VIDEO:
+            # gradio video need video path to show video
+            return None, save_file_path
+        else:
+            return frames[0], None
+
+    with gr.Blocks() as demo:
+        gr.Markdown("# 🚀 SGLang Diffusion Application")
+        with gr.Row():
+            gr.Textbox(label="Model", value=server_args.model_path)
+            gr.Textbox(label="Task name", value=task_name)
+
+        with gr.Row():
+            with gr.Column(scale=4):
+                prompt = gr.Textbox(label="Prompt", value="A curious raccoon")
+                negative_prompt = gr.Textbox(
+                    label="Negative_prompt",
+                    value="Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards",
+                )
+            with gr.Column(scale=1):
+                seed = gr.Number(label="seed", precision=0, value=1234)
+                run_btn = gr.Button("Generate", variant="primary", size="lg")
+
+        with gr.Row():
+            with gr.Column():
+                width = gr.Number(label="width", precision=0, value=720)
+                height = gr.Number(label="height", precision=0, value=480)
+                num_inference_steps = gr.Slider(
+                    minimum=0, maximum=50, value=20, step=1, label="num_inference_steps"
+                )
+                guidance_scale = gr.Slider(
+                    minimum=0.0, maximum=10, value=5, step=0.01, label="guidance_scale"
+                )
+                num_frames = gr.Slider(
+                    minimum=1,
+                    maximum=181,
+                    value=81,
+                    step=1,
+                    label="num_frames",
+                    visible=video_visible_only,
+                )
+                frames_per_second = gr.Slider(
+                    minimum=4,
+                    maximum=60,
+                    value=16,
+                    step=1,
+                    label="frames_per_second",
+                    visible=video_visible_only,
+                )
+                reference_image_paths_str = gr.Textbox(
+                    label="reference images",
+                    placeholder="Examples: 'image1.png, image2.png' or 'https://example.com/image1.png, https://example.com/image2.png'",
+                )
+                enable_teacache = gr.Checkbox(label="enable_teacache", value=False)
+
+            with gr.Column():
+                image_out = gr.Image(
+                    label="Generated Image", visible=image_visible_only, format="png"
+                )
+                video_out = gr.Video(
+                    label="Generated Video", visible=video_visible_only
+                )
+
+        run_btn.click(
+            fn=gradio_generate,
+            inputs=[
+                prompt,
+                negative_prompt,
+                reference_image_paths_str,
+                seed,
+                num_frames,
+                frames_per_second,
+                width,
+                height,
+                num_inference_steps,
+                guidance_scale,
+                enable_teacache,
+            ],
+            outputs=[image_out, video_out],
+        )
+
+        _, local_url, _ = demo.launch(
+            server_port=server_args.webui_port,
+            quiet=True,
+            prevent_thread_lock=True,
+            show_error=True,
+        )
+
+        # print banner
+        delimiter = "=" * 80
+        url = local_url or f"http://localhost:{server_args.webui_port}"
+        print(f"""
+{delimiter}
+\033[1mSGLang Diffusion WebUI available at:\033[0m \033[1;4;92m{url}\033[0m
+{delimiter}
+""")
+
+        demo.block_thread()