chore: import upstream snapshot with attribution

python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py

@@ -0,0 +1,252 @@
+import time
+from typing import Any, List, Optional, Union
+
+from fastapi import APIRouter, Body, HTTPException
+from pydantic import BaseModel, Field
+
+from sglang.multimodal_gen.registry import get_model_info
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    ListLorasReq,
+    MergeLoraWeightsReq,
+    SetLoraReq,
+    UnmergeLoraWeightsReq,
+    format_lora_message,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.srt.utils.json_response import orjson_response
+
+router = APIRouter(prefix="/v1")
+logger = init_logger(__name__)
+
+
+class ModelCard(BaseModel):
+    """Model cards."""
+
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "sglang"
+    root: Optional[str] = None
+    parent: Optional[str] = None
+    max_model_len: Optional[int] = None
+
+
+class DiffusionModelCard(ModelCard):
+    """Extended ModelCard with diffusion-specific fields."""
+
+    num_gpus: Optional[int] = None
+    task_type: Optional[str] = None
+    dit_precision: Optional[str] = None
+    vae_precision: Optional[str] = None
+    pipeline_name: Optional[str] = None
+    pipeline_class: Optional[str] = None
+
+
+async def _handle_lora_request(req: Any, success_msg: str, failure_msg: str):
+    try:
+        output: OutputBatch = await async_scheduler_client.forward(req)
+        if output.error is None:
+            return {"status": "ok", "message": success_msg}
+        else:
+            error_msg = output.error
+            raise HTTPException(status_code=500, detail=f"{failure_msg}: {error_msg}")
+    except Exception as e:
+        if isinstance(e, HTTPException):
+            raise
+        logger.error(f"Error during '{failure_msg}': {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/set_lora")
+async def set_lora(
+    lora_nickname: Union[str, List[str]] = Body(..., embed=True),
+    lora_path: Optional[Union[str, List[Optional[str]]]] = Body(None, embed=True),
+    target: Union[str, List[str]] = Body("all", embed=True),
+    strength: Union[float, List[float]] = Body(1.0, embed=True),
+    merge_mode: Optional[str] = Body(None, embed=True),
+):
+    """
+    Set LoRA adapter(s) for the specified transformer(s).
+    Supports both single LoRA (backward compatible) and multiple LoRA adapters.
+
+    Args:
+        lora_nickname: The nickname(s) of the adapter(s). Can be a string or a list of strings.
+        lora_path: Path(s) to the LoRA adapter(s) (local path or HF repo id).
+            Can be a string, None, or a list of strings/None. Must match the length of lora_nickname.
+        target: Which transformer(s) to apply the LoRA to. Can be a string or a list of strings.
+            If a list, must match the length of lora_nickname. Valid values:
+            - "all": Apply to all transformers (default)
+            - "transformer": Apply only to the primary transformer (high noise for Wan2.2)
+            - "transformer_2": Apply only to transformer_2 (low noise for Wan2.2)
+            - "critic": Apply only to the critic model
+        strength: LoRA strength(s) for merge, default 1.0. Can be a float or a list of floats.
+            If a list, must match the length of lora_nickname. Values < 1.0 reduce the effect,
+            values > 1.0 amplify the effect.
+        merge_mode: Optional LoRA merge mode: "auto", "merge", or "dynamic".
+    """
+    req = SetLoraReq(
+        lora_nickname=lora_nickname,
+        lora_path=lora_path,
+        target=target,
+        strength=strength,
+        merge_mode=merge_mode,
+    )
+    nickname_str, target_str, strength_str = format_lora_message(
+        lora_nickname, target, strength
+    )
+
+    return await _handle_lora_request(
+        req,
+        f"Successfully set LoRA adapter(s): {nickname_str} (target: {target_str}, strength: {strength_str})",
+        "Failed to set LoRA adapter",
+    )
+
+
+@router.post("/merge_lora_weights")
+async def merge_lora_weights(
+    target: str = Body("all", embed=True),
+    strength: float = Body(1.0, embed=True),
+):
+    """
+    Merge LoRA weights into the base model.
+
+    Args:
+        target: Which transformer(s) to merge. One of "all", "transformer",
+                "transformer_2", "critic".
+        strength: LoRA strength for merge, default 1.0. Values < 1.0 reduce the effect,
+            values > 1.0 amplify the effect.
+    """
+    req = MergeLoraWeightsReq(target=target, strength=strength)
+    return await _handle_lora_request(
+        req,
+        f"Successfully merged LoRA weights (target: {target}, strength: {strength})",
+        "Failed to merge LoRA weights",
+    )
+
+
+@router.post("/unmerge_lora_weights")
+async def unmerge_lora_weights(
+    target: str = Body("all", embed=True),
+):
+    """
+    Unmerge LoRA weights from the base model.
+
+    Args:
+        target: Which transformer(s) to unmerge. One of "all", "transformer",
+                "transformer_2", "critic".
+    """
+    req = UnmergeLoraWeightsReq(target=target)
+    return await _handle_lora_request(
+        req,
+        f"Successfully unmerged LoRA weights (target: {target})",
+        "Failed to unmerge LoRA weights",
+    )
+
+
+@router.get("/model_info")
+async def model_info():
+    """Get the model information."""
+    server_args = get_global_server_args()
+    if not server_args:
+        raise HTTPException(status_code=500, detail="Server args not initialized")
+
+    result = {
+        "model_path": server_args.model_path,
+    }
+    return result
+
+
+@router.get("/list_loras")
+async def list_loras():
+    """List loaded LoRA adapters and current application status per module."""
+    try:
+        req = ListLorasReq()
+        output: OutputBatch = await async_scheduler_client.forward(req)
+        if output.error is None:
+            return output.output or {}
+        else:
+            raise HTTPException(status_code=500, detail=output.error)
+    except Exception as e:
+        if isinstance(e, HTTPException):
+            raise
+        logger.error(f"Error during 'list_loras': {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/models")
+async def available_models():
+    """Show available models. OpenAI-compatible endpoint with extended diffusion info."""
+    server_args = get_global_server_args()
+    if not server_args:
+        raise HTTPException(status_code=500, detail="Server args not initialized")
+
+    model_info = get_model_info(
+        server_args.model_path,
+        backend=server_args.backend,
+        model_id=server_args.model_id,
+    )
+
+    card_kwargs = {
+        "id": server_args.model_path,
+        "root": server_args.model_path,
+        # Extended diffusion-specific fields
+        "num_gpus": server_args.num_gpus,
+        "task_type": server_args.pipeline_config.task_type.name,
+        "dit_precision": server_args.pipeline_config.dit_precision,
+        "vae_precision": server_args.pipeline_config.vae_precision,
+    }
+
+    if model_info:
+        card_kwargs["pipeline_name"] = model_info.pipeline_cls.pipeline_name
+        card_kwargs["pipeline_class"] = model_info.pipeline_cls.__name__
+
+    model_card = DiffusionModelCard(**card_kwargs)
+
+    # Return dict directly to preserve extended fields (ModelList strips them)
+    return {"object": "list", "data": [model_card.model_dump()]}
+
+
+@router.get("/models/{model:path}")
+async def retrieve_model(model: str):
+    """Retrieve a model instance. OpenAI-compatible endpoint with extended diffusion info."""
+    server_args = get_global_server_args()
+    if not server_args:
+        raise HTTPException(status_code=500, detail="Server args not initialized")
+
+    if model != server_args.model_path:
+        return orjson_response(
+            {
+                "error": {
+                    "message": f"The model '{model}' does not exist",
+                    "type": "invalid_request_error",
+                    "param": "model",
+                    "code": "model_not_found",
+                }
+            },
+            status_code=404,
+        )
+
+    model_info = get_model_info(
+        server_args.model_path,
+        backend=server_args.backend,
+        model_id=server_args.model_id,
+    )
+
+    card_kwargs = {
+        "id": model,
+        "root": model,
+        "num_gpus": server_args.num_gpus,
+        "task_type": server_args.pipeline_config.task_type.name,
+        "dit_precision": server_args.pipeline_config.dit_precision,
+        "vae_precision": server_args.pipeline_config.vae_precision,
+    }
+
+    if model_info:
+        card_kwargs["pipeline_name"] = model_info.pipeline_cls.pipeline_name
+        card_kwargs["pipeline_class"] = model_info.pipeline_cls.__name__
+
+    # Return dict to preserve extended fields
+    return DiffusionModelCard(**card_kwargs).model_dump()

python/sglang/multimodal_gen/runtime/entrypoints/openai/image_api.py

@@ -0,0 +1,447 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+import base64
+import contextlib
+import json
+import os
+import time
+from typing import Any, List, Optional
+
+from fastapi import (
+    APIRouter,
+    File,
+    Form,
+    HTTPException,
+    Path,
+    Query,
+    Request,
+    UploadFile,
+)
+from fastapi.responses import FileResponse
+
+from sglang.multimodal_gen.configs.sample.sampling_params import generate_request_id
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    ImageGenerationsRequest,
+    ImageResponse,
+    ImageResponseData,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.storage import cloud_storage
+from sglang.multimodal_gen.runtime.entrypoints.openai.stores import IMAGE_STORE
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    add_common_data_to_response,
+    build_sampling_params,
+    choose_output_image_ext,
+    flatten_extra_params,
+    merge_image_input_list,
+    process_generation_batch,
+    save_image_to_path,
+    temp_dir_if_disabled,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.srt.observability.trace import extract_trace_headers
+
+router = APIRouter(prefix="/v1/images", tags=["images"])
+
+
+def _get_extra_field(request, field_name):
+    """Get a field from model_extra, with fallback to nested extra_body dict."""
+    extra = request.model_extra or {}
+    value = extra.get(field_name)
+    if value is not None:
+        return value
+    if field_name == "use_guardrails" and extra.get("guardrails") is not None:
+        return extra["guardrails"]
+
+    for container_name in ("extra_body", "extra_json", "extra_args", "extra_params"):
+        value = _parse_extra_container(extra.get(container_name)).get(field_name)
+        if value is not None:
+            return value
+
+    return value
+
+
+def _parse_extra_container(value: Any) -> dict[str, Any]:
+    if isinstance(value, str):
+        try:
+            value = json.loads(value)
+        except Exception:
+            return {}
+    if isinstance(value, dict):
+        return flatten_extra_params(dict(value))
+    return {}
+
+
+def _read_b64_for_paths(paths: list[str]) -> list[str]:
+    """Read and base64-encode each file. Must be called before cloud upload deletes them."""
+    result = []
+    for path in paths:
+        with open(path, "rb") as f:
+            result.append(base64.b64encode(f.read()).decode("utf-8"))
+    return result
+
+
+def _build_image_response_kwargs(
+    save_file_path_list: list[str],
+    resp_format: str,
+    prompt: str,
+    request_id: str,
+    result: OutputBatch,
+    *,
+    b64_list: list[str] | None = None,
+    cloud_url: str | None = None,
+    fallback_url: str | None = None,
+    is_persistent: bool = True,
+) -> dict:
+    """Build ImageResponse data list.
+
+    For b64_json: uses pre-read b64_list (call _read_b64_for_paths first).
+    For url: uses cloud_url or fallback_url.
+    file_path is omitted when is_persistent=False to avoid exposing stale temp paths.
+    """
+    ret = None
+    if resp_format == "b64_json":
+        if not b64_list:
+            raise ValueError("b64_list required for b64_json response_format")
+        data = [
+            ImageResponseData(
+                b64_json=b64,
+                revised_prompt=prompt,
+                file_path=os.path.abspath(path) if is_persistent else None,
+            )
+            for b64, path in zip(b64_list, save_file_path_list)
+        ]
+        ret = {"data": data}
+    elif resp_format == "url":
+        url = cloud_url or fallback_url
+        if not url:
+            raise HTTPException(
+                status_code=400,
+                detail="response_format='url' requires cloud storage to be configured.",
+            )
+        ret = {
+            "data": [
+                ImageResponseData(
+                    url=url,
+                    revised_prompt=prompt,
+                    file_path=(
+                        os.path.abspath(save_file_path_list[0])
+                        if is_persistent
+                        else None
+                    ),
+                )
+            ],
+        }
+    else:
+        raise HTTPException(
+            status_code=400, detail=f"response_format={resp_format} is not supported"
+        )
+
+    ret = add_common_data_to_response(ret, request_id=request_id, result=result)
+
+    return ret
+
+
+@router.post("/generations", response_model=ImageResponse)
+async def generations(
+    request: ImageGenerationsRequest,
+    raw_request: Request,
+):
+    request_id = generate_request_id()
+    server_args = get_global_server_args()
+    is_cosmos3 = "cosmos3" in (server_args.model_path or "").lower()
+    ext = (
+        "png"
+        if is_cosmos3 and request.output_format is None
+        else choose_output_image_ext(request.output_format, request.background)
+    )
+
+    with temp_dir_if_disabled(server_args.output_path) as output_dir:
+        sampling = build_sampling_params(
+            request_id,
+            prompt=request.prompt,
+            size=request.size,
+            width=request.width,
+            height=request.height,
+            num_outputs_per_prompt=max(1, min(int(request.n or 1), 10)),
+            output_file_name=f"{request_id}.{ext}",
+            output_path=output_dir,
+            num_frames=1,
+            seed=request.seed,
+            generator_device=request.generator_device,
+            num_inference_steps=request.num_inference_steps,
+            guidance_scale=request.guidance_scale,
+            true_cfg_scale=request.true_cfg_scale,
+            negative_prompt=request.negative_prompt,
+            max_sequence_length=(
+                request.max_sequence_length
+                if request.max_sequence_length is not None
+                else _get_extra_field(request, "max_sequence_length")
+            ),
+            flow_shift=(
+                request.flow_shift
+                if request.flow_shift is not None
+                else _get_extra_field(request, "flow_shift")
+            ),
+            use_duration_template=_get_extra_field(request, "use_duration_template"),
+            use_resolution_template=_get_extra_field(
+                request, "use_resolution_template"
+            ),
+            use_system_prompt=_get_extra_field(request, "use_system_prompt"),
+            use_guardrails=_get_extra_field(request, "use_guardrails"),
+            enable_teacache=request.enable_teacache,
+            output_compression=request.output_compression,
+            output_quality=request.output_quality,
+            diffusers_kwargs=request.diffusers_kwargs,
+            enable_upscaling=request.enable_upscaling,
+            upscaling_model_path=request.upscaling_model_path,
+            upscaling_scale=request.upscaling_scale,
+            perf_dump_path=request.perf_dump_path,
+            use_pe=_get_extra_field(request, "use_pe"),
+            preset=_get_extra_field(request, "preset"),
+            progressive_mode=(
+                request.progressive_mode
+                if request.progressive_mode is not None
+                else _get_extra_field(request, "progressive_mode")
+            ),
+            progressive_levels=(
+                request.progressive_levels
+                if request.progressive_levels is not None
+                else _get_extra_field(request, "progressive_levels")
+            ),
+            progressive_delta=(
+                request.progressive_delta
+                if request.progressive_delta is not None
+                else _get_extra_field(request, "progressive_delta")
+            ),
+        )
+        trace_headers = extract_trace_headers(raw_request.headers)
+        batch = prepare_request(
+            server_args=server_args,
+            sampling_params=sampling,
+            external_trace_header=trace_headers,
+        )
+        # Add diffusers_kwargs if provided
+        if request.diffusers_kwargs:
+            batch.extra["diffusers_kwargs"] = request.diffusers_kwargs
+
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+        resp_format = (request.response_format or "b64_json").lower()
+        if (
+            is_cosmos3
+            and "response_format" not in request.model_fields_set
+            and request.response_format == "url"
+        ):
+            resp_format = "b64_json"
+
+        # read b64 before cloud upload may delete the local file
+        b64_list = (
+            _read_b64_for_paths(save_file_path_list)
+            if resp_format == "b64_json"
+            else None
+        )
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        is_persistent = server_args.output_path is not None
+        await IMAGE_STORE.upsert(
+            request_id,
+            {
+                "id": request_id,
+                "created_at": int(time.time()),
+                "file_path": None if cloud_url or not is_persistent else save_file_path,
+                "url": cloud_url,
+            },
+        )
+
+        response_kwargs = _build_image_response_kwargs(
+            save_file_path_list,
+            resp_format,
+            request.prompt,
+            request_id,
+            result,
+            b64_list=b64_list,
+            cloud_url=cloud_url,
+            fallback_url=f"/v1/images/{request_id}/content" if is_persistent else None,
+            is_persistent=is_persistent,
+        )
+
+    return ImageResponse(**response_kwargs)
+
+
+@router.post("/edits", response_model=ImageResponse)
+async def edits(
+    raw_request: Request,
+    image: Optional[List[UploadFile]] = File(None),
+    image_array: Optional[List[UploadFile]] = File(None, alias="image[]"),
+    url: Optional[List[str]] = Form(None),
+    url_array: Optional[List[str]] = Form(None, alias="url[]"),
+    prompt: str = Form(...),
+    mask: Optional[UploadFile] = File(None),
+    model: Optional[str] = Form(None),
+    n: Optional[int] = Form(1),
+    response_format: Optional[str] = Form(None),
+    size: Optional[str] = Form(None),
+    output_format: Optional[str] = Form(None),
+    background: Optional[str] = Form("auto"),
+    seed: Optional[int] = Form(None),
+    generator_device: Optional[str] = Form("cuda"),
+    user: Optional[str] = Form(None),
+    negative_prompt: Optional[str] = Form(None),
+    guidance_scale: Optional[float] = Form(None),
+    true_cfg_scale: Optional[float] = Form(None),
+    num_inference_steps: Optional[int] = Form(None),
+    output_quality: Optional[str] = Form("default"),
+    output_compression: Optional[int] = Form(None),
+    enable_teacache: Optional[bool] = Form(False),
+    enable_upscaling: Optional[bool] = Form(False),
+    upscaling_model_path: Optional[str] = Form(None),
+    upscaling_scale: Optional[int] = Form(4),
+    num_frames: int = Form(1),
+):
+    request_id = generate_request_id()
+    server_args = get_global_server_args()
+    # Resolve images from either `image` or `image[]` (OpenAI SDK sends `image[]` when list is provided)
+    images = image or image_array
+    urls = url or url_array
+
+    if (not images or len(images) == 0) and (not urls or len(urls) == 0):
+        raise HTTPException(
+            status_code=422, detail="Field 'image' or 'url' is required"
+        )
+
+    image_list = merge_image_input_list(images, urls)
+
+    with contextlib.ExitStack() as stack:
+        uploads_dir = stack.enter_context(
+            temp_dir_if_disabled(server_args.input_save_path)
+        )
+        output_dir = stack.enter_context(temp_dir_if_disabled(server_args.output_path))
+
+        input_paths = []
+        try:
+            for idx, img in enumerate(image_list):
+                filename = img.filename if hasattr(img, "filename") else f"image_{idx}"
+                input_path = await save_image_to_path(
+                    img,
+                    os.path.join(uploads_dir, f"{request_id}_{idx}_{filename}"),
+                    prefer_remote_source=server_args.input_save_path is None,
+                )
+                input_paths.append(input_path)
+        except Exception as e:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Failed to process image source: {str(e)}",
+            )
+
+        ext = choose_output_image_ext(output_format, background)
+        sampling = build_sampling_params(
+            request_id,
+            prompt=prompt,
+            size=size,
+            num_outputs_per_prompt=max(1, min(int(n or 1), 10)),
+            output_file_name=f"{request_id}.{ext}",
+            output_path=output_dir,
+            image_path=input_paths,
+            seed=seed,
+            generator_device=generator_device,
+            negative_prompt=negative_prompt,
+            guidance_scale=guidance_scale,
+            true_cfg_scale=true_cfg_scale,
+            num_inference_steps=num_inference_steps,
+            enable_teacache=enable_teacache,
+            num_frames=num_frames,
+            output_compression=output_compression,
+            output_quality=output_quality,
+            enable_upscaling=enable_upscaling,
+            upscaling_model_path=upscaling_model_path,
+            upscaling_scale=upscaling_scale,
+        )
+        trace_headers = extract_trace_headers(raw_request.headers)
+        batch = prepare_request(
+            server_args=server_args,
+            sampling_params=sampling,
+            external_trace_header=trace_headers,
+        )
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+        resp_format = (response_format or "b64_json").lower()
+
+        # read b64 before cloud upload may delete the local file
+        b64_list = (
+            _read_b64_for_paths(save_file_path_list)
+            if resp_format == "b64_json"
+            else None
+        )
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        is_persistent = server_args.output_path is not None
+        is_input_persistent = server_args.input_save_path is not None
+        await IMAGE_STORE.upsert(
+            request_id,
+            {
+                "id": request_id,
+                "created_at": int(time.time()),
+                "file_path": None if cloud_url or not is_persistent else save_file_path,
+                "url": cloud_url,
+                "input_image_paths": input_paths if is_input_persistent else None,
+                "num_input_images": len(input_paths),
+            },
+        )
+
+        response_kwargs = _build_image_response_kwargs(
+            save_file_path_list,
+            resp_format,
+            prompt,
+            request_id,
+            result,
+            b64_list=b64_list,
+            cloud_url=cloud_url,
+            fallback_url=f"/v1/images/{request_id}/content" if is_persistent else None,
+            is_persistent=is_persistent,
+        )
+
+    return ImageResponse(**response_kwargs)
+
+
+@router.get("/{image_id}/content")
+async def download_image_content(
+    image_id: str = Path(...), variant: Optional[str] = Query(None)
+):
+    item = await IMAGE_STORE.get(image_id)
+    if not item:
+        raise HTTPException(status_code=404, detail="Image not found")
+
+    if item.get("url"):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Image has been uploaded to cloud storage. Please use the cloud URL: {item.get('url')}",
+        )
+
+    file_path = item.get("file_path")
+    if not file_path:
+        raise HTTPException(
+            status_code=404,
+            detail="Image was not persisted on disk (output_path is disabled). Use b64_json response_format or configure cloud storage.",
+        )
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="Image is still being generated")
+
+    ext = os.path.splitext(file_path)[1].lower()
+    media_type = "image/jpeg"
+    if ext == ".png":
+        media_type = "image/png"
+    elif ext == ".webp":
+        media_type = "image/webp"
+
+    return FileResponse(
+        path=file_path, media_type=media_type, filename=os.path.basename(file_path)
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/mesh_api.py

@@ -0,0 +1,296 @@
+import asyncio
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from fastapi import (
+    APIRouter,
+    File,
+    Form,
+    HTTPException,
+    Path,
+    Query,
+    Request,
+    UploadFile,
+)
+from fastapi.responses import FileResponse
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    SamplingParams,
+    generate_request_id,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    MeshGenerationsRequest,
+    MeshListResponse,
+    MeshResponse,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.storage import cloud_storage
+from sglang.multimodal_gen.runtime.entrypoints.openai.stores import MESH_STORE
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    add_common_data_to_response,
+    merge_image_input_list,
+    process_generation_batch,
+    save_image_to_path,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+router = APIRouter(prefix="/v1/meshes", tags=["meshes"])
+
+
+def _normalize_format(fmt: Optional[str]) -> str:
+    fmt = (fmt or "glb").lower()
+    return fmt if fmt in ("glb", "obj") else "glb"
+
+
+def _build_sampling_params_from_request(
+    request_id: str, req: MeshGenerationsRequest, image_path: Optional[str] = None
+) -> SamplingParams:
+    ext = _normalize_format(req.output_format)
+
+    server_args = get_global_server_args()
+    sampling_kwargs: Dict[str, Any] = {
+        "request_id": request_id,
+        "prompt": req.prompt,
+        "num_frames": 1,
+        "image_path": [image_path] if image_path else None,
+        "save_output": True,
+        "output_file_name": f"{request_id}.{ext}",
+        "seed": req.seed,
+        "generator_device": req.generator_device,
+    }
+    if req.num_inference_steps is not None:
+        sampling_kwargs["num_inference_steps"] = req.num_inference_steps
+    if req.guidance_scale is not None:
+        sampling_kwargs["guidance_scale"] = req.guidance_scale
+    if req.negative_prompt is not None:
+        sampling_kwargs["negative_prompt"] = req.negative_prompt
+
+    return SamplingParams.from_user_sampling_params_args(
+        model_path=server_args.model_path,
+        server_args=server_args,
+        **sampling_kwargs,
+    )
+
+
+def _mesh_job_from_sampling(
+    request_id: str, req: MeshGenerationsRequest, sampling: SamplingParams
+) -> Dict[str, Any]:
+    return {
+        "id": request_id,
+        "object": "mesh",
+        "model": req.model or "",
+        "status": "queued",
+        "progress": 0,
+        "created_at": int(time.time()),
+        "format": _normalize_format(req.output_format),
+        "file_path": os.path.abspath(sampling.output_file_path()),
+    }
+
+
+async def _dispatch_job_async(job_id: str, batch: Req) -> None:
+    from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+
+    try:
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+
+        file_size = None
+        if os.path.exists(save_file_path):
+            file_size = os.path.getsize(save_file_path)
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        update_fields: Dict[str, Any] = {
+            "status": "completed",
+            "progress": 100,
+            "completed_at": int(time.time()),
+            "url": cloud_url,
+            "file_path": save_file_path if not cloud_url else None,
+            "file_size_bytes": file_size,
+        }
+        update_fields = add_common_data_to_response(
+            update_fields, request_id=job_id, result=result
+        )
+        await MESH_STORE.update_fields(job_id, update_fields)
+    except Exception as e:
+        logger.error(f"{e}")
+        await MESH_STORE.update_fields(
+            job_id, {"status": "failed", "error": {"message": str(e)}}
+        )
+
+
+@router.post("", response_model=MeshResponse)
+async def create_mesh(
+    request: Request,
+    image: Optional[List[UploadFile]] = File(None),
+    image_array: Optional[List[UploadFile]] = File(None, alias="image[]"),
+    url: Optional[List[str]] = Form(None),
+    url_array: Optional[List[str]] = Form(None, alias="url[]"),
+    prompt: Optional[str] = Form("generate 3d mesh"),
+    model: Optional[str] = Form(None),
+    seed: Optional[int] = Form(None),
+    generator_device: Optional[str] = Form("cuda"),
+    guidance_scale: Optional[float] = Form(None),
+    num_inference_steps: Optional[int] = Form(None),
+    negative_prompt: Optional[str] = Form(None),
+    output_format: Optional[str] = Form("glb"),
+):
+    content_type = request.headers.get("content-type", "").lower()
+    request_id = generate_request_id()
+    server_args = get_global_server_args()
+
+    input_path = None
+
+    if "multipart/form-data" in content_type:
+        images = image or image_array
+        urls = url or url_array
+        image_list = merge_image_input_list(images, urls)
+
+        if not image_list:
+            raise HTTPException(
+                status_code=422,
+                detail="Field 'image' or 'url' is required for mesh generation",
+            )
+
+        uploads_dir = os.path.join("outputs", "uploads")
+        os.makedirs(uploads_dir, exist_ok=True)
+        img = image_list[0]
+        filename = img.filename if hasattr(img, "filename") else "input_image"
+        try:
+            input_path = await save_image_to_path(
+                img, os.path.join(uploads_dir, f"{request_id}_{filename}")
+            )
+        except Exception as e:
+            raise HTTPException(
+                status_code=400, detail=f"Failed to process image source: {str(e)}"
+            )
+
+        req = MeshGenerationsRequest(
+            prompt=prompt or "generate 3d mesh",
+            model=model,
+            seed=seed,
+            generator_device=generator_device,
+            num_inference_steps=num_inference_steps,
+            negative_prompt=negative_prompt,
+            output_format=output_format,
+            **(
+                {"guidance_scale": guidance_scale} if guidance_scale is not None else {}
+            ),
+        )
+    else:
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        try:
+            payload: Dict[str, Any] = dict(body or {})
+
+            if payload.get("input_image"):
+                img_src = payload.pop("input_image")
+                uploads_dir = os.path.join("outputs", "uploads")
+                os.makedirs(uploads_dir, exist_ok=True)
+                input_path = await save_image_to_path(
+                    img_src,
+                    os.path.join(uploads_dir, f"{request_id}_input_image"),
+                )
+
+            req = MeshGenerationsRequest(**payload)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Invalid request body: {e}")
+
+    if not input_path:
+        raise HTTPException(
+            status_code=422,
+            detail="An input image is required for mesh generation",
+        )
+
+    sampling_params = _build_sampling_params_from_request(request_id, req, input_path)
+    job = _mesh_job_from_sampling(request_id, req, sampling_params)
+    await MESH_STORE.upsert(request_id, job)
+
+    batch = prepare_request(
+        server_args=server_args,
+        sampling_params=sampling_params,
+    )
+
+    asyncio.create_task(_dispatch_job_async(request_id, batch))
+    return MeshResponse(**job)
+
+
+@router.get("", response_model=MeshListResponse)
+async def list_meshes(
+    after: Optional[str] = Query(None),
+    limit: Optional[int] = Query(None, ge=1, le=100),
+    order: Optional[str] = Query("desc"),
+):
+    order = (order or "desc").lower()
+    if order not in ("asc", "desc"):
+        order = "desc"
+    jobs = await MESH_STORE.list_values()
+
+    reverse = order != "asc"
+    jobs.sort(key=lambda j: j.get("created_at", 0), reverse=reverse)
+
+    if after is not None:
+        try:
+            idx = next(i for i, j in enumerate(jobs) if j["id"] == after)
+            jobs = jobs[idx + 1 :]
+        except StopIteration:
+            jobs = []
+
+    if limit is not None:
+        jobs = jobs[:limit]
+    items = [MeshResponse(**j) for j in jobs]
+    return MeshListResponse(data=items)
+
+
+@router.get("/{mesh_id}", response_model=MeshResponse)
+async def retrieve_mesh(mesh_id: str = Path(...)):
+    job = await MESH_STORE.get(mesh_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Mesh not found")
+    return MeshResponse(**job)
+
+
+@router.delete("/{mesh_id}", response_model=MeshResponse)
+async def delete_mesh(mesh_id: str = Path(...)):
+    job = await MESH_STORE.pop(mesh_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Mesh not found")
+    job["status"] = "deleted"
+    return MeshResponse(**job)
+
+
+@router.get("/{mesh_id}/content")
+async def download_mesh_content(
+    mesh_id: str = Path(...), variant: Optional[str] = Query(None)
+):
+    job = await MESH_STORE.get(mesh_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Mesh not found")
+
+    if job.get("url"):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Mesh has been uploaded to cloud storage. Please use the cloud URL: {job.get('url')}",
+        )
+
+    file_path = job.get("file_path")
+    if not file_path or not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="Generation is still in-progress")
+
+    ext = os.path.splitext(file_path)[1].lower()
+    media_type = {
+        ".glb": "model/gltf-binary",
+        ".obj": "text/plain",
+    }.get(ext, "application/octet-stream")
+
+    return FileResponse(
+        path=file_path, media_type=media_type, filename=os.path.basename(file_path)
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py

@@ -0,0 +1,227 @@
+import time
+import uuid
+from abc import ABC
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# Image API protocol models
+class ImageResponseData(BaseModel):
+    b64_json: Optional[str] = None
+    url: Optional[str] = None
+    revised_prompt: Optional[str] = None
+    file_path: Optional[str] = None
+
+
+class ImageResponse(BaseModel):
+    id: str
+    created: int = Field(default_factory=lambda: int(time.time()))
+    data: List[ImageResponseData]
+    peak_memory_mb: Optional[float] = None
+    inference_time_s: Optional[float] = None
+
+
+class ImageGenerationsRequest(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    prompt: str
+    model: Optional[str] = None
+    n: Optional[int] = 1
+    quality: Optional[str] = "auto"
+    response_format: Optional[str] = "url"  # url | b64_json
+    size: Optional[str] = "1024x1024"  # e.g., 1024x1024
+    style: Optional[str] = "vivid"
+    background: Optional[str] = "auto"  # transparent | opaque | auto
+    output_format: Optional[str] = None  # png | jpeg | webp
+    user: Optional[str] = None
+    # SGLang extensions
+    width: Optional[int] = None
+    height: Optional[int] = None
+    num_inference_steps: Optional[int] = None
+    guidance_scale: Optional[float] = None
+    true_cfg_scale: Optional[float] = (
+        None  # for CFG vs guidance distillation (e.g., QwenImage)
+    )
+    seed: Optional[Union[int, List[int]]] = None
+    generator_device: Optional[str] = "cuda"
+    negative_prompt: Optional[str] = None
+    output_quality: Optional[str] = "default"
+    output_compression: Optional[int] = None
+    enable_teacache: Optional[bool] = False
+    max_sequence_length: Optional[int] = None
+    flow_shift: Optional[float] = None
+    # Upscaling
+    enable_upscaling: Optional[bool] = False
+    upscaling_model_path: Optional[str] = None
+    upscaling_scale: Optional[int] = 4
+    diffusers_kwargs: Optional[Dict[str, Any]] = None  # kwargs for diffusers backend
+    # Performance profiling
+    perf_dump_path: Optional[str] = None
+    # Progressive resolution generation
+    progressive_mode: Optional[str] = None
+    progressive_levels: Optional[int] = None
+    progressive_delta: Optional[float] = None
+
+
+# Video API protocol models
+class VideoResponse(BaseModel):
+    id: str
+    object: str = "video"
+    model: str = "sora-2"
+    status: str = "queued"
+    progress: int = 0
+    created_at: int = Field(default_factory=lambda: int(time.time()))
+    size: str = ""
+    seconds: str = "4"
+    quality: str = "standard"
+    url: Optional[str] = None
+    remixed_from_video_id: Optional[str] = None
+    completed_at: Optional[int] = None
+    expires_at: Optional[int] = None
+    error: Optional[Dict[str, Any]] = None
+    file_path: Optional[str] = None
+    file_paths: Optional[List[str]] = None
+    num_outputs: Optional[int] = None
+    peak_memory_mb: Optional[float] = None
+    inference_time_s: Optional[float] = None
+    action: Optional[Dict[str, Any]] = None
+
+
+class VideoGenerationsRequest(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    prompt: str
+    input_reference: Optional[str] = None
+    reference_url: Optional[str] = None
+    video_path: Optional[str] = None
+    video_url: Optional[str] = None
+    model: Optional[str] = None
+    n: Optional[int] = 1
+    num_outputs_per_prompt: Optional[int] = None
+    seconds: Optional[int] = 4
+    size: Optional[str] = ""
+    fps: Optional[int] = None
+    num_frames: Optional[int] = None
+    seed: Optional[Union[int, List[int]]] = None
+    generator_device: Optional[str] = "cuda"
+    # SGLang extensions
+    width: Optional[int] = None
+    height: Optional[int] = None
+    num_inference_steps: Optional[int] = None
+    guidance_scale: Optional[float] = None
+    guidance_scale_2: Optional[float] = None
+    true_cfg_scale: Optional[float] = (
+        None  # for CFG vs guidance distillation (e.g., QwenImage)
+    )
+    negative_prompt: Optional[str] = None
+    max_sequence_length: Optional[int] = None
+    flow_shift: Optional[float] = None
+    enable_teacache: Optional[bool] = False
+    # Frame interpolation
+    enable_frame_interpolation: Optional[bool] = False
+    frame_interpolation_exp: Optional[int] = 1  # 1=2×, 2=4×
+    frame_interpolation_scale: Optional[float] = 1.0
+    frame_interpolation_model_path: Optional[str] = None
+    # Upscaling
+    enable_upscaling: Optional[bool] = False
+    upscaling_model_path: Optional[str] = None
+    upscaling_scale: Optional[int] = 4
+    output_quality: Optional[str] = "default"
+    output_compression: Optional[int] = None
+    output_path: Optional[str] = None
+    diffusers_kwargs: Optional[Dict[str, Any]] = None  # kwargs for diffusers backend
+    # Performance profiling
+    perf_dump_path: Optional[str] = None
+
+
+class VideoListResponse(BaseModel):
+    data: List[VideoResponse]
+    object: str = "list"
+
+
+class VideoRemixRequest(BaseModel):
+    prompt: str
+
+
+class RealtimeVideoGenerationsRequest(VideoGenerationsRequest):
+    type: Literal["init"]
+    # WebSocket does not support multipart/form-data image uploads
+    first_frame: Optional[bytes | str] = None
+    condition_inputs: Optional[Dict[str, Any]] = None
+    max_chunks: Optional[int] = Field(default=None, ge=1)
+    seed: Optional[int] = 42
+    guidance_scale: Optional[float] = 1.0
+    size: Optional[str] = "832x480"
+    profile: Optional[bool] = False
+    num_profiled_timesteps: Optional[int] = None
+    profile_all_stages: Optional[bool] = False
+    realtime_output_format: Optional[Literal["raw", "webp", "jpeg"]] = None
+    realtime_preview_max_width: Optional[int] = None
+    realtime_output_pacing: Optional[bool] = False
+    realtime_causal_sink_size: Optional[int] = None
+    realtime_causal_kv_cache_num_frames: Optional[int] = None
+
+
+class RealtimeEvent(BaseModel):
+    type: Literal["event"]
+    kind: str
+    payload: Any = None
+    event_id: Optional[int] = None
+
+
+# Mesh API protocol models
+class MeshResponse(BaseModel):
+    id: str
+    object: str = "mesh"
+    model: str = ""
+    status: str = "queued"
+    progress: int = 0
+    created_at: int = Field(default_factory=lambda: int(time.time()))
+    format: str = "glb"
+    url: Optional[str] = None
+    completed_at: Optional[int] = None
+    expires_at: Optional[int] = None
+    error: Optional[Dict[str, Any]] = None
+    file_path: Optional[str] = None
+    file_size_bytes: Optional[int] = None
+    peak_memory_mb: Optional[float] = None
+    inference_time_s: Optional[float] = None
+
+
+class MeshGenerationsRequest(BaseModel):
+    prompt: str = "generate 3d mesh"
+    input_image: Optional[str] = None
+    model: Optional[str] = None
+    seed: Optional[Union[int, List[int]]] = None
+    generator_device: Optional[str] = "cuda"
+    num_inference_steps: Optional[int] = None
+    guidance_scale: Optional[float] = None
+    negative_prompt: Optional[str] = None
+    output_format: Optional[str] = "glb"
+
+
+class MeshListResponse(BaseModel):
+    data: List[MeshResponse]
+    object: str = "list"
+
+
+@dataclass
+class BaseReq(ABC):
+    rid: Optional[Union[str, List[str]]] = field(default=None, kw_only=True)
+    http_worker_ipc: Optional[str] = field(default=None, kw_only=True)
+
+    def regenerate_rid(self):
+        """Generate a new request ID and return it."""
+        if isinstance(self.rid, list):
+            self.rid = [uuid.uuid4().hex for _ in range(len(self.rid))]
+        else:
+            self.rid = uuid.uuid4().hex
+        return self.rid
+
+
+@dataclass
+class VertexGenerateReqInput(BaseReq):
+    instances: List[dict]
+    parameters: Optional[dict] = None

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/__init__.py

@@ -0,0 +1 @@
+# SPDX-License-Identifier: Apache-2.0

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/lingbot_world_realtime_adapter.py

@@ -0,0 +1,352 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+    BaseRealtimeModelAdapter,
+    RealtimeChunkInputs,
+    build_realtime_sampling_params,
+    save_realtime_first_frame,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.lingbot_world.constants import (
+    LINGBOT_CAMERA_ACTIONS_CONDITION,
+    LINGBOT_PROMPT_UPDATED_CONDITION,
+)
+from sglang.multimodal_gen.runtime.realtime.control_signals import (
+    ControlSignalQueue,
+    ParsedControlEventPayload,
+    parse_control_event_payload,
+)
+from sglang.multimodal_gen.runtime.realtime.states import (
+    RealtimeCameraControlState,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+        RealtimeChunkContext,
+    )
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+LINGBOT_REALTIME_DEFAULT_NUM_INFERENCE_STEPS = 4
+LINGBOT_REALTIME_MIN_CONDITION_CHUNKS = 2
+COMPOSITE_INPUT_EVENT_KIND = "composite_input"
+
+
+class LingBotWorldRealtimeState(RealtimeCameraControlState):
+    def __init__(self):
+        super().__init__(
+            min_pulse_items=1,
+            script_maxlen=512,
+            max_transitions=512,
+        )
+        self.prompt_queue = ControlSignalQueue(max_events={"prompt": 1})
+
+    def clear(self) -> None:
+        super().clear()
+        self.prompt_queue.clear()
+
+    def receive_prompt(self, prompt: str, *, event_id: int | None = None) -> None:
+        self.prompt_queue.push("prompt", prompt, event_id=event_id)
+
+    def parse_camera_control_event_payload(
+        self,
+        payload: Any,
+        *,
+        event_id: int | None,
+    ) -> ParsedControlEventPayload:
+        return parse_control_event_payload(
+            payload,
+            event_id=event_id,
+            kind="camera_actions",
+            normalize_state_payload=self._normalize_state_actions,
+            validate_script_payload=LingBotWorldRealtimeAdapter._validate_camera_actions,
+        )
+
+    def receive_parsed_camera_control_event_payload(
+        self,
+        parsed: ParsedControlEventPayload,
+        *,
+        event_id: int | None,
+    ) -> str:
+        if parsed.mode == "state":
+            transitions = parsed.payload
+            self.receive_camera_state_transitions(transitions)
+            return f"kind=camera_actions, mode=state, transitions={len(transitions)}"
+
+        camera_actions = parsed.payload
+        self.receive_camera_action_script(camera_actions, event_id=event_id)
+        return f"kind=camera_actions, mode=script, frames={len(camera_actions)}"
+
+    def receive_camera_control_event_payload(
+        self,
+        payload: Any,
+        *,
+        event_id: int | None,
+    ) -> str:
+        parsed = self.parse_camera_control_event_payload(payload, event_id=event_id)
+        return self.receive_parsed_camera_control_event_payload(
+            parsed, event_id=event_id
+        )
+
+    def sample_prompt(self) -> str:
+        prompt = self.prompt_queue.pop_latest("prompt")
+        if not isinstance(prompt, str):
+            raise ValueError("prompt event payload must be a string")
+        self.latest_sampled_event_id = self.prompt_queue.last_sampled_seq_id("prompt")
+        return prompt
+
+    def has_prompt(self) -> bool:
+        return self.prompt_queue.has_events("prompt")
+
+
+class LingBotWorldRealtimeAdapter(BaseRealtimeModelAdapter):
+    def create_state(self) -> LingBotWorldRealtimeState:
+        return LingBotWorldRealtimeState()
+
+    def _state(self, session: GenerateSession) -> LingBotWorldRealtimeState:
+        state = session.adapter_state
+        if not isinstance(state, LingBotWorldRealtimeState):
+            raise TypeError("LingBot realtime adapter state is not initialized")
+        return state
+
+    async def on_init(
+        self,
+        session: GenerateSession,
+        request: RealtimeVideoGenerationsRequest,
+    ) -> None:
+        condition_inputs = request.condition_inputs or {}
+        camera_actions = condition_inputs.get(LINGBOT_CAMERA_ACTIONS_CONDITION)
+        if camera_actions is not None:
+            state = self._state(session)
+            state.receive_camera_action_script(
+                self._validate_camera_actions(camera_actions)
+            )
+
+        await save_realtime_first_frame(session, request)
+
+    @staticmethod
+    def _validate_camera_actions(payload: Any) -> list[list[str]]:
+        if not isinstance(payload, list):
+            raise ValueError("camera_actions event payload must be list[list[str]]")
+        normalized = []
+        for frame_actions in payload:
+            if not isinstance(frame_actions, list):
+                raise ValueError("camera_actions event payload must be list[list[str]]")
+            normalized.append(list(frame_actions))
+        return normalized
+
+    def ingest_event(
+        self,
+        session: GenerateSession,
+        event: RealtimeEvent,
+    ) -> str:
+        state = self._state(session)
+        if event.kind == "camera_actions":
+            return self._ingest_camera_actions(state, event.payload, event.event_id)
+        elif event.kind == "prompt":
+            return self._ingest_prompt(state, event.payload, event.event_id)
+        elif event.kind == COMPOSITE_INPUT_EVENT_KIND:
+            return self._ingest_composite_input(state, event.payload, event.event_id)
+        raise ValueError(f"unsupported event kind: {event.kind}")
+
+    def _ingest_camera_actions(
+        self,
+        state: LingBotWorldRealtimeState,
+        payload: Any,
+        event_id: int | None,
+    ) -> str:
+        return state.receive_camera_control_event_payload(
+            payload,
+            event_id=event_id,
+        )
+
+    def _ingest_prompt(
+        self,
+        state: LingBotWorldRealtimeState,
+        payload: Any,
+        event_id: int | None,
+    ) -> str:
+        prompt = self._validate_prompt_payload(payload)
+        state.receive_prompt(prompt, event_id=event_id)
+        return f"kind=prompt, prompt_len={len(prompt)}"
+
+    @staticmethod
+    def _validate_prompt_payload(payload: Any) -> str:
+        if not isinstance(payload, str) or not payload:
+            raise ValueError("prompt event payload must be a non-empty string")
+        return payload
+
+    def _ingest_composite_input(
+        self,
+        state: LingBotWorldRealtimeState,
+        payload: Any,
+        event_id: int | None,
+    ) -> str:
+        if not isinstance(payload, dict):
+            raise ValueError("composite_input event payload must be a map")
+        input_types = payload.get("input_types")
+        if not isinstance(input_types, list) or not input_types:
+            raise ValueError(
+                "composite_input event payload requires non-empty input_types"
+            )
+
+        parsed_inputs = []
+        for input_type in input_types:
+            if not isinstance(input_type, str) or not input_type:
+                raise ValueError(
+                    "composite_input input_types must contain non-empty strings"
+                )
+            if input_type not in payload:
+                raise ValueError(f"composite_input event payload requires {input_type}")
+            parsed_inputs.append(
+                (
+                    input_type,
+                    self._parse_composite_input_item(
+                        state,
+                        input_type,
+                        payload[input_type],
+                        event_id,
+                    ),
+                )
+            )
+
+        input_logs = []
+        for input_type, parsed_payload in parsed_inputs:
+            input_logs.append(
+                self._ingest_parsed_composite_input_item(
+                    state,
+                    input_type,
+                    parsed_payload,
+                    event_id,
+                )
+            )
+        return f"kind=composite_input, inputs={input_logs}"
+
+    def _parse_composite_input_item(
+        self,
+        state: LingBotWorldRealtimeState,
+        input_type: str,
+        payload: Any,
+        event_id: int | None,
+    ) -> Any:
+        if input_type == "camera_actions":
+            return state.parse_camera_control_event_payload(
+                payload,
+                event_id=event_id,
+            )
+        if input_type == "prompt":
+            return self._validate_prompt_payload(payload)
+        raise ValueError(f"unsupported composite_input type: {input_type}")
+
+    def _ingest_parsed_composite_input_item(
+        self,
+        state: LingBotWorldRealtimeState,
+        input_type: str,
+        parsed_payload: Any,
+        event_id: int | None,
+    ) -> str:
+        if input_type == "camera_actions":
+            return state.receive_parsed_camera_control_event_payload(
+                parsed_payload,
+                event_id=event_id,
+            )
+        if input_type == "prompt":
+            state.receive_prompt(parsed_payload, event_id=event_id)
+            return f"kind=prompt, prompt_len={len(parsed_payload)}"
+        raise ValueError(f"unsupported composite_input type: {input_type}")
+
+    def sample_chunk_inputs(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> RealtimeChunkInputs:
+        """Samples user inputs (conditions) for the current RealtimeChunk from RealtimeStates"""
+        state = self._state(session)
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        prompt_updated = False
+        if chunk.index == 0:
+            prompt = request.prompt
+        elif state.has_prompt():
+            prompt = state.sample_prompt()
+            request.prompt = prompt
+            prompt_updated = True
+        else:
+            prompt = request.prompt
+
+        condition_inputs = {}
+        if prompt_updated:
+            condition_inputs[LINGBOT_PROMPT_UPDATED_CONDITION] = True
+        camera_actions = state.sample_camera_actions(chunk_size)
+        if camera_actions is not None:
+            condition_inputs[LINGBOT_CAMERA_ACTIONS_CONDITION] = camera_actions
+        return RealtimeChunkInputs(prompt=prompt, condition_inputs=condition_inputs)
+
+    def build_sampling_params(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_inputs: RealtimeChunkInputs,
+        chunk_size: int,
+    ):
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        num_frames = self._condition_num_frames(
+            request=request,
+            server_args=server_args,
+            chunk_size=chunk_size,
+        )
+
+        return build_realtime_sampling_params(
+            chunk.request_id,
+            request=request,
+            chunk_inputs=chunk_inputs,
+            num_frames=num_frames,
+            num_inference_steps=(
+                request.num_inference_steps
+                or LINGBOT_REALTIME_DEFAULT_NUM_INFERENCE_STEPS
+            ),
+            chunk_size=chunk_size,
+        )
+
+    @staticmethod
+    def _condition_num_frames(
+        *,
+        request: RealtimeVideoGenerationsRequest,
+        server_args: ServerArgs | None,
+        chunk_size: int,
+    ) -> int:
+        if server_args is None:
+            return int(request.num_frames or 0)
+
+        # encode one extra blank condition chunk so repeat-last never reuses
+        # the first-frame image mask on later realtime chunks
+        temporal_ratio = int(
+            server_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio
+        )
+        required_latent_frames = chunk_size * LINGBOT_REALTIME_MIN_CONDITION_CHUNKS
+        required_num_frames = (required_latent_frames - 1) * temporal_ratio + 1
+        return max(int(request.num_frames or 0), required_num_frames)
+
+    def get_realtime_event_id(self, session: GenerateSession) -> int | None:
+        return self._state(session).latest_sampled_event_id
+
+    def clear_state(self, session: GenerateSession) -> None:
+        state = session.adapter_state
+        if isinstance(state, LingBotWorldRealtimeState):
+            state.clear()

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/adapters/sana_wm_realtime_adapter.py

@@ -0,0 +1,264 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+    BaseRealtimeModelAdapter,
+    RealtimeChunkInputs,
+    build_realtime_sampling_params,
+    save_realtime_first_frame,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_wm.base import (
+    normalize_sana_wm_camera_actions,
+    parse_sana_wm_action_string,
+    snap_sana_wm_num_frames,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_wm.self_forcing import (
+    SanaWMSelfForcingSampler,
+)
+from sglang.multimodal_gen.runtime.realtime.states import (
+    RealtimeCameraControlState,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+        RealtimeChunkContext,
+    )
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
+        OutputBatch,
+    )
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+SANA_WM_DEFAULT_SIZE = "1280x704"
+SANA_WM_DEFAULT_NUM_FRAMES = 1081
+SANA_WM_DEFAULT_FPS = 16
+SANA_WM_DEFAULT_STEPS = 4
+SANA_WM_DEFAULT_GUIDANCE = 1.0
+SANA_WM_CONTROL_PULSE_FRAMES = 8
+
+
+def _normalize_sana_wm_state_actions(actions: list[Any]) -> list[str]:
+    return [str(action).lower() for action in actions]
+
+
+class SanaWMRealtimeAdapterState(RealtimeCameraControlState):
+    def __init__(self):
+        super().__init__(
+            min_pulse_items=SANA_WM_CONTROL_PULSE_FRAMES,
+            script_maxlen=2048,
+            max_transitions=512,
+            normalize_state_actions=_normalize_sana_wm_state_actions,
+        )
+        self.base_condition_inputs: dict[str, Any] = {}
+
+    def clear(self) -> None:
+        super().clear()
+        self.base_condition_inputs.clear()
+
+    def receive_camera_control_event_payload(
+        self,
+        payload: Any,
+        *,
+        event_id: int | None,
+    ) -> str:
+        return super().receive_camera_control_event_payload(
+            payload,
+            event_id=event_id,
+            validate_camera_actions=SanaWMRealtimeAdapter._validate_camera_actions,
+        )
+
+
+class SanaWMRealtimeAdapter(BaseRealtimeModelAdapter):
+    def create_state(self) -> SanaWMRealtimeAdapterState:
+        return SanaWMRealtimeAdapterState()
+
+    def _state(self, session: GenerateSession) -> SanaWMRealtimeAdapterState:
+        state = session.adapter_state
+        if not isinstance(state, SanaWMRealtimeAdapterState):
+            raise TypeError("SANA-WM realtime adapter state is not initialized")
+        return state
+
+    @staticmethod
+    def _validate_camera_actions(payload: Any) -> list[list[str]]:
+        return normalize_sana_wm_camera_actions(
+            payload, error_label="camera_actions event payload"
+        )
+
+    @staticmethod
+    def _raw_frame_count(result: OutputBatch) -> int | None:
+        if result.raw_frame_batches is None:
+            return None
+        return sum(len(frames) for frames in result.raw_frame_batches)
+
+    async def on_init(
+        self,
+        session: GenerateSession,
+        request: RealtimeVideoGenerationsRequest,
+    ) -> None:
+        request.size = request.size or SANA_WM_DEFAULT_SIZE
+        if request.num_frames is not None:
+            request.num_frames = int(request.num_frames)
+        else:
+            # Open-ended session: keep num_frames unset so prepare_next_request
+            # samples uniform action chunks (no front-loaded segmentation), and
+            # flag the stage explicitly via condition_inputs —
+            # build_sampling_params strips None fields, so the per-chunk batch
+            # would otherwise carry the SamplingParams default num_frames.
+            request.condition_inputs = {
+                **(request.condition_inputs or {}),
+                "sana_wm_open_ended": True,
+            }
+        request.fps = int(request.fps or SANA_WM_DEFAULT_FPS)
+        request.num_inference_steps = int(
+            request.num_inference_steps or SANA_WM_DEFAULT_STEPS
+        )
+        request.guidance_scale = float(
+            request.guidance_scale or SANA_WM_DEFAULT_GUIDANCE
+        )
+        if request.negative_prompt is None:
+            request.negative_prompt = ""
+        if request.generator_device is None:
+            request.generator_device = "cuda"
+
+        state = self._state(session)
+        condition_inputs = dict(request.condition_inputs or {})
+        camera_actions = condition_inputs.pop("camera_actions", None)
+        action = condition_inputs.pop("action", None)
+        if camera_actions is not None and action is not None:
+            raise ValueError("pass only one of camera_actions or action")
+        if camera_actions is not None:
+            state.receive_camera_control_event_payload(camera_actions, event_id=None)
+        if action is not None:
+            if not isinstance(action, str) or not action:
+                raise ValueError("action condition input must be a non-empty string")
+            state.receive_camera_action_script(
+                parse_sana_wm_action_string(action), event_id=None
+            )
+        state.base_condition_inputs = condition_inputs
+
+        await save_realtime_first_frame(
+            session,
+            request,
+            required_error="SANA-WM realtime requires first_frame",
+            cache_remote_urls=True,
+        )
+
+    def ingest_event(
+        self,
+        session: GenerateSession,
+        event: RealtimeEvent,
+    ) -> str:
+        state = self._state(session)
+        if event.kind == "camera_actions":
+            return state.receive_camera_control_event_payload(
+                event.payload,
+                event_id=event.event_id,
+            )
+        if event.kind == "action":
+            if not isinstance(event.payload, str) or not event.payload:
+                raise ValueError("action event payload must be a non-empty string")
+            camera_actions = parse_sana_wm_action_string(event.payload)
+            state.receive_camera_action_script(camera_actions, event_id=event.event_id)
+            return f"kind=action, frames={len(camera_actions)}"
+        raise ValueError(f"unsupported event kind: {event.kind}")
+
+    def sample_chunk_inputs(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> RealtimeChunkInputs:
+        action_chunk_size = self._action_chunk_size(
+            session,
+            server_args,
+            chunk,
+            chunk_size,
+        )
+        state = self._state(session)
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        condition_inputs = dict(state.base_condition_inputs) if chunk.index == 0 else {}
+        camera_actions = state.sample_camera_actions(action_chunk_size)
+        if camera_actions is not None:
+            condition_inputs["camera_actions"] = camera_actions
+        return RealtimeChunkInputs(
+            prompt=request.prompt,
+            condition_inputs=condition_inputs,
+        )
+
+    def build_sampling_params(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_inputs: RealtimeChunkInputs,
+        chunk_size: int,
+    ):
+        request = session.request
+        if request is None:
+            raise ValueError("realtime request is not initialized")
+
+        return build_realtime_sampling_params(
+            chunk.request_id,
+            request=request,
+            chunk_inputs=chunk_inputs,
+            num_frames=request.num_frames,
+            num_inference_steps=request.num_inference_steps,
+            chunk_size=chunk_size,
+        )
+
+    def _action_chunk_size(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> int:
+        temporal_compression = int(
+            server_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio
+        )
+        # Match action sampling to the latent span used by the batch path. Chunk
+        # 0 may carry a front-loaded remainder, so a fixed nfpb*tc action count
+        # would read static-padded camera poses and drift from batch output.
+        action_chunk_size = chunk_size * temporal_compression
+        req_num_frames = (
+            session.request.num_frames if session.request is not None else None
+        )
+        if req_num_frames is not None:
+            snapped = snap_sana_wm_num_frames(
+                int(req_num_frames), stride=temporal_compression
+            )
+            latent_t = (snapped - 1) // temporal_compression + 1
+            segments = SanaWMSelfForcingSampler.create_autoregressive_segments(
+                latent_t, chunk_size
+            )
+            idx = int(chunk.index)
+            if 0 <= idx and idx + 1 < len(segments):
+                action_chunk_size = (
+                    segments[idx + 1] - segments[idx]
+                ) * temporal_compression
+        return action_chunk_size
+
+    def get_realtime_event_id(self, session: GenerateSession) -> int | None:
+        return self._state(session).latest_sampled_event_id
+
+    def on_chunk_complete(self, session: GenerateSession, result: OutputBatch) -> None:
+        if session.request is not None and self._raw_frame_count(result) == 0:
+            session.request.max_chunks = session.generate_chunk_cnt + 1
+        session.generate_chunk_completed()
+
+    def clear_state(self, session: GenerateSession) -> None:
+        state = session.adapter_state
+        if isinstance(state, SanaWMRealtimeAdapterState):
+            state.clear()

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/generate_session.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+from uuid import uuid4
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.realtime.session import (
+    RealtimeSession,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+        BaseRealtimeModelAdapter,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeChunkContext:
+    session_id: str
+    index: int
+    request_id: str
+
+
+class GenerateSession:
+    """A realtime generation session"""
+
+    def __init__(self):
+        self.id = uuid4().hex
+        self.request: RealtimeVideoGenerationsRequest | None = None
+        self.input_temp_dir: str | None = None
+        self.generate_chunk_cnt = 0
+        self.current_chunk: RealtimeChunkContext | None = None
+        self.realtime_session = RealtimeSession()
+        self.adapter: BaseRealtimeModelAdapter | None = None
+        self.adapter_state: Any = None
+        self.output_pace_next_send_at: float | None = None
+        self.output_pace_last_event_id: int | None = None
+
+    def set_adapter(self, adapter: BaseRealtimeModelAdapter):
+        self.adapter = adapter
+        self.adapter_state = adapter.create_state()
+
+    def set_request(self, request: RealtimeVideoGenerationsRequest):
+        self.request = request
+
+    def dispose(self):
+        if self.adapter is not None:
+            self.adapter.dispose(self)
+        self.request = None
+        self.input_temp_dir = None
+        self.generate_chunk_cnt = 0
+        self.current_chunk = None
+        self.adapter = None
+        self.adapter_state = None
+        self.output_pace_next_send_at = None
+        self.output_pace_last_event_id = None
+        self.realtime_session.dispose()
+
+    def new_chunk(self) -> RealtimeChunkContext:
+        if self.current_chunk is not None:
+            raise RuntimeError("previous realtime chunk is still active")
+        chunk = RealtimeChunkContext(
+            session_id=self.id,
+            index=self.generate_chunk_cnt,
+            request_id=f"{self.id}_{uuid4().hex}",
+        )
+        self.current_chunk = chunk
+        return chunk
+
+    def generate_chunk_completed(self):
+        self.generate_chunk_cnt += 1
+        self.current_chunk = None
+
+    def reached_max_chunks(self) -> bool:
+        return (
+            self.request is not None
+            and self.request.max_chunks is not None
+            and self.generate_chunk_cnt >= self.request.max_chunks
+        )

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/realtime_adapter.py

@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import hashlib
+import os
+import tempfile
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from fastapi import WebSocket
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_output_adapter import (
+    RawRGBRealtimeOutputAdapter,
+    RealtimeFrameSendStats,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    build_sampling_params,
+    save_image_to_path,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    prepare_request,
+)
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+        RealtimeChunkContext,
+    )
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
+        OutputBatch,
+        Req,
+    )
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+@dataclass(slots=True)
+class RealtimeChunkInputs:
+    """Sampled from realtime control state, consumed by the Req"""
+
+    prompt: str
+    condition_inputs: dict[str, Any] = field(default_factory=dict)
+
+
+async def save_realtime_first_frame(
+    session: GenerateSession,
+    request: RealtimeVideoGenerationsRequest,
+    *,
+    required_error: str | None = None,
+    cache_remote_urls: bool = False,
+) -> None:
+    first_frame = request.first_frame
+    if first_frame is None:
+        if required_error is not None:
+            raise ValueError(required_error)
+        return
+
+    server_args = get_global_server_args()
+    if server_args.input_save_path is not None:
+        uploads_dir = server_args.input_save_path
+        os.makedirs(uploads_dir, exist_ok=True)
+    else:
+        if session.input_temp_dir is None:
+            session.input_temp_dir = tempfile.mkdtemp(prefix="sglang_input_")
+        uploads_dir = session.input_temp_dir
+
+    if (
+        cache_remote_urls
+        and isinstance(first_frame, str)
+        and first_frame.lower().startswith(("http://", "https://"))
+    ):
+        suffix = os.path.splitext(first_frame.split("?", 1)[0])[1]
+        digest = hashlib.sha256(first_frame.encode("utf-8")).hexdigest()[:16]
+        target_path = os.path.join(uploads_dir, f"realtime_ref_{digest}{suffix}")
+        if os.path.exists(target_path):
+            request.first_frame = target_path
+            return
+    else:
+        target_path = os.path.join(uploads_dir, f"{session.id}_first_frame")
+
+    request.first_frame = await save_image_to_path(first_frame, target_path)
+
+
+def build_realtime_sampling_params(
+    request_id: str,
+    *,
+    request: RealtimeVideoGenerationsRequest,
+    chunk_inputs: RealtimeChunkInputs,
+    num_frames: int | None,
+    num_inference_steps: int | None,
+    chunk_size: int,
+):
+    return build_sampling_params(
+        request_id,
+        prompt=chunk_inputs.prompt,
+        size=request.size,
+        num_frames=num_frames,
+        fps=request.fps,
+        image_path=request.first_frame,
+        output_file_name=request_id,
+        save_output=False,
+        seed=request.seed,
+        generator_device=request.generator_device,
+        num_inference_steps=num_inference_steps,
+        guidance_scale=request.guidance_scale,
+        guidance_scale_2=request.guidance_scale_2,
+        negative_prompt=request.negative_prompt,
+        enable_teacache=request.enable_teacache,
+        enable_frame_interpolation=request.enable_frame_interpolation,
+        frame_interpolation_exp=request.frame_interpolation_exp,
+        frame_interpolation_scale=request.frame_interpolation_scale,
+        frame_interpolation_model_path=request.frame_interpolation_model_path,
+        enable_upscaling=request.enable_upscaling,
+        upscaling_model_path=request.upscaling_model_path,
+        upscaling_scale=request.upscaling_scale,
+        diffusers_kwargs=request.diffusers_kwargs,
+        profile=request.profile,
+        num_profiled_timesteps=request.num_profiled_timesteps,
+        profile_all_stages=request.profile_all_stages,
+        perf_dump_path=request.perf_dump_path,
+        output_path=request.output_path,
+        output_compression=request.output_compression,
+        output_quality=request.output_quality,
+        condition_inputs=chunk_inputs.condition_inputs,
+        realtime_chunk_size=chunk_size,
+    )
+
+
+class BaseRealtimeModelAdapter:
+    def __init__(self):
+        self.output_adapter = RawRGBRealtimeOutputAdapter()
+
+    async def on_init(
+        self,
+        session: GenerateSession,
+        request: RealtimeVideoGenerationsRequest,
+    ) -> None:
+        raise NotImplementedError
+
+    def create_state(self) -> Any:
+        """create a state for managing runtime states"""
+        raise NotImplementedError
+
+    def ingest_event(
+        self,
+        session: GenerateSession,
+        event: RealtimeEvent,
+    ) -> str:
+        """
+        Ingest a realtime endpoint event and install it into the model's realtime control queues
+        """
+        raise NotImplementedError
+
+    async def wait_for_next_chunk(self, session: GenerateSession) -> None:
+        del session
+
+    def get_chunk_size(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+    ) -> int:
+        del session, chunk
+        arch_config = server_args.pipeline_config.dit_config.arch_config
+        return int(getattr(arch_config, "num_frames_per_block", 3))
+
+    def sample_chunk_inputs(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_size: int,
+    ) -> RealtimeChunkInputs:
+        raise NotImplementedError
+
+    def build_sampling_params(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+        chunk_inputs: RealtimeChunkInputs,
+        chunk_size: int,
+    ):
+        raise NotImplementedError
+
+    def get_realtime_event_id(self, session: GenerateSession) -> int | None:
+        del session
+        return None
+
+    def prepare_next_request(
+        self,
+        session: GenerateSession,
+        server_args: ServerArgs,
+        chunk: RealtimeChunkContext,
+    ) -> Req:
+        chunk_size = self.get_chunk_size(session, server_args, chunk)
+        chunk_inputs = self.sample_chunk_inputs(
+            session,
+            server_args,
+            chunk,
+            chunk_size,
+        )
+        sampling_params = self.build_sampling_params(
+            session,
+            server_args,
+            chunk,
+            chunk_inputs,
+            chunk_size,
+        )
+        batch = prepare_request(
+            server_args=server_args,
+            sampling_params=sampling_params,
+        )
+        self.apply_realtime_request_fields(
+            batch,
+            session,
+            chunk,
+            event_id=self.get_realtime_event_id(session),
+        )
+        return batch
+
+    def apply_realtime_request_fields(
+        self,
+        batch: Req,
+        session: GenerateSession,
+        chunk: RealtimeChunkContext,
+        *,
+        event_id: int | None,
+    ) -> None:
+        batch.realtime_session_id = session.id
+        batch.return_raw_frames = True
+        batch.block_idx = chunk.index
+        batch.realtime_event_id = event_id
+        if session.request is None:
+            return
+        batch.realtime_output_format = session.request.realtime_output_format
+        batch.realtime_preview_max_width = session.request.realtime_preview_max_width
+        batch.realtime_output_pacing = bool(session.request.realtime_output_pacing)
+        batch.realtime_causal_sink_size = session.request.realtime_causal_sink_size
+        batch.realtime_causal_kv_cache_num_frames = (
+            session.request.realtime_causal_kv_cache_num_frames
+        )
+
+    async def send_output(
+        self,
+        ws: WebSocket,
+        session: GenerateSession,
+        result: OutputBatch,
+        batch: Req,
+    ) -> RealtimeFrameSendStats:
+        """send the generate output (usually frames) back via websocket"""
+        return await self.output_adapter.send(ws, session, result, batch)
+
+    def on_chunk_complete(self, session: GenerateSession, result: OutputBatch) -> None:
+        del result
+        session.generate_chunk_completed()
+
+    def clear_state(self, session: GenerateSession) -> None:
+        del session
+
+    def dispose(self, session: GenerateSession) -> None:
+        self.clear_state(session)
+        self.output_adapter.reset()

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/realtime_output_adapter.py

@@ -0,0 +1,610 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import asyncio
+import io
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+import msgspec.msgpack
+from fastapi import WebSocket
+from PIL import Image
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.timer import (
+    RealtimeStageTimer,
+)
+from sglang.multimodal_gen.runtime.utils.realtime_video import (
+    JPEG_FRAME_CONTENT_TYPE,
+    RAW_RGB_CHANNELS,
+    RAW_RGB_CONTENT_TYPE,
+    WEBP_FRAME_CONTENT_TYPE,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+        GenerateSession,
+    )
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
+        OutputBatch,
+        Req,
+    )
+
+
+class RealtimeFrameBatchHeader(TypedDict, total=False):
+    type: str
+    request_id: str
+    chunk_index: int
+    content_type: str
+    num_frames: int
+    total_size: int
+    format: str
+    width: int
+    height: int
+    channels: int
+    bytes_per_frame: int
+    raw_size: int
+    encoding: str
+    delta_reference: str
+    payload_lengths: list[int]
+    event_id: int
+    frame_batch_index: int
+    num_frame_batches: int
+    is_final_frame_batch: bool
+
+
+class RealtimeFrameBatchMessage(RealtimeFrameBatchHeader, total=False):
+    payload: bytes
+
+
+class RealtimeFrameSendStats(TypedDict):
+    header_pack_ms: float
+    header_write_ms: float
+    raw_payload_build_ms: float
+    raw_write_ms: float
+    ws_write_ms: float
+    pace_wait_ms: float
+    raw_bytes: int
+    ws_payload_bytes: int
+    num_frames: int
+    num_batches: int
+    frame_shape: tuple[int, int, int] | None
+    content_type: str
+
+
+def empty_frame_send_stats(content_type: str = "") -> RealtimeFrameSendStats:
+    return {
+        "header_pack_ms": 0.0,
+        "header_write_ms": 0.0,
+        "raw_payload_build_ms": 0.0,
+        "raw_write_ms": 0.0,
+        "ws_write_ms": 0.0,
+        "pace_wait_ms": 0.0,
+        "raw_bytes": 0,
+        "ws_payload_bytes": 0,
+        "num_frames": 0,
+        "num_batches": 0,
+        "frame_shape": None,
+        "content_type": content_type,
+    }
+
+
+def _raw_rgb_frame_metadata(batch: Req) -> dict[str, int | str]:
+    frame_width = batch.width
+    frame_height = batch.height
+    if frame_width is None or frame_height is None:
+        return {}
+
+    frame_width = int(frame_width)
+    frame_height = int(frame_height)
+    if batch.enable_upscaling:
+        upscaling_scale = int(batch.upscaling_scale or 1)
+        frame_width *= upscaling_scale
+        frame_height *= upscaling_scale
+
+    return {
+        "format": "rgb24",
+        "width": frame_width,
+        "height": frame_height,
+        "channels": RAW_RGB_CHANNELS,
+        "bytes_per_frame": frame_width * frame_height * RAW_RGB_CHANNELS,
+    }
+
+
+def _frame_shape_from_metadata(
+    metadata: dict[str, int | str] | None,
+) -> tuple[int, int, int] | None:
+    if not metadata:
+        return None
+    return (
+        int(metadata["height"]),
+        int(metadata["width"]),
+        int(metadata["channels"]),
+    )
+
+
+RAW_RGB_FRAMES_PER_WS_MESSAGE = 16
+ENCODED_PREVIEW_FRAMES_PER_WS_MESSAGE = 6
+FRAME_BATCH_PACK_OFFLOAD_BYTES = 64 * 1024
+WEBP_DEFAULT_QUALITY = 90
+JPEG_DEFAULT_QUALITY = 95
+JPEG_SUBSAMPLING = 0
+RAW_LOSSLESS_OUTPUT_FORMAT = "raw"
+ENCODED_PREVIEW_FORMATS = {"webp", "jpeg"}
+
+
+@dataclass(frozen=True)
+class _TransportPayload:
+    content_type: str
+    payload: bytes
+    metadata: dict[str, int | str | bool | list[int]]
+
+
+def _split_frame_batch(
+    frames: list[bytes],
+    frames_per_message: int = RAW_RGB_FRAMES_PER_WS_MESSAGE,
+) -> list[list[bytes]]:
+    if not frames:
+        return [frames]
+    return [
+        frames[i : i + frames_per_message]
+        for i in range(0, len(frames), frames_per_message)
+    ]
+
+
+def _encode_rgb_frame_to_webp(
+    frame: bytes,
+    *,
+    width: int,
+    height: int,
+    quality: int,
+    preview_max_width: int | None,
+) -> bytes:
+    buffer = io.BytesIO()
+    image = _resize_preview_image(
+        Image.frombytes("RGB", (width, height), frame),
+        preview_max_width=preview_max_width,
+    )
+    image.save(
+        buffer,
+        format="WEBP",
+        quality=quality,
+        method=0,
+    )
+    return buffer.getvalue()
+
+
+def _encode_rgb_frame_to_jpeg(
+    frame: bytes,
+    *,
+    width: int,
+    height: int,
+    quality: int,
+    preview_max_width: int | None,
+) -> bytes:
+    buffer = io.BytesIO()
+    image = _resize_preview_image(
+        Image.frombytes("RGB", (width, height), frame),
+        preview_max_width=preview_max_width,
+    )
+    image.save(
+        buffer,
+        format="JPEG",
+        quality=quality,
+        subsampling=JPEG_SUBSAMPLING,
+    )
+    return buffer.getvalue()
+
+
+def _preview_dimensions(
+    *,
+    width: int,
+    height: int,
+    preview_max_width: int | None,
+) -> tuple[int, int]:
+    if (
+        preview_max_width is None
+        or preview_max_width <= 0
+        or width <= preview_max_width
+    ):
+        return width, height
+    preview_width = int(preview_max_width)
+    preview_height = max(1, round(height * preview_width / width))
+    return preview_width, preview_height
+
+
+def _resize_preview_image(
+    image: Image.Image,
+    *,
+    preview_max_width: int | None,
+) -> Image.Image:
+    width, height = image.size
+    preview_width, preview_height = _preview_dimensions(
+        width=width,
+        height=height,
+        preview_max_width=preview_max_width,
+    )
+    if (preview_width, preview_height) == image.size:
+        return image
+    return image.resize((preview_width, preview_height), Image.Resampling.BICUBIC)
+
+
+def _pack_frame_batch_message(
+    header: RealtimeFrameBatchHeader,
+    payload: bytes,
+) -> bytes:
+    message: RealtimeFrameBatchMessage = {
+        **header,
+        "type": "frame_batch",
+        "payload": payload,
+    }
+    return msgspec.msgpack.encode(message)
+
+
+def _pack_frame_batch_header(header: RealtimeFrameBatchHeader) -> bytes:
+    return msgspec.msgpack.encode(header)
+
+
+def _build_transport_payload(
+    transport_frames: list[bytes],
+    *,
+    content_type: str,
+    metadata: dict[str, int | str],
+    output_format: str | None,
+    transport_quality: int | None,
+    preview_max_width: int | None,
+) -> _TransportPayload:
+    payload_content_type = content_type
+    payload_metadata: dict[str, int | str | bool | list[int]] = {}
+    raw_payload = b""
+
+    if (
+        output_format in ENCODED_PREVIEW_FORMATS
+        and content_type == RAW_RGB_CONTENT_TYPE
+        and transport_frames
+    ):
+        if output_format == "webp":
+            encoded_frames = [
+                _encode_rgb_frame_to_webp(
+                    frame,
+                    width=int(metadata["width"]),
+                    height=int(metadata["height"]),
+                    quality=int(transport_quality or WEBP_DEFAULT_QUALITY),
+                    preview_max_width=preview_max_width,
+                )
+                for frame in transport_frames
+            ]
+            payload_content_type = WEBP_FRAME_CONTENT_TYPE
+        else:
+            encoded_frames = [
+                _encode_rgb_frame_to_jpeg(
+                    frame,
+                    width=int(metadata["width"]),
+                    height=int(metadata["height"]),
+                    quality=int(transport_quality or JPEG_DEFAULT_QUALITY),
+                    preview_max_width=preview_max_width,
+                )
+                for frame in transport_frames
+            ]
+            payload_content_type = JPEG_FRAME_CONTENT_TYPE
+        raw_payload = b"".join(encoded_frames)
+        preview_width, preview_height = _preview_dimensions(
+            width=int(metadata["width"]),
+            height=int(metadata["height"]),
+            preview_max_width=preview_max_width,
+        )
+        payload_metadata = {
+            "format": output_format,
+            "encoding": output_format,
+            "source_width": int(metadata["width"]),
+            "source_height": int(metadata["height"]),
+            "preview_width": preview_width,
+            "preview_height": preview_height,
+            "width": preview_width,
+            "height": preview_height,
+            "payload_lengths": [len(frame) for frame in encoded_frames],
+        }
+    elif content_type == RAW_RGB_CONTENT_TYPE and transport_frames:
+        raw_payload = b"".join(transport_frames)
+        payload_metadata = {
+            "raw_size": len(raw_payload),
+            "encoding": RAW_LOSSLESS_OUTPUT_FORMAT,
+        }
+    else:
+        raw_payload = b"".join(transport_frames)
+
+    return _TransportPayload(
+        content_type=payload_content_type,
+        payload=raw_payload,
+        metadata=payload_metadata,
+    )
+
+
+def _should_build_payload_off_loop(
+    *,
+    content_type: str,
+    output_format: str | None,
+    transport_frames: list[bytes],
+) -> bool:
+    if content_type != RAW_RGB_CONTENT_TYPE or not transport_frames:
+        return False
+    return output_format in ENCODED_PREVIEW_FORMATS or output_format is None
+
+
+def _is_encoded_preview_transport(
+    *,
+    content_type: str,
+    output_format: str | None,
+) -> bool:
+    return (
+        output_format in ENCODED_PREVIEW_FORMATS
+        and content_type == RAW_RGB_CONTENT_TYPE
+    )
+
+
+async def _build_encoded_preview_payloads(
+    split_batches: list[list[bytes]],
+    *,
+    content_type: str,
+    metadata: dict[str, int | str],
+    output_format: str,
+    transport_quality: int | None,
+    preview_max_width: int | None,
+    event_id: int | None,
+) -> list[_TransportPayload]:
+    return list(
+        await asyncio.gather(
+            *(
+                _build_encoded_preview_payload(
+                    transport_frames,
+                    metadata=metadata,
+                    output_format=output_format,
+                    transport_quality=transport_quality,
+                    preview_max_width=preview_max_width,
+                )
+                for transport_frames in split_batches
+            )
+        )
+    )
+
+
+async def _build_encoded_preview_payload(
+    transport_frames: list[bytes],
+    *,
+    metadata: dict[str, int | str],
+    output_format: str,
+    transport_quality: int | None,
+    preview_max_width: int | None,
+) -> _TransportPayload:
+    width = int(metadata["width"])
+    height = int(metadata["height"])
+    if output_format == "webp":
+        encoded_frames = list(
+            await asyncio.gather(
+                *(
+                    asyncio.to_thread(
+                        _encode_rgb_frame_to_webp,
+                        frame,
+                        width=width,
+                        height=height,
+                        quality=int(transport_quality or WEBP_DEFAULT_QUALITY),
+                        preview_max_width=preview_max_width,
+                    )
+                    for frame in transport_frames
+                )
+            )
+        )
+        payload_content_type = WEBP_FRAME_CONTENT_TYPE
+    else:
+        encoded_frames = list(
+            await asyncio.gather(
+                *(
+                    asyncio.to_thread(
+                        _encode_rgb_frame_to_jpeg,
+                        frame,
+                        width=width,
+                        height=height,
+                        quality=int(transport_quality or JPEG_DEFAULT_QUALITY),
+                        preview_max_width=preview_max_width,
+                    )
+                    for frame in transport_frames
+                )
+            )
+        )
+        payload_content_type = JPEG_FRAME_CONTENT_TYPE
+
+    preview_width, preview_height = _preview_dimensions(
+        width=width,
+        height=height,
+        preview_max_width=preview_max_width,
+    )
+    return _TransportPayload(
+        content_type=payload_content_type,
+        payload=b"".join(encoded_frames),
+        metadata={
+            "format": output_format,
+            "encoding": output_format,
+            "source_width": width,
+            "source_height": height,
+            "preview_width": preview_width,
+            "preview_height": preview_height,
+            "width": preview_width,
+            "height": preview_height,
+            "payload_lengths": [len(frame) for frame in encoded_frames],
+        },
+    )
+
+
+class RawRGBRealtimeOutputAdapter:
+    """send raw RGB over WebSocket using lossless transport"""
+
+    def __init__(self) -> None:
+        pass
+
+    def reset(self) -> None:
+        pass
+
+    async def send(
+        self,
+        ws: WebSocket,
+        session: GenerateSession,
+        result: OutputBatch,
+        batch: Req,
+    ) -> RealtimeFrameSendStats:
+        """send frames through ws"""
+        content_type = result.raw_frame_content_type
+        if result.raw_frame_batches is None:
+            return empty_frame_send_stats(content_type)
+        if batch.block_idx == 0:
+            self.reset()
+
+        frame_metadata = (
+            result.raw_frame_metadata or _raw_rgb_frame_metadata(batch)
+            if content_type == RAW_RGB_CONTENT_TYPE
+            else {}
+        )
+        output_format = getattr(batch, "realtime_output_format", None)
+        preview_max_width = getattr(batch, "realtime_preview_max_width", None)
+        stats = await self._send_frame_batches(
+            ws,
+            result.raw_frame_batches,
+            content_type=content_type,
+            chunk_index_start=batch.block_idx,
+            request_id=batch.request_id,
+            event_id=getattr(batch, "realtime_event_id", None),
+            frame_metadata=frame_metadata,
+            output_format=output_format,
+            transport_quality=getattr(batch, "output_compression", None),
+            preview_max_width=preview_max_width,
+        )
+        stats["frame_shape"] = _frame_shape_from_metadata(frame_metadata)
+        return stats
+
+    async def _send_frame_batches(
+        self,
+        ws: WebSocket,
+        frame_batches: list[list[bytes]],
+        *,
+        content_type: str,
+        chunk_index_start: int,
+        request_id: str,
+        event_id: int | None = None,
+        frame_metadata: dict[str, int | str] | None = None,
+        output_format: str | None = None,
+        transport_quality: int | None = None,
+        preview_max_width: int | None = None,
+    ) -> RealtimeFrameSendStats:
+        chunk_index = chunk_index_start
+        metadata = frame_metadata or {}
+        stats = empty_frame_send_stats(content_type)
+        for frames in frame_batches:
+            split_batches = (
+                _split_frame_batch(frames, ENCODED_PREVIEW_FRAMES_PER_WS_MESSAGE)
+                if _is_encoded_preview_transport(
+                    content_type=content_type,
+                    output_format=output_format,
+                )
+                else (
+                    _split_frame_batch(frames)
+                    if content_type == RAW_RGB_CONTENT_TYPE
+                    else [frames]
+                )
+            )
+            num_frame_batches = len(split_batches)
+            encoded_preview_payloads: list[_TransportPayload] | None = None
+            if _is_encoded_preview_transport(
+                content_type=content_type,
+                output_format=output_format,
+            ):
+                timer = RealtimeStageTimer()
+                encoded_preview_payloads = await _build_encoded_preview_payloads(
+                    split_batches,
+                    content_type=content_type,
+                    metadata=metadata,
+                    output_format=output_format,
+                    transport_quality=transport_quality,
+                    preview_max_width=preview_max_width,
+                    event_id=event_id,
+                )
+                stats["raw_payload_build_ms"] += timer.mark_ms()
+            for frame_batch_index, transport_frames in enumerate(split_batches):
+                timer = RealtimeStageTimer()
+                transport_metadata = metadata
+                if encoded_preview_payloads is not None:
+                    transport_payload = encoded_preview_payloads[frame_batch_index]
+                else:
+                    if _should_build_payload_off_loop(
+                        content_type=content_type,
+                        output_format=output_format,
+                        transport_frames=transport_frames,
+                    ):
+                        transport_payload = await asyncio.to_thread(
+                            _build_transport_payload,
+                            transport_frames,
+                            content_type=content_type,
+                            metadata=metadata,
+                            output_format=output_format,
+                            transport_quality=transport_quality,
+                            preview_max_width=preview_max_width,
+                        )
+                    else:
+                        transport_payload = _build_transport_payload(
+                            transport_frames,
+                            content_type=content_type,
+                            metadata=metadata,
+                            output_format=output_format,
+                            transport_quality=transport_quality,
+                            preview_max_width=preview_max_width,
+                        )
+                    stats["raw_payload_build_ms"] += timer.mark_ms()
+
+                header: RealtimeFrameBatchHeader = {
+                    "type": "frame_batch_header",
+                    "request_id": request_id,
+                    "chunk_index": chunk_index,
+                    "content_type": transport_payload.content_type,
+                    "num_frames": len(transport_frames),
+                    "total_size": len(transport_payload.payload),
+                    "frame_batch_index": frame_batch_index,
+                    "num_frame_batches": num_frame_batches,
+                    "is_final_frame_batch": frame_batch_index == num_frame_batches - 1,
+                }
+                if event_id is not None:
+                    header["event_id"] = event_id
+                header.update(transport_metadata)
+                header.update(transport_payload.metadata)
+
+                if len(transport_payload.payload) >= FRAME_BATCH_PACK_OFFLOAD_BYTES:
+                    header_payload = _pack_frame_batch_header(header)
+                    stats["header_pack_ms"] += timer.mark_ms()
+
+                    await ws.send_bytes(header_payload)
+                    stats["header_write_ms"] += timer.mark_ms()
+
+                    await ws.send_bytes(transport_payload.payload)
+                    stats["raw_write_ms"] += timer.mark_ms()
+
+                    stats["ws_payload_bytes"] += len(header_payload) + len(
+                        transport_payload.payload
+                    )
+                else:
+                    message_payload = _pack_frame_batch_message(
+                        header,
+                        transport_payload.payload,
+                    )
+                    stats["header_pack_ms"] += timer.mark_ms()
+
+                    stats["header_write_ms"] += timer.mark_ms()
+                    await ws.send_bytes(message_payload)
+                    stats["raw_write_ms"] += timer.mark_ms()
+
+                    stats["ws_payload_bytes"] += len(message_payload)
+
+                stats["raw_bytes"] += sum(len(frame) for frame in transport_frames)
+                stats["num_frames"] += len(transport_frames)
+                stats["num_batches"] += 1
+                stats["content_type"] = transport_payload.content_type
+            chunk_index += 1
+
+        stats["ws_write_ms"] = stats["header_write_ms"] + stats["raw_write_ms"]
+        return stats

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/realtime_video_api.py

@@ -0,0 +1,500 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import shutil
+import time
+from typing import TYPE_CHECKING
+
+import msgspec.msgpack
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    RealtimeEvent,
+    RealtimeVideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.generate_session import (
+    GenerateSession,
+    RealtimeChunkContext,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_output_adapter import (
+    RealtimeFrameSendStats,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.registry import (
+    get_realtime_model_adapter,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.timer import (
+    RealtimeStageTimer,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    process_generation_batch,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    ReleaseRealtimeSessionReq,
+)
+from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+
+logger = init_logger(__name__)
+router = APIRouter(prefix="/v1/realtime_video", tags=["realtime"])
+_ACTIVE_SESSION_IDS: set[str] = set()
+_ACTIVE_SESSION_WAIT_SECONDS = 1.0
+_ACTIVE_SESSION_WAIT_INTERVAL_SECONDS = 0.1
+
+
+def _transport_ms(value: float) -> int:
+    return max(0, int(value + 0.5))
+
+
+async def _wait_for_active_session_slot(
+    *,
+    timeout_s: float = _ACTIVE_SESSION_WAIT_SECONDS,
+    interval_s: float = _ACTIVE_SESSION_WAIT_INTERVAL_SECONDS,
+) -> bool:
+    deadline = time.monotonic() + timeout_s
+    while _ACTIVE_SESSION_IDS and time.monotonic() < deadline:
+        await asyncio.sleep(interval_s)
+    return not _ACTIVE_SESSION_IDS
+
+
+def _log_realtime_chunk_timing(
+    session: GenerateSession,
+    chunk: RealtimeChunkContext,
+    batch: "Req",
+    request_prepare_ms: float,
+    scheduler_forward_ms: float,
+    chunk_total_ms: float,
+    send_stats: RealtimeFrameSendStats,
+) -> None:
+    logger.info(
+        "realtime chunk timing: session_id=%s request_id=%s "
+        "chunk_idx=%s event_id=%s condition_kinds=%s "
+        "request_prepare=%.2fms scheduler_forward=%.2fms "
+        "output_pace=%.2fms "
+        "header_pack=%.2fms "
+        "header_write=%.2fms raw_payload_build=%.2fms raw_write=%.2fms "
+        "ws_write=%.2fms chunk_total=%.2fms batches=%d frames=%d "
+        "frame_shape=%s raw_bytes=%d ws_payload_bytes=%d content_type=%s",
+        session.id,
+        chunk.request_id,
+        batch.block_idx,
+        getattr(batch, "realtime_event_id", None),
+        sorted(batch.condition_inputs) if batch.condition_inputs else [],
+        request_prepare_ms,
+        scheduler_forward_ms,
+        send_stats["pace_wait_ms"],
+        send_stats["header_pack_ms"],
+        send_stats["header_write_ms"],
+        send_stats["raw_payload_build_ms"],
+        send_stats["raw_write_ms"],
+        send_stats["ws_write_ms"],
+        chunk_total_ms,
+        send_stats["num_batches"],
+        send_stats["num_frames"],
+        send_stats["frame_shape"],
+        send_stats["raw_bytes"],
+        send_stats["ws_payload_bytes"],
+        send_stats["content_type"],
+    )
+
+
+async def _send_realtime_chunk_stats(
+    ws: WebSocket,
+    session: GenerateSession,
+    chunk: RealtimeChunkContext,
+    batch: "Req",
+    request_prepare_ms: float,
+    scheduler_forward_ms: float,
+    chunk_total_ms: float,
+    send_stats: RealtimeFrameSendStats,
+) -> None:
+    await ws.send_bytes(
+        msgspec.msgpack.encode(
+            {
+                "type": "chunk_stats",
+                "session_id": session.id,
+                "request_id": chunk.request_id,
+                "chunk_index": batch.block_idx,
+                "event_id": getattr(batch, "realtime_event_id", None),
+                "request_prepare_ms": _transport_ms(request_prepare_ms),
+                "scheduler_forward_ms": _transport_ms(scheduler_forward_ms),
+                "pace_wait_ms": _transport_ms(send_stats["pace_wait_ms"]),
+                "header_write_ms": _transport_ms(send_stats["header_write_ms"]),
+                "raw_payload_build_ms": _transport_ms(
+                    send_stats["raw_payload_build_ms"]
+                ),
+                "raw_write_ms": _transport_ms(send_stats["raw_write_ms"]),
+                "ws_write_ms": _transport_ms(send_stats["ws_write_ms"]),
+                "chunk_total_ms": _transport_ms(chunk_total_ms),
+                "num_batches": send_stats["num_batches"],
+                "num_frames": send_stats["num_frames"],
+                "raw_bytes": send_stats["raw_bytes"],
+                "ws_payload_bytes": send_stats["ws_payload_bytes"],
+                "content_type": send_stats["content_type"],
+            }
+        )
+    )
+
+
+async def _generate_loop(ws: WebSocket, session: GenerateSession):
+    adapter = session.adapter
+    if adapter is None:
+        raise ValueError("realtime adapter is not initialized")
+
+    pending_send_task = None
+    while not session.reached_max_chunks():
+        try:
+            if pending_send_task is not None and pending_send_task.done():
+                await pending_send_task
+                pending_send_task = None
+
+            # send to scheduler and generate video chunk
+            server_args = get_global_server_args()
+
+            await adapter.wait_for_next_chunk(session)
+
+            timer = RealtimeStageTimer()
+            chunk_started = time.perf_counter()
+
+            chunk = session.new_chunk()
+            batch = adapter.prepare_next_request(
+                session,
+                server_args,
+                chunk,
+            )
+            if batch.condition_inputs:
+                logger.debug(
+                    "consume realtime conditions, session_id=%s, block_idx=%s, kinds=%s",
+                    session.id,
+                    batch.block_idx,
+                    sorted(batch.condition_inputs),
+                )
+            request_prepare_ms = timer.mark_ms()
+
+            _, result = await process_generation_batch(async_scheduler_client, batch)
+            scheduler_forward_ms = timer.mark_ms()
+
+            # finish
+            adapter.on_chunk_complete(session, result)
+            if pending_send_task is not None:
+                await pending_send_task
+            if getattr(batch, "realtime_output_pacing", False):
+                await _send_output_and_log(
+                    ws,
+                    session,
+                    chunk,
+                    batch,
+                    result,
+                    request_prepare_ms,
+                    scheduler_forward_ms,
+                    chunk_started,
+                )
+                pending_send_task = None
+            else:
+                pending_send_task = asyncio.create_task(
+                    _send_output_and_log(
+                        ws,
+                        session,
+                        chunk,
+                        batch,
+                        result,
+                        request_prepare_ms,
+                        scheduler_forward_ms,
+                        chunk_started,
+                    )
+                )
+
+        except asyncio.CancelledError:
+            if pending_send_task is not None:
+                pending_send_task.cancel()
+                await _await_realtime_task(pending_send_task)
+            logger.info("generation completed, session_id=%s", session.id)
+            break
+        except WebSocketDisconnect:
+            if pending_send_task is not None:
+                pending_send_task.cancel()
+                await _await_realtime_task(pending_send_task)
+            logger.info(
+                "client disconnected during generation, session_id=%s", session.id
+            )
+            break
+        except Exception as e:
+            if pending_send_task is not None:
+                pending_send_task.cancel()
+                await _await_realtime_task(pending_send_task)
+            err_msg = str(e).splitlines()[0]
+            logger.error("error during generate loop: %s", err_msg)
+            try:
+                await write_error_msg(f"error during generate loop: {err_msg}", ws)
+            except Exception as send_error:
+                logger.error(
+                    "error during sending complete msg: %s",
+                    send_error,
+                )
+            break
+    else:
+        if pending_send_task is not None:
+            await pending_send_task
+        logger.info(
+            "generation reached max chunks, session_id=%s, max_chunks=%s",
+            session.id,
+            session.request.max_chunks if session.request is not None else None,
+        )
+
+
+async def _send_output_and_log(
+    ws: WebSocket,
+    session: GenerateSession,
+    chunk: RealtimeChunkContext,
+    batch: "Req",
+    result,
+    request_prepare_ms: float,
+    scheduler_forward_ms: float,
+    chunk_started: float,
+) -> RealtimeFrameSendStats:
+    if session.adapter is None:
+        raise ValueError("realtime adapter is not initialized")
+    pace_wait_ms = await _wait_for_realtime_output_slot(session, batch, result)
+    send_stats = await session.adapter.send_output(
+        ws,
+        session,
+        result,
+        batch,
+    )
+    send_stats["pace_wait_ms"] = pace_wait_ms
+    chunk_total_ms = (time.perf_counter() - chunk_started) * 1000
+    _log_realtime_chunk_timing(
+        session,
+        chunk,
+        batch,
+        request_prepare_ms,
+        scheduler_forward_ms,
+        chunk_total_ms,
+        send_stats,
+    )
+    await _send_realtime_chunk_stats(
+        ws,
+        session,
+        chunk,
+        batch,
+        request_prepare_ms,
+        scheduler_forward_ms,
+        chunk_total_ms,
+        send_stats,
+    )
+    return send_stats
+
+
+def _result_num_frames(result) -> int:
+    if result.raw_frame_batches is None:
+        return 0
+    return sum(len(frames) for frames in result.raw_frame_batches)
+
+
+def _output_pacing_fps(batch: "Req") -> float:
+    fps = float(batch.fps or 0)
+    if batch.enable_frame_interpolation:
+        fps *= 2 ** int(batch.frame_interpolation_exp or 1)
+    return fps
+
+
+async def _wait_for_realtime_output_slot(
+    session: GenerateSession,
+    batch: "Req",
+    result,
+) -> float:
+    if not getattr(batch, "realtime_output_pacing", False):
+        return 0.0
+
+    frame_count = _result_num_frames(result)
+    output_fps = _output_pacing_fps(batch)
+    if frame_count <= 0 or output_fps <= 0:
+        return 0.0
+
+    now = time.perf_counter()
+    next_send_at = session.output_pace_next_send_at
+    if next_send_at is None:
+        next_send_at = now
+    if (
+        batch.realtime_event_id is not None
+        and batch.realtime_event_id != session.output_pace_last_event_id
+    ):
+        next_send_at = min(next_send_at, now)
+        session.output_pace_last_event_id = batch.realtime_event_id
+
+    wait_s = max(0.0, next_send_at - now)
+    if wait_s > 0:
+        await asyncio.sleep(wait_s)
+
+    send_started_at = time.perf_counter()
+    session.output_pace_next_send_at = (
+        max(next_send_at, send_started_at) + frame_count / output_fps
+    )
+    return wait_s * 1000
+
+
+async def _await_realtime_task(task: asyncio.Task | None) -> None:
+    if task is None:
+        return
+    try:
+        await task
+    except (asyncio.CancelledError, WebSocketDisconnect):
+        pass
+    except Exception as e:
+        logger.debug("realtime task exited with error: %s", e)
+
+
+async def _listen_events(ws: WebSocket, session: GenerateSession):
+    """listen for user events: usually condition inputs"""
+    async for message in ws.iter_bytes():
+        data = None
+        try:
+            data = msgspec.msgpack.decode(message)
+            if not isinstance(data, dict):
+                raise ValueError("realtime event must be a map")
+            realtime_event = RealtimeEvent.model_validate(data)
+            if session.adapter is None:
+                raise ValueError("realtime adapter is not initialized")
+            event_log = session.adapter.ingest_event(session, realtime_event)
+            logger.info(
+                "receive realtime event, session_id=%s, event_id=%s, %s",
+                session.id,
+                realtime_event.event_id,
+                event_log,
+            )
+        except Exception as e:
+            event_kind = data.get("kind") if isinstance(data, dict) else None
+            logger.warning("invalid event, kind=%s, error=%s", event_kind, e)
+            await write_error_msg("invalid event", ws)
+            continue
+
+
+async def _listen_generate_request(ws: WebSocket, session: GenerateSession):
+    while True:
+        try:
+            data = msgspec.msgpack.decode(await ws.receive_bytes())
+            if not isinstance(data, dict):
+                raise ValueError("generate request must be a map")
+
+            realtime_req = RealtimeVideoGenerationsRequest.model_validate(data)
+            adapter = get_realtime_model_adapter(get_global_server_args())
+            session.set_adapter(adapter)
+            await adapter.on_init(session, realtime_req)
+
+            # Keep session state update atomic with validated request.
+            session.set_request(realtime_req)
+            break
+        except WebSocketDisconnect:
+            raise
+        except Exception as e:
+            logger.warning(
+                "invalid generate request, session_id=%s, error=%s",
+                session.id,
+                e,
+            )
+            await write_error_msg("invalid generate request", ws)
+            continue
+
+
+async def _cleanup_realtime_session(
+    session: GenerateSession,
+    generate_task: asyncio.Task | None,
+    listen_task: asyncio.Task | None,
+) -> None:
+    logger.info("terminating session, session_id=%s", session.id)
+    for task in (generate_task, listen_task):
+        if task and not task.done():
+            task.cancel()
+    for task in (generate_task, listen_task):
+        if task is None:
+            continue
+        await _await_realtime_task(task)
+    try:
+        await async_scheduler_client.forward(
+            ReleaseRealtimeSessionReq(session_id=session.id)
+        )
+    except Exception as e:
+        logger.warning(
+            "failed to release realtime session on scheduler, session_id=%s, error=%s",
+            session.id,
+            e,
+        )
+    if session.input_temp_dir is not None:
+        shutil.rmtree(session.input_temp_dir, ignore_errors=True)
+    session.dispose()
+
+
+async def _close_realtime_websocket(
+    websocket: WebSocket,
+    *,
+    code: int,
+    reason: str,
+) -> None:
+    try:
+        await websocket.close(code=code, reason=reason)
+    except (RuntimeError, WebSocketDisconnect):
+        pass
+
+
+async def _wait_for_server_warmup(websocket: WebSocket) -> None:
+    warmup_done = getattr(websocket.app.state, "server_warmup_done", None)
+    if warmup_done is not None and not warmup_done.is_set():
+        await warmup_done.wait()
+
+
+@router.websocket("/generate")
+async def generate(websocket: WebSocket):
+    """endpoint for creating a new realtime session"""
+    await websocket.accept()
+    await _wait_for_server_warmup(websocket)
+    if _ACTIVE_SESSION_IDS and not await _wait_for_active_session_slot():
+        logger.warning(
+            "reject realtime session because another session is active: %s",
+            sorted(_ACTIVE_SESSION_IDS),
+        )
+        try:
+            await write_error_msg(
+                "another realtime session is already active", websocket
+            )
+        finally:
+            await websocket.close(code=1008)
+        return
+
+    session = GenerateSession()
+    _ACTIVE_SESSION_IDS.add(session.id)
+    generate_task = None
+    listen_task = None
+    try:
+        # receive new generate request
+        await _listen_generate_request(websocket, session)
+
+        # continuously generate video chunk
+        generate_task = asyncio.create_task(_generate_loop(websocket, session))
+        # continuously listen for user events
+        listen_task = asyncio.create_task(_listen_events(websocket, session))
+
+        wait_tasks = [generate_task, listen_task]
+        await asyncio.wait(wait_tasks, return_when=asyncio.FIRST_COMPLETED)
+        if generate_task.done() and session.reached_max_chunks():
+            await _close_realtime_websocket(
+                websocket,
+                code=1000,
+                reason="generation complete",
+            )
+
+    except WebSocketDisconnect:
+        logger.info("client disconnected, session_id=%s", session.id)
+    finally:
+        try:
+            await _cleanup_realtime_session(session, generate_task, listen_task)
+        finally:
+            _ACTIVE_SESSION_IDS.discard(session.id)
+
+
+async def write_error_msg(error_msg: str, websocket: WebSocket):
+    await websocket.send_bytes(
+        msgspec.msgpack.encode({"type": "error", "content": error_msg})
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/registry.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.realtime_adapter import (
+    BaseRealtimeModelAdapter,
+)
+
+if TYPE_CHECKING:
+    from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+_REALTIME_ADAPTER_REGISTRY: dict[type, type[BaseRealtimeModelAdapter]] = {}
+_BUILTIN_ADAPTERS_REGISTERED = False
+
+
+def register_realtime_model_adapter(
+    pipeline_config_cls: type,
+    adapter_cls: type[BaseRealtimeModelAdapter],
+) -> None:
+    _REALTIME_ADAPTER_REGISTRY[pipeline_config_cls] = adapter_cls
+
+
+def _register_builtin_realtime_model_adapters() -> None:
+    global _BUILTIN_ADAPTERS_REGISTERED
+    if _BUILTIN_ADAPTERS_REGISTERED:
+        return
+
+    from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import (
+        LingBotWorldCausalDMDConfig,
+    )
+    from sglang.multimodal_gen.configs.pipeline_configs.sana_wm import (
+        SanaWMRealtimeConfig,
+    )
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.adapters.lingbot_world_realtime_adapter import (
+        LingBotWorldRealtimeAdapter,
+    )
+    from sglang.multimodal_gen.runtime.entrypoints.openai.realtime.adapters.sana_wm_realtime_adapter import (
+        SanaWMRealtimeAdapter,
+    )
+
+    register_realtime_model_adapter(
+        LingBotWorldCausalDMDConfig,
+        LingBotWorldRealtimeAdapter,
+    )
+    register_realtime_model_adapter(
+        SanaWMRealtimeConfig,
+        SanaWMRealtimeAdapter,
+    )
+    _BUILTIN_ADAPTERS_REGISTERED = True
+
+
+def get_realtime_model_adapter(
+    server_args: ServerArgs,
+) -> BaseRealtimeModelAdapter:
+    _register_builtin_realtime_model_adapters()
+
+    pipeline_config = server_args.pipeline_config
+    for config_cls in type(pipeline_config).__mro__:
+        adapter_cls = _REALTIME_ADAPTER_REGISTRY.get(config_cls)
+        if adapter_cls is not None:
+            return adapter_cls()
+
+    raise ValueError(
+        "Realtime video is not supported for pipeline config "
+        f"{type(pipeline_config).__name__}; no realtime adapter is registered."
+    )

python/sglang/multimodal_gen/runtime/entrypoints/openai/realtime/timer.py

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import time
+
+
+class RealtimeStageTimer:
+    __slots__ = ("_last", "_start")
+
+    def __init__(self):
+        now = time.perf_counter()
+        self._start = now
+        self._last = now
+
+    def mark_ms(self) -> float:
+        now = time.perf_counter()
+        elapsed_ms = (now - self._last) * 1000.0
+        self._last = now
+        return elapsed_ms
+
+    def total_ms(self) -> float:
+        return (time.perf_counter() - self._start) * 1000.0

python/sglang/multimodal_gen/runtime/entrypoints/openai/storage.py

@@ -0,0 +1,109 @@
+import asyncio
+import os
+from typing import Optional
+
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+
+class CloudStorage:
+    def __init__(self):
+        self.enabled = os.getenv("SGLANG_CLOUD_STORAGE_TYPE", "").lower() == "s3"
+        if not self.enabled:
+            return
+
+        try:
+            import boto3
+        except ImportError:
+            logger.error(
+                "boto3 is not installed. Please install it with `pip install boto3` to use cloud storage."
+            )
+            self.enabled = False
+            return
+
+        self.bucket_name = os.getenv("SGLANG_S3_BUCKET_NAME")
+        if not self.bucket_name:
+            self.enabled = False
+            return
+
+        endpoint_url = os.getenv("SGLANG_S3_ENDPOINT_URL") or None
+        region_name = os.getenv("SGLANG_S3_REGION_NAME") or None
+
+        self.client = boto3.client(
+            "s3",
+            aws_access_key_id=os.getenv("SGLANG_S3_ACCESS_KEY_ID"),
+            aws_secret_access_key=os.getenv("SGLANG_S3_SECRET_ACCESS_KEY"),
+            endpoint_url=endpoint_url,
+            region_name=region_name,
+        )
+        self.endpoint_url = endpoint_url
+        self.region_name = region_name
+
+    def is_enabled(self) -> bool:
+        return self.enabled
+
+    async def upload_file(self, local_path: str, destination_key: str) -> Optional[str]:
+        if not self.is_enabled():
+            return None
+
+        def _sync_upload():
+            """Synchronous part of the upload to run in a thread."""
+            ext = os.path.splitext(local_path)[1].lower()
+            content_type = {
+                ".png": "image/png",
+                ".jpg": "image/jpeg",
+                ".jpeg": "image/jpeg",
+                ".webp": "image/webp",
+                ".mp4": "video/mp4",
+                ".glb": "model/gltf-binary",
+                ".obj": "text/plain",
+            }.get(ext, "application/octet-stream")
+
+            # Use the client created once in __init__
+            self.client.upload_file(
+                local_path,
+                self.bucket_name,
+                destination_key,
+                ExtraArgs={"ContentType": content_type},
+            )
+
+        try:
+            # Offload the blocking I/O call to a thread executor
+            await asyncio.get_running_loop().run_in_executor(None, _sync_upload)
+        except Exception as e:
+            # If upload fails, log the error and return None for fallback
+            logger.error(f"Upload failed for {destination_key}: {e}")
+            return None
+
+        # Simplified URL generation with a default region
+        if self.endpoint_url:
+            url = (
+                f"{self.endpoint_url.rstrip('/')}/{self.bucket_name}/{destination_key}"
+            )
+        else:
+            region = self.region_name or "us-east-1"
+            url = f"https://{self.bucket_name}.s3.{region}.amazonaws.com/{destination_key}"
+
+        logger.info(f"Uploaded {local_path} to {url}")
+        return url
+
+    async def upload_and_cleanup(self, file_path: str) -> Optional[str]:
+        """Helper to upload a file and delete the local copy if successful."""
+        if not self.is_enabled():
+            return None
+
+        key = os.path.basename(file_path)
+        url = await self.upload_file(file_path, key)
+
+        if url:
+            try:
+                # pass if removal fails
+                os.remove(file_path)
+            except OSError as e:
+                logger.warning(f"Failed to remove temporary file {file_path}: {e}")
+        return url
+
+
+# Global instance
+cloud_storage = CloudStorage()

python/sglang/multimodal_gen/runtime/entrypoints/openai/stores.py

@@ -0,0 +1,48 @@
+import asyncio
+from typing import Any, Dict, List, Optional
+
+
+class AsyncDictStore:
+    """A small async-safe in-memory key-value store for dict items.
+
+    This encapsulates the usual pattern of a module-level dict guarded by
+    an asyncio.Lock and provides simple CRUD methods that are safe to call
+    concurrently from FastAPI request handlers and background tasks.
+    """
+
+    def __init__(self) -> None:
+        self._items: Dict[str, Dict[str, Any]] = {}
+        self._lock = asyncio.Lock()
+
+    async def upsert(self, key: str, value: Dict[str, Any]) -> None:
+        async with self._lock:
+            self._items[key] = value
+
+    async def update_fields(
+        self, key: str, updates: Dict[str, Any]
+    ) -> Optional[Dict[str, Any]]:
+        async with self._lock:
+            item = self._items.get(key)
+            if item is None:
+                return None
+            item.update(updates)
+            return item
+
+    async def get(self, key: str) -> Optional[Dict[str, Any]]:
+        async with self._lock:
+            return self._items.get(key)
+
+    async def pop(self, key: str) -> Optional[Dict[str, Any]]:
+        async with self._lock:
+            return self._items.pop(key, None)
+
+    async def list_values(self) -> List[Dict[str, Any]]:
+        async with self._lock:
+            return list(self._items.values())
+
+
+# Global stores shared by OpenAI entrypoints
+# [request_id, dict]
+VIDEO_STORE = AsyncDictStore()
+IMAGE_STORE = AsyncDictStore()
+MESH_STORE = AsyncDictStore()

python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py

@@ -0,0 +1,452 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+import asyncio
+import inspect
+import json
+import os
+import shutil
+import tempfile
+import time
+from contextlib import contextmanager
+from typing import Any, Generator, List, Optional, Union
+
+import httpx
+from fastapi import HTTPException, UploadFile
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    DataType,
+    SamplingParams,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import (
+    ListLorasReq,
+    MergeLoraWeightsReq,
+    SetLoraReq,
+    ShutdownReq,
+    UnmergeLoraWeightsReq,
+    format_lora_message,
+    save_outputs,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
+from sglang.multimodal_gen.runtime.scheduler_client import AsyncSchedulerClient
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.common import parse_size
+from sglang.multimodal_gen.runtime.utils.image_io import save_base64_image_to_path
+from sglang.multimodal_gen.runtime.utils.logging_utils import (
+    init_logger,
+    log_batch_completion,
+    log_generation_timer,
+)
+from sglang.multimodal_gen.runtime.utils.trace_wrapper import trace_req
+
+# re-export LoRA protocol types for backward compatibility
+__all__ = [
+    "SetLoraReq",
+    "MergeLoraWeightsReq",
+    "UnmergeLoraWeightsReq",
+    "ListLorasReq",
+    "ShutdownReq",
+    "format_lora_message",
+]
+
+logger = init_logger(__name__)
+
+OUTPUT_QUALITY_MAPPER = {"maximum": 100, "high": 90, "medium": 55, "low": 35}
+DEFAULT_FPS = 24
+DEFAULT_VIDEO_SECONDS = 4
+
+
+def _bad_request(message: str) -> HTTPException:
+    return HTTPException(status_code=400, detail=message)
+
+
+def _parse_size_or_raise(size: str) -> tuple[int, int]:
+    width, height = parse_size(size)
+    if width is None or height is None or width <= 0 or height <= 0:
+        raise _bad_request("size must be formatted as positive WIDTHxHEIGHT")
+    return width, height
+
+
+def _validate_positive_int(kwargs: dict[str, Any], name: str) -> None:
+    value = kwargs.get(name)
+    if value is not None and int(value) <= 0:
+        raise _bad_request(f"{name} must be positive")
+
+
+def flatten_extra_params(payload: Any) -> dict[str, Any]:
+    """Promote vLLM-Omni-style extra_params into regular request fields."""
+    if not isinstance(payload, dict):
+        return {}
+
+    extra_params = payload.pop("extra_params", None)
+    if isinstance(extra_params, str):
+        try:
+            extra_params = json.loads(extra_params)
+        except Exception:
+            extra_params = None
+    if not isinstance(extra_params, dict):
+        if "guardrails" in payload:
+            payload.setdefault("use_guardrails", payload["guardrails"])
+        return payload
+
+    for key, value in extra_params.items():
+        payload.setdefault(key, value)
+    if "guardrails" in extra_params:
+        payload.setdefault("use_guardrails", extra_params["guardrails"])
+
+    return payload
+
+
+@contextmanager
+def temp_dir_if_disabled(
+    configured_path: str | None,
+) -> Generator[str, None, None]:
+    """Yield *configured_path* when it is set, otherwise create a temporary
+    directory that is automatically removed when the context exits."""
+    if configured_path is not None:
+        os.makedirs(configured_path, exist_ok=True)
+        yield configured_path
+    else:
+        tmp = tempfile.mkdtemp(prefix="sglang_")
+        try:
+            yield tmp
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+
+
+def choose_output_image_ext(
+    output_format: Optional[str], background: Optional[str]
+) -> str:
+    fmt = (output_format or "").lower()
+    if fmt in {"png", "webp", "jpeg", "jpg"}:
+        return "jpg" if fmt == "jpeg" else fmt
+    if (background or "auto").lower() == "transparent":
+        return "png"
+    return "jpg"
+
+
+def build_sampling_params(request_id: str, **kwargs) -> SamplingParams:
+    """Build SamplingParams from request parameters.
+
+    Handles size parsing, output_quality resolution, and None filtering before
+    delegating to SamplingParams.from_user_sampling_params_args. Callers pass
+    only the parameters they have; None values are stripped automatically so
+    that SamplingParams defaults apply.
+    """
+    server_args = get_global_server_args()
+
+    # pop HTTP-layer params that aren't SamplingParams fields
+    output_quality = kwargs.pop("output_quality", None)
+
+    has_explicit_compression = kwargs.get("output_compression") is not None
+
+    # parse "WxH" size string if provided
+    size = kwargs.pop("size", None)
+    if size:
+        w, h = _parse_size_or_raise(size)
+        # treat None dimensions as unset so parsed size can fill them
+        if kwargs.get("width") is None:
+            kwargs["width"] = w
+        if kwargs.get("height") is None:
+            kwargs["height"] = h
+
+    for name in (
+        "width",
+        "height",
+        "num_frames",
+        "num_inference_steps",
+        "num_outputs_per_prompt",
+    ):
+        _validate_positive_int(kwargs, name)
+
+    # filter out None values to let SamplingParams defaults apply
+    kwargs = {k: v for k, v in kwargs.items() if v is not None}
+    kwargs.setdefault("save_output", True)
+
+    sampling_params = SamplingParams.from_user_sampling_params_args(
+        model_path=server_args.model_path,
+        server_args=server_args,
+        request_id=request_id,
+        **kwargs,
+    )
+
+    # resolve output_quality → output_compression with the correct data_type.
+    # SamplingParams.__post_init__ may have resolved with the wrong data_type
+    # (default VIDEO) before _adjust() set the correct one.
+    if not has_explicit_compression and output_quality is not None:
+        resolved = adjust_output_quality(output_quality, sampling_params.data_type)
+        if resolved is not None:
+            sampling_params.output_compression = resolved
+
+    return sampling_params
+
+
+async def save_image_to_path(
+    image: Union[UploadFile, bytes, str],
+    target_path: str,
+    *,
+    prefer_remote_source: bool = False,
+) -> str:
+    input_path = await _maybe_url_image(
+        image, target_path, prefer_remote_source=prefer_remote_source
+    )
+    if input_path is None:
+        input_path = await _save_upload_to_path(image, target_path)
+    return input_path
+
+
+# Helpers
+async def _save_upload_to_path(
+    upload: Union[UploadFile, bytes], target_path: str
+) -> str:
+    os.makedirs(os.path.dirname(target_path), exist_ok=True)
+    if isinstance(upload, bytes):
+        content = upload
+    elif isinstance(upload, (bytearray, memoryview)):
+        content = bytes(upload)
+    else:
+        read = getattr(upload, "read", None)
+        if not callable(read):
+            raise TypeError(f"Unsupported image upload type: {type(upload).__name__}")
+        content = read()
+        if inspect.isawaitable(content):
+            content = await content
+        if isinstance(content, (bytearray, memoryview)):
+            content = bytes(content)
+        if not isinstance(content, bytes):
+            raise TypeError(
+                f"Image upload read() returned {type(content).__name__}, expected bytes"
+            )
+    with open(target_path, "wb") as f:
+        f.write(content)
+    return target_path
+
+
+async def _maybe_url_image(
+    img_url: str,
+    target_path: str,
+    *,
+    prefer_remote_source: bool = False,
+) -> str | None:
+    if not isinstance(img_url, str):
+        return None
+
+    if img_url.lower().startswith(("http://", "https://")):
+        # Only bypass persistence when the caller explicitly disables input saves.
+        # Otherwise keep the prefetch outside the measured server stages.
+        if prefer_remote_source:
+            return img_url
+        # download image from URL and persist on disk
+        input_path = await _save_url_image_to_path(img_url, target_path)
+        return input_path
+    elif img_url.startswith("data:image"):
+        if prefer_remote_source:
+            return img_url
+        # encode image base64 url and persist on disk
+        input_path = save_base64_image_to_path(img_url, target_path)
+        return input_path
+    else:
+        raise ValueError("Unsupported image url format")
+
+
+async def _save_url_image_to_path(image_url: str, target_path: str) -> str:
+    """Download image from URL and save to target path."""
+
+    def _is_retryable_download_error(error: Exception) -> bool:
+        if isinstance(error, httpx.HTTPStatusError):
+            status_code = error.response.status_code
+            # Retry on rate limit and transient server-side failures.
+            return status_code == 429 or 500 <= status_code < 600
+        # Retry on transient network/protocol issues.
+        return isinstance(
+            error,
+            (
+                httpx.TimeoutException,
+                httpx.NetworkError,
+                httpx.RemoteProtocolError,
+            ),
+        )
+
+    os.makedirs(os.path.dirname(target_path), exist_ok=True)
+
+    max_attempts = 3
+    backoff_seconds = 0.2
+    last_error: Exception | None = None
+
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            for attempt in range(1, max_attempts + 1):
+                try:
+                    response = await client.get(image_url, timeout=10.0)
+                    response.raise_for_status()
+
+                    # Determine file extension from content type or URL after downloading
+                    if not os.path.splitext(target_path)[1]:
+                        content_type = response.headers.get("content-type", "").lower()
+
+                        url_path = image_url.split("?")[0]
+                        _, url_ext = os.path.splitext(url_path)
+                        url_ext = url_ext.lower()
+
+                        if url_ext in {
+                            ".jpg",
+                            ".jpeg",
+                            ".png",
+                            ".webp",
+                            ".gif",
+                            ".bmp",
+                        }:
+                            ext = ".jpg" if url_ext == ".jpeg" else url_ext
+                        elif content_type.startswith("image/"):
+                            if "jpeg" in content_type or "jpg" in content_type:
+                                ext = ".jpg"
+                            elif "png" in content_type:
+                                ext = ".png"
+                            elif "webp" in content_type:
+                                ext = ".webp"
+                            else:
+                                ext = ".jpg"  # Default to jpg
+                        elif content_type == "application/octet-stream":
+                            # for octet-stream, if we couldn't get it from URL, default to jpg
+                            ext = ".jpg"
+                        else:
+                            raise ValueError(
+                                f"URL does not point to an image. Content-Type: {content_type}"
+                            )
+                        target_path = f"{target_path}{ext}"
+
+                    with open(target_path, "wb") as f:
+                        f.write(response.content)
+
+                    return target_path
+                except Exception as e:
+                    last_error = e
+                    if attempt == max_attempts or not _is_retryable_download_error(e):
+                        raise
+                    wait_s = backoff_seconds * (2 ** (attempt - 1))
+                    logger.warning(
+                        "Retrying image download (%s/%s) for %s after %.1fs due to: %s",
+                        attempt,
+                        max_attempts,
+                        image_url,
+                        wait_s,
+                        e,
+                    )
+                    await asyncio.sleep(wait_s)
+    except Exception as e:
+        final_error = last_error or e
+        raise Exception(
+            f"Failed to download image from URL {image_url}: {str(final_error)}"
+        )
+
+
+async def process_generation_batch(
+    scheduler_client: AsyncSchedulerClient,
+    batch,
+) -> tuple[list[str], OutputBatch]:
+    total_start_time = time.perf_counter()
+    with trace_req(batch.trace_ctx), log_generation_timer(logger, batch.prompt):
+        result = await scheduler_client.forward([batch])
+
+        if (
+            result.output is None
+            and result.output_file_paths is None
+            and result.raw_frame_batches is None
+        ):
+            error_msg = result.error or "Unknown error"
+            raise RuntimeError(
+                f"Model generation returned no output. Error from scheduler: {error_msg}"
+            )
+
+        save_file_path_list = []
+        if result.output_file_paths:
+            save_file_path_list = result.output_file_paths
+        elif result.output is not None:
+            num_outputs = len(result.output)
+            save_file_path_list = save_outputs(
+                result.output,
+                batch.data_type,
+                batch.fps,
+                batch.save_output,
+                lambda idx: str(batch.output_file_path(num_outputs, idx)),
+                audio=result.audio,
+                audio_sample_rate=result.audio_sample_rate,
+                output_compression=batch.output_compression,
+                enable_frame_interpolation=batch.enable_frame_interpolation,
+                frame_interpolation_exp=batch.frame_interpolation_exp,
+                frame_interpolation_scale=batch.frame_interpolation_scale,
+                frame_interpolation_model_path=batch.frame_interpolation_model_path,
+                enable_upscaling=batch.enable_upscaling,
+                upscaling_model_path=batch.upscaling_model_path,
+                upscaling_scale=batch.upscaling_scale,
+            )
+
+    total_time = time.perf_counter() - total_start_time
+    if get_global_server_args().batching_max_size > 1:
+        log_batch_completion(
+            logger,
+            len(save_file_path_list),
+            total_time,
+        )
+
+    if result.peak_memory_mb and result.peak_memory_mb > 0:
+        logger.info(f"Peak memory usage: {result.peak_memory_mb:.2f} MB")
+
+    return save_file_path_list, result
+
+
+def merge_image_input_list(*inputs: Union[List, Any, None]) -> List:
+    """
+    Merge multiple image input sources into a single list.
+
+    This function handles both single items and lists of items, merging them
+    into a single flattened list. Useful for processing images, URLs, or other
+    multimedia inputs that can come as either single items or lists.
+
+    Args:
+        *inputs: Variable number of inputs, each can be None, single item, or list
+
+    Returns:
+        List: Flattened list of all non-None inputs
+
+    Example:
+        >>> merge_image_input_list(["img1", "img2"], "img3", None)
+        ["img1", "img2", "img3"]
+    """
+    result = []
+    for input_item in inputs:
+        if input_item is not None:
+            if isinstance(input_item, list):
+                result.extend(input_item)
+            else:
+                result.append(input_item)
+    return result
+
+
+def add_common_data_to_response(
+    response: dict, request_id: str, result: OutputBatch
+) -> dict:
+    if result.peak_memory_mb and result.peak_memory_mb > 0:
+        response["peak_memory_mb"] = result.peak_memory_mb
+
+    if result.metrics and result.metrics.total_duration_s > 0:
+        response["inference_time_s"] = result.metrics.total_duration_s
+
+    response["id"] = request_id
+
+    if result.action_pred is not None:
+        t = result.action_pred
+        response["action"] = {
+            "data": t.tolist(),
+            "shape": list(t.shape),
+            "dtype": str(t.dtype).replace("torch.", ""),
+            "raw_action_dim": result.action_raw_action_dim,
+            "action_mode": result.action_mode,
+            "domain_id": result.action_domain_id,
+        }
+
+    return response
+
+
+def adjust_output_quality(output_quality: str, data_type: DataType = None) -> int:
+    if output_quality == "default":
+        return 50 if data_type == DataType.VIDEO else 75
+    return OUTPUT_QUALITY_MAPPER.get(output_quality, None)

python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py

@@ -0,0 +1,741 @@
+# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
+
+import asyncio
+import json
+import os
+import shutil
+import tempfile
+import time
+from typing import Any, Dict, Optional
+
+from fastapi import (
+    APIRouter,
+    File,
+    Form,
+    HTTPException,
+    Path,
+    Query,
+    Request,
+    UploadFile,
+)
+from fastapi.responses import FileResponse
+
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+    SamplingParams,
+    generate_request_id,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+    VideoGenerationsRequest,
+    VideoListResponse,
+    VideoResponse,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.storage import cloud_storage
+from sglang.multimodal_gen.runtime.entrypoints.openai.stores import VIDEO_STORE
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+    DEFAULT_FPS,
+    DEFAULT_VIDEO_SECONDS,
+    add_common_data_to_response,
+    build_sampling_params,
+    flatten_extra_params,
+    merge_image_input_list,
+    process_generation_batch,
+    save_image_to_path,
+)
+from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.server_args import get_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+from sglang.srt.observability.trace import extract_trace_headers
+
+logger = init_logger(__name__)
+router = APIRouter(prefix="/v1/videos", tags=["videos"])
+
+_VIDEO_EXTENSIONS = {
+    ".avi",
+    ".gif",
+    ".m4v",
+    ".mkv",
+    ".mov",
+    ".mp4",
+    ".mpeg",
+    ".mpg",
+    ".webm",
+}
+
+
+def _extra_value(request: VideoGenerationsRequest, name: str) -> Any:
+    return (request.model_extra or {}).get(name)
+
+
+def _request_value(request: VideoGenerationsRequest, name: str) -> Any:
+    value = getattr(request, name, None)
+    if value is not None:
+        return value
+    return _extra_value(request, name)
+
+
+def _parse_form_extra_value(value: Any) -> Any:
+    if not isinstance(value, str):
+        return value
+    try:
+        return json.loads(value)
+    except Exception:
+        return value
+
+
+def _is_probably_video_source(source: Any) -> bool:
+    content_type = (getattr(source, "content_type", "") or "").lower()
+    if content_type.startswith("video/"):
+        return True
+
+    if isinstance(source, str):
+        if source.lower().startswith("data:video"):
+            return True
+        source_name = source
+    else:
+        source_name = getattr(source, "filename", None)
+
+    if not source_name:
+        return False
+    source_name = str(source_name).split("?", 1)[0].split("#", 1)[0]
+    return os.path.splitext(source_name)[1].lower() in _VIDEO_EXTENSIONS
+
+
+def _is_cosmos3_server(server_args) -> bool:
+    from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config
+
+    return isinstance(server_args.pipeline_config, Cosmos3Config)
+
+
+def _normalize_optional_string(value: Any) -> Any:
+    if isinstance(value, str) and not value.strip():
+        return None
+    return value
+
+
+def _coerce_optional_int_list(value: Any) -> list[int] | None:
+    value = _parse_form_extra_value(value)
+    if value is None:
+        return None
+    if isinstance(value, str) and not value.strip():
+        return None
+    if isinstance(value, (list, tuple)):
+        return [int(item) for item in value]
+    return [int(value)]
+
+
+def _resolve_video_path(req: VideoGenerationsRequest) -> str | None:
+    video_path = _request_value(req, "video_path") or _request_value(req, "video_url")
+    if video_path:
+        return str(video_path)
+
+    input_reference = _request_value(req, "input_reference")
+    if _is_probably_video_source(input_reference):
+        return str(input_reference)
+
+    reference_url = _request_value(req, "reference_url")
+    if _is_probably_video_source(reference_url):
+        return str(reference_url)
+
+    return None
+
+
+def _resolve_image_path(
+    req: VideoGenerationsRequest, video_path: str | None
+) -> str | None:
+    image_path = _request_value(req, "input_reference")
+    if video_path and image_path == video_path:
+        return None
+    if _is_probably_video_source(image_path):
+        return None
+    return image_path
+
+
+def _resolve_sound_duration(
+    req: VideoGenerationsRequest, *, num_frames: int, fps: int
+) -> float | None:
+    generate_sound = _request_value(req, "generate_sound")
+    sound_duration = _request_value(req, "sound_duration")
+
+    if generate_sound is False:
+        return 0.0
+    if sound_duration is not None:
+        return float(sound_duration)
+    if generate_sound is True:
+        return float(num_frames) / float(fps)
+    return None
+
+
+def _cosmos3_sampling_param_kwargs(
+    req: VideoGenerationsRequest, *, num_frames: int, fps: int
+) -> Dict[str, Any]:
+    """Map HTTP/API aliases to Cosmos3SamplingParams field names."""
+    kwargs: Dict[str, Any] = {}
+
+    sound_duration = _resolve_sound_duration(req, num_frames=num_frames, fps=fps)
+    if sound_duration is not None:
+        kwargs["sound_duration"] = sound_duration
+
+    condition_frame_indexes = _request_value(req, "condition_frame_indexes")
+    if condition_frame_indexes is None:
+        condition_frame_indexes = _request_value(req, "condition_frame_indexes_vision")
+    condition_frame_indexes = _coerce_optional_int_list(condition_frame_indexes)
+    if condition_frame_indexes is not None:
+        kwargs["condition_frame_indexes"] = condition_frame_indexes
+
+    for name in (
+        "condition_video_keep",
+        "action_mode",
+        "domain_id",
+        "domain_name",
+        "raw_action_dim",
+        "action_fps",
+        "action",
+        "action_view_point",
+        "action_normalization",
+    ):
+        value = _parse_form_extra_value(_request_value(req, name))
+        value = _normalize_optional_string(value)
+        if value is not None:
+            kwargs[name] = value
+
+    return kwargs
+
+
+def _build_video_sampling_params(request_id: str, request: VideoGenerationsRequest):
+    """Resolve video-specific defaults (fps, seconds → num_frames) then
+    delegate to the shared build_sampling_params."""
+    server_args = get_global_server_args()
+    seconds = request.seconds if request.seconds is not None else DEFAULT_VIDEO_SECONDS
+    fps = request.fps if request.fps is not None else DEFAULT_FPS
+    num_frames = request.num_frames if request.num_frames is not None else fps * seconds
+    num_outputs = request.num_outputs_per_prompt
+    if num_outputs is None:
+        num_outputs = request.n or 1
+    video_path = _resolve_video_path(request)
+    image_path = _resolve_image_path(request, video_path)
+    cosmos3_kwargs = {}
+    if _is_cosmos3_server(server_args):
+        cosmos3_kwargs = _cosmos3_sampling_param_kwargs(
+            request, num_frames=num_frames, fps=fps
+        )
+        if server_args.pipeline_config.action_stats_path is not None:
+            cosmos3_kwargs["action_stats_path"] = (
+                server_args.pipeline_config.action_stats_path
+            )
+
+    return build_sampling_params(
+        request_id,
+        prompt=request.prompt,
+        num_outputs_per_prompt=max(1, min(int(num_outputs), 10)),
+        size=request.size,
+        width=request.width,
+        height=request.height,
+        num_frames=num_frames,
+        fps=fps,
+        image_path=image_path,
+        video_path=video_path,
+        output_file_name=request_id,
+        seed=request.seed,
+        generator_device=request.generator_device,
+        num_inference_steps=request.num_inference_steps,
+        guidance_scale=request.guidance_scale,
+        guidance_scale_2=request.guidance_scale_2,
+        negative_prompt=request.negative_prompt,
+        max_sequence_length=request.max_sequence_length,
+        flow_shift=request.flow_shift,
+        use_duration_template=_extra_value(request, "use_duration_template"),
+        use_resolution_template=_extra_value(request, "use_resolution_template"),
+        use_system_prompt=_extra_value(request, "use_system_prompt"),
+        use_guardrails=_extra_value(request, "use_guardrails"),
+        enable_teacache=request.enable_teacache,
+        enable_frame_interpolation=request.enable_frame_interpolation,
+        frame_interpolation_exp=request.frame_interpolation_exp,
+        frame_interpolation_scale=request.frame_interpolation_scale,
+        frame_interpolation_model_path=request.frame_interpolation_model_path,
+        enable_upscaling=request.enable_upscaling,
+        upscaling_model_path=request.upscaling_model_path,
+        upscaling_scale=request.upscaling_scale,
+        output_path=request.output_path,
+        output_compression=request.output_compression,
+        output_quality=request.output_quality,
+        perf_dump_path=request.perf_dump_path,
+        diffusers_kwargs=request.diffusers_kwargs,
+        **cosmos3_kwargs,
+    )
+
+
+# extract metadata which http_server needs to know
+def _video_job_from_sampling(
+    request_id: str, req: VideoGenerationsRequest, sampling: SamplingParams
+) -> Dict[str, Any]:
+    size_str = f"{sampling.width}x{sampling.height}"
+    seconds = int(round((sampling.num_frames or 0) / float(sampling.fps or 24)))
+    return {
+        "id": request_id,
+        "object": "video",
+        "model": req.model or "sora-2",
+        "status": "queued",
+        "progress": 0,
+        "created_at": int(time.time()),
+        "size": size_str,
+        "seconds": str(seconds),
+        "quality": "standard",
+        "file_path": os.path.abspath(sampling.output_file_path()),
+    }
+
+
+async def _save_first_input_image(
+    image_sources,
+    request_id: str,
+    uploads_dir: str,
+    *,
+    prefer_remote_source: bool = False,
+) -> str | None:
+    """Save the first input image from a list of sources and return its path."""
+    image_list = merge_image_input_list(image_sources)
+    if not image_list:
+        return None
+    image = image_list[0]
+
+    os.makedirs(uploads_dir, exist_ok=True)
+
+    filename = image.filename if hasattr(image, "filename") else "url_image"
+    target_path = os.path.join(uploads_dir, f"{request_id}_{filename}")
+    return await save_image_to_path(
+        image, target_path, prefer_remote_source=prefer_remote_source
+    )
+
+
+async def _dispatch_job_async(
+    job_id: str,
+    batch: Req,
+    *,
+    temp_dirs: list[str] | None = None,
+    output_persistent: bool = True,
+) -> None:
+    from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
+
+    try:
+        save_file_path_list, result = await process_generation_batch(
+            async_scheduler_client, batch
+        )
+        save_file_path = save_file_path_list[0]
+
+        cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
+
+        persistent_path = (
+            save_file_path if not cloud_url and output_persistent else None
+        )
+        update_fields = {
+            "status": "completed",
+            "progress": 100,
+            "completed_at": int(time.time()),
+            "url": cloud_url,
+            "file_path": persistent_path,
+            "file_paths": (
+                [os.path.abspath(path) for path in save_file_path_list]
+                if output_persistent
+                else None
+            ),
+            "num_outputs": len(save_file_path_list),
+        }
+        update_fields = add_common_data_to_response(
+            update_fields, request_id=job_id, result=result
+        )
+        await VIDEO_STORE.update_fields(job_id, update_fields)
+    except Exception as e:
+        logger.error(f"{e}")
+        await VIDEO_STORE.update_fields(
+            job_id, {"status": "failed", "error": {"message": str(e)}}
+        )
+    finally:
+        for td in temp_dirs or []:
+            shutil.rmtree(td, ignore_errors=True)
+
+
+# TODO: support image to video generation
+@router.post("", response_model=VideoResponse)
+async def create_video(
+    request: Request,
+    # multipart/form-data fields (optional; used only when content-type is multipart)
+    prompt: Optional[str] = Form(None),
+    input_reference: Optional[UploadFile] = File(None),
+    reference_url: Optional[str] = Form(None),
+    video_reference: Optional[UploadFile] = File(None),
+    video_url: Optional[str] = Form(None),
+    video_path: Optional[str] = Form(None),
+    model: Optional[str] = Form(None),
+    n: Optional[int] = Form(1),
+    num_outputs_per_prompt: Optional[int] = Form(None),
+    seconds: Optional[int] = Form(None),
+    size: Optional[str] = Form(None),
+    fps: Optional[int] = Form(None),
+    num_frames: Optional[int] = Form(None),
+    seed: Optional[int] = Form(None),
+    generator_device: Optional[str] = Form("cuda"),
+    negative_prompt: Optional[str] = Form(None),
+    guidance_scale: Optional[float] = Form(None),
+    num_inference_steps: Optional[int] = Form(None),
+    max_sequence_length: Optional[int] = Form(None),
+    flow_shift: Optional[float] = Form(None),
+    enable_teacache: Optional[bool] = Form(None),
+    enable_frame_interpolation: Optional[bool] = Form(None),
+    frame_interpolation_exp: Optional[int] = Form(None),
+    frame_interpolation_scale: Optional[float] = Form(None),
+    frame_interpolation_model_path: Optional[str] = Form(None),
+    enable_upscaling: Optional[bool] = Form(None),
+    upscaling_model_path: Optional[str] = Form(None),
+    upscaling_scale: Optional[int] = Form(None),
+    output_quality: Optional[str] = Form(None),
+    output_compression: Optional[int] = Form(None),
+    output_path: Optional[str] = Form(None),
+    extra_params: Optional[str] = Form(None),
+    extra_body: Optional[str] = Form(None),
+):
+    content_type = request.headers.get("content-type", "").lower()
+    request_id = generate_request_id()
+
+    server_args = get_global_server_args()
+    task_type = server_args.pipeline_config.task_type
+
+    # Resolve input upload directory (may be a temp dir when saving is disabled)
+    temp_dirs: list[str] = []
+    if server_args.input_save_path is not None:
+        uploads_dir = server_args.input_save_path
+        os.makedirs(uploads_dir, exist_ok=True)
+    else:
+        uploads_dir = tempfile.mkdtemp(prefix="sglang_input_")
+        temp_dirs.append(uploads_dir)
+
+    # Resolve output directory
+    effective_output_path = server_args.output_path
+    output_persistent = True
+    if "multipart/form-data" not in content_type:
+        # JSON body may carry a per-request output_path; checked after parsing below
+        pass
+
+    if "multipart/form-data" in content_type:
+        if not prompt:
+            raise HTTPException(status_code=400, detail="prompt is required")
+
+        video_input_path = None
+        image_sources = merge_image_input_list(input_reference, reference_url)
+        if video_reference is not None:
+            video_input_path = await _save_first_input_image(
+                video_reference,
+                request_id,
+                uploads_dir,
+                prefer_remote_source=server_args.input_save_path is None,
+            )
+        elif video_path or video_url:
+            video_input_path = video_path or video_url
+        elif input_reference is not None and _is_probably_video_source(input_reference):
+            video_input_path = await _save_first_input_image(
+                input_reference,
+                request_id,
+                uploads_dir,
+                prefer_remote_source=server_args.input_save_path is None,
+            )
+            image_sources = merge_image_input_list(reference_url)
+        elif reference_url and _is_probably_video_source(reference_url):
+            video_input_path = reference_url
+            image_sources = merge_image_input_list(input_reference)
+
+        # Validate image input based on model task type
+        if task_type.requires_image_input() and not image_sources:
+            raise HTTPException(
+                status_code=400,
+                detail="input_reference or reference_url is required for image-to-video generation",
+            )
+        input_path = None
+        if image_sources:
+            try:
+                input_path = await _save_first_input_image(
+                    image_sources,
+                    request_id,
+                    uploads_dir,
+                    prefer_remote_source=server_args.input_save_path is None,
+                )
+            except Exception as e:
+                raise HTTPException(
+                    status_code=400, detail=f"Failed to process image source: {str(e)}"
+                )
+
+        # Parse extra_body JSON (if provided in multipart form) to get fps/num_frames overrides
+        extra_from_form: Dict[str, Any] = {}
+        if extra_body:
+            try:
+                extra_from_form = flatten_extra_params(json.loads(extra_body))
+            except Exception:
+                extra_from_form = {}
+        if extra_params:
+            try:
+                extra_from_form.update(
+                    flatten_extra_params({"extra_params": json.loads(extra_params)})
+                )
+            except Exception:
+                pass
+
+        def form_value(name: str, value: Any) -> Any:
+            selected = value if value is not None else extra_from_form.get(name)
+            return _parse_form_extra_value(selected)
+
+        raw_form = await request.form()
+        for key in (
+            "use_duration_template",
+            "use_resolution_template",
+            "use_system_prompt",
+            "use_guardrails",
+            "guardrails",
+            "video_path",
+            "video_url",
+            "generate_sound",
+            "sound_duration",
+            "condition_frame_indexes",
+            "action_mode",
+            "domain_id",
+            "domain_name",
+            "raw_action_dim",
+            "action_fps",
+            "action",
+            "action_view_point",
+            "action_normalization",
+            "condition_frame_indexes_vision",
+            "condition_video_keep",
+        ):
+            if key in raw_form and key not in extra_from_form:
+                extra_from_form[key] = _parse_form_extra_value(raw_form[key])
+        flatten_extra_params(extra_from_form)
+
+        request_field_names = set(VideoGenerationsRequest.model_fields)
+        extra_request_fields = {
+            key: value
+            for key, value in extra_from_form.items()
+            if key not in request_field_names
+        }
+        fps_val = form_value("fps", fps)
+        num_frames_val = form_value("num_frames", num_frames)
+
+        req = VideoGenerationsRequest(
+            prompt=prompt,
+            input_reference=input_path,
+            video_path=form_value("video_path", video_input_path),
+            video_url=form_value("video_url", video_url),
+            model=form_value("model", model),
+            n=form_value("n", n),
+            num_outputs_per_prompt=form_value(
+                "num_outputs_per_prompt", num_outputs_per_prompt
+            ),
+            seconds=form_value("seconds", seconds) or 4,
+            size=form_value("size", size),
+            fps=fps_val,
+            num_frames=num_frames_val,
+            seed=form_value("seed", seed),
+            generator_device=form_value("generator_device", generator_device),
+            negative_prompt=form_value("negative_prompt", negative_prompt),
+            num_inference_steps=form_value("num_inference_steps", num_inference_steps),
+            guidance_scale=form_value("guidance_scale", guidance_scale),
+            max_sequence_length=form_value("max_sequence_length", max_sequence_length),
+            flow_shift=form_value("flow_shift", flow_shift),
+            enable_teacache=form_value("enable_teacache", enable_teacache),
+            enable_frame_interpolation=form_value(
+                "enable_frame_interpolation", enable_frame_interpolation
+            ),
+            frame_interpolation_exp=form_value(
+                "frame_interpolation_exp", frame_interpolation_exp
+            ),
+            frame_interpolation_scale=form_value(
+                "frame_interpolation_scale", frame_interpolation_scale
+            ),
+            frame_interpolation_model_path=form_value(
+                "frame_interpolation_model_path", frame_interpolation_model_path
+            ),
+            enable_upscaling=form_value("enable_upscaling", enable_upscaling),
+            upscaling_model_path=form_value(
+                "upscaling_model_path", upscaling_model_path
+            ),
+            upscaling_scale=form_value("upscaling_scale", upscaling_scale),
+            output_compression=form_value("output_compression", output_compression),
+            output_quality=form_value("output_quality", output_quality),
+            output_path=form_value("output_path", output_path),
+            diffusers_kwargs=form_value("diffusers_kwargs", None),
+            **extra_request_fields,
+        )
+    else:
+        try:
+            body = await request.json()
+        except Exception:
+            body = {}
+        try:
+            # If client uses extra_body, merge it into the top-level payload
+            payload: Dict[str, Any] = dict(body or {})
+            extra = payload.pop("extra_body", None)
+            if isinstance(extra, str):
+                extra = json.loads(extra)
+            if isinstance(extra, dict):
+                payload.update(flatten_extra_params(extra))
+            # openai may turn extra_body to extra_json
+            extra_json = payload.pop("extra_json", None)
+            if isinstance(extra_json, str):
+                extra_json = json.loads(extra_json)
+            if isinstance(extra_json, dict):
+                payload.update(flatten_extra_params(extra_json))
+            flatten_extra_params(payload)
+            # Validate image input based on model task type
+            if payload.get("video_url") and not payload.get("video_path"):
+                payload["video_path"] = payload["video_url"]
+            if _is_probably_video_source(payload.get("reference_url")):
+                payload.setdefault("video_path", payload.get("reference_url"))
+            if _is_probably_video_source(payload.get("input_reference")):
+                payload.setdefault("video_path", payload.get("input_reference"))
+
+            has_image_input = (
+                payload.get("reference_url")
+                and not _is_probably_video_source(payload.get("reference_url"))
+            ) or (
+                payload.get("input_reference")
+                and not _is_probably_video_source(payload.get("input_reference"))
+            )
+            if task_type.requires_image_input() and not has_image_input:
+                raise HTTPException(
+                    status_code=400,
+                    detail="input_reference or reference_url is required for image-to-video generation",
+                )
+            # for non-multipart/form-data type
+            if payload.get("reference_url") and not _is_probably_video_source(
+                payload.get("reference_url")
+            ):
+                try:
+                    input_path = await _save_first_input_image(
+                        payload.get("reference_url"),
+                        request_id,
+                        uploads_dir,
+                        prefer_remote_source=server_args.input_save_path is None,
+                    )
+                except Exception as e:
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Failed to process image source: {str(e)}",
+                    )
+                payload["input_reference"] = input_path
+            req = VideoGenerationsRequest(**payload)
+        except Exception as e:
+            raise HTTPException(status_code=400, detail=f"Invalid request body: {e}")
+
+    # Resolve per-request output_path override
+    effective_output_path = req.output_path or server_args.output_path
+    if effective_output_path is None:
+        output_tmp = tempfile.mkdtemp(prefix="sglang_output_")
+        temp_dirs.append(output_tmp)
+        effective_output_path = output_tmp
+        output_persistent = False
+
+    # Inject resolved output_path so _build_video_sampling_params picks it up
+    req.output_path = effective_output_path
+
+    logger.debug(f"Server received from create_video endpoint: req={req}")
+
+    try:
+        sampling_params = _build_video_sampling_params(request_id, req)
+    except (ValueError, TypeError) as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    job = _video_job_from_sampling(request_id, req, sampling_params)
+    await VIDEO_STORE.upsert(request_id, job)
+
+    # Build Req for scheduler
+    trace_headers = extract_trace_headers(request.headers)
+    batch = prepare_request(
+        server_args=server_args,
+        sampling_params=sampling_params,
+        external_trace_header=trace_headers,
+    )
+    # Add diffusers_kwargs if provided
+    if req.diffusers_kwargs:
+        batch.extra["diffusers_kwargs"] = req.diffusers_kwargs
+        if "max_sequence_length" in req.diffusers_kwargs:
+            batch.max_sequence_length = req.diffusers_kwargs["max_sequence_length"]
+        if "flow_shift" in req.diffusers_kwargs:
+            batch.flow_shift = req.diffusers_kwargs["flow_shift"]
+    # Enqueue the job asynchronously and return immediately
+    asyncio.create_task(
+        _dispatch_job_async(
+            request_id,
+            batch,
+            temp_dirs=temp_dirs or None,
+            output_persistent=output_persistent,
+        )
+    )
+    return VideoResponse(**job)
+
+
+@router.get("", response_model=VideoListResponse)
+async def list_videos(
+    after: Optional[str] = Query(None),
+    limit: Optional[int] = Query(None, ge=1, le=100),
+    order: Optional[str] = Query("desc"),
+):
+    # Normalize order
+    order = (order or "desc").lower()
+    if order not in ("asc", "desc"):
+        order = "desc"
+    jobs = await VIDEO_STORE.list_values()
+
+    reverse = order != "asc"
+    jobs.sort(key=lambda j: j.get("created_at", 0), reverse=reverse)
+
+    if after is not None:
+        try:
+            idx = next(i for i, j in enumerate(jobs) if j["id"] == after)
+            jobs = jobs[idx + 1 :]
+        except StopIteration:
+            jobs = []
+
+    if limit is not None:
+        jobs = jobs[:limit]
+    items = [VideoResponse(**j) for j in jobs]
+    return VideoListResponse(data=items)
+
+
+@router.get("/{video_id}", response_model=VideoResponse)
+async def retrieve_video(video_id: str = Path(...)):
+    job = await VIDEO_STORE.get(video_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Video not found")
+    return VideoResponse(**job)
+
+
+# TODO: support aborting a job.
+@router.delete("/{video_id}", response_model=VideoResponse)
+async def delete_video(video_id: str = Path(...)):
+    job = await VIDEO_STORE.pop(video_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Video not found")
+    # Mark as deleted in response semantics
+    job["status"] = "deleted"
+    return VideoResponse(**job)
+
+
+@router.get("/{video_id}/content")
+async def download_video_content(
+    video_id: str = Path(...), variant: Optional[str] = Query(None)
+):
+    job = await VIDEO_STORE.get(video_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Video not found")
+
+    if job.get("url"):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Video has been uploaded to cloud storage. Please use the cloud URL: {job.get('url')}",
+        )
+
+    file_path = job.get("file_path")
+    if not file_path or not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="Generation is still in-progress")
+
+    media_type = "video/mp4"  # default variant
+    return FileResponse(
+        path=file_path, media_type=media_type, filename=os.path.basename(file_path)
+    )