# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo import asyncio import base64 import os import signal import uuid from contextlib import asynccontextmanager, suppress from typing import TYPE_CHECKING import httpx import torch from fastapi import APIRouter, FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, video_api from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( VertexGenerateReqInput, ) from sglang.multimodal_gen.runtime.entrypoints.openai.realtime import ( realtime_video_api, ) from sglang.multimodal_gen.runtime.entrypoints.openai.utils import build_sampling_params from sglang.multimodal_gen.runtime.entrypoints.post_training import ( rollout_api, weights_api, ) from sglang.multimodal_gen.runtime.entrypoints.utils import ( prepare_request, save_outputs, ) from sglang.multimodal_gen.runtime.entrypoints.vla import api as vla_api from sglang.multimodal_gen.runtime.entrypoints.vla import openpi from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client from sglang.multimodal_gen.runtime.server_args import ServerArgs, get_global_server_args from sglang.multimodal_gen.runtime.server_warmup import ( run_async_client_warmup, should_run_synthetic_server_warmup, ) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.srt.utils.json_response import orjson_response from sglang.version import __version__ if TYPE_CHECKING: from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req logger = init_logger(__name__) VERTEX_ROUTE = os.environ.get("AIP_PREDICT_ROUTE", "/vertex_generate") SERVER_WARMUP_BYPASS_PATHS = ( "/health", "/health_generate", "/model_info", "/server_info", ) async def _wait_until_http_ready(server_args: ServerArgs) -> None: """for server warmup""" health_url = f"{server_args.url()}/health" # Probe the local server directly: a loopback readiness check must never be # routed through an HTTP proxy. trust_env=False also avoids crashing startup # on a malformed proxy env var, since httpx parses *_PROXY/NO_PROXY when the # client is constructed (raising httpx.InvalidURL before any request). See #28493. async with httpx.AsyncClient(trust_env=False) as client: for _ in range(120): try: response = await client.get(health_url, timeout=5.0) if response.status_code == 200: return except httpx.HTTPError: pass await asyncio.sleep(1.0) raise RuntimeError(f"HTTP server did not become ready at {health_url}") async def _run_server_warmup_after_http_ready( server_args: ServerArgs, warmup_done: asyncio.Event ) -> None: try: if not should_run_synthetic_server_warmup(server_args): warmup_done.set() return await _wait_until_http_ready(server_args) await run_async_client_warmup( server_args, async_scheduler_client.forward, fail_open=server_args.warmup_resolutions is None, ) logger.info("The server is fired up and ready to roll!") warmup_done.set() except asyncio.CancelledError: raise except Exception as e: logger.error("Server warmup failed; aborting startup: %s", e, exc_info=True) os.kill(os.getpid(), signal.SIGTERM) @asynccontextmanager async def lifespan(app: FastAPI): from sglang.multimodal_gen.runtime.scheduler_client import ( async_scheduler_client, run_zeromq_broker, ) # 1. Initialize the singleton client that connects to the backend Scheduler server_args = app.state.server_args async_scheduler_client.initialize(server_args) warmup_done = asyncio.Event() app.state.server_warmup_done = warmup_done # 2. Start the ZMQ Broker in the background to handle offline requests broker_task = asyncio.create_task(run_zeromq_broker(server_args)) warmup_task = None if server_args.server_warmup: warmup_task = asyncio.create_task( _run_server_warmup_after_http_ready(server_args, warmup_done) ) else: warmup_done.set() try: yield finally: if warmup_task is not None and not warmup_task.done(): warmup_task.cancel() with suppress(asyncio.CancelledError): await warmup_task # On shutdown logger.info("FastAPI app is shutting down...") broker_task.cancel() async_scheduler_client.close() # Health router health_router = APIRouter() @health_router.get("/health") async def health(): return {"status": "ok"} @health_router.get("/models", deprecated=True) async def get_models(request: Request): """ Get information about the model served by this server. .. deprecated:: Use /v1/models instead for OpenAI-compatible model discovery. This endpoint will be removed in a future version. """ from sglang.multimodal_gen.registry import get_model_info server_args: ServerArgs = request.app.state.server_args model_info = get_model_info(server_args.model_path, model_id=server_args.model_id) response = { "model_path": server_args.model_path, "num_gpus": server_args.num_gpus, "task_type": server_args.pipeline_config.task_type.name, "dit_precision": server_args.pipeline_config.dit_precision, "vae_precision": server_args.pipeline_config.vae_precision, } if model_info: response["pipeline_name"] = model_info.pipeline_cls.pipeline_name response["pipeline_class"] = model_info.pipeline_cls.__name__ return response @health_router.get("/server_info") async def server_info_endpoint(request: Request): """Get server information. Returns fields compatible with the LLM engine's /server_info so that the model gateway can discover diffusion workers. """ server_args: ServerArgs = request.app.state.server_args return { "model_path": server_args.model_path, "served_model_name": server_args.model_id or server_args.model_path, "tp_size": server_args.tp_size, "dp_size": server_args.dp_size, "version": __version__, } @health_router.get("/model_info") async def model_info_endpoint(request: Request): """Get model information. Returns fields compatible with the LLM engine's /model_info so that the model gateway can detect capabilities for diffusion workers. """ from sglang.multimodal_gen.registry import get_model_info server_args: ServerArgs = request.app.state.server_args task_type = server_args.pipeline_config.task_type try: registry_info = get_model_info( server_args.model_path, backend=server_args.backend, model_id=server_args.model_id, ) except Exception: logger.warning("Failed to resolve model info from registry", exc_info=True) registry_info = None return { # Fields consumed by the model gateway for worker discovery "model_path": server_args.model_path, "is_generation": True, "model_type": "diffusion", "architectures": ( [registry_info.pipeline_cls.__name__] if registry_info else None ), # Fields matching the LLM engine's /model_info shape "has_image_understanding": task_type.accepts_image_input(), "has_audio_understanding": False, # Diffusion-specific fields "task_type": task_type.name, "is_image_gen": task_type.is_image_gen(), } @health_router.get("/health_generate") async def health_generate(): # TODO : health generate endpoint return {"status": "ok"} @health_router.get("/stats") async def stats_endpoint(request: Request): """Get runtime statistics including disagg pipeline metrics. Returns queue depth, request counts, latency, throughput, etc. Sends a GetDisaggStatsReq to the scheduler via ZMQ and returns the result. """ from sglang.multimodal_gen.runtime.entrypoints.utils import GetDisaggStatsReq server_args: ServerArgs = request.app.state.server_args response: dict = { "status": "ok", "model_path": server_args.model_path, } # Query the scheduler for disagg metrics try: stats_response = await async_scheduler_client.forward(GetDisaggStatsReq()) if hasattr(stats_response, "output") and stats_response.output is not None: response["disagg"] = stats_response.output except Exception as e: response["disagg"] = {"error": str(e)} return response def make_serializable(obj): """Recursively converts Tensors to None for JSON serialization.""" if isinstance(obj, torch.Tensor): return None if isinstance(obj, dict): return {k: make_serializable(v) for k, v in obj.items()} if isinstance(obj, list): return [make_serializable(v) for v in obj] return obj def encode_video_to_base64(file_path: str): if not os.path.exists(file_path): return None with open(file_path, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") async def forward_to_scheduler( req_obj: "Req", sp: SamplingParams, ): """Forwards request to scheduler and processes the result.""" try: response = await async_scheduler_client.forward(req_obj) if response.output is None and response.output_file_paths is None: raise RuntimeError("Model generation returned no output.") if response.output_file_paths: output_file_path = response.output_file_paths[0] else: output_file_path = sp.output_file_path() save_outputs( [response.output[0]], sp.data_type, sp.fps, True, lambda _idx: output_file_path, audio=response.audio, audio_sample_rate=response.audio_sample_rate, enable_frame_interpolation=sp.enable_frame_interpolation, frame_interpolation_exp=sp.frame_interpolation_exp, frame_interpolation_scale=sp.frame_interpolation_scale, frame_interpolation_model_path=sp.frame_interpolation_model_path, enable_upscaling=sp.enable_upscaling, upscaling_model_path=sp.upscaling_model_path, upscaling_scale=sp.upscaling_scale, ) if hasattr(response, "model_dump"): data = response.model_dump() else: data = response if isinstance(response, dict) else vars(response) if output_file_path: logger.info("Processing output file: %s", output_file_path) b64_video = encode_video_to_base64(output_file_path) if b64_video: data["output"] = b64_video data.pop("video_data", None) data.pop("video_tensor", None) return make_serializable(data) except Exception as e: logger.error("Error during generation: %s", e, exc_info=True) return {"error": str(e)} vertex_router = APIRouter() @vertex_router.post(VERTEX_ROUTE) async def vertex_generate(vertex_req: VertexGenerateReqInput): if not vertex_req.instances: return orjson_response({"predictions": []}) server_args = get_global_server_args() params = vertex_req.parameters or {} futures = [] for inst in vertex_req.instances: rid = f"vertex_{uuid.uuid4()}" sp = build_sampling_params( rid, prompt=inst.get("prompt") or inst.get("text"), image_path=inst.get("image") or inst.get("image_url"), num_frames=params.get("num_frames"), fps=params.get("fps"), width=params.get("width"), height=params.get("height"), guidance_scale=params.get("guidance_scale"), save_output=params.get("save_output"), ) backend_req = prepare_request(server_args, sampling_params=sp) futures.append(forward_to_scheduler(backend_req, sp)) results = await asyncio.gather(*futures) return orjson_response({"predictions": results}) def create_app(server_args: ServerArgs): """ Create and configure the FastAPI application instance. """ app = FastAPI(lifespan=lifespan) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) @app.middleware("http") async def wait_for_server_warmup(request: Request, call_next): warmup_done = getattr(request.app.state, "server_warmup_done", None) if ( warmup_done is not None and not warmup_done.is_set() and request.url.path not in SERVER_WARMUP_BYPASS_PATHS ): await warmup_done.wait() return await call_next(request) app.include_router(health_router) app.include_router(vertex_router) from sglang.multimodal_gen.runtime.entrypoints.openai import common_api, mesh_api app.include_router(common_api.router) app.include_router(image_api.router) app.include_router(video_api.router) app.include_router(realtime_video_api.router) if server_args.pipeline_config.task_type.is_action_gen(): app.include_router(vla_api.router) app.include_router(openpi.router) app.include_router(mesh_api.router) app.include_router(weights_api.router) app.include_router(rollout_api.router) app.state.server_args = server_args return app