chore: import upstream snapshot with attribution
PR Test (NPU) / check-changes (push) Has been cancelled
PR Test (NPU) / pr-gate (push) Has been cancelled
PR Test (NPU) / set-image-config (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-4-npu-a3 (push) Has been cancelled
PR Test (NPU) / stage-b-test-16-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-1-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-2-npu-a3 (push) Has been cancelled
PR Test (Arm64) / pr-gate (push) Has been cancelled
PR Test (Arm64) / check-changes (push) Has been cancelled
PR Test (Arm64) / build-test (push) Has been cancelled
PR Test (sgl-router) / gate (push) Has been cancelled
PR Test (sgl-router) / tier-1 — lint (push) Has been cancelled
PR Test (sgl-router) / tier-2 — build + test (push) Has been cancelled
PR Test (sgl-router) / tier-3 — docker (placeholder) (push) Has been cancelled
PR Test (sgl-router) / tier-3 — k8s integration (push) Has been cancelled
PR Test (sgl-router) / tier-3 — e2e (push) Has been cancelled
PR Test (sgl-router) / finish (push) Has been cancelled
PR Test (NPU) / single-node-poc (map[name:qwen3_6_27b_w8a8_1p_in64k_out1k_50ms runner:linux-aarch64-a3-2 test_case:test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type:perf]) (push) Has been cancelled
PR Test (NPU) / pr-test-npu-finish (push) Has been cancelled
PR Test (Xeon) / pr-gate (push) Has been cancelled
PR Test (Xeon) / check-changes (push) Has been cancelled
PR Test (Xeon) / build-test (, xeon-gnr, base-b-test-cpu) (push) Has been cancelled
PR Test (XPU) / check-changes (push) Has been cancelled
PR Test (XPU) / pr-gate (push) Has been cancelled
PR Test (XPU) / stage-a-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / wait-for-stage-a (push) Has been cancelled
PR Test (XPU) / stage-b-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / finish (push) Has been cancelled
CI Model Inventory / build-inventory (push) Has been cancelled
Lint / lint (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Compilation Check (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Manual Policy (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Request Processing (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Summary (push) Has been cancelled
PR Test (SMG) / build-wheel (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on windows (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (x86_64 - auto) (push) Has been cancelled
PR Test (SMG) / python-unit-tests (push) Has been cancelled
PR Test (SMG) / unit-tests (push) Has been cancelled
PR Test (SMG) / benchmarks (push) Has been cancelled
PR Test (SMG) / chat-completions (push) Has been cancelled
PR Test (SMG) / chat-completions-4gpu (push) Has been cancelled
PR Test (SMG) / e2e (push) Has been cancelled
PR Test (SMG) / docker-build-test (push) Has been cancelled
PR Test (SMG) / k8s-integration (push) Has been cancelled
PR Test (SMG) / finish (push) Has been cancelled
PR Test (SMG) / summarize-benchmarks (push) Has been cancelled
Release SGLang Model Gateway Docker Image / publish (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Build SDist (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Upload to PyPI (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (aarch64, 12.9, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (x86_64, 12.9, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu129 (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (aarch64, 13.0, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (x86_64, 13.0, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu130 (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 700) (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 720) (push) Has been cancelled
Release SGLang Kernels / release-rocm700 (push) Has been cancelled
Release SGLang Kernels / release-rocm720 (push) Has been cancelled
Release SGLang Kernels / build-musa43 (43, 3.10) (push) Has been cancelled
Release SGLang Kernels / release-musa43 (push) Has been cancelled
PR Test (NPU) / check-changes (push) Has been cancelled
PR Test (NPU) / pr-gate (push) Has been cancelled
PR Test (NPU) / set-image-config (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-1-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (0) (push) Has been cancelled
PR Test (NPU) / stage-b-test-2-npu-a2 (1) (push) Has been cancelled
PR Test (NPU) / stage-b-test-4-npu-a3 (push) Has been cancelled
PR Test (NPU) / stage-b-test-16-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-1-npu-a3 (push) Has been cancelled
PR Test (NPU) / multimodal-gen-test-2-npu-a3 (push) Has been cancelled
PR Test (Arm64) / pr-gate (push) Has been cancelled
PR Test (Arm64) / check-changes (push) Has been cancelled
PR Test (Arm64) / build-test (push) Has been cancelled
PR Test (sgl-router) / gate (push) Has been cancelled
PR Test (sgl-router) / tier-1 — lint (push) Has been cancelled
PR Test (sgl-router) / tier-2 — build + test (push) Has been cancelled
PR Test (sgl-router) / tier-3 — docker (placeholder) (push) Has been cancelled
PR Test (sgl-router) / tier-3 — k8s integration (push) Has been cancelled
PR Test (sgl-router) / tier-3 — e2e (push) Has been cancelled
PR Test (sgl-router) / finish (push) Has been cancelled
PR Test (NPU) / single-node-poc (map[name:qwen3_6_27b_w8a8_1p_in64k_out1k_50ms runner:linux-aarch64-a3-2 test_case:test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type:perf]) (push) Has been cancelled
PR Test (NPU) / pr-test-npu-finish (push) Has been cancelled
PR Test (Xeon) / pr-gate (push) Has been cancelled
PR Test (Xeon) / check-changes (push) Has been cancelled
PR Test (Xeon) / build-test (, xeon-gnr, base-b-test-cpu) (push) Has been cancelled
PR Test (XPU) / check-changes (push) Has been cancelled
PR Test (XPU) / pr-gate (push) Has been cancelled
PR Test (XPU) / stage-a-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / wait-for-stage-a (push) Has been cancelled
PR Test (XPU) / stage-b-test-1-gpu-xpu (push) Has been cancelled
PR Test (XPU) / finish (push) Has been cancelled
CI Model Inventory / build-inventory (push) Has been cancelled
Lint / lint (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Compilation Check (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Manual Policy (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark - Request Processing (push) Has been cancelled
PR Benchmark (SMG Components) / Benchmark Summary (push) Has been cancelled
PR Test (SMG) / build-wheel (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on windows (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (x86_64 - auto) (push) Has been cancelled
PR Test (SMG) / python-unit-tests (push) Has been cancelled
PR Test (SMG) / unit-tests (push) Has been cancelled
PR Test (SMG) / benchmarks (push) Has been cancelled
PR Test (SMG) / chat-completions (push) Has been cancelled
PR Test (SMG) / chat-completions-4gpu (push) Has been cancelled
PR Test (SMG) / e2e (push) Has been cancelled
PR Test (SMG) / docker-build-test (push) Has been cancelled
PR Test (SMG) / k8s-integration (push) Has been cancelled
PR Test (SMG) / finish (push) Has been cancelled
PR Test (SMG) / summarize-benchmarks (push) Has been cancelled
Release SGLang Model Gateway Docker Image / publish (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on macos (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - auto) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (aarch64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / build on linux (x86_64 - musllinux_1_1) (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Build SDist (push) Has been cancelled
Release SGLang Model Gateway to PyPI / Upload to PyPI (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (aarch64, 12.9, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu129-matrix (x86_64, 12.9, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu129 (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (aarch64, 13.0, 3.10, arm-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / build-cu130-matrix (x86_64, 13.0, 3.10, x64-kernel-build-node) (push) Has been cancelled
Release SGLang Kernels / release-cu130 (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 700) (push) Has been cancelled
Release SGLang Kernels / build-rocm-matrix (3.10, 720) (push) Has been cancelled
Release SGLang Kernels / release-rocm700 (push) Has been cancelled
Release SGLang Kernels / release-rocm720 (push) Has been cancelled
Release SGLang Kernels / build-musa43 (43, 3.10) (push) Has been cancelled
Release SGLang Kernels / release-musa43 (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,741 @@
|
||||
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import (
|
||||
APIRouter,
|
||||
File,
|
||||
Form,
|
||||
HTTPException,
|
||||
Path,
|
||||
Query,
|
||||
Request,
|
||||
UploadFile,
|
||||
)
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
SamplingParams,
|
||||
generate_request_id,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
VideoGenerationsRequest,
|
||||
VideoListResponse,
|
||||
VideoResponse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.storage import cloud_storage
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.stores import VIDEO_STORE
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
||||
DEFAULT_FPS,
|
||||
DEFAULT_VIDEO_SECONDS,
|
||||
add_common_data_to_response,
|
||||
build_sampling_params,
|
||||
flatten_extra_params,
|
||||
merge_image_input_list,
|
||||
process_generation_batch,
|
||||
save_image_to_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import prepare_request
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.srt.observability.trace import extract_trace_headers
|
||||
|
||||
logger = init_logger(__name__)
|
||||
router = APIRouter(prefix="/v1/videos", tags=["videos"])
|
||||
|
||||
_VIDEO_EXTENSIONS = {
|
||||
".avi",
|
||||
".gif",
|
||||
".m4v",
|
||||
".mkv",
|
||||
".mov",
|
||||
".mp4",
|
||||
".mpeg",
|
||||
".mpg",
|
||||
".webm",
|
||||
}
|
||||
|
||||
|
||||
def _extra_value(request: VideoGenerationsRequest, name: str) -> Any:
|
||||
return (request.model_extra or {}).get(name)
|
||||
|
||||
|
||||
def _request_value(request: VideoGenerationsRequest, name: str) -> Any:
|
||||
value = getattr(request, name, None)
|
||||
if value is not None:
|
||||
return value
|
||||
return _extra_value(request, name)
|
||||
|
||||
|
||||
def _parse_form_extra_value(value: Any) -> Any:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
|
||||
def _is_probably_video_source(source: Any) -> bool:
|
||||
content_type = (getattr(source, "content_type", "") or "").lower()
|
||||
if content_type.startswith("video/"):
|
||||
return True
|
||||
|
||||
if isinstance(source, str):
|
||||
if source.lower().startswith("data:video"):
|
||||
return True
|
||||
source_name = source
|
||||
else:
|
||||
source_name = getattr(source, "filename", None)
|
||||
|
||||
if not source_name:
|
||||
return False
|
||||
source_name = str(source_name).split("?", 1)[0].split("#", 1)[0]
|
||||
return os.path.splitext(source_name)[1].lower() in _VIDEO_EXTENSIONS
|
||||
|
||||
|
||||
def _is_cosmos3_server(server_args) -> bool:
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config
|
||||
|
||||
return isinstance(server_args.pipeline_config, Cosmos3Config)
|
||||
|
||||
|
||||
def _normalize_optional_string(value: Any) -> Any:
|
||||
if isinstance(value, str) and not value.strip():
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _coerce_optional_int_list(value: Any) -> list[int] | None:
|
||||
value = _parse_form_extra_value(value)
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str) and not value.strip():
|
||||
return None
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [int(item) for item in value]
|
||||
return [int(value)]
|
||||
|
||||
|
||||
def _resolve_video_path(req: VideoGenerationsRequest) -> str | None:
|
||||
video_path = _request_value(req, "video_path") or _request_value(req, "video_url")
|
||||
if video_path:
|
||||
return str(video_path)
|
||||
|
||||
input_reference = _request_value(req, "input_reference")
|
||||
if _is_probably_video_source(input_reference):
|
||||
return str(input_reference)
|
||||
|
||||
reference_url = _request_value(req, "reference_url")
|
||||
if _is_probably_video_source(reference_url):
|
||||
return str(reference_url)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_image_path(
|
||||
req: VideoGenerationsRequest, video_path: str | None
|
||||
) -> str | None:
|
||||
image_path = _request_value(req, "input_reference")
|
||||
if video_path and image_path == video_path:
|
||||
return None
|
||||
if _is_probably_video_source(image_path):
|
||||
return None
|
||||
return image_path
|
||||
|
||||
|
||||
def _resolve_sound_duration(
|
||||
req: VideoGenerationsRequest, *, num_frames: int, fps: int
|
||||
) -> float | None:
|
||||
generate_sound = _request_value(req, "generate_sound")
|
||||
sound_duration = _request_value(req, "sound_duration")
|
||||
|
||||
if generate_sound is False:
|
||||
return 0.0
|
||||
if sound_duration is not None:
|
||||
return float(sound_duration)
|
||||
if generate_sound is True:
|
||||
return float(num_frames) / float(fps)
|
||||
return None
|
||||
|
||||
|
||||
def _cosmos3_sampling_param_kwargs(
|
||||
req: VideoGenerationsRequest, *, num_frames: int, fps: int
|
||||
) -> Dict[str, Any]:
|
||||
"""Map HTTP/API aliases to Cosmos3SamplingParams field names."""
|
||||
kwargs: Dict[str, Any] = {}
|
||||
|
||||
sound_duration = _resolve_sound_duration(req, num_frames=num_frames, fps=fps)
|
||||
if sound_duration is not None:
|
||||
kwargs["sound_duration"] = sound_duration
|
||||
|
||||
condition_frame_indexes = _request_value(req, "condition_frame_indexes")
|
||||
if condition_frame_indexes is None:
|
||||
condition_frame_indexes = _request_value(req, "condition_frame_indexes_vision")
|
||||
condition_frame_indexes = _coerce_optional_int_list(condition_frame_indexes)
|
||||
if condition_frame_indexes is not None:
|
||||
kwargs["condition_frame_indexes"] = condition_frame_indexes
|
||||
|
||||
for name in (
|
||||
"condition_video_keep",
|
||||
"action_mode",
|
||||
"domain_id",
|
||||
"domain_name",
|
||||
"raw_action_dim",
|
||||
"action_fps",
|
||||
"action",
|
||||
"action_view_point",
|
||||
"action_normalization",
|
||||
):
|
||||
value = _parse_form_extra_value(_request_value(req, name))
|
||||
value = _normalize_optional_string(value)
|
||||
if value is not None:
|
||||
kwargs[name] = value
|
||||
|
||||
return kwargs
|
||||
|
||||
|
||||
def _build_video_sampling_params(request_id: str, request: VideoGenerationsRequest):
|
||||
"""Resolve video-specific defaults (fps, seconds → num_frames) then
|
||||
delegate to the shared build_sampling_params."""
|
||||
server_args = get_global_server_args()
|
||||
seconds = request.seconds if request.seconds is not None else DEFAULT_VIDEO_SECONDS
|
||||
fps = request.fps if request.fps is not None else DEFAULT_FPS
|
||||
num_frames = request.num_frames if request.num_frames is not None else fps * seconds
|
||||
num_outputs = request.num_outputs_per_prompt
|
||||
if num_outputs is None:
|
||||
num_outputs = request.n or 1
|
||||
video_path = _resolve_video_path(request)
|
||||
image_path = _resolve_image_path(request, video_path)
|
||||
cosmos3_kwargs = {}
|
||||
if _is_cosmos3_server(server_args):
|
||||
cosmos3_kwargs = _cosmos3_sampling_param_kwargs(
|
||||
request, num_frames=num_frames, fps=fps
|
||||
)
|
||||
if server_args.pipeline_config.action_stats_path is not None:
|
||||
cosmos3_kwargs["action_stats_path"] = (
|
||||
server_args.pipeline_config.action_stats_path
|
||||
)
|
||||
|
||||
return build_sampling_params(
|
||||
request_id,
|
||||
prompt=request.prompt,
|
||||
num_outputs_per_prompt=max(1, min(int(num_outputs), 10)),
|
||||
size=request.size,
|
||||
width=request.width,
|
||||
height=request.height,
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
image_path=image_path,
|
||||
video_path=video_path,
|
||||
output_file_name=request_id,
|
||||
seed=request.seed,
|
||||
generator_device=request.generator_device,
|
||||
num_inference_steps=request.num_inference_steps,
|
||||
guidance_scale=request.guidance_scale,
|
||||
guidance_scale_2=request.guidance_scale_2,
|
||||
negative_prompt=request.negative_prompt,
|
||||
max_sequence_length=request.max_sequence_length,
|
||||
flow_shift=request.flow_shift,
|
||||
use_duration_template=_extra_value(request, "use_duration_template"),
|
||||
use_resolution_template=_extra_value(request, "use_resolution_template"),
|
||||
use_system_prompt=_extra_value(request, "use_system_prompt"),
|
||||
use_guardrails=_extra_value(request, "use_guardrails"),
|
||||
enable_teacache=request.enable_teacache,
|
||||
enable_frame_interpolation=request.enable_frame_interpolation,
|
||||
frame_interpolation_exp=request.frame_interpolation_exp,
|
||||
frame_interpolation_scale=request.frame_interpolation_scale,
|
||||
frame_interpolation_model_path=request.frame_interpolation_model_path,
|
||||
enable_upscaling=request.enable_upscaling,
|
||||
upscaling_model_path=request.upscaling_model_path,
|
||||
upscaling_scale=request.upscaling_scale,
|
||||
output_path=request.output_path,
|
||||
output_compression=request.output_compression,
|
||||
output_quality=request.output_quality,
|
||||
perf_dump_path=request.perf_dump_path,
|
||||
diffusers_kwargs=request.diffusers_kwargs,
|
||||
**cosmos3_kwargs,
|
||||
)
|
||||
|
||||
|
||||
# extract metadata which http_server needs to know
|
||||
def _video_job_from_sampling(
|
||||
request_id: str, req: VideoGenerationsRequest, sampling: SamplingParams
|
||||
) -> Dict[str, Any]:
|
||||
size_str = f"{sampling.width}x{sampling.height}"
|
||||
seconds = int(round((sampling.num_frames or 0) / float(sampling.fps or 24)))
|
||||
return {
|
||||
"id": request_id,
|
||||
"object": "video",
|
||||
"model": req.model or "sora-2",
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"created_at": int(time.time()),
|
||||
"size": size_str,
|
||||
"seconds": str(seconds),
|
||||
"quality": "standard",
|
||||
"file_path": os.path.abspath(sampling.output_file_path()),
|
||||
}
|
||||
|
||||
|
||||
async def _save_first_input_image(
|
||||
image_sources,
|
||||
request_id: str,
|
||||
uploads_dir: str,
|
||||
*,
|
||||
prefer_remote_source: bool = False,
|
||||
) -> str | None:
|
||||
"""Save the first input image from a list of sources and return its path."""
|
||||
image_list = merge_image_input_list(image_sources)
|
||||
if not image_list:
|
||||
return None
|
||||
image = image_list[0]
|
||||
|
||||
os.makedirs(uploads_dir, exist_ok=True)
|
||||
|
||||
filename = image.filename if hasattr(image, "filename") else "url_image"
|
||||
target_path = os.path.join(uploads_dir, f"{request_id}_{filename}")
|
||||
return await save_image_to_path(
|
||||
image, target_path, prefer_remote_source=prefer_remote_source
|
||||
)
|
||||
|
||||
|
||||
async def _dispatch_job_async(
|
||||
job_id: str,
|
||||
batch: Req,
|
||||
*,
|
||||
temp_dirs: list[str] | None = None,
|
||||
output_persistent: bool = True,
|
||||
) -> None:
|
||||
from sglang.multimodal_gen.runtime.scheduler_client import async_scheduler_client
|
||||
|
||||
try:
|
||||
save_file_path_list, result = await process_generation_batch(
|
||||
async_scheduler_client, batch
|
||||
)
|
||||
save_file_path = save_file_path_list[0]
|
||||
|
||||
cloud_url = await cloud_storage.upload_and_cleanup(save_file_path)
|
||||
|
||||
persistent_path = (
|
||||
save_file_path if not cloud_url and output_persistent else None
|
||||
)
|
||||
update_fields = {
|
||||
"status": "completed",
|
||||
"progress": 100,
|
||||
"completed_at": int(time.time()),
|
||||
"url": cloud_url,
|
||||
"file_path": persistent_path,
|
||||
"file_paths": (
|
||||
[os.path.abspath(path) for path in save_file_path_list]
|
||||
if output_persistent
|
||||
else None
|
||||
),
|
||||
"num_outputs": len(save_file_path_list),
|
||||
}
|
||||
update_fields = add_common_data_to_response(
|
||||
update_fields, request_id=job_id, result=result
|
||||
)
|
||||
await VIDEO_STORE.update_fields(job_id, update_fields)
|
||||
except Exception as e:
|
||||
logger.error(f"{e}")
|
||||
await VIDEO_STORE.update_fields(
|
||||
job_id, {"status": "failed", "error": {"message": str(e)}}
|
||||
)
|
||||
finally:
|
||||
for td in temp_dirs or []:
|
||||
shutil.rmtree(td, ignore_errors=True)
|
||||
|
||||
|
||||
# TODO: support image to video generation
|
||||
@router.post("", response_model=VideoResponse)
|
||||
async def create_video(
|
||||
request: Request,
|
||||
# multipart/form-data fields (optional; used only when content-type is multipart)
|
||||
prompt: Optional[str] = Form(None),
|
||||
input_reference: Optional[UploadFile] = File(None),
|
||||
reference_url: Optional[str] = Form(None),
|
||||
video_reference: Optional[UploadFile] = File(None),
|
||||
video_url: Optional[str] = Form(None),
|
||||
video_path: Optional[str] = Form(None),
|
||||
model: Optional[str] = Form(None),
|
||||
n: Optional[int] = Form(1),
|
||||
num_outputs_per_prompt: Optional[int] = Form(None),
|
||||
seconds: Optional[int] = Form(None),
|
||||
size: Optional[str] = Form(None),
|
||||
fps: Optional[int] = Form(None),
|
||||
num_frames: Optional[int] = Form(None),
|
||||
seed: Optional[int] = Form(None),
|
||||
generator_device: Optional[str] = Form("cuda"),
|
||||
negative_prompt: Optional[str] = Form(None),
|
||||
guidance_scale: Optional[float] = Form(None),
|
||||
num_inference_steps: Optional[int] = Form(None),
|
||||
max_sequence_length: Optional[int] = Form(None),
|
||||
flow_shift: Optional[float] = Form(None),
|
||||
enable_teacache: Optional[bool] = Form(None),
|
||||
enable_frame_interpolation: Optional[bool] = Form(None),
|
||||
frame_interpolation_exp: Optional[int] = Form(None),
|
||||
frame_interpolation_scale: Optional[float] = Form(None),
|
||||
frame_interpolation_model_path: Optional[str] = Form(None),
|
||||
enable_upscaling: Optional[bool] = Form(None),
|
||||
upscaling_model_path: Optional[str] = Form(None),
|
||||
upscaling_scale: Optional[int] = Form(None),
|
||||
output_quality: Optional[str] = Form(None),
|
||||
output_compression: Optional[int] = Form(None),
|
||||
output_path: Optional[str] = Form(None),
|
||||
extra_params: Optional[str] = Form(None),
|
||||
extra_body: Optional[str] = Form(None),
|
||||
):
|
||||
content_type = request.headers.get("content-type", "").lower()
|
||||
request_id = generate_request_id()
|
||||
|
||||
server_args = get_global_server_args()
|
||||
task_type = server_args.pipeline_config.task_type
|
||||
|
||||
# Resolve input upload directory (may be a temp dir when saving is disabled)
|
||||
temp_dirs: list[str] = []
|
||||
if server_args.input_save_path is not None:
|
||||
uploads_dir = server_args.input_save_path
|
||||
os.makedirs(uploads_dir, exist_ok=True)
|
||||
else:
|
||||
uploads_dir = tempfile.mkdtemp(prefix="sglang_input_")
|
||||
temp_dirs.append(uploads_dir)
|
||||
|
||||
# Resolve output directory
|
||||
effective_output_path = server_args.output_path
|
||||
output_persistent = True
|
||||
if "multipart/form-data" not in content_type:
|
||||
# JSON body may carry a per-request output_path; checked after parsing below
|
||||
pass
|
||||
|
||||
if "multipart/form-data" in content_type:
|
||||
if not prompt:
|
||||
raise HTTPException(status_code=400, detail="prompt is required")
|
||||
|
||||
video_input_path = None
|
||||
image_sources = merge_image_input_list(input_reference, reference_url)
|
||||
if video_reference is not None:
|
||||
video_input_path = await _save_first_input_image(
|
||||
video_reference,
|
||||
request_id,
|
||||
uploads_dir,
|
||||
prefer_remote_source=server_args.input_save_path is None,
|
||||
)
|
||||
elif video_path or video_url:
|
||||
video_input_path = video_path or video_url
|
||||
elif input_reference is not None and _is_probably_video_source(input_reference):
|
||||
video_input_path = await _save_first_input_image(
|
||||
input_reference,
|
||||
request_id,
|
||||
uploads_dir,
|
||||
prefer_remote_source=server_args.input_save_path is None,
|
||||
)
|
||||
image_sources = merge_image_input_list(reference_url)
|
||||
elif reference_url and _is_probably_video_source(reference_url):
|
||||
video_input_path = reference_url
|
||||
image_sources = merge_image_input_list(input_reference)
|
||||
|
||||
# Validate image input based on model task type
|
||||
if task_type.requires_image_input() and not image_sources:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="input_reference or reference_url is required for image-to-video generation",
|
||||
)
|
||||
input_path = None
|
||||
if image_sources:
|
||||
try:
|
||||
input_path = await _save_first_input_image(
|
||||
image_sources,
|
||||
request_id,
|
||||
uploads_dir,
|
||||
prefer_remote_source=server_args.input_save_path is None,
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Failed to process image source: {str(e)}"
|
||||
)
|
||||
|
||||
# Parse extra_body JSON (if provided in multipart form) to get fps/num_frames overrides
|
||||
extra_from_form: Dict[str, Any] = {}
|
||||
if extra_body:
|
||||
try:
|
||||
extra_from_form = flatten_extra_params(json.loads(extra_body))
|
||||
except Exception:
|
||||
extra_from_form = {}
|
||||
if extra_params:
|
||||
try:
|
||||
extra_from_form.update(
|
||||
flatten_extra_params({"extra_params": json.loads(extra_params)})
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def form_value(name: str, value: Any) -> Any:
|
||||
selected = value if value is not None else extra_from_form.get(name)
|
||||
return _parse_form_extra_value(selected)
|
||||
|
||||
raw_form = await request.form()
|
||||
for key in (
|
||||
"use_duration_template",
|
||||
"use_resolution_template",
|
||||
"use_system_prompt",
|
||||
"use_guardrails",
|
||||
"guardrails",
|
||||
"video_path",
|
||||
"video_url",
|
||||
"generate_sound",
|
||||
"sound_duration",
|
||||
"condition_frame_indexes",
|
||||
"action_mode",
|
||||
"domain_id",
|
||||
"domain_name",
|
||||
"raw_action_dim",
|
||||
"action_fps",
|
||||
"action",
|
||||
"action_view_point",
|
||||
"action_normalization",
|
||||
"condition_frame_indexes_vision",
|
||||
"condition_video_keep",
|
||||
):
|
||||
if key in raw_form and key not in extra_from_form:
|
||||
extra_from_form[key] = _parse_form_extra_value(raw_form[key])
|
||||
flatten_extra_params(extra_from_form)
|
||||
|
||||
request_field_names = set(VideoGenerationsRequest.model_fields)
|
||||
extra_request_fields = {
|
||||
key: value
|
||||
for key, value in extra_from_form.items()
|
||||
if key not in request_field_names
|
||||
}
|
||||
fps_val = form_value("fps", fps)
|
||||
num_frames_val = form_value("num_frames", num_frames)
|
||||
|
||||
req = VideoGenerationsRequest(
|
||||
prompt=prompt,
|
||||
input_reference=input_path,
|
||||
video_path=form_value("video_path", video_input_path),
|
||||
video_url=form_value("video_url", video_url),
|
||||
model=form_value("model", model),
|
||||
n=form_value("n", n),
|
||||
num_outputs_per_prompt=form_value(
|
||||
"num_outputs_per_prompt", num_outputs_per_prompt
|
||||
),
|
||||
seconds=form_value("seconds", seconds) or 4,
|
||||
size=form_value("size", size),
|
||||
fps=fps_val,
|
||||
num_frames=num_frames_val,
|
||||
seed=form_value("seed", seed),
|
||||
generator_device=form_value("generator_device", generator_device),
|
||||
negative_prompt=form_value("negative_prompt", negative_prompt),
|
||||
num_inference_steps=form_value("num_inference_steps", num_inference_steps),
|
||||
guidance_scale=form_value("guidance_scale", guidance_scale),
|
||||
max_sequence_length=form_value("max_sequence_length", max_sequence_length),
|
||||
flow_shift=form_value("flow_shift", flow_shift),
|
||||
enable_teacache=form_value("enable_teacache", enable_teacache),
|
||||
enable_frame_interpolation=form_value(
|
||||
"enable_frame_interpolation", enable_frame_interpolation
|
||||
),
|
||||
frame_interpolation_exp=form_value(
|
||||
"frame_interpolation_exp", frame_interpolation_exp
|
||||
),
|
||||
frame_interpolation_scale=form_value(
|
||||
"frame_interpolation_scale", frame_interpolation_scale
|
||||
),
|
||||
frame_interpolation_model_path=form_value(
|
||||
"frame_interpolation_model_path", frame_interpolation_model_path
|
||||
),
|
||||
enable_upscaling=form_value("enable_upscaling", enable_upscaling),
|
||||
upscaling_model_path=form_value(
|
||||
"upscaling_model_path", upscaling_model_path
|
||||
),
|
||||
upscaling_scale=form_value("upscaling_scale", upscaling_scale),
|
||||
output_compression=form_value("output_compression", output_compression),
|
||||
output_quality=form_value("output_quality", output_quality),
|
||||
output_path=form_value("output_path", output_path),
|
||||
diffusers_kwargs=form_value("diffusers_kwargs", None),
|
||||
**extra_request_fields,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
body = {}
|
||||
try:
|
||||
# If client uses extra_body, merge it into the top-level payload
|
||||
payload: Dict[str, Any] = dict(body or {})
|
||||
extra = payload.pop("extra_body", None)
|
||||
if isinstance(extra, str):
|
||||
extra = json.loads(extra)
|
||||
if isinstance(extra, dict):
|
||||
payload.update(flatten_extra_params(extra))
|
||||
# openai may turn extra_body to extra_json
|
||||
extra_json = payload.pop("extra_json", None)
|
||||
if isinstance(extra_json, str):
|
||||
extra_json = json.loads(extra_json)
|
||||
if isinstance(extra_json, dict):
|
||||
payload.update(flatten_extra_params(extra_json))
|
||||
flatten_extra_params(payload)
|
||||
# Validate image input based on model task type
|
||||
if payload.get("video_url") and not payload.get("video_path"):
|
||||
payload["video_path"] = payload["video_url"]
|
||||
if _is_probably_video_source(payload.get("reference_url")):
|
||||
payload.setdefault("video_path", payload.get("reference_url"))
|
||||
if _is_probably_video_source(payload.get("input_reference")):
|
||||
payload.setdefault("video_path", payload.get("input_reference"))
|
||||
|
||||
has_image_input = (
|
||||
payload.get("reference_url")
|
||||
and not _is_probably_video_source(payload.get("reference_url"))
|
||||
) or (
|
||||
payload.get("input_reference")
|
||||
and not _is_probably_video_source(payload.get("input_reference"))
|
||||
)
|
||||
if task_type.requires_image_input() and not has_image_input:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="input_reference or reference_url is required for image-to-video generation",
|
||||
)
|
||||
# for non-multipart/form-data type
|
||||
if payload.get("reference_url") and not _is_probably_video_source(
|
||||
payload.get("reference_url")
|
||||
):
|
||||
try:
|
||||
input_path = await _save_first_input_image(
|
||||
payload.get("reference_url"),
|
||||
request_id,
|
||||
uploads_dir,
|
||||
prefer_remote_source=server_args.input_save_path is None,
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Failed to process image source: {str(e)}",
|
||||
)
|
||||
payload["input_reference"] = input_path
|
||||
req = VideoGenerationsRequest(**payload)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid request body: {e}")
|
||||
|
||||
# Resolve per-request output_path override
|
||||
effective_output_path = req.output_path or server_args.output_path
|
||||
if effective_output_path is None:
|
||||
output_tmp = tempfile.mkdtemp(prefix="sglang_output_")
|
||||
temp_dirs.append(output_tmp)
|
||||
effective_output_path = output_tmp
|
||||
output_persistent = False
|
||||
|
||||
# Inject resolved output_path so _build_video_sampling_params picks it up
|
||||
req.output_path = effective_output_path
|
||||
|
||||
logger.debug(f"Server received from create_video endpoint: req={req}")
|
||||
|
||||
try:
|
||||
sampling_params = _build_video_sampling_params(request_id, req)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
job = _video_job_from_sampling(request_id, req, sampling_params)
|
||||
await VIDEO_STORE.upsert(request_id, job)
|
||||
|
||||
# Build Req for scheduler
|
||||
trace_headers = extract_trace_headers(request.headers)
|
||||
batch = prepare_request(
|
||||
server_args=server_args,
|
||||
sampling_params=sampling_params,
|
||||
external_trace_header=trace_headers,
|
||||
)
|
||||
# Add diffusers_kwargs if provided
|
||||
if req.diffusers_kwargs:
|
||||
batch.extra["diffusers_kwargs"] = req.diffusers_kwargs
|
||||
if "max_sequence_length" in req.diffusers_kwargs:
|
||||
batch.max_sequence_length = req.diffusers_kwargs["max_sequence_length"]
|
||||
if "flow_shift" in req.diffusers_kwargs:
|
||||
batch.flow_shift = req.diffusers_kwargs["flow_shift"]
|
||||
# Enqueue the job asynchronously and return immediately
|
||||
asyncio.create_task(
|
||||
_dispatch_job_async(
|
||||
request_id,
|
||||
batch,
|
||||
temp_dirs=temp_dirs or None,
|
||||
output_persistent=output_persistent,
|
||||
)
|
||||
)
|
||||
return VideoResponse(**job)
|
||||
|
||||
|
||||
@router.get("", response_model=VideoListResponse)
|
||||
async def list_videos(
|
||||
after: Optional[str] = Query(None),
|
||||
limit: Optional[int] = Query(None, ge=1, le=100),
|
||||
order: Optional[str] = Query("desc"),
|
||||
):
|
||||
# Normalize order
|
||||
order = (order or "desc").lower()
|
||||
if order not in ("asc", "desc"):
|
||||
order = "desc"
|
||||
jobs = await VIDEO_STORE.list_values()
|
||||
|
||||
reverse = order != "asc"
|
||||
jobs.sort(key=lambda j: j.get("created_at", 0), reverse=reverse)
|
||||
|
||||
if after is not None:
|
||||
try:
|
||||
idx = next(i for i, j in enumerate(jobs) if j["id"] == after)
|
||||
jobs = jobs[idx + 1 :]
|
||||
except StopIteration:
|
||||
jobs = []
|
||||
|
||||
if limit is not None:
|
||||
jobs = jobs[:limit]
|
||||
items = [VideoResponse(**j) for j in jobs]
|
||||
return VideoListResponse(data=items)
|
||||
|
||||
|
||||
@router.get("/{video_id}", response_model=VideoResponse)
|
||||
async def retrieve_video(video_id: str = Path(...)):
|
||||
job = await VIDEO_STORE.get(video_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Video not found")
|
||||
return VideoResponse(**job)
|
||||
|
||||
|
||||
# TODO: support aborting a job.
|
||||
@router.delete("/{video_id}", response_model=VideoResponse)
|
||||
async def delete_video(video_id: str = Path(...)):
|
||||
job = await VIDEO_STORE.pop(video_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Video not found")
|
||||
# Mark as deleted in response semantics
|
||||
job["status"] = "deleted"
|
||||
return VideoResponse(**job)
|
||||
|
||||
|
||||
@router.get("/{video_id}/content")
|
||||
async def download_video_content(
|
||||
video_id: str = Path(...), variant: Optional[str] = Query(None)
|
||||
):
|
||||
job = await VIDEO_STORE.get(video_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Video not found")
|
||||
|
||||
if job.get("url"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Video has been uploaded to cloud storage. Please use the cloud URL: {job.get('url')}",
|
||||
)
|
||||
|
||||
file_path = job.get("file_path")
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=404, detail="Generation is still in-progress")
|
||||
|
||||
media_type = "video/mp4" # default variant
|
||||
return FileResponse(
|
||||
path=file_path, media_type=media_type, filename=os.path.basename(file_path)
|
||||
)
|
||||
Reference in New Issue
Block a user