chore: import upstream snapshot with attribution

python/sglang/multimodal_gen/benchmarks/bench_offline_throughput.py

@@ -0,0 +1,540 @@
+"""
+Benchmark offline throughput for multimodal generation models (Image/Video Generation).
+
+This script benchmarks generation throughput without running a server, using low-level APIs.
+It provides detailed metrics on throughput, latency, and resource utilization.
+
+# Usage Examples
+
+## Text-to-Video with VBench dataset
+python -m sglang.multimodal_gen.benchmarks.bench_offline_throughput \\
+    --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \\
+    --dataset vbench \\
+    --num-prompts 20 \\
+    --batch-size 1 \\
+    --width 512 --height 512 --num-frames 16
+
+## Random dataset for stress testing
+python -m sglang.multimodal_gen.benchmarks.bench_offline_throughput \\
+    --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers \\
+    --dataset random \\
+    --num-prompts 100 \\
+    --batch-size 1 \\
+    --num-inference-steps 20 \\
+    --output-file results.json
+"""
+
+import argparse
+import dataclasses
+import json
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+
+import torch
+from tqdm import tqdm
+
+from sglang.multimodal_gen.benchmarks.datasets import RandomDataset, VBenchDataset
+from sglang.multimodal_gen.runtime.entrypoints.diffusion_generator import DiffGenerator
+from sglang.multimodal_gen.runtime.server_args import ServerArgs, set_global_server_args
+from sglang.multimodal_gen.runtime.utils.logging_utils import (
+    configure_logger,
+    init_logger,
+)
+from sglang.multimodal_gen.test.test_utils import print_divider, print_value_formatted
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class BatchOutput:
+    """Container for batch generation results."""
+
+    latency: float = 0.0
+    latency_per_sample: float = 0.0
+    num_samples: int = 0
+    total_frames: int = 0
+    peak_memory_mb: float = 0.0
+    success: bool = False
+    error: str = ""
+
+
+@dataclass
+class BenchArgs:
+    """Benchmark configuration for multimodal generation."""
+
+    # Diffusion Model Configuration
+    num_inference_steps: int = 20
+    guidance_scale: float = 7.5
+    seed: int = 42
+    disable_safety_checker: bool = False
+
+    # Output Configuration
+    width: int = 32
+    height: int = 32
+    num_frames: int = 1
+    fps: int = 24
+
+    # Dataset & Benchmark
+    dataset: str = "random"
+    dataset_path: str = ""
+    task_name: str = "unknown"
+    num_prompts: int = 10
+    num_outputs_per_prompt: int = 1
+    batch_size: int = 1
+    random_request_config: str = None
+    random_request_seed: int = 42
+
+    # Benchmark Execution
+    skip_warmup: bool = False
+    output_file: str = ""
+    disable_tqdm: bool = False
+
+    # Profiling
+    profile: bool = False
+    num_profiled_timesteps: int = 5
+    profile_all_stages: bool = False
+
+    @staticmethod
+    def add_cli_args(parser: argparse.ArgumentParser):
+        """Add benchmark-specific CLI arguments."""
+        # Diffusion Model Configuration
+        parser.add_argument(
+            "--num-inference-steps",
+            type=int,
+            default=20,
+            help="Number of denoising steps",
+        )
+        parser.add_argument(
+            "--guidance-scale",
+            type=float,
+            default=7.5,
+            help="Classifier-free guidance scale",
+        )
+        parser.add_argument("--seed", type=int, default=42, help="Random seed")
+        parser.add_argument(
+            "--disable-safety-checker",
+            action="store_true",
+            help="Disable NSFW detection",
+        )
+
+        # Output Configuration
+        parser.add_argument("--width", type=int, default=32, help="Image/video width")
+        parser.add_argument("--height", type=int, default=32, help="Image/video height")
+        parser.add_argument(
+            "--num-frames", type=int, default=1, help="Number of frames for video"
+        )
+        parser.add_argument("--fps", type=int, default=24, help="FPS for video")
+
+        # Dataset & Benchmark
+        parser.add_argument(
+            "--dataset",
+            type=str,
+            default="random",
+            choices=["vbench", "random"],
+            help="Dataset to use",
+        )
+        parser.add_argument(
+            "--dataset-path",
+            type=str,
+            default="",
+            help="Path to dataset (prompts file or image directory)",
+        )
+        parser.add_argument(
+            "--task-name",
+            type=str,
+            default="unknown",
+            help="Task name for benchmark identification",
+        )
+        parser.add_argument(
+            "--num-prompts",
+            type=int,
+            default=10,
+            help="Total number of prompts to benchmark",
+        )
+        parser.add_argument(
+            "--num-outputs-per-prompt",
+            type=int,
+            default=1,
+            help="Number of generated outputs requested per prompt",
+        )
+        parser.add_argument(
+            "--batch-size",
+            type=int,
+            default=1,
+            help="Batch size per generation call (currently only bs=1 is supported)",
+        )
+
+        parser.add_argument(
+            "--random-request-config",
+            type=str,
+            default=None,
+            help=(
+                "JSON string defining random request profiles. "
+                "Each profile may contain: width, height, num_inference_steps, etc. "
+                "The 'weight' field controls sampling probability (relative weight)."
+            ),
+        )
+        parser.add_argument(
+            "--random-request-seed",
+            type=int,
+            default=42,
+            help="Random seed for sampling request profiles (default: 42).",
+        )
+
+        # Benchmark Execution
+        parser.add_argument(
+            "--skip-warmup", action="store_true", help="Skip warmup batch"
+        )
+        parser.add_argument(
+            "--output-file",
+            type=str,
+            default="",
+            help="Output JSON file for results (append mode)",
+        )
+        parser.add_argument(
+            "--disable-tqdm",
+            action="store_true",
+            help="Disable progress bar",
+        )
+        parser.add_argument(
+            "--profile",
+            action="store_true",
+            help=(
+                "Enable PyTorch profiler for diffusion generation. "
+                "Set SGLANG_DIFFUSION_TORCH_PROFILER_DIR to control trace output directory."
+            ),
+        )
+        parser.add_argument(
+            "--num-profiled-timesteps",
+            type=int,
+            default=5,
+            help=(
+                "Number of denoising timesteps to profile after warmup. "
+                "Use -1 to profile all denoising timesteps."
+            ),
+        )
+        parser.add_argument(
+            "--profile-all-stages",
+            action="store_true",
+            help="Profile all diffusion pipeline stages instead of only denoising steps.",
+        )
+
+    @classmethod
+    def from_cli_args(cls, args: argparse.Namespace):
+        """Create BenchArgs from parsed CLI arguments."""
+        attrs = [attr.name for attr in dataclasses.fields(cls)]
+        return cls(**{attr: getattr(args, attr) for attr in attrs})
+
+
+def initialize_engine(server_args: ServerArgs) -> DiffGenerator:
+    """Initialize diffusion pipeline engine."""
+    logger.info("Initializing engine...")
+    engine = DiffGenerator.from_server_args(server_args, local_mode=True)
+    logger.info("Engine initialized successfully")
+    return engine
+
+
+def generate_batch(
+    engine: DiffGenerator,
+    bench_args: BenchArgs,
+    prompts: List[str],
+    user_sampling_params: List[Dict[str, Any]],
+) -> BatchOutput:
+    """Generate batch of images/videos synchronously."""
+    assert len(user_sampling_params) == len(prompts), (
+        f"user_sampling_params length ({len(user_sampling_params)}) must match "
+        f"prompts length ({len(prompts)})"
+    )
+
+    output = BatchOutput()
+    start_time = time.perf_counter()
+
+    torch.get_device_module().reset_peak_memory_stats()
+
+    for prompt, params in zip(prompts, user_sampling_params):
+        try:
+            sampling_params_kwargs = dict(params)
+            sampling_params_kwargs["prompt"] = prompt
+            result = engine.generate(sampling_params_kwargs=sampling_params_kwargs)
+
+            if result is not None:
+                if isinstance(result, list):
+                    output.total_frames += len(result)
+                else:
+                    output.total_frames += 1
+            output.num_samples += 1
+        except Exception as e:
+            logger.error(f"Generation failed for prompt '{prompt[:50]}...': {e}")
+            output.error = str(e)
+
+    output.latency = time.perf_counter() - start_time
+    output.latency_per_sample = output.latency / len(prompts) if prompts else 0.0
+    output.success = output.num_samples > 0
+    output.peak_memory_mb = torch.get_device_module().max_memory_allocated() / (
+        1024 * 1024
+    )
+
+    logger.debug(
+        f"Batch generated: {output.num_samples}/{len(prompts)} samples in {output.latency:.2f}s"
+    )
+
+    return output
+
+
+def calculate_metrics(
+    outputs: List[BatchOutput],
+    total_duration: float,
+    resolution: Tuple[int, int, int],
+    num_requests: int,
+    all_sampling_params: Optional[List[Dict[str, Any]]] = None,
+) -> Dict[str, Any]:
+    """Calculate generation-specific throughput metrics."""
+    successful = [o for o in outputs if o.success]
+    num_success = sum(o.num_samples for o in successful)
+    total_frames = sum(o.total_frames for o in successful)
+    peak_memory = max((o.peak_memory_mb for o in outputs), default=0)
+
+    width, height, frames = resolution
+    if all_sampling_params:
+        total_pixels = sum(
+            p.get("width", width)
+            * p.get("height", height)
+            * p.get("num_frames", frames)
+            for p in all_sampling_params[:num_success]
+        )
+    else:
+        total_pixels = num_success * width * height * frames
+
+    metrics = {
+        "num_requests": num_requests,
+        "successful_requests": num_success,
+        "failed_requests": num_requests - num_success,
+        "total_duration_seconds": total_duration,
+        "total_frames_generated": total_frames,
+        "total_pixels_generated": total_pixels,
+        "images_per_second": num_success / total_duration if total_duration > 0 else 0,
+        "frames_per_second": total_frames / total_duration if total_duration > 0 else 0,
+        "megapixels_per_second": (
+            total_pixels / (total_duration * 1e6) if total_duration > 0 else 0
+        ),
+        "requests_per_second": (
+            num_success / total_duration if total_duration > 0 else 0
+        ),
+        "latency_per_request_seconds": (
+            total_duration / num_success if num_success > 0 else 0
+        ),
+        "peak_memory_mb": peak_memory,
+    }
+
+    return metrics
+
+
+def throughput_test(
+    server_args: ServerArgs,
+    bench_args: BenchArgs,
+) -> Dict[str, Any]:
+    """Main throughput benchmark function."""
+    configure_logger(server_args=server_args)
+    logger.info("Starting offline throughput benchmark...")
+
+    engine = initialize_engine(server_args)
+
+    if bench_args.random_request_config and bench_args.dataset != "random":
+        raise ValueError(
+            "--random-request-config can only be used with --dataset random"
+        )
+
+    if bench_args.num_outputs_per_prompt != 1:
+        raise ValueError(
+            "bench_offline_throughput currently supports only --num-outputs-per-prompt 1"
+        )
+
+    logger.info(f"Loading {bench_args.dataset} dataset...")
+    if bench_args.dataset == "vbench":
+        bench_args.task_name = str(engine.server_args.pipeline_config.task_type)
+        dataset = VBenchDataset(bench_args)
+    elif bench_args.dataset == "random":
+        dataset = RandomDataset(bench_args)
+    else:
+        raise ValueError(f"Unknown dataset: {bench_args.dataset}")
+
+    _sampling_params = {
+        "guidance_scale": bench_args.guidance_scale,
+        "num_inference_steps": bench_args.num_inference_steps,
+        "height": bench_args.height,
+        "width": bench_args.width,
+        "num_frames": bench_args.num_frames,
+        "num_outputs_per_prompt": bench_args.num_outputs_per_prompt,
+        "seed": bench_args.seed,
+        "profile": bench_args.profile,
+        "num_profiled_timesteps": bench_args.num_profiled_timesteps,
+        "profile_all_stages": bench_args.profile_all_stages,
+    }
+    if bench_args.disable_safety_checker:
+        _sampling_params["safety_checker"] = None
+
+    total_count = min(bench_args.num_prompts, len(dataset))
+    all_prompts = [dataset[i].prompt for i in range(total_count)]
+
+    if bench_args.random_request_config:
+        all_sampling_params = []
+        for i in range(total_count):
+            params = dict(_sampling_params)
+            params.update(dataset.get_sampling_params(i))
+            all_sampling_params.append(params)
+    else:
+        all_sampling_params = [_sampling_params] * total_count
+
+    if not bench_args.skip_warmup:
+        logger.info("Running warmup batch...")
+        warmup_count = min(bench_args.batch_size, total_count)
+        warmup_prompts = all_prompts[:warmup_count]
+        warmup_sampling_params = [
+            {**p, "profile": False} for p in all_sampling_params[:warmup_count]
+        ]
+        generate_batch(engine, bench_args, warmup_prompts, warmup_sampling_params)
+
+    logger.info(f"Running benchmark with {bench_args.num_prompts} prompts...")
+    outputs: List[BatchOutput] = []
+
+    start_time = time.perf_counter()
+
+    num_batches = (total_count + bench_args.batch_size - 1) // bench_args.batch_size
+    pbar = tqdm(
+        total=num_batches,
+        disable=bench_args.disable_tqdm,
+        desc="Benchmark",
+    )
+
+    for batch_start in range(0, total_count, bench_args.batch_size):
+        batch_end = min(batch_start + bench_args.batch_size, total_count)
+        batch_prompts = all_prompts[batch_start:batch_end]
+        batch_sampling_params = all_sampling_params[batch_start:batch_end]
+
+        batch_output = generate_batch(
+            engine, bench_args, batch_prompts, batch_sampling_params
+        )
+        outputs.append(batch_output)
+
+        pbar.update(1)
+
+    pbar.close()
+    total_duration = time.perf_counter() - start_time
+
+    resolution = (bench_args.width, bench_args.height, bench_args.num_frames)
+    metrics = calculate_metrics(
+        outputs,
+        total_duration,
+        resolution=resolution,
+        num_requests=total_count,
+        all_sampling_params=all_sampling_params,
+    )
+
+    display_results(
+        metrics,
+        bench_args,
+        model_path=server_args.model_path,
+    )
+
+    if bench_args.output_file:
+        save_results(metrics, bench_args, server_args)
+
+    return metrics
+
+
+def display_results(
+    metrics: Dict[str, Any],
+    bench_args: BenchArgs,
+    model_path: str,
+):
+    """Display benchmark results in console."""
+    print(
+        "\n{s:{c}^{n}}".format(s=" Offline Throughput Benchmark Result ", n=110, c="=")
+    )
+    print_value_formatted("Model:", model_path)
+    print_value_formatted("Dataset:", bench_args.dataset)
+    print_value_formatted(
+        "Resolution:",
+        f"{bench_args.width}x{bench_args.height}x{bench_args.num_frames}",
+    )
+    print_value_formatted("Num Inference Steps:", bench_args.num_inference_steps)
+    print_divider(75)
+    print_value_formatted("Total Requests:", metrics["num_requests"])
+    print_value_formatted("Successful Requests:", metrics["successful_requests"])
+    print_value_formatted("Failed Requests:", metrics["failed_requests"])
+    print_value_formatted(
+        "Total Duration (seconds):", metrics["total_duration_seconds"]
+    )
+    print_divider(75)
+    print_value_formatted("Frames Generated:", metrics["total_frames_generated"])
+    print_value_formatted(
+        "Megapixels Generated:", metrics["total_pixels_generated"] / 1e6
+    )
+    print_divider(75)
+    print_value_formatted(
+        "Frame Throughput (frames/sec):", metrics["frames_per_second"]
+    )
+    print_value_formatted("MP Throughput (MP/sec):", metrics["megapixels_per_second"])
+    print_value_formatted("Requests Per Second:", metrics["requests_per_second"])
+    print_value_formatted(
+        "Latency Per Request (sec):", metrics["latency_per_request_seconds"]
+    )
+    print_value_formatted("Peak Memory (MB):", metrics["peak_memory_mb"])
+    print_divider(110, "=")
+
+
+def save_results(
+    metrics: Dict[str, Any],
+    bench_args: BenchArgs,
+    server_args: ServerArgs,
+):
+    """Save benchmark results to JSON file."""
+    result = {
+        "metadata": {
+            "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
+            "model_path": server_args.model_path,
+            "task_type": bench_args.task_name,
+            "backend": "engine",
+        },
+        "configuration": {
+            "num_inference_steps": bench_args.num_inference_steps,
+            "guidance_scale": bench_args.guidance_scale,
+            "seed": bench_args.seed,
+            "batch_size": bench_args.batch_size,
+            "num_prompts": bench_args.num_prompts,
+            "resolution": f"{bench_args.width}x{bench_args.height}x{bench_args.num_frames}",
+            "dataset": bench_args.dataset,
+        },
+        "results": metrics,
+    }
+
+    with open(bench_args.output_file, "a") as f:
+        f.write(json.dumps(result) + "\n")
+
+    logger.info(f"Results saved to {bench_args.output_file}")
+
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(
+        description="Offline throughput benchmark for multimodal generation models"
+    )
+
+    ServerArgs.add_cli_args(parser)
+    BenchArgs.add_cli_args(parser)
+
+    args, unknown_args = parser.parse_known_args()
+
+    server_args = ServerArgs.from_cli_args(args, unknown_args)
+    bench_args = BenchArgs.from_cli_args(args)
+
+    set_global_server_args(server_args)
+
+    result = throughput_test(server_args, bench_args)
+
+    return result
+
+
+if __name__ == "__main__":
+    main()

python/sglang/multimodal_gen/benchmarks/bench_serving.py

@@ -0,0 +1,842 @@
+"""
+Benchmark online serving for diffusion models (Image/Video Generation).
+
+
+Usage:
+    # launch a server and benchmark on it
+
+    # T2V or T2I or any other multimodal generation model
+    sglang serve --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers --num-gpus 1 --port 1231
+
+    # benchmark it and make sure the port is the same as the server's port
+    python3 -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --num-prompts 20 --port 1231
+
+    # benchmark with SLO metrics enabled
+    python3 -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --num-prompts 20 --port 1231 --slo --slo-scale 3.0 --warmup-requests 2
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import time
+from dataclasses import replace
+from typing import Any, Dict, List, Optional
+
+import aiohttp
+import numpy as np
+import requests
+from tqdm.asyncio import tqdm
+
+from sglang.multimodal_gen.benchmarks.datasets import (
+    RandomDataset,
+    RequestFuncInput,
+    RequestFuncOutput,
+    VBenchDataset,
+)
+from sglang.multimodal_gen.runtime.utils.logging_utils import (
+    configure_logger,
+    init_logger,
+)
+from sglang.multimodal_gen.test.test_utils import print_divider, print_value_formatted
+from sglang.srt.utils.network import NetworkAddress
+
+logger = init_logger(__name__)
+
+# Patch size used for computing area units (e.g. in latent diffusion models).
+PATCH_SIZE = 16
+PATCH_AREA = PATCH_SIZE * PATCH_SIZE
+
+
+def _get_response_output_count(resp_json: Dict[str, Any]) -> int:
+    if isinstance(resp_json.get("num_outputs"), int):
+        return resp_json["num_outputs"]
+    if isinstance(resp_json.get("data"), list):
+        return len(resp_json["data"])
+    if isinstance(resp_json.get("file_paths"), list):
+        return len(resp_json["file_paths"])
+    if isinstance(resp_json.get("urls"), list):
+        return len(resp_json["urls"])
+    if resp_json.get("file_path") or resp_json.get("url"):
+        return 1
+    return 0
+
+
+def _compute_scale_factor(req: RequestFuncInput, args) -> Optional[float]:
+    """Computes the composite scale factor (area × frames × steps) for a request."""
+    width = req.width or args.width
+    height = req.height or args.height
+    if None in (width, height):
+        return None
+    frames = req.num_frames or args.num_frames
+    steps = req.num_inference_steps or args.num_inference_steps
+
+    frame_scale = frames if isinstance(frames, int) and frames > 0 else 1
+    step_scale = steps if isinstance(steps, int) and steps > 0 else 1
+
+    area_units = max((float(width) * float(height)) / float(PATCH_AREA), 1.0)
+    return area_units * float(frame_scale) * float(step_scale)
+
+
+def _compute_expected_latency_ms_from_base(
+    req: RequestFuncInput, args, base_time_ms: Optional[float]
+) -> Optional[float]:
+    """Scales latency linearly by pixel area, frame count, and inference steps."""
+    if base_time_ms is None:
+        return None
+    scale = _compute_scale_factor(req, args)
+    if scale is None:
+        return None
+    return float(base_time_ms) * scale
+
+
+def _infer_slo_base_time_ms_from_warmups(
+    warmup_pairs: List[tuple], args
+) -> Optional[float]:
+    """Derives median base latency from successful warmup runs."""
+    candidates_ms: List[float] = []
+    for req, out in warmup_pairs:
+        if not out.success or out.latency <= 0:
+            logger.warning(
+                f"Skipping warmup result: success={out.success}, latency={out.latency:.3f}"
+            )
+            continue
+
+        scale = _compute_scale_factor(req, args)
+        if scale is None or scale <= 0:
+            continue
+
+        candidates_ms.append((out.latency * 1000.0) / scale)
+
+    return float(np.median(candidates_ms)) if candidates_ms else None
+
+
+def _populate_slo_ms_from_warmups(
+    requests_list: List[RequestFuncInput], warmup_pairs: List[tuple], args
+) -> List[RequestFuncInput]:
+    """Assigns estimated SLO targets to requests lacking them."""
+    if not any(req.slo_ms is None for req in requests_list):
+        return requests_list
+
+    base_time_ms = _infer_slo_base_time_ms_from_warmups(warmup_pairs, args)
+    if base_time_ms is None:
+        return requests_list
+
+    slo_scale = float(getattr(args, "slo_scale", 3.0))
+    if slo_scale <= 0:
+        raise ValueError(f"slo_scale must be positive, got {slo_scale}.")
+
+    updated: List[RequestFuncInput] = []
+    for req in requests_list:
+        if req.slo_ms is not None:
+            updated.append(req)
+            continue
+        expected_ms = _compute_expected_latency_ms_from_base(req, args, base_time_ms)
+        if expected_ms is not None:
+            # Create a new RequestFuncInput with updated slo_ms
+            updated.append(replace(req, slo_ms=expected_ms * slo_scale))
+        else:
+            updated.append(req)
+
+    return updated
+
+
+async def async_request_image_sglang(
+    input: RequestFuncInput,
+    session: aiohttp.ClientSession,
+    pbar: Optional[tqdm] = None,
+) -> RequestFuncOutput:
+    output = RequestFuncOutput()
+    output.start_time = time.perf_counter()
+
+    # Check if we need to use multipart (for image edits with input images)
+    if input.image_paths and len(input.image_paths) > 0:
+        # Use multipart/form-data for image edits
+        data = aiohttp.FormData()
+        data.add_field("model", input.model)
+        data.add_field("prompt", input.prompt)
+        data.add_field("response_format", "b64_json")
+        data.add_field("n", str(input.num_outputs_per_prompt))
+
+        if input.width and input.height:
+            data.add_field("size", f"{input.width}x{input.height}")
+
+        # Merge extra parameters
+        for key, value in input.extra_body.items():
+            data.add_field(key, str(value))
+
+        # Add image file(s)
+        for idx, img_path in enumerate(input.image_paths):
+            if os.path.exists(img_path):
+                data.add_field(
+                    "image",
+                    open(img_path, "rb"),
+                    filename=os.path.basename(img_path),
+                    content_type="application/octet-stream",
+                )
+            else:
+                output.error = f"Image file not found: {img_path}"
+                output.success = False
+                if pbar:
+                    pbar.update(1)
+                return output
+
+        try:
+            async with session.post(input.api_url, data=data) as response:
+                if response.status == 200:
+                    resp_json = await response.json()
+                    output.response_body = resp_json
+                    output.success = True
+                    output.output_count = _get_response_output_count(resp_json)
+                    if "peak_memory_mb" in resp_json:
+                        output.peak_memory_mb = resp_json["peak_memory_mb"]
+                else:
+                    output.error = f"HTTP {response.status}: {await response.text()}"
+                    output.success = False
+        except Exception as e:
+            output.error = str(e)
+            output.success = False
+    else:
+        # Use JSON for text-to-image generation
+        payload = {
+            "model": input.model,
+            "prompt": input.prompt,
+            "n": input.num_outputs_per_prompt,
+            "response_format": "b64_json",
+        }
+
+        if input.width and input.height:
+            payload["size"] = f"{input.width}x{input.height}"
+        if input.num_inference_steps:
+            payload["num_inference_steps"] = input.num_inference_steps
+
+        # Merge extra parameters
+        payload.update(input.extra_body)
+
+        try:
+            async with session.post(input.api_url, json=payload) as response:
+                if response.status == 200:
+                    resp_json = await response.json()
+                    output.response_body = resp_json
+                    output.success = True
+                    output.output_count = _get_response_output_count(resp_json)
+                    if "peak_memory_mb" in resp_json:
+                        output.peak_memory_mb = resp_json["peak_memory_mb"]
+                else:
+                    output.error = f"HTTP {response.status}: {await response.text()}"
+                    output.success = False
+        except Exception as e:
+            output.error = str(e)
+            output.success = False
+
+    output.latency = time.perf_counter() - output.start_time
+
+    # Check SLO if defined
+    if input.slo_ms is not None and output.success:
+        output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
+
+    if pbar:
+        pbar.update(1)
+    return output
+
+
+async def async_request_video_sglang(
+    input: RequestFuncInput,
+    session: aiohttp.ClientSession,
+    pbar: Optional[tqdm] = None,
+) -> RequestFuncOutput:
+    output = RequestFuncOutput()
+    output.start_time = time.perf_counter()
+
+    # 1. Submit Job
+    job_id = None
+    # Check if we need to upload images (Multipart) or just send JSON
+    if input.image_paths and len(input.image_paths) > 0:
+        # Use multipart/form-data
+        data = aiohttp.FormData()
+        data.add_field("model", input.model)
+        data.add_field("prompt", input.prompt)
+        data.add_field("num_outputs_per_prompt", str(input.num_outputs_per_prompt))
+
+        if input.width and input.height:
+            data.add_field("size", f"{input.width}x{input.height}")
+
+        # Add extra body fields to form data if possible, or assume simple key-values
+        # Note: Nested dicts in extra_body might need JSON serialization if API expects it stringified
+        if input.extra_body:
+            data.add_field("extra_body", json.dumps(input.extra_body))
+
+        # Explicitly add fps/num_frames if they are not in extra_body (bench_serving logic overrides)
+        if input.num_frames:
+            data.add_field("num_frames", str(input.num_frames))
+        if input.fps:
+            data.add_field("fps", str(input.fps))
+
+        # Add image file
+        # Currently only support single image upload as 'input_reference' per API spec
+        img_path = input.image_paths[0]
+        if os.path.exists(img_path):
+            data.add_field(
+                "input_reference",
+                open(img_path, "rb"),
+                filename=os.path.basename(img_path),
+                content_type="application/octet-stream",
+            )
+        else:
+            output.error = f"Image file not found: {img_path}"
+            output.success = False
+            if pbar:
+                pbar.update(1)
+            return output
+
+        try:
+            async with session.post(input.api_url, data=data) as response:
+                if response.status == 200:
+                    resp_json = await response.json()
+                    job_id = resp_json.get("id")
+                else:
+                    output.error = (
+                        f"Submit failed HTTP {response.status}: {await response.text()}"
+                    )
+                    output.success = False
+                    if pbar:
+                        pbar.update(1)
+                    return output
+        except Exception as e:
+            output.error = f"Submit exception: {str(e)}"
+            output.success = False
+            if pbar:
+                pbar.update(1)
+            return output
+
+    else:
+        # Use JSON
+        payload: Dict[str, Any] = {
+            "model": input.model,
+            "prompt": input.prompt,
+            "num_outputs_per_prompt": input.num_outputs_per_prompt,
+        }
+        if input.width and input.height:
+            payload["size"] = f"{input.width}x{input.height}"
+        if input.num_frames:
+            payload["num_frames"] = input.num_frames
+        if input.num_inference_steps:
+            payload["num_inference_steps"] = input.num_inference_steps
+        if input.fps:
+            payload["fps"] = input.fps
+
+        payload.update(input.extra_body)
+
+        try:
+            async with session.post(input.api_url, json=payload) as response:
+                if response.status == 200:
+                    resp_json = await response.json()
+                    job_id = resp_json.get("id")
+                else:
+                    output.error = (
+                        f"Submit failed HTTP {response.status}: {await response.text()}"
+                    )
+                    output.success = False
+                    if pbar:
+                        pbar.update(1)
+                    return output
+        except Exception as e:
+            output.error = f"Submit exception: {str(e)}"
+            output.success = False
+            if pbar:
+                pbar.update(1)
+            return output
+
+    if not job_id:
+        output.error = "No job_id returned"
+        output.success = False
+        if pbar:
+            pbar.update(1)
+        return output
+
+    # 2. Poll for completion
+    # Assuming the API returns a 'status' field.
+    # We construct the check URL. Assuming api_url is like .../v1/videos
+    # The check url should be .../v1/videos/{id}
+    check_url = f"{input.api_url}/{job_id}"
+
+    while True:
+        try:
+            async with session.get(check_url) as response:
+                if response.status == 200:
+                    status_data = await response.json()
+                    status = status_data.get("status")
+                    if status == "completed":
+                        output.success = True
+                        output.response_body = status_data
+                        output.output_count = _get_response_output_count(status_data)
+                        if "peak_memory_mb" in status_data:
+                            output.peak_memory_mb = status_data["peak_memory_mb"]
+                        break
+                    elif status == "failed":
+                        output.success = False
+                        output.error = f"Job failed: {status_data.get('error')}"
+                        break
+                    else:
+                        # queued or processing
+                        await asyncio.sleep(1.0)
+                else:
+                    output.success = False
+                    output.error = (
+                        f"Poll failed HTTP {response.status}: {await response.text()}"
+                    )
+                    break
+        except Exception as e:
+            output.success = False
+            output.error = f"Poll exception: {str(e)}"
+            break
+
+    output.latency = time.perf_counter() - output.start_time
+
+    # Check SLO if defined
+    if input.slo_ms is not None and output.success:
+        output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
+
+    if pbar:
+        pbar.update(1)
+    return output
+
+
+def calculate_metrics(
+    outputs: List[RequestFuncOutput],
+    total_duration: float,
+    requests_list: List[RequestFuncInput],
+    args,
+    slo_enabled: bool,
+):
+    success_outputs = [o for o in outputs if o.success]
+    error_outputs = [o for o in outputs if not o.success]
+
+    num_success = len(success_outputs)
+    latencies = [o.latency for o in success_outputs]
+    completed_outputs = sum(o.output_count for o in success_outputs)
+    peak_memories = [
+        o.peak_memory_mb
+        for o in success_outputs
+        if o.peak_memory_mb is not None and o.peak_memory_mb > 0
+    ]
+
+    metrics = {
+        "duration": total_duration,
+        "completed_requests": num_success,
+        "completed_outputs": completed_outputs,
+        "failed_requests": len(error_outputs),
+        "throughput_qps": num_success / total_duration if total_duration > 0 else 0,
+        "output_throughput_ops": (
+            completed_outputs / total_duration if total_duration > 0 else 0
+        ),
+        "latency_mean": np.mean(latencies) if latencies else 0,
+        "latency_median": np.median(latencies) if latencies else 0,
+        "latency_p50": np.percentile(latencies, 50) if latencies else 0,
+        "latency_p90": np.percentile(latencies, 90) if latencies else 0,
+        "latency_p95": np.percentile(latencies, 95) if latencies else 0,
+        "latency_p99": np.percentile(latencies, 99) if latencies else 0,
+        "num_outputs_per_prompt": args.num_outputs_per_prompt,
+        "peak_memory_mb_max": max(peak_memories) if peak_memories else 0,
+        "peak_memory_mb_mean": np.mean(peak_memories) if peak_memories else 0,
+        "peak_memory_mb_median": np.median(peak_memories) if peak_memories else 0,
+    }
+
+    if slo_enabled:
+        slo_defined_total = 0
+        slo_met_success = 0
+
+        for req, out in zip(requests_list, outputs):
+            if req.slo_ms is None:
+                continue
+            slo_defined_total += 1
+            if out.slo_achieved:
+                slo_met_success += 1
+
+        slo_attain_all = (
+            (slo_met_success / slo_defined_total) if slo_defined_total > 0 else 0.0
+        )
+
+        metrics.update(
+            {
+                "slo_attainment_rate": slo_attain_all,
+                "slo_met_success": slo_met_success,
+                "slo_scale": getattr(args, "slo_scale", 3.0),
+            }
+        )
+
+    return metrics
+
+
+def wait_for_service(base_url: str, timeout: int = 1200) -> None:
+    logger.info(f"Waiting for service at {base_url}...")
+    start_time = time.time()
+    while True:
+        try:
+            # Try /health endpoint first
+            resp = requests.get(f"{base_url}/health", timeout=1)
+            if resp.status_code == 200:
+                logger.info("Service is ready.")
+                break
+        except requests.exceptions.RequestException:
+            pass
+
+        if time.time() - start_time > timeout:
+            raise TimeoutError(
+                f"Service at {base_url} did not start within {timeout} seconds."
+            )
+
+        time.sleep(1)
+
+
+async def benchmark(args):
+    from huggingface_hub import model_info
+
+    # Construct base_url if not provided
+    if args.base_url is None:
+        args.base_url = NetworkAddress(args.host, args.port).to_url()
+
+    # Wait for service
+    wait_for_service(args.base_url)
+
+    # Fetch model info
+    try:
+        resp = requests.get(f"{args.base_url}/v1/model_info", timeout=5)
+        if resp.status_code == 200:
+            info = resp.json()
+            if "model_path" in info and info["model_path"]:
+                args.model = info["model_path"]
+                logger.info(f"Updated model name from server: {args.model}")
+    except Exception as e:
+        logger.info(f"Failed to fetch model info: {e}. Using default: {args.model}")
+
+    valid_tasks = (
+        "text-to-video",
+        "image-to-video",
+        "video-to-video",
+        "text-to-image",
+        "image-to-image",
+    )
+
+    # Resolve task_name with priority: args.task > local config > HF pipeline_tag
+    if args.task:
+        task_name = args.task
+        logger.info(f"Using task from --task: {task_name}")
+    elif os.path.exists(args.model):
+        config_path = os.path.join(args.model, "config.json")
+        if os.path.exists(config_path):
+            with open(config_path, "r") as f:
+                config = json.load(f)
+            task_name = config.get("pipeline_tag", "text-to-image")
+            logger.info(f"Inferred task from local config.json: {task_name}")
+        else:
+            task_name = "text-to-image"
+            logger.info(f"No config.json found, defaulting task to: {task_name}")
+    else:
+        task_name = model_info(args.model).pipeline_tag
+        logger.info(f"Inferred task from HuggingFace pipeline_tag: {task_name}")
+
+    if task_name not in valid_tasks:
+        raise ValueError(
+            f"Task '{task_name}' is not a valid multimodal generation task. "
+            f"Use --task to specify one of: {', '.join(valid_tasks)}"
+        )
+
+    if task_name in ("text-to-video", "image-to-video", "video-to-video"):
+        api_url = f"{args.base_url}/v1/videos"
+        request_func = async_request_video_sglang
+    else:  # text-to-image or image-to-image
+        api_url = (
+            f"{args.base_url}/v1/images/edits"
+            if task_name == "image-to-image"
+            else f"{args.base_url}/v1/images/generations"
+        )
+        request_func = async_request_image_sglang
+
+    setattr(args, "task_name", task_name)
+
+    if args.random_request_config and args.dataset != "random":
+        raise ValueError(
+            "--random-request-config can only be used with --dataset random"
+        )
+
+    if args.dataset == "vbench":
+        dataset = VBenchDataset(args, api_url, args.model)
+    elif args.dataset == "random":
+        dataset = RandomDataset(args, api_url, args.model)
+    else:
+        raise ValueError(f"Unknown dataset: {args.dataset}")
+
+    logger.info("Loading requests...")
+    requests_list = dataset.get_requests()
+    logger.info(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
+
+    # Limit concurrency
+    if args.max_concurrency is not None:
+        semaphore = asyncio.Semaphore(args.max_concurrency)
+    else:
+        semaphore = None
+
+    async def limited_request_func(req, session, pbar):
+        if semaphore:
+            async with semaphore:
+                return await request_func(req, session, pbar)
+        else:
+            return await request_func(req, session, pbar)
+
+    async with aiohttp.ClientSession() as session:
+        # Run warmup requests
+        warmup_pairs: List[tuple] = []
+        if args.warmup_requests and requests_list:
+            # The server always overrides warmup requests to use
+            # num_inference_steps=1 (see Req.set_as_warmup), so we match
+            # that here to keep the benchmark's SLO estimation consistent.
+            warmup_steps = 1
+            logger.info(
+                f"Running {args.warmup_requests} warmup request(s) with "
+                f"num_inference_steps={warmup_steps}..."
+            )
+            for i in range(args.warmup_requests):
+                warm_req = requests_list[i % len(requests_list)]
+                warm_req = replace(
+                    warm_req,
+                    num_inference_steps=warmup_steps,
+                )
+                warm_out = await limited_request_func(warm_req, session, None)
+                warmup_pairs.append((warm_req, warm_out))
+                logger.info(
+                    f"Warmup {i+1}/{args.warmup_requests}: "
+                    f"latency={warm_out.latency:.2f}s, success={warm_out.success}"
+                )
+
+        # Populate SLO values from warmups if enabled
+        if args.slo:
+            requests_list = _populate_slo_ms_from_warmups(
+                requests_list=requests_list, warmup_pairs=warmup_pairs, args=args
+            )
+
+        # Run benchmark
+        pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
+        start_time = time.perf_counter()
+        tasks = []
+        for req in requests_list:
+            if args.request_rate != float("inf"):
+                # Poisson process: inter-arrival times follow exponential distribution
+                interval = np.random.exponential(1.0 / args.request_rate)
+                await asyncio.sleep(interval)
+
+            task = asyncio.create_task(limited_request_func(req, session, pbar))
+            tasks.append(task)
+
+        outputs = await asyncio.gather(*tasks)
+        total_duration = time.perf_counter() - start_time
+
+        pbar.close()
+
+    # Calculate metrics
+    metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
+
+    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
+
+    # Section 1: Configuration
+    print_value_formatted("Task:", task_name)
+    print_value_formatted("Model:", args.model)
+    print_value_formatted("Dataset:", args.dataset)
+
+    # Section 2: Execution & Traffic
+    print_divider(50)
+    print_value_formatted("Benchmark duration (s):", metrics["duration"])
+    print_value_formatted("Request rate:", str(args.request_rate))
+    print_value_formatted(
+        "Max request concurrency:",
+        str(args.max_concurrency) if args.max_concurrency else "not set",
+    )
+    print_value_formatted(
+        "Successful requests:",
+        f"{metrics['completed_requests']}/{len(requests_list)}",
+    )
+    print_value_formatted("Completed outputs:", metrics["completed_outputs"])
+    print_value_formatted("Outputs per prompt:", metrics["num_outputs_per_prompt"])
+
+    # Section 3: Performance Metrics
+    print_divider(50)
+
+    print_value_formatted("Request throughput (req/s):", metrics["throughput_qps"])
+    print_value_formatted(
+        "Output throughput (outputs/s):", metrics["output_throughput_ops"]
+    )
+
+    print_value_formatted("Latency Mean (s):", metrics["latency_mean"])
+    print_value_formatted("Latency Median (s):", metrics["latency_median"])
+    print_value_formatted("Latency P90 (s):", metrics["latency_p90"])
+    print_value_formatted("Latency P95 (s):", metrics["latency_p95"])
+    print_value_formatted("Latency P99 (s):", metrics["latency_p99"])
+
+    if metrics["peak_memory_mb_max"] > 0:
+        print_divider(50)
+        print_value_formatted("Peak Memory Max (MB):", metrics["peak_memory_mb_max"])
+        print_value_formatted("Peak Memory Mean (MB):", metrics["peak_memory_mb_mean"])
+        print_value_formatted(
+            "Peak Memory Median (MB):", metrics["peak_memory_mb_median"]
+        )
+
+    if args.slo and "slo_attainment_rate" in metrics:
+        print_divider(50)
+        print(
+            "{:<40} {:<15.2%}".format(
+                "SLO Attainment Rate:", metrics["slo_attainment_rate"]
+            )
+        )
+        print("{:<40} {:<15}".format("SLO Met (Success):", metrics["slo_met_success"]))
+        print("{:<40} {:<15.2f}".format("SLO Scale:", metrics["slo_scale"]))
+
+    print_divider(60)
+
+    if args.output_file:
+        with open(args.output_file, "w") as f:
+            json.dump(metrics, f, indent=2)
+        print(f"Metrics saved to {args.output_file}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Benchmark serving for diffusion models."
+    )
+    parser.add_argument(
+        "--backend",
+        type=str,
+        default=None,
+        help="DEPRECATED: --task is deprecated and will be ignored. The task will be inferred from --model.",
+    )
+    parser.add_argument(
+        "--base-url",
+        type=str,
+        default=None,
+        help="Base URL of the server (e.g., http://localhost:30000). Overrides host/port.",
+    )
+    parser.add_argument("--host", type=str, default="localhost", help="Server host.")
+    parser.add_argument("--port", type=int, default=30000, help="Server port.")
+    parser.add_argument("--model", type=str, default="default", help="Model name.")
+    parser.add_argument(
+        "--dataset",
+        type=str,
+        default="vbench",
+        choices=["vbench", "random"],
+        help="Dataset to use.",
+    )
+    parser.add_argument(
+        "--task",
+        type=str,
+        choices=[
+            "text-to-video",
+            "image-to-video",
+            "text-to-image",
+            "image-to-image",
+            "video-to-video",
+        ],
+        default=None,
+        help="The task will be inferred from huggingface pipeline_tag. When huggingface pipeline_tag is not provided, --task will be used.",
+    )
+    parser.add_argument(
+        "--dataset-path",
+        type=str,
+        default=None,
+        help="Path to local dataset file (optional).",
+    )
+    parser.add_argument(
+        "--num-prompts", type=int, default=10, help="Number of prompts to benchmark."
+    )
+    parser.add_argument(
+        "--num-outputs-per-prompt",
+        type=int,
+        default=1,
+        help="Number of generated outputs requested per prompt.",
+    )
+    parser.add_argument(
+        "--max-concurrency",
+        type=int,
+        default=1,
+        help="Maximum number of concurrent requests, default to `1`. This can be used "
+        "to help simulate an environment where a higher level component "
+        "is enforcing a maximum number of concurrent requests. While the "
+        "--request-rate argument controls the rate at which requests are "
+        "initiated, this argument will control how many are actually allowed "
+        "to execute at a time. This means that when used in combination, the "
+        "actual request rate may be lower than specified with --request-rate, "
+        "if the server is not processing requests fast enough to keep up.",
+    )
+    parser.add_argument(
+        "--request-rate",
+        type=float,
+        default=float("inf"),
+        help="Number of requests per second. If this is inf, then all the requests are sent at time 0. "
+        "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.",
+    )
+    parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
+    parser.add_argument("--height", type=int, default=None, help="Image/Video height.")
+    parser.add_argument(
+        "--random-request-config",
+        type=str,
+        default=None,
+        help=(
+            "JSON string defining random request profiles. "
+            "Each profile may contain: width, height, num_inference_steps, "
+            "num_outputs_per_prompt, etc. "
+            "The 'weight' field controls sampling probability (relative weight). "
+            "Example: "
+            '[{"width":512,"height":512,"num_inference_steps":20,"weight":0.15},'
+            '{"width":768,"height":768,"num_inference_steps":20,"weight":0.85}]'
+        ),
+    )
+    parser.add_argument(
+        "--random-request-seed",
+        type=int,
+        default=42,
+        help="Random seed for sampling request profiles (default: 42).",
+    )
+    parser.add_argument(
+        "--num-frames", type=int, default=None, help="Number of frames (for video)."
+    )
+    parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
+    parser.add_argument(
+        "--output-file", type=str, default=None, help="Output JSON file for metrics."
+    )
+    parser.add_argument(
+        "--disable-tqdm", action="store_true", help="Disable progress bar."
+    )
+    parser.add_argument(
+        "--log-level",
+        type=str,
+        default="INFO",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        help="Log level.",
+    )
+    parser.add_argument(
+        "--slo",
+        action="store_true",
+        help="Enable SLO calculation. Uses trace-provided slo_ms or infers from warmups.",
+    )
+    parser.add_argument(
+        "--slo-scale",
+        type=float,
+        default=3.0,
+        help="SLO target multiplier: slo_ms = estimated_exec_time_ms * slo_scale (default: 3).",
+    )
+    parser.add_argument(
+        "--warmup-requests",
+        type=int,
+        default=1,
+        help="Number of warmup requests to run before measurement.",
+    )
+    parser.add_argument(
+        "--num-inference-steps",
+        type=int,
+        default=None,
+        help="Number of inference steps for diffusion models.",
+    )
+
+    args = parser.parse_args()
+
+    configure_logger(args)
+
+    asyncio.run(benchmark(args))

python/sglang/multimodal_gen/benchmarks/compare_perf.py

@@ -0,0 +1,301 @@
+import argparse
+import json
+import os
+import re
+from datetime import datetime
+from typing import Any, Dict, List, Tuple
+
+
+def calculate_diff(base: float, new: float) -> Tuple[float, float]:
+    """Returns (diff, diff_percent)."""
+    diff = new - base
+    if base == 0:
+        percent = 0.0
+    else:
+        percent = (diff / base) * 100
+    return diff, percent
+
+
+def calculate_upper_bound(baseline: float, rel_tol: float, min_abs_tol: float) -> float:
+    """Calculates the upper bound for performance regression check."""
+    rel_limit = baseline * (1 + rel_tol)
+    abs_limit = baseline + min_abs_tol
+    return max(rel_limit, abs_limit)
+
+
+def calculate_lower_bound(baseline: float, rel_tol: float, min_abs_tol: float) -> float:
+    """Calculates the lower bound for performance improvement check."""
+    rel_lower = baseline * (1 - rel_tol)
+    abs_lower = baseline - min_abs_tol
+    return min(rel_lower, abs_lower)
+
+
+def get_perf_status_emoji(
+    baseline: float,
+    new: float,
+    rel_tol: float = 0.1,
+    min_abs_tol: float = 120.0,
+) -> str:
+    """
+    Determines the status emoji based on performance difference.
+
+    Logic:
+      Upper bound (Slower): max(baseline * (1 + rel_tol), baseline + min_abs_tol)
+      Lower bound (Faster): min(baseline * (1 - rel_tol), baseline - min_abs_tol)
+    """
+    upper_bound = calculate_upper_bound(baseline, rel_tol, min_abs_tol)
+    lower_bound = calculate_lower_bound(baseline, rel_tol, min_abs_tol)
+
+    if new > upper_bound:
+        return "🔴"
+    elif new < lower_bound:
+        return "🟢"
+    else:
+        return "⚪️"
+
+
+def consolidate_steps(
+    steps_list: List[Dict[str, Any]],
+) -> Tuple[Dict[str, float], List[str], Dict[str, int]]:
+    """
+    Aggregates specific repeating steps (like denoising_step_*) into groups.
+    Returns:
+        - aggregated_durations: {name: duration_ms}
+        - ordered_names: list of names in execution order
+        - counts: {name: count_of_steps_aggregated}
+    """
+    durations = {}
+    counts = {}
+    ordered_names = []
+    seen_names = set()
+
+    # Regex for steps to group
+    # Group "denoising_step_0", "denoising_step_1" -> "Denoising Loop"
+    denoise_pattern = re.compile(r"^denoising_step_(\d+)$")
+    denoising_group_name = "Denoising Loop"
+
+    for step in steps_list:
+        name = step.get("name", "unknown")
+        dur = step.get("duration_ms", 0.0)
+
+        match = denoise_pattern.match(name)
+        if match:
+            key = denoising_group_name
+            if key not in durations:
+                durations[key] = 0.0
+                counts[key] = 0
+                if key not in seen_names:
+                    ordered_names.append(key)
+                    seen_names.add(key)
+            durations[key] += dur
+            counts[key] += 1
+        else:
+            # Standard stage (preserve order)
+            if name not in durations:
+                durations[name] = 0.0
+                counts[name] = 0
+                if name not in seen_names:
+                    ordered_names.append(name)
+                    seen_names.add(name)
+            durations[name] += dur
+            counts[name] += 1
+
+    return durations, ordered_names, counts
+
+
+def _load_benchmark_file(file_path: str) -> Dict[str, Any]:
+    """Loads a benchmark JSON file."""
+    with open(file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _get_status_emoji_from_diff_percent(diff_pct):
+    if diff_pct < -2.0:
+        return "✅"
+    elif diff_pct > 2.0:
+        return "❌"
+    else:
+        return "⚪️"
+
+
+def _print_single_comparison_report(
+    others_data, base_e2e, combined_order, base_durations, others_processed, base_counts
+):
+    new_data = others_data[0]
+    new_e2e = new_data.get("total_duration_ms", 0)
+    diff_ms, diff_pct = calculate_diff(base_e2e, new_e2e)
+    status = _get_status_emoji_from_diff_percent(diff_pct)
+
+    print("#### 1. High-level Summary")
+    print("| Metric | Baseline | New | Diff | Status |")
+    print("| :--- | :--- | :--- | :--- | :--- |")
+    print(
+        f"| **E2E Latency** | {base_e2e:.2f} ms | {new_e2e:.2f} ms | **{diff_ms:+.2f} ms ({diff_pct:+.1f}%)** | {status} |"
+    )
+    print(
+        f"| **Throughput** | {1000 / base_e2e if base_e2e else 0:.2f} req/s | {1000 / new_e2e if new_e2e else 0:.2f} req/s | - | - |"
+    )
+    print("\n")
+
+    print("#### 2. Stage Breakdown")
+    print("| Stage Name | Baseline (ms) | New (ms) | Diff (ms) | Diff (%) | Status |")
+    print("| :--- | :--- | :--- | :--- | :--- | :--- |")
+
+    new_durations, _, new_counts = others_processed[0]
+
+    for stage in combined_order:
+        b_val = base_durations.get(stage, 0.0)
+        n_val = new_durations.get(stage, 0.0)
+        b_count = base_counts.get(stage, 1)
+        n_count = new_counts.get(stage, 1)
+
+        s_diff, s_pct = calculate_diff(b_val, n_val)
+
+        count_str = ""
+        if stage == "Denoising Loop":
+            count_str = (
+                f" ({n_count} steps)"
+                if n_count == b_count
+                else f" ({b_count}->{n_count} steps)"
+            )
+
+        status_emoji = get_perf_status_emoji(b_val, n_val)
+        print(
+            f"| {stage}{count_str} | {b_val:.2f} | {n_val:.2f} | {s_diff:+.2f} | {s_pct:+.1f}% | {status_emoji} |"
+        )
+
+
+def _print_multi_comparison_report(
+    base_e2e,
+    others_data,
+    other_labels,
+    combined_order,
+    base_durations,
+    others_processed,
+):
+    print("#### 1. High-level Summary")
+    header = "| Metric | Baseline | " + " | ".join(other_labels) + " |"
+    sep = "| :--- | :--- | " + " | ".join([":---"] * len(other_labels)) + " |"
+    print(header)
+    print(sep)
+
+    # E2E Row
+    row_e2e = f"| **E2E Latency** | {base_e2e:.2f} ms |"
+    for i, d in enumerate(others_data):
+        val = d.get("total_duration_ms", 0)
+        diff_ms, diff_pct = calculate_diff(base_e2e, val)
+
+        status = _get_status_emoji_from_diff_percent(diff_pct)
+
+        row_e2e += f" {val:.2f} ms ({diff_pct:+.1f}%) {status} |"
+    print(row_e2e)
+    print("\n")
+
+    print("#### 2. Stage Breakdown")
+    # Header: Stage | Baseline | Label1 | Label2 ...
+    header = "| Stage Name | Baseline | " + " | ".join(other_labels) + " |"
+    sep = "| :--- | :--- | " + " | ".join([":---"] * len(other_labels)) + " |"
+    print(header)
+    print(sep)
+
+    for stage in combined_order:
+        b_val = base_durations.get(stage, 0.0)
+        row_str = f"| {stage} | {b_val:.2f} |"
+
+        for i, (n_durations, _, n_counts) in enumerate(others_processed):
+            n_val = n_durations.get(stage, 0.0)
+            _, s_pct = calculate_diff(b_val, n_val)
+            status_emoji = get_perf_status_emoji(b_val, n_val)
+
+            row_str += f" {n_val:.2f} ({s_pct:+.1f}%) {status_emoji} |"
+        print(row_str)
+
+
+def compare_benchmarks(file_paths: List[str], output_format: str = "markdown"):
+    """
+    Compares benchmark JSON files and prints a report.
+    First file is baseline, others will be compared against it.
+    """
+    if len(file_paths) < 2:
+        print("Error: Need at least 2 files to compare.")
+        return
+
+    try:
+        data_list = [_load_benchmark_file(f) for f in file_paths]
+    except Exception as e:
+        print(f"Error loading benchmark files: {e}")
+        return
+
+    base_data = data_list[0]
+    others_data = data_list[1:]
+
+    # Use filenames as labels if multiple comparisons, else just "New"
+    other_labels = [os.path.basename(p) for p in file_paths[1:]]
+
+    base_e2e = base_data.get("total_duration_ms", 0)
+
+    base_durations, base_order, base_counts = consolidate_steps(
+        base_data.get("steps", [])
+    )
+
+    others_processed = []
+    for d in others_data:
+        dur, order, counts = consolidate_steps(d.get("steps", []))
+        others_processed.append((dur, order, counts))
+
+    combined_order = []
+    # Collect all unique stages maintaining order from newest to baseline
+    for _, order, _ in reversed(others_processed):
+        for name in order:
+            if name not in combined_order:
+                combined_order.append(name)
+    for name in base_order:
+        if name not in combined_order:
+            combined_order.append(name)
+
+    if output_format == "markdown":
+        print("### Performance Comparison Report\n")
+
+        if len(others_data) == 1:
+            _print_single_comparison_report(
+                others_data,
+                base_e2e,
+                combined_order,
+                base_durations,
+                others_processed,
+                base_counts,
+            )
+        else:
+            _print_multi_comparison_report(
+                base_e2e,
+                others_data,
+                other_labels,
+                combined_order,
+                base_durations,
+                others_processed,
+            )
+
+        print("\n")
+        # Metadata
+        print("<details>")
+        print("<summary>Metadata</summary>\n")
+        print(f"- Baseline Commit: `{base_data.get('commit_hash', 'N/A')}`")
+        for i, d in enumerate(others_data):
+            label = "New" if len(others_data) == 1 else other_labels[i]
+            print(f"- {label} Commit: `{d.get('commit_hash', 'N/A')}`")
+        print(f"- Timestamp: {datetime.now().isoformat()}")
+        print("</details>")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Compare sglang-diffusion performance JSON files."
+    )
+    parser.add_argument(
+        "files",
+        nargs="+",
+        help="List of JSON files. First is baseline, others are compared against it.",
+    )
+    args = parser.parse_args()
+
+    compare_benchmarks(args.files)

python/sglang/multimodal_gen/benchmarks/datasets.py

@@ -0,0 +1,361 @@
+import glob
+import json
+import os
+import random
+import re
+import subprocess
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import requests
+from PIL import Image
+
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class RequestFuncInput:
+    prompt: str
+    api_url: str = ""
+    model: str = ""
+    num_outputs_per_prompt: int = 1
+    width: Optional[int] = None
+    height: Optional[int] = None
+    num_frames: Optional[int] = None
+    fps: Optional[int] = None
+    extra_body: Dict[str, Any] = field(default_factory=dict)
+    image_paths: Optional[List[str]] = None
+    request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    slo_ms: Optional[float] = None
+    num_inference_steps: Optional[int] = None
+
+
+@dataclass
+class RequestFuncOutput:
+    success: bool = False
+    latency: float = 0.0
+    error: str = ""
+    start_time: float = 0.0
+    response_body: Dict[str, Any] = field(default_factory=dict)
+    peak_memory_mb: float = 0.0
+    slo_achieved: Optional[bool] = None
+    output_count: int = 0
+
+
+def is_dir_not_empty(path: str) -> bool:
+    return os.path.isdir(path) and bool(os.listdir(path))
+
+
+class BaseDataset(ABC):
+    def __init__(self, args, api_url: str = "", model: str = ""):
+        self.args = args
+        self.api_url = api_url
+        self.model = model
+        self.items: List[Dict[str, Any]] = []
+
+    @abstractmethod
+    def __len__(self) -> int:
+        pass
+
+    @abstractmethod
+    def __getitem__(self, idx: int) -> RequestFuncInput:
+        pass
+
+    def get_requests(self) -> List[RequestFuncInput]:
+        return [self[i] for i in range(len(self))]
+
+
+class VBenchDataset(BaseDataset):
+    """
+    Dataset loader for VBench prompts.
+    Supports t2v, i2v.
+    """
+
+    T2V_PROMPT_URL = "https://raw.githubusercontent.com/Vchitect/VBench/master/prompts/prompts_per_dimension/subject_consistency.txt"
+    I2V_DOWNLOAD_SCRIPT_URL = "https://raw.githubusercontent.com/Vchitect/VBench/master/vbench2_beta_i2v/download_data.sh"
+
+    def __init__(self, args, api_url: str = "", model: str = ""):
+        super().__init__(args, api_url, model)
+        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "sglang")
+        self.items = self._load_data()
+
+    @staticmethod
+    def _normalize_task_name(task_name: Any) -> Any:
+        """Normalize enum-style task values to legacy benchmark task-name strings."""
+        enum_to_task_name = {
+            "T2V": "text-to-video",
+            "I2V": "image-to-video",
+            "TI2V": "image-to-video",
+            "T2I": "text-to-image",
+            "I2I": "image-to-image",
+            "TI2I": "image-to-image",
+        }
+
+        # Handle Enum-like objects, e.g., ModelTaskType.T2I
+        enum_name = getattr(task_name, "name", None)
+        if isinstance(enum_name, str):
+            return enum_to_task_name.get(enum_name, task_name)
+
+        # Handle direct string inputs or enum string repr
+        if isinstance(task_name, str):
+            if task_name in enum_to_task_name:
+                return enum_to_task_name[task_name]
+            if "." in task_name:
+                suffix = task_name.split(".")[-1]
+                return enum_to_task_name.get(suffix, task_name)
+
+        return task_name
+
+    def _load_data(self) -> List[Dict[str, Any]]:
+        task_name = self._normalize_task_name(self.args.task_name)
+
+        if task_name in ("text-to-video", "text-to-image", "video-to-video"):
+            return self._load_t2v_prompts()
+        elif task_name in ("image-to-video", "image-to-image"):
+            return self._load_i2v_data()
+        else:
+            raise ValueError(
+                f"Illegal task name is found in VBenchDataset {self.args.task_name}"
+            )
+
+    def _download_file(self, url: str, dest_path: str) -> None:
+        """Download a file from URL to destination path."""
+        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+        resp = requests.get(url)
+        resp.raise_for_status()
+        with open(dest_path, "w") as f:
+            f.write(resp.text)
+
+    def _load_t2v_prompts(self) -> List[Dict[str, Any]]:
+        path = self.args.dataset_path
+
+        if not path:
+            path = os.path.join(self.cache_dir, "vbench_subject_consistency.txt")
+            if not os.path.exists(path):
+                logger.info(f"Downloading VBench T2V prompts to {path}...")
+                try:
+                    self._download_file(self.T2V_PROMPT_URL, path)
+                except Exception as e:
+                    logger.info(f"Failed to download VBench prompts: {e}")
+                    return [{"prompt": "A cat sitting on a bench"}] * 50
+
+        prompts = []
+        with open(path, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    prompts.append({"prompt": line})
+
+        return self._resize_data(prompts)
+
+    def _auto_download_i2v_dataset(self) -> Optional[str]:
+        """Auto-download VBench I2V dataset and return the dataset directory."""
+        vbench_i2v_dir = os.path.join(self.cache_dir, "vbench_i2v", "vbench2_beta_i2v")
+        info_json_path = os.path.join(vbench_i2v_dir, "data", "i2v-bench-info.json")
+        crop_dir = os.path.join(vbench_i2v_dir, "data", "crop")
+        origin_dir = os.path.join(vbench_i2v_dir, "data", "origin")
+
+        if (
+            os.path.exists(info_json_path)
+            and is_dir_not_empty(crop_dir)
+            and is_dir_not_empty(origin_dir)
+        ):
+            return vbench_i2v_dir
+
+        logger.info(f"Downloading VBench I2V dataset to {vbench_i2v_dir}...")
+        try:
+            cache_root = os.path.join(self.cache_dir, "vbench_i2v")
+            script_path = os.path.join(cache_root, "download_data.sh")
+
+            self._download_file(self.I2V_DOWNLOAD_SCRIPT_URL, script_path)
+            os.chmod(script_path, 0o755)
+
+            logger.info("Executing download_data.sh (this may take a while)...")
+
+            result = subprocess.run(
+                ["bash", script_path],
+                cwd=cache_root,
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode != 0:
+                raise RuntimeError(f"Download script failed: {result.stderr}")
+            missing_packages = re.findall(r"(\S+): command not found", result.stderr)
+            if missing_packages:
+                missing_packages = list(set(missing_packages))
+                package_list = ", ".join(f"'{cmd}'" for cmd in missing_packages)
+                raise RuntimeError(
+                    f"Download script failed because the following commands are not installed: {package_list}.\n"
+                    "Please install them (e.g., on Ubuntu: `sudo apt install ...`) and try again."
+                )
+            logger.info(
+                f"Successfully downloaded VBench I2V dataset to {vbench_i2v_dir}"
+            )
+        except Exception as e:
+            logger.info(f"Failed to download VBench I2V dataset: {e}")
+            logger.info("Please manually download following instructions at:")
+            logger.info(
+                "https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v#22-download"
+            )
+            return None
+
+        return vbench_i2v_dir if os.path.exists(info_json_path) else None
+
+    def _load_from_i2v_json(self, json_path: str) -> List[Dict[str, Any]]:
+        """Load I2V data from i2v-bench-info.json format."""
+        with open(json_path, "r") as f:
+            items = json.load(f)
+
+        base_dir = os.path.dirname(
+            os.path.dirname(json_path)
+        )  # Go up to vbench2_beta_i2v
+        origin_dir = os.path.join(base_dir, "data", "origin")
+
+        data = []
+        for item in items:
+            img_path = os.path.join(origin_dir, item.get("file_name", ""))
+            if os.path.exists(img_path):
+                data.append({"prompt": item.get("caption", ""), "image_path": img_path})
+            else:
+                logger.warning(f"Image not found: {img_path}")
+
+        logger.info(f"Loaded {len(data)} I2V samples from VBench I2V dataset")
+        return data
+
+    def _scan_directory_for_images(self, path: str) -> List[Dict[str, Any]]:
+        """Scan directory for image files."""
+        exts = ["*.jpg", "*.jpeg", "*.png", "*.webp"]
+        files = []
+
+        for ext in exts:
+            files.extend(glob.glob(os.path.join(path, ext)))
+            files.extend(glob.glob(os.path.join(path, ext.upper())))
+
+            origin_dir = os.path.join(path, "data", "origin")
+            if os.path.exists(origin_dir):
+                files.extend(glob.glob(os.path.join(origin_dir, ext)))
+                files.extend(glob.glob(os.path.join(origin_dir, ext.upper())))
+
+        return [
+            {"prompt": os.path.splitext(os.path.basename(f))[0], "image_path": f}
+            for f in files
+        ]
+
+    def _create_dummy_data(self) -> List[Dict[str, Any]]:
+        """Create dummy data with a placeholder image in cache directory."""
+        logger.info("No I2V data found. Using dummy placeholders.")
+
+        dummy_image = os.path.join(self.cache_dir, "dummy_image.jpg")
+        if not os.path.exists(dummy_image):
+            os.makedirs(self.cache_dir, exist_ok=True)
+            img = Image.new("RGB", (100, 100), color="red")
+            img.save(dummy_image)
+            logger.info(f"Created dummy image at {dummy_image}")
+
+        return [{"prompt": "A moving cat", "image_path": dummy_image}] * 10
+
+    def _load_i2v_data(self) -> List[Dict[str, Any]]:
+        """Load I2V data from VBench I2V dataset or user-provided path."""
+        path = self.args.dataset_path
+        if not path:
+            path = self._auto_download_i2v_dataset()
+            if not path:
+                return self._resize_data(self._create_dummy_data())
+
+        info_json_candidates = [
+            os.path.join(path, "data", "i2v-bench-info.json"),
+            path if path.endswith(".json") else None,
+        ]
+
+        for json_path in info_json_candidates:
+            if json_path and os.path.exists(json_path):
+                try:
+                    return self._resize_data(self._load_from_i2v_json(json_path))
+                except Exception as e:
+                    logger.info(f"Failed to load {json_path}: {e}")
+
+        if os.path.isdir(path):
+            data = self._scan_directory_for_images(path)
+            if data:
+                return self._resize_data(data)
+
+        return self._resize_data(self._create_dummy_data())
+
+    def _resize_data(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Resize data to match num_prompts."""
+        if not self.args.num_prompts:
+            return data
+
+        if len(data) < self.args.num_prompts:
+            factor = (self.args.num_prompts // len(data)) + 1
+            data = data * factor
+
+        return data[: self.args.num_prompts]
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    def __getitem__(self, idx: int) -> RequestFuncInput:
+        item = self.items[idx]
+        return RequestFuncInput(
+            prompt=item.get("prompt", ""),
+            api_url=self.api_url,
+            model=self.model,
+            num_outputs_per_prompt=self.args.num_outputs_per_prompt,
+            width=self.args.width,
+            height=self.args.height,
+            num_frames=self.args.num_frames,
+            fps=self.args.fps,
+            num_inference_steps=self.args.num_inference_steps,
+            image_paths=[item["image_path"]] if "image_path" in item else None,
+        )
+
+
+class RandomDataset(BaseDataset):
+    def __init__(self, args, api_url: str = "", model: str = ""):
+        super().__init__(args, api_url, model)
+        self.num_prompts = args.num_prompts or 100
+
+        self.random_request_config = args.random_request_config
+        if self.random_request_config:
+            self.random_request_config = json.loads(self.random_request_config)
+            weights = [p.pop("weight") for p in self.random_request_config]
+            seed = args.random_request_seed
+            rng = random.Random(seed)
+            self._sampled_requests = rng.choices(
+                self.random_request_config, weights=weights, k=self.num_prompts
+            )
+        else:
+            self._sampled_requests = None
+
+    def get_sampling_params(self, idx: int) -> dict:
+        """Return the per-request sampling profile dict, or empty dict if not mix-diffusion."""
+        if self._sampled_requests:
+            return self._sampled_requests[idx]
+        return {}
+
+    def __len__(self) -> int:
+        return self.num_prompts
+
+    def __getitem__(self, idx: int) -> RequestFuncInput:
+        profile = self._sampled_requests[idx] if self._sampled_requests else {}
+
+        return RequestFuncInput(
+            prompt=f"Random prompt {idx} for benchmarking diffusion models",
+            api_url=self.api_url,
+            model=self.model,
+            num_outputs_per_prompt=profile.get(
+                "num_outputs_per_prompt", self.args.num_outputs_per_prompt
+            ),
+            width=profile.get("width", self.args.width),
+            height=profile.get("height", self.args.height),
+            num_frames=profile.get("num_frames", self.args.num_frames),
+            num_inference_steps=profile.get(
+                "num_inference_steps", self.args.num_inference_steps
+            ),
+            fps=profile.get("fps", self.args.fps),
+        )