vllm-project--vllm-omni/tests/model_extras/test_model_extras.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

from __future__ import annotations

import pytest
from PIL import Image

from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_extras import (
    build_image_to_image_prompt,
    build_image_to_video_prompt,
    build_text_to_image_prompt,
    get_extra_body_params,
    get_extra_output_params,
    should_init_extra_args_for_non_diffusion_stages,
)


@pytest.mark.core_model
@pytest.mark.cpu
def test_bagel_extra_registry_declares_request_and_response_params() -> None:
    assert get_extra_body_params("BagelPipeline") == frozenset(
        {
            "cfg_text_scale",
            "cfg_img_scale",
            "cfg_interval",
            "cfg_renorm_type",
            "cfg_renorm_min",
            "negative_prompt",
            "think",
            "max_think_tokens",
            "do_sample",
            "text_temperature",
            "timestep_shift",
        }
    )
    assert get_extra_output_params("BagelPipeline") == frozenset({"text_output", "think_text"})
    assert should_init_extra_args_for_non_diffusion_stages("BagelPipeline") is True


@pytest.mark.core_model
@pytest.mark.cpu
def test_sensenova_extra_registry_declares_request_and_response_params() -> None:
    assert get_extra_body_params("SenseNovaU1Pipeline") == frozenset(
        {
            "think",
            "cfg_scale",
            "cfg_norm",
            "timestep_shift",
            "t_eps",
            "img_cfg_scale",
            "max_tokens",
        }
    )
    assert get_extra_output_params("SenseNovaU1Pipeline") == frozenset({"think_text"})
    assert should_init_extra_args_for_non_diffusion_stages("SenseNovaU1Pipeline") is False


@pytest.mark.core_model
@pytest.mark.cpu
def test_cosmos3_extra_registry_declares_request_and_response_params() -> None:
    assert get_extra_body_params("Cosmos3OmniDiffusersPipeline") == frozenset(
        {
            "flow_shift",
            "max_sequence_length",
            "use_resolution_template",
            "use_duration_template",
            "use_system_prompt",
            "system_prompt",
            "negative_prompt",
            "guardrails",
            "condition_frame_indexes_vision",
            "condition_video_keep",
            "generate_sound",
            "sound_gen",
            "sound_duration",
            "audio_duration",
            "action_mode",
            "action",
            "domain_name",
            "domain_id",
            "raw_action_dim",
            "action_chunk_size",
            "action_space",
            "action_fps",
            "image_height",
            "image_width",
            "history_length",
            "conditioning_fps",
            "resolution",
            "image_size",
            "use_state",
            "observation",
            "robot_obs",
            "deterministic_seed",
            "session_id",
        }
    )
    assert get_extra_output_params("Cosmos3OmniDiffusersPipeline") == frozenset(
        {
            "action",
            "raw_action_dim",
            "domain_id",
            "action_mode",
        }
    )
    assert should_init_extra_args_for_non_diffusion_stages("Cosmos3OmniDiffusersPipeline") is False


@pytest.mark.core_model
@pytest.mark.cpu
def test_magi_human_extra_registry_declares_request_and_response_params() -> None:
    assert get_extra_body_params("MagiHumanPipeline") == frozenset(
        {
            "seconds",
            "audio_path",
            "image_path",
            "sr_height",
            "sr_width",
            "sr_num_inference_steps",
        }
    )
    assert get_extra_output_params("MagiHumanPipeline") == frozenset()
    assert should_init_extra_args_for_non_diffusion_stages("MagiHumanPipeline") is False


@pytest.mark.core_model
@pytest.mark.cpu
def test_ming_flash_omni_extra_registry_declares_request_and_response_params() -> None:
    assert get_extra_body_params("MingImagePipeline") == frozenset(
        {
            "height",
            "width",
            "steps",
            "cfg",
            "seed",
            "byte5_text",
            "negative_prompt",
        }
    )
    assert get_extra_output_params("MingImagePipeline") == frozenset()
    assert should_init_extra_args_for_non_diffusion_stages("MingImagePipeline") is True


@pytest.mark.core_model
@pytest.mark.cpu
def test_ming_flash_omni_declared_extra_args_route_into_sampling_params() -> None:
    params = OmniDiffusionSamplingParams()
    declared = get_extra_body_params("MingImagePipeline")
    apply_declared_extra_args(
        params,
        declared,
        {
            "steps": 6,
            "cfg": 1.5,
            "byte5_text": ["理解与生成统一"],
            "negative_prompt": "ugly, blurry",
            "unknown": "ignored",
        },
    )
    assert params.extra_args == {
        "steps": 6,
        "cfg": 1.5,
        "byte5_text": ["理解与生成统一"],
        "negative_prompt": "ugly, blurry",
    }


@pytest.mark.core_model
@pytest.mark.cpu
def test_ming_flash_omni_text_to_image_prompt_builder() -> None:
    assert build_text_to_image_prompt(
        "MingImagePipeline",
        prompt="Please draw a cute cat.",
        negative_prompt="ugly, blurry",
        height=512,
        width=768,
    ) == {
        "prompt": "Please draw a cute cat.",
        "modalities": ["image"],
        "mm_processor_kwargs": {
            "modalities": ["image"],
            "target_h": 512,
            "target_w": 768,
        },
        "negative_prompt": "ugly, blurry",
    }
    # target_h/w are omitted when height/width are not supplied so the
    # pipeline's 1024x1024 default still applies.
    assert build_text_to_image_prompt(
        "MingImagePipeline",
        prompt="Draw a poster.",
        negative_prompt=None,
    ) == {
        "prompt": "Draw a poster.",
        "modalities": ["image"],
        "mm_processor_kwargs": {"modalities": ["image"]},
    }


@pytest.mark.core_model
@pytest.mark.cpu
def test_ming_flash_omni_image_to_image_prompt_builder() -> None:
    dummy_image = Image.new("RGB", (64, 64))
    result = build_image_to_image_prompt(
        "MingImagePipeline",
        prompt="Change the background to a sandy beach at sunset.",
        negative_prompt=None,
        input_image=dummy_image,
        height=256,
        width=256,
    )
    assert result["modalities"] == ["img2img"]
    assert result["multi_modal_data"]["img2img"] is dummy_image
    assert result["mm_processor_kwargs"] == {
        "modalities": ["img2img"],
        "target_h": 256,
        "target_w": 256,
    }
    assert "negative_prompt" not in result


@pytest.mark.core_model
@pytest.mark.cpu
def test_cosmos3_text_to_image_prompt_builder_selects_image_modality() -> None:
    assert build_text_to_image_prompt(
        "Cosmos3OmniDiffusersPipeline",
        prompt="a red sports car at golden hour",
        negative_prompt="blurry, distorted",
        height=1024,
        width=1024,
    ) == {
        "prompt": "a red sports car at golden hour",
        "modalities": ["image"],
        "negative_prompt": "blurry, distorted",
    }
    assert build_text_to_image_prompt(
        "Cosmos3OmniDiffusersPipeline",
        prompt="a red sports car",
        negative_prompt=None,
    ) == {"prompt": "a red sports car", "modalities": ["image"]}


@pytest.mark.core_model
@pytest.mark.cpu
def test_audiox_extra_registry_declares_request_and_response_params() -> None:
    assert get_extra_body_params("AudioXPipeline") == frozenset(
        {
            "audiox_task",
            "seconds_start",
            "seconds_total",
            "sigma_min",
            "sigma_max",
            "cfg_rescale",
            "video_path",
            "audio_path",
        }
    )
    assert get_extra_output_params("AudioXPipeline") == frozenset({"audiox_task"})
    assert should_init_extra_args_for_non_diffusion_stages("AudioXPipeline") is False


@pytest.mark.core_model
@pytest.mark.cpu
def test_audiox_declared_extra_args_route_into_sampling_params() -> None:
    params = OmniDiffusionSamplingParams()
    declared = get_extra_body_params("AudioXPipeline")
    apply_declared_extra_args(
        params,
        declared,
        {
            "audiox_task": "t2a",
            "seconds_total": 10.0,
            "sigma_min": 0.03,
            "unknown": "ignored",
        },
    )
    assert params.extra_args == {
        "audiox_task": "t2a",
        "seconds_total": 10.0,
        "sigma_min": 0.03,
    }


@pytest.mark.core_model
@pytest.mark.cpu
def test_helios_extra_registry_declares_request_and_response_params() -> None:
    expected_body = frozenset(
        {
            "is_enable_stage2",
            "pyramid_num_stages",
            "pyramid_num_inference_steps_list",
            "is_amplify_first_chunk",
            "is_skip_first_chunk",
            "use_cfg_zero_star",
            "use_zero_init",
            "zero_steps",
            "image",
            "video",
            "add_noise_to_image_latents",
            "image_noise_sigma_min",
            "image_noise_sigma_max",
            "add_noise_to_video_latents",
            "video_noise_sigma_min",
            "video_noise_sigma_max",
        }
    )
    # Both the base and pyramid class names resolve to the same declaration.
    for cls in ("HeliosPipeline", "HeliosPyramidPipeline"):
        assert get_extra_body_params(cls) == expected_body
        assert get_extra_output_params(cls) == frozenset()
        assert should_init_extra_args_for_non_diffusion_stages(cls) is False


@pytest.mark.core_model
@pytest.mark.cpu
def test_vace_extra_registry_has_no_pipeline_params() -> None:
    assert get_extra_body_params("WanVACEPipeline") == frozenset()
    assert get_extra_output_params("WanVACEPipeline") == frozenset()
    assert should_init_extra_args_for_non_diffusion_stages("WanVACEPipeline") is False


@pytest.mark.core_model
@pytest.mark.cpu
def test_unknown_pipeline_has_empty_extra_registry() -> None:
    assert get_extra_body_params("UnknownPipeline") == frozenset()
    assert get_extra_output_params("UnknownPipeline") == frozenset()
    assert should_init_extra_args_for_non_diffusion_stages("UnknownPipeline") is False


@pytest.mark.core_model
@pytest.mark.cpu
def test_bagel_text_to_image_prompt_builder() -> None:
    assert build_text_to_image_prompt(
        "BagelPipeline",
        prompt="a cat",
        negative_prompt="blurry",
        height=512,
        width=768,
    ) == {
        "prompt": "<|im_start|>a cat<|im_end|>",
        "modalities": ["image"],
        "mm_processor_kwargs": {
            "target_h": 512,
            "target_w": 768,
            "modalities": ["image"],
        },
        "negative_prompt": "blurry",
    }


@pytest.mark.core_model
@pytest.mark.cpu
def test_bagel_image_to_image_prompt_builder() -> None:
    dummy_image = Image.new("RGB", (64, 64))
    result = build_image_to_image_prompt(
        "BagelPipeline",
        prompt="paint it",
        negative_prompt="ugly",
        input_image=dummy_image,
        height=256,
        width=256,
    )
    assert result["prompt"] == "<|fim_middle|><|im_start|>paint it<|im_end|>"
    assert result["modalities"] == ["img2img"]
    assert result["multi_modal_data"]["img2img"] is dummy_image
    assert result["mm_processor_kwargs"]["target_h"] == 256
    assert result["mm_processor_kwargs"]["target_w"] == 256
    assert result["negative_prompt"] == "ugly"


@pytest.mark.core_model
@pytest.mark.cpu
def test_unknown_pipeline_uses_default_text_to_image_prompt() -> None:
    assert build_text_to_image_prompt(
        "UnknownPipeline",
        prompt="a cat",
        negative_prompt=None,
        height=512,
        width=512,
    ) == {"prompt": "a cat"}


@pytest.mark.core_model
@pytest.mark.cpu
def test_unknown_pipeline_uses_default_image_to_image_prompt() -> None:
    dummy_image = Image.new("RGB", (64, 64))
    result = build_image_to_image_prompt(
        "UnknownPipeline",
        prompt="edit",
        negative_prompt=None,
        input_image=dummy_image,
    )
    assert result == {
        "prompt": "edit",
        "multi_modal_data": {"image": dummy_image},
    }


def _build_vace_prompt(media_inputs: dict[str, object], *, num_frames: int = 5) -> dict:
    return build_image_to_video_prompt(
        "WanVACEPipeline",
        prompt="a bird flying",
        negative_prompt=None,
        media_inputs=media_inputs,
        height=16,
        width=320,
        num_frames=num_frames,
    )


@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize(
    "media_inputs",
    [
        {"image": Image.new("RGB", (320, 16), "red")},
        {"last_image": Image.new("RGB", (320, 16), "blue")},
        {
            "image": Image.new("RGB", (320, 16), "red"),
            "last_image": Image.new("RGB", (320, 16), "blue"),
        },
        {"image": Image.new("RGB", (320, 16), "red"), "mask": Image.new("L", (320, 16), 0)},
        {"reference_images": [Image.new("RGB", (64, 64), "red")]},
    ],
    ids=["i2v", "v2lf", "flf2v", "inpaint", "r2v"],
)
def test_vace_image_to_video_prompt_builder(media_inputs: dict[str, object]) -> None:
    result = _build_vace_prompt(media_inputs)
    mmd = result["multi_modal_data"]
    if "reference_images" in mmd:
        assert mmd["reference_images"] is media_inputs["reference_images"]
    else:
        assert len(mmd["video"]) == len(mmd["mask"]) == 5


@pytest.mark.core_model
@pytest.mark.cpu
@pytest.mark.parametrize(
    ("media_inputs", "message"),
    [
        ({}, "requires a conditioning media input"),
        ({"mask": Image.new("L", (320, 16))}, "mask input requires an image"),
        ({"control_image": Image.new("RGB", (320, 16))}, "Unsupported VACE media input"),
    ],
)
def test_vace_rejects_invalid_media_combinations(media_inputs: dict[str, object], message: str) -> None:
    with pytest.raises(ValueError, match=message):
        _build_vace_prompt(media_inputs)


@pytest.mark.core_model
@pytest.mark.cpu
def test_declared_extra_args_apply_to_existing_sampling_params() -> None:
    params = OmniDiffusionSamplingParams(extra_args={"existing": 1})

    declared_extra_params: frozenset[str] = frozenset({"cfg_text_scale", "think"})
    apply_declared_extra_args(
        params,
        declared_extra_params,
        {
            "cfg_text_scale": 4.0,
            "think": False,
            "unknown": "ignored",
        },
    )

    assert params.extra_args == {
        "existing": 1,
        "cfg_text_scale": 4.0,
        "think": False,
    }


@pytest.mark.core_model
@pytest.mark.cpu
def test_mammothmoda2_extra_registry_declares_request_and_response_params() -> None:
    assert get_extra_body_params("MammothModa2DiTPipeline") == frozenset(
        {
            "text_guidance_scale",
            "cfg_range",
            "num_inference_steps",
        }
    )
    assert get_extra_output_params("MammothModa2DiTPipeline") == frozenset()
    assert should_init_extra_args_for_non_diffusion_stages("MammothModa2DiTPipeline") is True


@pytest.mark.core_model
@pytest.mark.cpu
def test_mammothmoda2_text_to_image_prompt_builder() -> None:
    # Image dims are converted to the AR grid (width/16 x height/16); the negative
    # prompt is ignored (MammothModa2 t2i uses CFG, not an explicit negative path).
    assert build_text_to_image_prompt(
        "MammothModa2DiTPipeline",
        prompt="a cat",
        negative_prompt="blurry",
        height=512,
        width=768,
    ) == {
        "prompt": (
            "<|im_start|>system\nYou are a helpful image generator.<|im_end|>\n"
            "<|im_start|>user\na cat<|im_end|>\n"
            "<|im_start|>assistant\n"
            "<|image start|>48*32<|image token|>"
        ),
        "additional_information": {
            "omni_task": ["t2i"],
            "ar_width": [48],
            "ar_height": [32],
            "image_height": [512],
            "image_width": [768],
        },
    }