sgl-project--sglang/python/sglang/multimodal_gen/configs/models/dits/zimage.py

# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo

# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
from typing import Tuple

from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
from sglang.multimodal_gen.configs.models.fsdp import is_zimage_layer


@dataclass
class ZImageArchConfig(DiTArchConfig):
    all_patch_size: Tuple[int, ...] = (2,)
    all_f_patch_size: Tuple[int, ...] = (1,)
    in_channels: int = 16
    out_channels: int | None = None
    dim: int = 3840
    num_layers: int = 30
    n_refiner_layers: int = 2
    num_attention_heads: int = 30
    n_kv_heads: int = 30
    norm_eps: float = 1e-5
    qk_norm: bool = True
    cap_feat_dim: int = 2560
    rope_theta: float = 256.0
    t_scale: float = 1000.0
    axes_dims: Tuple[int, int, int] = (32, 48, 48)
    axes_lens: Tuple[int, int, int] = (1024, 512, 512)

    _fsdp_shard_conditions: list = field(default_factory=lambda: [is_zimage_layer])

    stacked_params_mapping: list[tuple[str, str, str]] = field(
        default_factory=lambda: [
            # (param_name, shard_name, shard_id)
            (".feed_forward.w13", ".feed_forward.w1", "gate"),
            (".feed_forward.w13", ".feed_forward.w3", "up"),
        ]
    )

    param_names_mapping: dict = field(
        default_factory=lambda: {
            r"(.*)\.attention\.to_q\.weight$": (r"\1.attention.to_qkv.weight", 0, 3),
            r"(.*)\.attention\.to_k\.weight$": (r"\1.attention.to_qkv.weight", 1, 3),
            r"(.*)\.attention\.to_v\.weight$": (r"\1.attention.to_qkv.weight", 2, 3),
            r"(.*)\.attention\.to_q\.weight_scale_inv$": (
                r"\1.attention.to_qkv.weight_scale_inv",
                0,
                3,
            ),
            r"(.*)\.attention\.to_k\.weight_scale_inv$": (
                r"\1.attention.to_qkv.weight_scale_inv",
                1,
                3,
            ),
            r"(.*)\.attention\.to_v\.weight_scale_inv$": (
                r"\1.attention.to_qkv.weight_scale_inv",
                2,
                3,
            ),
            r"(.*)\.attention\.to_q\.(lora_A|lora_B)$": (
                r"\1.attention.to_qkv.\2",
                0,
                3,
            ),
            r"(.*)\.attention\.to_k\.(lora_A|lora_B)$": (
                r"\1.attention.to_qkv.\2",
                1,
                3,
            ),
            r"(.*)\.attention\.to_v\.(lora_A|lora_B)$": (
                r"\1.attention.to_qkv.\2",
                2,
                3,
            ),
            r"(.*)\.feed_forward\.w1\.weight$": (r"\1.feed_forward.w13.weight", 0, 2),
            r"(.*)\.feed_forward\.w3\.weight$": (r"\1.feed_forward.w13.weight", 1, 2),
            r"(.*)\.feed_forward\.w1\.(lora_A|lora_B)$": (
                r"\1.feed_forward.w13.\2",
                0,
                2,
            ),
            r"(.*)\.feed_forward\.w3\.(lora_A|lora_B)$": (
                r"\1.feed_forward.w13.\2",
                1,
                2,
            ),
        }
    )

    def __post_init__(self):
        super().__post_init__()
        self.out_channels = self.out_channels or self.in_channels
        self.num_channels_latents = self.in_channels
        self.hidden_size = self.dim


@dataclass
class ZImageDitConfig(DiTConfig):
    arch_config: ZImageArchConfig = field(default_factory=ZImageArchConfig)

    prefix: str = "zimage"