sgl-project--sglang/python/sglang/multimodal_gen/configs/models/dits/ltx_2.py

# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
from enum import Enum

from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
from sglang.multimodal_gen.configs.models.fsdp import is_blocks_or_transformer_blocks


class LTXModelType(Enum):
    """
    Model type enum mirroring upstream `LTXModelType`.

    Upstream reference:
      - `LTX-2/packages/ltx-core/src/ltx_core/model/transformer/model.py::LTXModelType`
    """

    AudioVideo = "ltx av model"
    VideoOnly = "ltx video only model"
    AudioOnly = "ltx audio only model"

    def is_video_enabled(self) -> bool:
        return self in (LTXModelType.AudioVideo, LTXModelType.VideoOnly)

    def is_audio_enabled(self) -> bool:
        return self in (LTXModelType.AudioVideo, LTXModelType.AudioOnly)


class LTX2RopeType(str, Enum):
    """
    Minimal RoPE type enum mirroring LTX-2 upstream `LTXRopeType`.

    Upstream reference:
      - `LTX-2/packages/ltx-core/src/ltx_core/model/transformer/rope.py::LTXRopeType`
    """

    INTERLEAVED = "interleaved"
    SPLIT = "split"


class LTX2AttentionFunction(str, Enum):
    """
    Placeholder enum for upstream `AttentionFunction.DEFAULT`.

    Upstream reference:
      - `LTX-2/packages/ltx-core/src/ltx_core/model/transformer/attention.py`
    """

    DEFAULT = "default"


@dataclass
class LTX2ArchConfig(DiTArchConfig):
    """Architecture configuration for LTX-2 Video Transformer."""

    _fsdp_shard_conditions: list = field(
        default_factory=lambda: [is_blocks_or_transformer_blocks]
    )

    param_names_mapping: dict = field(
        default_factory=lambda: {
            # Parameter name mappings from HuggingFace checkpoint keys to SGLang module names.
            # We use upstream variable names (patchify_proj, adaln_single) but HF uses different keys.
            #
            # HF key -> SGLang key (upstream naming)
            r"^model\.diffusion_model\.(.*)$": r"\1",
            r"^proj_in\.(.*)$": r"patchify_proj.\1",
            r"^time_embed\.(.*)$": r"adaln_single.\1",
            r"^audio_proj_in\.(.*)$": r"audio_patchify_proj.\1",
            r"^audio_time_embed\.(.*)$": r"audio_adaln_single.\1",
            # FeedForward
            r"(.*)ff\.net\.0\.proj\.(.*)$": r"\1ff.proj_in.\2",
            r"(.*)ff\.net\.2\.(.*)$": r"\1ff.proj_out.\2",
            # Attention Norms
            r"(.*)\.norm_q\.(.*)$": r"\1.q_norm.\2",
            r"(.*)\.norm_k\.(.*)$": r"\1.k_norm.\2",
            # Scale Shift Tables (Global)
            r"^av_cross_attn_video_scale_shift\.(.*)$": r"av_ca_video_scale_shift_adaln_single.\1",
            r"^av_cross_attn_audio_scale_shift\.(.*)$": r"av_ca_audio_scale_shift_adaln_single.\1",
            r"^av_cross_attn_video_a2v_gate\.(.*)$": r"av_ca_a2v_gate_adaln_single.\1",
            r"^av_cross_attn_audio_v2a_gate\.(.*)$": r"av_ca_v2a_gate_adaln_single.\1",
            # Scale Shift Tables (Block Level)
            # HF: scale_shift_table_a2v_ca_video -> SGLang: video_a2v_cross_attn_scale_shift_table
            r"(.*)scale_shift_table_a2v_ca_video": r"\1video_a2v_cross_attn_scale_shift_table",
            r"(.*)scale_shift_table_a2v_ca_audio": r"\1audio_a2v_cross_attn_scale_shift_table",
        }
    )

    reverse_param_names_mapping: dict = field(
        default_factory=lambda: {
            # Reverse mapping: SGLang module names -> HF checkpoint keys (for saving).
            r"^patchify_proj\.(.*)$": r"proj_in.\1",
            r"^adaln_single\.(.*)$": r"time_embed.\1",
            r"^audio_patchify_proj\.(.*)$": r"audio_proj_in.\1",
            r"^audio_adaln_single\.(.*)$": r"audio_time_embed.\1",
            # FeedForward
            r"(.*)ff\.proj_in\.(.*)$": r"\1ff.net.0.proj.\2",
            r"(.*)ff\.proj_out\.(.*)$": r"\1ff.net.2.\2",
            # Attention Norms
            r"(.*)\.q_norm\.(.*)$": r"\1.norm_q.\2",
            r"(.*)\.k_norm\.(.*)$": r"\1.norm_k.\2",
            # Scale Shift Tables (Global)
            r"^av_ca_video_scale_shift_adaln_single\.(.*)$": r"av_cross_attn_video_scale_shift.\1",
            r"^av_ca_audio_scale_shift_adaln_single\.(.*)$": r"av_cross_attn_audio_scale_shift.\1",
            r"^av_ca_a2v_gate_adaln_single\.(.*)$": r"av_cross_attn_video_a2v_gate.\1",
            r"^av_ca_v2a_gate_adaln_single\.(.*)$": r"av_cross_attn_audio_v2a_gate.\1",
            # Scale Shift Tables (Block Level)
            # SGLang: video_a2v_cross_attn_scale_shift_table -> HF: scale_shift_table_a2v_ca_video
            r"(.*)video_a2v_cross_attn_scale_shift_table": r"\1scale_shift_table_a2v_ca_video",
            r"(.*)audio_a2v_cross_attn_scale_shift_table": r"\1scale_shift_table_a2v_ca_audio",
        }
    )

    lora_param_names_mapping: dict = field(
        default_factory=lambda: {
            # LoRA parameter name mappings from official repo format to HF format.
            # This is applied before param_names_mapping when loading LoRA adapters.
            # Will be populated if LoRA adapters use different naming conventions.
        }
    )

    # Model type and attention configuration
    model_type: LTXModelType = LTXModelType.AudioVideo
    attention_type: LTX2AttentionFunction = LTX2AttentionFunction.DEFAULT
    rope_type: LTX2RopeType = LTX2RopeType.INTERLEAVED
    double_precision_rope: bool = False
    quantize_video_rope_coords_to_hidden_dtype: bool = False
    apply_gated_attention: bool = False
    cross_attention_adaln: bool = False
    caption_proj_before_connector: bool = False

    # Video parameters
    num_attention_heads: int = 32
    attention_head_dim: int = 128
    in_channels: int = 128
    out_channels: int = 128
    num_layers: int = 48
    cross_attention_dim: int = 4096
    norm_eps: float = 1e-6
    caption_channels: int = 3840
    positional_embedding_theta: float = 10000.0
    positional_embedding_max_pos: list[int] | None = None
    timestep_scale_multiplier: int = 1000
    use_middle_indices_grid: bool = True

    # Audio parameters
    audio_num_attention_heads: int = 32
    audio_attention_head_dim: int = 64
    audio_in_channels: int = 128
    audio_out_channels: int = 128
    audio_cross_attention_dim: int = 2048
    audio_positional_embedding_max_pos: list[int] | None = None
    av_ca_timestep_scale_multiplier: int = 1

    # 2.3 connector-related fields may show up in transformer/config.json.
    connector_attention_head_dim: int = 128
    connector_num_attention_heads: int = 30
    connector_num_layers: int = 2
    audio_connector_attention_head_dim: int = 128
    audio_connector_num_attention_heads: int = 30
    audio_connector_num_layers: int = 2

    # SGLang-specific parameters
    patch_size: tuple[int, int, int] = (1, 2, 2)
    text_len: int = 512
    enable_packed_qkv_input_a2a: bool = False

    def __post_init__(self):
        super().__post_init__()
        # Video derived values
        self.hidden_size = self.num_attention_heads * self.attention_head_dim
        self.num_channels_latents = self.out_channels
        if self.positional_embedding_max_pos is None:
            self.positional_embedding_max_pos = [20, 2048, 2048]

        # Audio derived values
        self.audio_hidden_size = (
            self.audio_num_attention_heads * self.audio_attention_head_dim
        )
        if self.audio_positional_embedding_max_pos is None:
            self.audio_positional_embedding_max_pos = [20]


@dataclass
class LTX2Config(DiTConfig):
    """Configuration for LTX-2 Video Transformer."""

    arch_config: LTX2ArchConfig = field(default_factory=LTX2ArchConfig)

    prefix: str = "ltx2"
    torch_compile_mode: str = "default"