invoke-ai--invokeai/invokeai/backend/z_image/z_image_patchify_utils.py

# Copyright (c) 2024, Lincoln D. Stein and the InvokeAI Development Team
"""Utility functions for Z-Image patchify operations."""

from typing import List, Tuple

import torch

# Sequence must be multiple of this value (from diffusers transformer_z_image)
SEQ_MULTI_OF = 32


def create_coordinate_grid(
    size: Tuple[int, ...],
    start: Tuple[int, ...] | None = None,
    device: torch.device | None = None,
) -> torch.Tensor:
    """Create a coordinate grid for position embeddings.

    Args:
        size: Size of the grid (e.g., (F, H, W))
        start: Starting coordinates (default: all zeros)
        device: Target device

    Returns:
        Coordinate grid tensor of shape (*size, len(size))
    """
    if start is None:
        start = tuple(0 for _ in size)

    axes = [
        torch.arange(x0, x0 + span, dtype=torch.int32, device=device) for x0, span in zip(start, size, strict=False)
    ]
    grids = torch.meshgrid(axes, indexing="ij")
    return torch.stack(grids, dim=-1)


def patchify_control_context(
    all_image: List[torch.Tensor],
    patch_size: int,
    f_patch_size: int,
    cap_seq_len: int,
) -> Tuple[List[torch.Tensor], List[Tuple[int, int, int]], List[torch.Tensor], List[torch.Tensor]]:
    """Patchify control images without embedding.

    This function extracts patches from control images for control context processing.
    It handles padding and position ID creation for the control signal.

    Args:
        all_image: List of control image tensors [C, F, H, W]
        patch_size: Spatial patch size (height and width)
        f_patch_size: Frame patch size
        cap_seq_len: Caption sequence length (for position ID offset)

    Returns:
        Tuple of:
        - all_image_out: List of patchified image tensors
        - all_image_size: List of (F, H, W) tuples
        - all_image_pos_ids: List of position ID tensors
        - all_image_pad_mask: List of padding mask tensors
    """
    pH = pW = patch_size
    pF = f_patch_size
    device = all_image[0].device

    all_image_out: List[torch.Tensor] = []
    all_image_size: List[Tuple[int, int, int]] = []
    all_image_pos_ids: List[torch.Tensor] = []
    all_image_pad_mask: List[torch.Tensor] = []

    # Calculate padded caption length for position offset
    cap_padding_len = (-cap_seq_len) % SEQ_MULTI_OF
    cap_padded_len = cap_seq_len + cap_padding_len

    for image in all_image:
        C, F, H, W = image.size()
        all_image_size.append((F, H, W))
        F_tokens, H_tokens, W_tokens = F // pF, H // pH, W // pW

        # Patchify: [C, F, H, W] -> [(F_tokens*H_tokens*W_tokens), (pF*pH*pW*C)]
        # Step 1: Rearrange to put spatial dims together for proper patching
        # [C, F, H, W] -> [F, H, W, C]
        image = image.permute(1, 2, 3, 0).contiguous()

        # Step 2: Split H and W into tokens and patch sizes
        # [F, H, W, C] -> [F, H_tokens, pH, W_tokens, pW, C]
        image = image.view(F, H_tokens, pH, W_tokens, pW, C)

        # Step 3: Rearrange to group patches and features
        # [F, H_tokens, pH, W_tokens, pW, C] -> [F, H_tokens, W_tokens, pH, pW, C]
        image = image.permute(0, 1, 3, 2, 4, 5).contiguous()

        # Step 4: For F > 1, we'd need to handle F similarly, but for F=1 this is simpler
        # Final reshape: [F*H_tokens*W_tokens, pH*pW*C]
        num_patches = F_tokens * H_tokens * W_tokens
        patch_features = pF * pH * pW * C
        image = image.reshape(num_patches, patch_features)

        image_ori_len = len(image)
        image_padding_len = (-image_ori_len) % SEQ_MULTI_OF

        # Create position IDs
        image_ori_pos_ids = create_coordinate_grid(
            size=(F_tokens, H_tokens, W_tokens),
            start=(cap_padded_len + 1, 0, 0),
            device=device,
        ).flatten(0, 2)

        image_padding_pos_ids = (
            create_coordinate_grid(
                size=(1, 1, 1),
                start=(0, 0, 0),
                device=device,
            )
            .flatten(0, 2)
            .repeat(image_padding_len, 1)
        )
        image_padded_pos_ids = torch.cat([image_ori_pos_ids, image_padding_pos_ids], dim=0)
        all_image_pos_ids.append(image_padded_pos_ids)

        # Padding mask
        all_image_pad_mask.append(
            torch.cat(
                [
                    torch.zeros((image_ori_len,), dtype=torch.bool, device=device),
                    torch.ones((image_padding_len,), dtype=torch.bool, device=device),
                ],
                dim=0,
            )
        )

        # Padded feature
        image_padded_feat = torch.cat([image, image[-1:].repeat(image_padding_len, 1)], dim=0)
        all_image_out.append(image_padded_feat)

    return all_image_out, all_image_size, all_image_pos_ids, all_image_pad_mask