# Copyright 2024 NVIDIA CORPORATION & AFFILIATES # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # SPDX-License-Identifier: Apache-2.0 import math import re from collections.abc import Iterable from functools import lru_cache from itertools import repeat import torch import torch.nn as nn import torch.nn.functional as F from torch.nn.attention.flex_attention import create_block_mask from torch.utils.checkpoint import checkpoint def _ntuple(n): def parse(x): if isinstance(x, Iterable) and not isinstance(x, str): return x return tuple(repeat(x, n)) return parse to_2tuple = _ntuple(2) to_3tuple = _ntuple(3) def set_grad_checkpoint(model, gc_step=1): assert isinstance(model, nn.Module) def set_attr(module): module.grad_checkpointing = True module.grad_checkpointing_step = gc_step model.apply(set_attr) def set_fp32_attention(model): assert isinstance(model, nn.Module) def set_attr(module): module.fp32_attention = True model.apply(set_attr) def auto_grad_checkpoint(module, *args, **kwargs): if getattr(module, "grad_checkpointing", False): if isinstance(module, Iterable): gc_step = module[0].grad_checkpointing_step return checkpoint_sequential(module, gc_step, *args, **kwargs) else: return checkpoint(module, *args, **kwargs) return module(*args, **kwargs) def checkpoint_sequential(functions, step, input, *args, **kwargs): # Hack for keyword-only parameter in a python 2.7-compliant way preserve = kwargs.pop("preserve_rng_state", True) if kwargs: raise ValueError("Unexpected keyword arguments: " + ",".join(arg for arg in kwargs)) def run_function(start, end, functions): def forward(input): for j in range(start, end + 1): input = functions[j](input, *args) return input return forward if isinstance(functions, torch.nn.Sequential): functions = list(functions.children()) # the last chunk has to be non-volatile end = -1 segment = len(functions) // step for start in range(0, step * (segment - 1), step): end = start + step - 1 input = checkpoint(run_function(start, end, functions), input, preserve_rng_state=preserve) return run_function(end + 1, len(functions) - 1, functions)(input) def prepare_prompt_ar(prompt, ratios, device="cpu", show=True): # get aspect_ratio or ar aspect_ratios = re.findall(r"--aspect_ratio\s+(\d+:\d+)", prompt) ars = re.findall(r"--ar\s+(\d+:\d+)", prompt) custom_hw = re.findall(r"--hw\s+(\d+:\d+)", prompt) if show: print("aspect_ratios:", aspect_ratios, "ars:", ars, "hws:", custom_hw) prompt_clean = prompt.split("--aspect_ratio")[0].split("--ar")[0].split("--hw")[0] if len(aspect_ratios) + len(ars) + len(custom_hw) == 0 and show: print( "Wrong prompt format. Set to default ar: 1. change your prompt into format '--ar h:w or --hw h:w' for correct generating" ) if len(aspect_ratios) != 0: ar = float(aspect_ratios[0].split(":")[0]) / float(aspect_ratios[0].split(":")[1]) elif len(ars) != 0: ar = float(ars[0].split(":")[0]) / float(ars[0].split(":")[1]) else: ar = 1.0 closest_ratio = min(ratios.keys(), key=lambda ratio: abs(float(ratio) - ar)) if len(custom_hw) != 0: custom_hw = [float(custom_hw[0].split(":")[0]), float(custom_hw[0].split(":")[1])] else: custom_hw = ratios[closest_ratio] default_hw = ratios[closest_ratio] prompt_show = f"prompt: {prompt_clean.strip()}\nSize: --ar {closest_ratio}, --bin hw {ratios[closest_ratio]}, --custom hw {custom_hw}" return ( prompt_clean, prompt_show, torch.tensor(default_hw, device=device)[None], torch.tensor([float(closest_ratio)], device=device)[None], torch.tensor(custom_hw, device=device)[None], ) def resize_and_crop_tensor(samples: torch.Tensor, new_width: int, new_height: int) -> torch.Tensor: orig_height, orig_width = samples.shape[2], samples.shape[3] # Check if resizing is needed if orig_height != new_height or orig_width != new_width: ratio = max(new_height / orig_height, new_width / orig_width) resized_width = int(orig_width * ratio) resized_height = int(orig_height * ratio) # Resize samples = F.interpolate(samples, size=(resized_height, resized_width), mode="bilinear", align_corners=False) # Center Crop start_x = (resized_width - new_width) // 2 end_x = start_x + new_width start_y = (resized_height - new_height) // 2 end_y = start_y + new_height samples = samples[:, :, start_y:end_y, start_x:end_x] return samples def val2list(x: list or tuple or any, repeat_time=1) -> list: # type: ignore """Repeat `val` for `repeat_time` times and return the list or val if list/tuple.""" if isinstance(x, (list, tuple)): return list(x) return [x for _ in range(repeat_time)] def val2tuple(x: list or tuple or any, min_len: int = 1, idx_repeat: int = -1) -> tuple: # type: ignore """Return tuple with min_len by repeating element at idx_repeat.""" # convert to list first x = val2list(x) # repeat elements if necessary if len(x) > 0: x[idx_repeat:idx_repeat] = [x[idx_repeat] for _ in range(min_len - len(x))] return tuple(x) def get_same_padding(kernel_size: int or tuple[int, ...]) -> int or tuple[int, ...]: if isinstance(kernel_size, tuple): return tuple([get_same_padding(ks) for ks in kernel_size]) else: assert kernel_size % 2 > 0, f"kernel size {kernel_size} should be odd number" return kernel_size // 2 def get_weight_dtype(mixed_precision): if mixed_precision in ["fp16", "float16"]: return torch.float16 elif mixed_precision in ["bf16", "bfloat16"]: return torch.bfloat16 elif mixed_precision in ["fp32", "float32", "float"]: return torch.float32 else: raise ValueError(f"weigh precision {mixed_precision} is not defined") @lru_cache def create_block_mask_cached(score_mod, B, H, M, N, device="cuda", _compile=False): block_mask = create_block_mask(score_mod, B, H, M, N, device=device, _compile=_compile) return block_mask def generate_temporal_head_mask_mod( context_length: int = 226, prompt_length: int = 226, num_frames: int = 13, token_per_frame: int = 1350, mul: int = 2 ): def round_to_multiple(idx): return math.ceil(idx / 128) * 128 def temporal_mask_mod(b, h, q_idx, kv_idx): two_frame = round_to_multiple(mul * token_per_frame) temporal_head_mask = torch.abs(q_idx - kv_idx) <= two_frame # return temporal_head_mask first_frame_mask = kv_idx < token_per_frame video_mask = first_frame_mask | temporal_head_mask return video_mask return temporal_mask_mod