"""Flux2 Klein Denoising Function. This module provides the denoising function for FLUX.2 Klein models, which use Qwen3 as the text encoder instead of CLIP+T5. """ import inspect import math from typing import Any, Callable import numpy as np import torch from tqdm import tqdm from invokeai.backend.rectified_flow.rectified_flow_inpaint_extension import RectifiedFlowInpaintExtension from invokeai.backend.stable_diffusion.diffusers_pipeline import PipelineIntermediateState def denoise( model: torch.nn.Module, # model input img: torch.Tensor, img_ids: torch.Tensor, txt: torch.Tensor, txt_ids: torch.Tensor, # sampling parameters timesteps: list[float], step_callback: Callable[[PipelineIntermediateState], None], guidance: float, cfg_scale: list[float], # Negative conditioning for CFG neg_txt: torch.Tensor | None = None, neg_txt_ids: torch.Tensor | None = None, # Scheduler for stepping (e.g., FlowMatchEulerDiscreteScheduler, FlowMatchHeunDiscreteScheduler) scheduler: Any = None, # Dynamic shifting parameter for FLUX.2 Klein (computed from image resolution) mu: float | None = None, # Inpainting extension for merging latents during denoising inpaint_extension: RectifiedFlowInpaintExtension | None = None, # Reference image conditioning (multi-reference image editing) img_cond_seq: torch.Tensor | None = None, img_cond_seq_ids: torch.Tensor | None = None, ) -> torch.Tensor: """Denoise latents using a FLUX.2 Klein transformer model. This is a simplified denoise function for FLUX.2 Klein models that uses the diffusers Flux2Transformer2DModel interface. All current FLUX.2 Klein variants (4B, 4B Base, 9B, 9B Base) have guidance_embeds=False in their HF transformer config (or absent/zeroed projection weights), so the guidance value is passed but effectively ignored by the model. The argument is retained for node-graph compatibility and future variants that may ship trained guidance projections. CFG is applied externally using negative conditioning when cfg_scale != 1.0. Args: model: The Flux2Transformer2DModel from diffusers. img: Packed latent image tensor of shape (B, seq_len, channels). img_ids: Image position IDs tensor. txt: Text encoder hidden states (Qwen3 embeddings). txt_ids: Text position IDs tensor. timesteps: List of timesteps for denoising schedule (linear sigmas from 1.0 to 1/n). step_callback: Callback function for progress updates. guidance: Guidance strength. Inert for all current FLUX.2 Klein variants (their guidance_embeds projection weights are absent/zero). cfg_scale: List of CFG scale values per step. neg_txt: Negative text embeddings for CFG (optional). neg_txt_ids: Negative text position IDs (optional). scheduler: Optional diffusers scheduler (Euler, Heun, LCM). If None, uses manual Euler. mu: Dynamic shifting parameter computed from image resolution. Required when scheduler has use_dynamic_shifting=True. Returns: Denoised latent tensor. """ total_steps = len(timesteps) - 1 # Store original sequence length for extracting output later (before concatenating reference images) original_seq_len = img.shape[1] # Concatenate reference image conditioning if provided (multi-reference image editing) if img_cond_seq is not None and img_cond_seq_ids is not None: img = torch.cat([img, img_cond_seq], dim=1) img_ids = torch.cat([img_ids, img_cond_seq_ids], dim=1) # The transformer forward() requires a guidance tensor even when guidance_embeds=False, # because the Flux2TimestepGuidanceEmbeddings forward signature takes it unconditionally. # All current Klein variants have guidance_embeds=False, so the value is ignored internally. guidance_vec = torch.full((img.shape[0],), guidance, device=img.device, dtype=img.dtype) # Use scheduler if provided use_scheduler = scheduler is not None if use_scheduler: # Set up scheduler with sigmas and mu for dynamic shifting # Convert timesteps (0-1 range) to sigmas for the scheduler # The scheduler will apply dynamic shifting internally using mu (if enabled in scheduler config) sigmas = np.array(timesteps[:-1], dtype=np.float32) # Exclude final 0.0 # Check if scheduler supports sigmas parameter using inspect.signature # FlowMatchHeunDiscreteScheduler and FlowMatchLCMScheduler don't support sigmas set_timesteps_sig = inspect.signature(scheduler.set_timesteps) supports_sigmas = "sigmas" in set_timesteps_sig.parameters if supports_sigmas and mu is not None: # Pass mu if provided - it will only be used if scheduler has use_dynamic_shifting=True scheduler.set_timesteps(sigmas=sigmas.tolist(), mu=mu, device=img.device) elif supports_sigmas: scheduler.set_timesteps(sigmas=sigmas.tolist(), device=img.device) else: # Scheduler doesn't support sigmas (e.g., Heun, LCM) - use num_inference_steps # # Important for img2img callers: if the initial latent/noise blend was # computed from a separate pre-scheduler schedule, that preblend may not # match this scheduler's true first step exactly. scheduler_kwargs: dict[str, Any] = {"num_inference_steps": len(sigmas), "device": img.device} if mu is not None and "mu" in set_timesteps_sig.parameters: scheduler_kwargs["mu"] = mu scheduler.set_timesteps(**scheduler_kwargs) num_scheduler_steps = len(scheduler.timesteps) is_heun = hasattr(scheduler, "state_in_first_order") user_step = 0 pbar = tqdm(total=total_steps, desc="Denoising") for step_index in range(num_scheduler_steps): timestep = scheduler.timesteps[step_index] # Convert scheduler timestep (0-1000) to normalized (0-1) for the model t_curr = timestep.item() / scheduler.config.num_train_timesteps t_vec = torch.full((img.shape[0],), t_curr, dtype=img.dtype, device=img.device) # Track if we're in first or second order step (for Heun) in_first_order = scheduler.state_in_first_order if is_heun else True # Run the transformer model (matching diffusers: guidance=guidance, return_dict=False) output = model( hidden_states=img, encoder_hidden_states=txt, timestep=t_vec, img_ids=img_ids, txt_ids=txt_ids, guidance=guidance_vec, return_dict=False, ) # Extract the sample from the output (return_dict=False returns tuple) pred = output[0] if isinstance(output, tuple) else output step_cfg_scale = cfg_scale[min(user_step, len(cfg_scale) - 1)] # Apply CFG if scale is not 1.0 if not math.isclose(step_cfg_scale, 1.0): if neg_txt is None: raise ValueError("Negative text conditioning is required when cfg_scale is not 1.0.") neg_output = model( hidden_states=img, encoder_hidden_states=neg_txt, timestep=t_vec, img_ids=img_ids, txt_ids=neg_txt_ids if neg_txt_ids is not None else txt_ids, guidance=guidance_vec, return_dict=False, ) neg_pred = neg_output[0] if isinstance(neg_output, tuple) else neg_output pred = neg_pred + step_cfg_scale * (pred - neg_pred) # Use scheduler.step() for the update step_output = scheduler.step(model_output=pred, timestep=timestep, sample=img) img = step_output.prev_sample # Get t_prev for inpainting (next sigma value) if step_index + 1 < len(scheduler.sigmas): t_prev = scheduler.sigmas[step_index + 1].item() else: t_prev = 0.0 # Apply inpainting merge at each step if inpaint_extension is not None: # Separate the generated latents from the reference conditioning gen_img = img[:, :original_seq_len, :] ref_img = img[:, original_seq_len:, :] # Merge only the generated part gen_img = inpaint_extension.merge_intermediate_latents_with_init_latents(gen_img, t_prev) # Concatenate back together img = torch.cat([gen_img, ref_img], dim=1) # For Heun, only increment user step after second-order step completes if is_heun: if not in_first_order: user_step += 1 if user_step <= total_steps: pbar.update(1) preview_img = img - t_curr * pred if inpaint_extension is not None: preview_img = inpaint_extension.merge_intermediate_latents_with_init_latents( preview_img, 0.0 ) # Extract only the generated image portion for preview (exclude reference images) callback_latents = ( preview_img[:, :original_seq_len, :] if img_cond_seq is not None else preview_img ) step_callback( PipelineIntermediateState( step=user_step, order=2, total_steps=total_steps, timestep=int(t_curr * 1000), latents=callback_latents, ), ) else: user_step += 1 if user_step <= total_steps: pbar.update(1) preview_img = img - t_curr * pred if inpaint_extension is not None: preview_img = inpaint_extension.merge_intermediate_latents_with_init_latents(preview_img, 0.0) # Extract only the generated image portion for preview (exclude reference images) callback_latents = preview_img[:, :original_seq_len, :] if img_cond_seq is not None else preview_img step_callback( PipelineIntermediateState( step=user_step, order=1, total_steps=total_steps, timestep=int(t_curr * 1000), latents=callback_latents, ), ) pbar.close() else: # Manual Euler stepping (original behavior) for step_index, (t_curr, t_prev) in tqdm(list(enumerate(zip(timesteps[:-1], timesteps[1:], strict=True)))): t_vec = torch.full((img.shape[0],), t_curr, dtype=img.dtype, device=img.device) # Run the transformer model (matching diffusers: guidance=guidance, return_dict=False) output = model( hidden_states=img, encoder_hidden_states=txt, timestep=t_vec, img_ids=img_ids, txt_ids=txt_ids, guidance=guidance_vec, return_dict=False, ) # Extract the sample from the output (return_dict=False returns tuple) pred = output[0] if isinstance(output, tuple) else output step_cfg_scale = cfg_scale[step_index] # Apply CFG if scale is not 1.0 if not math.isclose(step_cfg_scale, 1.0): if neg_txt is None: raise ValueError("Negative text conditioning is required when cfg_scale is not 1.0.") neg_output = model( hidden_states=img, encoder_hidden_states=neg_txt, timestep=t_vec, img_ids=img_ids, txt_ids=neg_txt_ids if neg_txt_ids is not None else txt_ids, guidance=guidance_vec, return_dict=False, ) neg_pred = neg_output[0] if isinstance(neg_output, tuple) else neg_output pred = neg_pred + step_cfg_scale * (pred - neg_pred) # Euler step preview_img = img - t_curr * pred img = img + (t_prev - t_curr) * pred # Apply inpainting merge at each step if inpaint_extension is not None: # Separate the generated latents from the reference conditioning gen_img = img[:, :original_seq_len, :] ref_img = img[:, original_seq_len:, :] # Merge only the generated part gen_img = inpaint_extension.merge_intermediate_latents_with_init_latents(gen_img, t_prev) # Concatenate back together img = torch.cat([gen_img, ref_img], dim=1) # Handling preview images preview_gen = preview_img[:, :original_seq_len, :] preview_gen = inpaint_extension.merge_intermediate_latents_with_init_latents(preview_gen, 0.0) # Extract only the generated image portion for preview (exclude reference images) callback_latents = preview_img[:, :original_seq_len, :] if img_cond_seq is not None else preview_img step_callback( PipelineIntermediateState( step=step_index + 1, order=1, total_steps=total_steps, timestep=int(t_curr), latents=callback_latents, ), ) # Extract only the generated image portion (exclude concatenated reference images) if img_cond_seq is not None: img = img[:, :original_seq_len, :] return img