invoke-ai--invokeai/invokeai/backend/image_util/pbr_maps/pbr_maps.py

# Original: https://github.com/joeyballentine/Material-Map-Generator
# Adopted and optimized for Invoke AI

import pathlib
from typing import Any, Literal

import cv2
import numpy as np
import numpy.typing as npt
import torch
from PIL import Image
from safetensors.torch import load_file

from invokeai.backend.image_util.pbr_maps.architecture.pbr_rrdb_net import PBR_RRDB_Net
from invokeai.backend.image_util.pbr_maps.utils.image_ops import crop_seamless, esrgan_launcher_split_merge

NORMAL_MAP_MODEL = (
    "https://huggingface.co/InvokeAI/pbr-material-maps/resolve/main/normal_map_generator.safetensors?download=true"
)
OTHER_MAP_MODEL = (
    "https://huggingface.co/InvokeAI/pbr-material-maps/resolve/main/franken_map_generator.safetensors?download=true"
)


class PBRMapsGenerator:
    def __init__(self, normal_map_model: PBR_RRDB_Net, other_map_model: PBR_RRDB_Net, device: torch.device) -> None:
        self.normal_map_model = normal_map_model
        self.other_map_model = other_map_model
        self.device = device

    @staticmethod
    def load_model(model_path: pathlib.Path, device: torch.device) -> PBR_RRDB_Net:
        state_dict = load_file(model_path.as_posix(), device=device.type)

        model = PBR_RRDB_Net(
            3,
            3,
            32,
            12,
            gc=32,
            upscale=1,
            norm_type=None,
            act_type="leakyrelu",
            mode="CNA",
            res_scale=1,
            upsample_mode="upconv",
        )

        model.load_state_dict(state_dict, strict=False)

        del state_dict
        if torch.cuda.is_available() and device.type == "cuda":
            torch.cuda.empty_cache()

        model.eval()

        for _, v in model.named_parameters():
            v.requires_grad = False

        return model.to(device)

    def process(self, img: npt.NDArray[Any], model: PBR_RRDB_Net):
        img = img.astype(np.float32) / np.iinfo(img.dtype).max
        img = img[..., ::-1].copy()
        tensor_img = torch.tensor(img).permute(2, 0, 1).unsqueeze(0).to(self.device)

        with torch.no_grad():
            output = model(tensor_img).data.squeeze(0).float().cpu().clamp_(0, 1).numpy()
            output = output[[2, 1, 0], :, :]
            output = np.transpose(output, (1, 2, 0))
            output = (output * 255.0).round()
            return output

    def _cv2_to_pil(self, image: npt.NDArray[Any]):
        return Image.fromarray(cv2.cvtColor(image.astype(np.uint8), cv2.COLOR_RGB2BGR))

    def generate_maps(
        self,
        image: Image.Image,
        tile_size: int = 512,
        border_mode: Literal["none", "seamless", "mirror", "replicate"] = "none",
    ):
        """
        Generate PBR texture maps (normal, roughness, and displacement) from an input image.
        The image can optionally be padded before inference to control how borders are treated,
        which can help create seamless or edge‑consistent textures.

        Args:
            image: Source image used to generate the PBR maps.
            tile_size: Maximum tile size used for tiled inference. If the image is larger than
                this size in either dimension, it will be split into tiles for processing and
                then merged.

            border_mode: Strategy for padding the image before inference:
                - "none": No padding is applied; the image is processed as‑is.
                - "seamless": Pads the image using wrap‑around tiling
                  (`cv2.BORDER_WRAP`) to help produce seamless textures.
                - "mirror": Pads the image by mirroring border pixels
                  (`cv2.BORDER_REFLECT_101`) to reduce edge artifacts.
                - "replicate": Pads the image by replicating the edge pixels outward
                  (`cv2.BORDER_REPLICATE`).

        Returns:
            A tuple of three PIL Images:
                - normal_map: RGB normal map generated from the input.
                - roughness: Single‑channel roughness map extracted from the second model output.
                - displacement: Single‑channel displacement (height) map extracted from the
                  second model output.
        """

        models = [self.normal_map_model, self.other_map_model]
        np_image = np.array(image).astype(np.uint8)

        match border_mode:
            case "seamless":
                np_image = cv2.copyMakeBorder(np_image, 16, 16, 16, 16, cv2.BORDER_WRAP)
            case "mirror":
                np_image = cv2.copyMakeBorder(np_image, 16, 16, 16, 16, cv2.BORDER_REFLECT_101)
            case "replicate":
                np_image = cv2.copyMakeBorder(np_image, 16, 16, 16, 16, cv2.BORDER_REPLICATE)
            case "none":
                pass

        img_height, img_width = np_image.shape[:2]

        # Checking whether to perform tiled inference
        do_split = img_height > tile_size or img_width > tile_size

        if do_split:
            rlts = esrgan_launcher_split_merge(np_image, self.process, models, scale_factor=1, tile_size=tile_size)
        else:
            rlts = [self.process(np_image, model) for model in models]

        if border_mode != "none":
            rlts = [crop_seamless(rlt) for rlt in rlts]

        normal_map = self._cv2_to_pil(rlts[0])
        roughness = self._cv2_to_pil(rlts[1][:, :, 1])
        displacement = self._cv2_to_pil(rlts[1][:, :, 0])

        return normal_map, roughness, displacement