nvlabs--longlive/fouroversix/scripts/resources.py

from __future__ import annotations

import configparser
import os
import shutil
import subprocess
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING

import modal
import tomllib

if TYPE_CHECKING:
    from collections.abc import Callable

FOUROVERSIX_CACHE_PATH = Path("/fouroversix")
FOUROVERSIX_INSTALL_PATH = Path("/root/fouroversix")
KERNEL_DEV_MODE = os.getenv("KERNEL_DEV_MODE", "0") == "1"

app = modal.App("fouroversix")
cache_volume = modal.Volume.from_name("fouroversix", create_if_missing=True)
hf_secret = modal.Secret.from_name("huggingface-secret")
wandb_secret = modal.Secret.from_name("wandb-secret")


class Dependency(str, Enum):
    """Dependencies to add to the base image."""

    awq = "awq"
    fast_hadamard_transform = "fast_hadamard_transform"
    flame = "flame"
    flash_attention = "flash_attention"
    fouroversix = "fouroversix"
    fp_quant = "fp_quant"
    qutlass = "qutlass"
    spinquant = "spinquant"
    transformer_engine = "transformer_engine"


class Submodule(str, Enum):
    """Submodules of Four Over Six to add to the base image."""

    cutlass = "cutlass"
    flame = "flame"
    fast_hadamard_transform = "fast_hadamard_transform"
    fp_quant = "fp_quant"
    llm_awq = "llm_awq"
    qutlass = "qutlass"
    spinquant = "spinquant"

    def has_untracked_or_unstaged_changes(self) -> bool:
        """Check if the submodule has untracked or unstaged changes."""

        git_status = subprocess.run(
            [
                "git",
                "-C",
                self.get_local_path(),
                "status",
                "--porcelain",
            ],
            check=False,
            stderr=subprocess.DEVNULL,
            stdout=subprocess.PIPE,
            text=True,
        )

        return bool(git_status.stdout.strip())

    def get_install_path(self) -> str:
        """Get the path where this submodule will be installed in the Modal image."""
        return f"{FOUROVERSIX_INSTALL_PATH}/{self.get_local_path()}"

    def get_local_path(self) -> str:
        """Get the path of the submodule relative to the root directory."""
        return f"third_party/{self.value.replace('_', '-')}"

    def get_remote_url(self) -> str:
        """Get the remote URL of the submodule."""

        gitmodules_path = Path(__file__).parent.parent / ".gitmodules"

        if not gitmodules_path.exists():
            gitmodules_path = FOUROVERSIX_INSTALL_PATH / ".gitmodules"

        with gitmodules_path.open() as f:
            # Remove leading whitespace to make it a valid INI file
            gitmodules_contents = "\n".join(line.lstrip() for line in f.readlines())

        config = configparser.ConfigParser()
        config.read_string(gitmodules_contents)

        for section in config.sections():
            if config[section]["path"] == self.get_local_path():
                url = config[section]["url"]
                break

        if url.startswith("https://"):
            return url

        msg = f"Unsupported remote URL format: {url}"
        raise ValueError(msg)


cuda_version_to_image_tag = {
    "12.8": "nvcr.io/nvidia/cuda-dl-base:25.03-cuda12.8-devel-ubuntu24.04",
    "12.9": "nvcr.io/nvidia/cuda-dl-base:25.06-cuda12.9-devel-ubuntu24.04",
    "13.0": "nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04",
    "13.1": "nvcr.io/nvidia/cuda-dl-base:25.12-cuda13.1-devel-ubuntu24.04",
}


def add_submodule(img: modal.Image, submodule: Submodule) -> modal.Image:
    if submodule.has_untracked_or_unstaged_changes():
        # Submodule has uncommitted changes, build image with local copy
        return img.add_local_dir(
            submodule.get_local_path(),
            submodule.get_install_path(),
            copy=True,
        )

    # Submodule has no uncommitted changes, download from remote to save time
    return img.run_commands(
        f"git clone {submodule.get_remote_url()} {submodule.get_install_path()}",
    )


def install_flash_attn() -> None:
    subprocess.run(
        ["pip", "install", "flash-attn", "--no-build-isolation"],
        check=False,
    )


def install_fouroversix() -> None:
    subprocess.run(
        [
            "pip",
            "install",
            "--no-deps",
            "--no-build-isolation",
            "-e",
            FOUROVERSIX_INSTALL_PATH.as_posix(),
        ],
        check=False,
    )


def install_fouroversix_non_editable() -> None:
    shutil.copytree(
        FOUROVERSIX_CACHE_PATH / "build",
        FOUROVERSIX_INSTALL_PATH / "build",
    )
    subprocess.run(
        ["python", "setup.py", "build_ext", "--inplace"],
        check=False,
    )
    shutil.copytree(
        FOUROVERSIX_INSTALL_PATH / "build",
        FOUROVERSIX_CACHE_PATH / "build",
        dirs_exist_ok=True,
    )
    subprocess.run(
        [
            "pip",
            "install",
            "--no-deps",
            "--no-build-isolation",
            FOUROVERSIX_INSTALL_PATH.as_posix(),
        ],
        check=False,
    )


def install_qutlass() -> None:
    subprocess.run(
        [
            "pip",
            "install",
            "--no-build-isolation",
            Submodule.qutlass.get_install_path(),
        ],
        check=False,
    )


def get_image(  # noqa: C901, PLR0912
    dependencies: list[Dependency] | None = None,
    *,
    cuda_version: str = "12.9",
    deploy: bool = False,
    extra_env: dict[str, str] | None = None,
    extra_pip_dependencies: list[str] | None = None,
    include_tests: bool = False,
    python_version: str = "3.13",
    pytorch_version: str = "2.10.0",
    run_before_copy: Callable[[modal.Image], modal.Image] | None = None,
) -> modal.Image:
    if dependencies is None:
        dependencies = [Dependency.fouroversix]

    pyproject_path = Path(__file__).parent.parent / "pyproject.toml"

    if not pyproject_path.exists():
        pyproject_path = Path(__file__).parent.parent / "fouroversix" / "pyproject.toml"

    with pyproject_path.open("rb") as f:
        pyproject_data = tomllib.load(f)

    img = (
        modal.Image.from_registry(
            cuda_version_to_image_tag[cuda_version],
            add_python=python_version,
        )
        .entrypoint([])
        .apt_install("clang", "git")
        .uv_pip_install(*pyproject_data["build-system"]["requires"], "numpy")
        .uv_pip_install(
            f"torch=={pytorch_version}",
            extra_index_url=(
                f"https://download.pytorch.org/whl/cu{cuda_version.replace('.', '')}"
            ),
        )
    )

    for dependency in dependencies:
        if dependency == Dependency.awq:
            img = add_submodule(img, Submodule.llm_awq).run_commands(
                f"pip install --no-deps -e {Submodule.llm_awq.get_install_path()}",
            )

        if dependency == Dependency.fast_hadamard_transform:
            img = add_submodule(img, Submodule.fast_hadamard_transform).run_commands(
                f"pip install {Submodule.fast_hadamard_transform.get_install_path()} "
                "--no-build-isolation",
            )

        if dependency == Dependency.flame:
            img = (
                img.apt_install("pciutils")
                .uv_pip_install(
                    "flash-linear-attention",
                    "ninja",
                    "psutil",
                    "git+https://github.com/pytorch/torchtitan.git@0b44d4c",
                    "tyro",
                    "wheel",
                )
                .run_commands(
                    "git clone https://github.com/fla-org/flame.git "
                    f"{FOUROVERSIX_INSTALL_PATH}/third_party/flame",
                    f"pip install -e {FOUROVERSIX_INSTALL_PATH}/third_party/flame",
                )
            )

        if dependency == Dependency.flash_attention:
            img = img.run_function(
                install_flash_attn,
                cpu=64,
                memory=128 * 1024,
                gpu="B200",
            )

        if dependency == Dependency.fouroversix:
            img = (
                add_submodule(
                    img.env(
                        {"CUDA_ARCHS": "100", "FORCE_BUILD": "1", "MAX_JOBS": "32"},
                    ),
                    Submodule.cutlass,
                )
                .add_local_file(
                    "pyproject.toml",
                    f"{FOUROVERSIX_INSTALL_PATH}/pyproject.toml",
                    copy=True,
                )
                .uv_pip_install(
                    *pyproject_data["project"]["optional-dependencies"]["evals"],
                )
                .add_local_file(
                    "setup.py",
                    f"{FOUROVERSIX_INSTALL_PATH}/setup.py",
                    copy=True,
                )
                .add_local_file(
                    "src/fouroversix/__init__.py",
                    f"{FOUROVERSIX_INSTALL_PATH}/src/fouroversix/__init__.py",
                    copy=True,
                )
            )

            if KERNEL_DEV_MODE:
                img = (
                    img.add_local_file(
                        "README.md",
                        f"{FOUROVERSIX_INSTALL_PATH}/README.md",
                        copy=True,
                    )
                    .add_local_file(
                        "LICENSE.md",
                        f"{FOUROVERSIX_INSTALL_PATH}/LICENSE.md",
                        copy=True,
                    )
                    .workdir(FOUROVERSIX_INSTALL_PATH)
                )

            img = img.add_local_dir(
                "src/fouroversix/csrc",
                f"{FOUROVERSIX_INSTALL_PATH}/src/fouroversix/csrc",
                copy=True,
            )

            if not KERNEL_DEV_MODE:
                img = img.run_function(install_fouroversix, cpu=32, memory=64 * 1024)

        if dependency == Dependency.fp_quant:
            img = add_submodule(img, Submodule.fp_quant).run_commands(
                f"pip install {Submodule.fp_quant.get_install_path()}/inference_lib",
            )

        if dependency == Dependency.qutlass:
            img = (
                add_submodule(img.apt_install("cmake"), Submodule.qutlass)
                .env({"MAX_JOBS": "32"})
                .run_function(install_qutlass, gpu="B200", cpu=32, memory=64 * 1024)
            )

        if dependency == Dependency.spinquant:
            img = add_submodule(img, Submodule.spinquant)

        if dependency == Dependency.transformer_engine:
            img = img.uv_pip_install(
                "transformer_engine[pytorch]",
                extra_options="--no-build-isolation",
            )

    if extra_pip_dependencies is not None:
        img = img.uv_pip_install(*extra_pip_dependencies)

    img = img.env({"HF_HOME": FOUROVERSIX_CACHE_PATH.as_posix(), **(extra_env or {})})

    if run_before_copy is not None:
        img = run_before_copy(img)

    # Add source files after all dependencies are added so we can avoid rebuilding when
    # they change
    for dependency in dependencies:
        if dependency == Dependency.flame:
            img = (
                img.add_local_dir(
                    "third_party/flame/custom_models",
                    f"{FOUROVERSIX_INSTALL_PATH}/third_party/flame/custom_models",
                )
                .add_local_dir(
                    "scripts/train/configs",
                    f"{FOUROVERSIX_INSTALL_PATH}/scripts/train/configs",
                )
                .add_local_file(
                    "third_party/flame/train.sh",
                    f"{FOUROVERSIX_INSTALL_PATH}/third_party/flame/train.sh",
                )
            )

        if dependency == Dependency.fouroversix:
            img = img.add_local_dir(
                "src",
                f"{FOUROVERSIX_INSTALL_PATH}/src",
                copy=deploy or KERNEL_DEV_MODE,
                ignore=lambda p: p.suffix == ".so",
            ).add_local_file(
                ".gitmodules",
                f"{FOUROVERSIX_INSTALL_PATH}/.gitmodules",
                copy=deploy or KERNEL_DEV_MODE,
            )

            if KERNEL_DEV_MODE:
                img = img.run_function(
                    install_fouroversix_non_editable,
                    cpu=32,
                    memory=64 * 1024,
                    volumes={FOUROVERSIX_CACHE_PATH.as_posix(): cache_volume},
                )

    if include_tests:
        img = img.add_local_dir("tests", f"{FOUROVERSIX_INSTALL_PATH}/tests")

    return img