unslothai--unsloth/tests/studio/install/test_gpu_detection_followups.py

"""GPU-detection follow-ups to PR 6174: NVIDIA precedence + /proc/driver/nvidia/gpus fallback ported to install_llama_prebuilt.py and setup.sh. Mocks/source-level only, no GPU."""

import importlib.util
import sys
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest


PACKAGE_ROOT = Path(__file__).resolve().parents[3]

# Load studio/install_llama_prebuilt.py the same way the sibling suite does.
_MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py"
_SPEC = importlib.util.spec_from_file_location(
    "studio_install_llama_prebuilt_followups", _MODULE_PATH
)
assert _SPEC is not None and _SPEC.loader is not None
prebuilt_mod = importlib.util.module_from_spec(_SPEC)
sys.modules[_SPEC.name] = prebuilt_mod
_SPEC.loader.exec_module(prebuilt_mod)

detect_host = prebuilt_mod.detect_host
_apply_host_overrides = prebuilt_mod._apply_host_overrides

SETUP_SH = PACKAGE_ROOT / "studio" / "setup.sh"


def _make_run_capture(rocminfo_stdout: str = ""):
    """Fake run_capture: rocminfo returns rocminfo_stdout, everything else empty."""

    def _run_capture(cmd, *args, **kwargs):
        exe = str(cmd[0]) if cmd else ""
        result = MagicMock()
        if exe.endswith("rocminfo"):
            result.returncode = 0
            result.stdout = rocminfo_stdout
        else:
            result.returncode = 1
            result.stdout = ""
        result.stderr = ""
        return result

    return _run_capture


def _run_detect_host(
    *,
    machine: str = "x86_64",
    system: str = "Linux",
    which_map: dict | None = None,
    proc_dir_entries: list | None = None,
    rocminfo_stdout: str = "",
    env: dict | None = None,
):
    """Drive detect_host() against a fully synthetic host."""
    which_map = which_map or {}
    proc_dir_entries = proc_dir_entries if proc_dir_entries is not None else []

    real_isdir = prebuilt_mod.os.path.isdir
    real_listdir = prebuilt_mod.os.listdir
    proc_path = "/proc/driver/nvidia/gpus"

    def fake_isdir(p):
        if str(p) == proc_path:
            return bool(proc_dir_entries)
        return real_isdir(p)

    def fake_listdir(p):
        if str(p) == proc_path:
            if not proc_dir_entries:
                raise OSError("no such dir")
            return list(proc_dir_entries)
        return real_listdir(p)

    patches = [
        patch.object(prebuilt_mod.platform, "system", return_value = system),
        patch.object(prebuilt_mod.platform, "machine", return_value = machine),
        patch.object(prebuilt_mod.platform, "mac_ver", return_value = ("", ("", "", ""), "")),
        patch.object(prebuilt_mod.shutil, "which", side_effect = lambda n: which_map.get(n)),
        patch.object(prebuilt_mod, "run_capture", side_effect = _make_run_capture(rocminfo_stdout)),
        patch.object(prebuilt_mod.os.path, "isdir", side_effect = fake_isdir),
        patch.object(prebuilt_mod.os, "listdir", side_effect = fake_listdir),
        patch.object(prebuilt_mod.os, "access", return_value = False),
        patch.dict(prebuilt_mod.os.environ, env or {}, clear = False),
    ]
    for p in patches:
        p.start()
    try:
        # Don't let the host's CUDA_VISIBLE_DEVICES leak in unless the scenario sets it.
        if env is None or "CUDA_VISIBLE_DEVICES" not in env:
            prebuilt_mod.os.environ.pop("CUDA_VISIBLE_DEVICES", None)
        return detect_host()
    finally:
        for p in patches:
            p.stop()


# ── install_llama_prebuilt.detect_host(): /proc NVIDIA fallback ──────────────


class TestDetectHostProcFallback:
    def test_proc_fallback_marks_physical_nvidia_when_smi_absent(self):
        """No nvidia-smi, but /proc/driver/nvidia/gpus is populated -> NVIDIA."""
        host = _run_detect_host(
            which_map = {},  # nvidia-smi resolves to None
            proc_dir_entries = ["0000:01:00.0"],
        )
        assert host.has_physical_nvidia is True

    def test_proc_fallback_has_usable_nvidia_when_devices_visible(self):
        """Default CUDA_VISIBLE_DEVICES (unset) -> visible tokens non-empty -> usable."""
        host = _run_detect_host(
            which_map = {},
            proc_dir_entries = ["0000:01:00.0"],
        )
        assert host.has_usable_nvidia is True

    def test_proc_fallback_not_usable_when_devices_hidden(self):
        """CUDA_VISIBLE_DEVICES='' hides all GPUs -> physical yes, usable no."""
        host = _run_detect_host(
            which_map = {},
            proc_dir_entries = ["0000:01:00.0"],
            env = {"CUDA_VISIBLE_DEVICES": ""},
        )
        assert host.has_physical_nvidia is True
        assert host.has_usable_nvidia is False

    def test_empty_proc_dir_does_not_mark_nvidia(self):
        """A driver dir that exists but is empty must not assert a GPU."""
        host = _run_detect_host(which_map = {}, proc_dir_entries = [])
        assert host.has_physical_nvidia is False

    def test_proc_fallback_is_linux_only(self):
        """The /proc fallback must not run on Windows (path is Linux-only)."""
        host = _run_detect_host(
            system = "Windows",
            machine = "amd64",
            which_map = {},
            proc_dir_entries = ["0000:01:00.0"],
        )
        assert host.has_physical_nvidia is False


# ── install_llama_prebuilt.detect_host(): NVIDIA precedence over ROCm ────────


class TestDetectHostNvidiaPrecedence:
    def test_rocm_probe_skipped_when_proc_nvidia_present(self):
        """rocminfo reports gfx1100, but a proc-detected NVIDIA GPU wins."""
        host = _run_detect_host(
            which_map = {"rocminfo": "/usr/bin/rocminfo"},
            proc_dir_entries = ["0000:01:00.0"],
            rocminfo_stdout = "  Name:                    gfx1100\n",
        )
        assert host.has_usable_nvidia is True
        assert host.has_rocm is False

    def test_rocm_detected_when_no_nvidia(self):
        """With no NVIDIA signal at all, rocminfo gfx1100 -> has_rocm True."""
        host = _run_detect_host(
            which_map = {"rocminfo": "/usr/bin/rocminfo"},
            proc_dir_entries = [],
            rocminfo_stdout = "  Name:                    gfx1100\n",
        )
        assert host.has_usable_nvidia is False
        assert host.has_rocm is True


# ── _apply_host_overrides: forwarded --rocm-gfx / --has-rocm still win ───────


class TestOverridesStillWin:
    def test_forwarded_gfx_forces_rocm_on_non_nvidia_host(self):
        host = _run_detect_host(which_map = {}, proc_dir_entries = [])
        assert host.has_rocm is False
        overridden = _apply_host_overrides(host, override_rocm_gfx = "gfx1100")
        assert overridden.has_rocm is True
        assert overridden.rocm_gfx_target == "gfx1100"

    def test_override_has_rocm_forces_rocm(self):
        host = _run_detect_host(which_map = {}, proc_dir_entries = [])
        overridden = _apply_host_overrides(host, override_has_rocm = True)
        assert overridden.has_rocm is True

    def test_force_cpu_drops_nvidia_attributes(self):
        host = _run_detect_host(which_map = {}, proc_dir_entries = ["0000:01:00.0"])
        assert host.has_usable_nvidia is True
        overridden = _apply_host_overrides(host, force_cpu = True)
        assert overridden.has_usable_nvidia is False
        assert overridden.has_physical_nvidia is False
        assert overridden.has_rocm is False


# ── setup.sh source-level guarantees ────────────────────────────────────────


class TestSetupShHardening:
    @pytest.fixture(scope = "class")
    def setup_src(self) -> str:
        return SETUP_SH.read_text(encoding = "utf-8")

    def test_has_usable_nvidia_helper_exists(self, setup_src):
        assert "_setup_has_usable_nvidia_gpu()" in setup_src

    def test_helper_uses_proc_fallback(self, setup_src):
        start = setup_src.find("_setup_has_usable_nvidia_gpu()")
        end = setup_src.find("\n}", start)
        body = setup_src[start:end]
        assert (
            "/proc/driver/nvidia/gpus" in body
        ), "_setup_has_usable_nvidia_gpu must fall back to /proc/driver/nvidia/gpus"

    def test_gpu_summary_uses_helper(self, setup_src):
        assert "if _setup_has_usable_nvidia_gpu; then" in setup_src

    def test_timeout_wrapper_exists(self, setup_src):
        start = setup_src.find("_setup_run_smi()")
        assert start >= 0, "_setup_run_smi timeout wrapper must exist"
        end = setup_src.find("\n}", start)
        body = setup_src[start:end]
        assert "timeout 10" in body
        assert "command -v timeout" in body

    def test_cuda_source_build_gated_on_usable_nvidia(self, setup_src):
        """The nvcc source-build search must be gated on _setup_nvidia_usable."""
        anchor = setup_src.find('NVCC_PATH=""\n')
        assert anchor >= 0
        window = setup_src[anchor : anchor + 700]
        assert (
            'if [ "$_setup_nvidia_usable" = true ]' in window
        ), "CUDA toolkit search must require a usable NVIDIA GPU, not just nvcc"

    def test_nvidia_helper_honours_hidden_cvd(self, setup_src):
        """_setup_has_usable_nvidia_gpu must consult the hidden-CVD helper so CVD ""/-1 suppresses NVIDIA before AMD gating."""
        assert "_setup_cvd_hides_nvidia()" in setup_src
        start = setup_src.find("_setup_has_usable_nvidia_gpu() {")
        end = setup_src.find("\n}", start)
        body = setup_src[start:end]
        assert "_setup_cvd_hides_nvidia" in body

    def test_rocm_source_build_gated_on_amd_detected(self, setup_src):
        """The hipcc source-build search must be gated on _setup_amd_detected."""
        anchor = setup_src.find('ROCM_HIPCC=""')
        assert anchor >= 0
        window = setup_src[anchor : anchor + 400]
        assert (
            '[ "$_setup_amd_detected" = true ]' in window
        ), "ROCm toolkit search must require a detected AMD GPU, not just hipcc"

    def test_compute_cap_probe_timeout_wrapped(self, setup_src):
        # nvidia-smi is now a variable ($_smi_bin), so check the wrapper precedes
        # the probe rather than matching a literal. The string also appears in a
        # comment, so scan all occurrences and accept if any is wrapped.
        wrapped = False
        start = 0
        while True:
            idx = setup_src.find("--query-gpu=compute_cap", start)
            if idx < 0:
                break
            if "_setup_run_smi" in setup_src[max(0, idx - 80) : idx]:
                wrapped = True
                break
            start = idx + 1
        assert wrapped, "compute_cap probe must be wrapped in _setup_run_smi (timeout-bounded)"

    def test_driver_version_probe_timeout_wrapped(self, setup_src):
        start = setup_src.find("_cuda_driver_max_version()")
        end = setup_src.find("\n}", start)
        body = setup_src[start:end]
        assert "_setup_run_smi nvidia-smi" in body


# TEST: install.sh -- UNSLOTH_TORCH_BACKEND classified on the final path segment


class TestBackendExportLeafClassification:
    """A mirror base path containing "rocm"/"gfx" must not mislabel a cu*/cpu index; classification uses TORCH_INDEX_URL's leaf only."""

    @pytest.fixture(scope = "class")
    def install_src(self) -> str:
        return (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")

    def test_export_block_uses_leaf(self, install_src):
        anchor = install_src.find("_torch_index_leaf=")
        assert anchor >= 0, "backend export must classify on the final path segment"
        window = install_src[anchor : anchor + 500]
        assert 'export UNSLOTH_TORCH_BACKEND="rocm"' in window
        assert 'export UNSLOTH_TORCH_BACKEND="cpu"' in window
        assert 'export UNSLOTH_TORCH_BACKEND="cuda"' in window

    def test_leaf_classification_behaviour(self, tmp_path):
        import subprocess as sp

        script = tmp_path / "leaf.sh"
        src = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        anchor = src.find("_torch_index_leaf=")
        block = src[anchor : src.find("esac", anchor) + 4]
        # Drive the extracted block with adversarial mirror URLs.
        script.write_text(
            "#!/bin/sh\n"
            'TORCH_INDEX_URL="$1"\n' + block + "\n"
            'printf "%s" "$UNSLOTH_TORCH_BACKEND"\n'
        )
        cases = {
            "https://download.pytorch.org/whl/cu128": "cuda",
            "https://download.pytorch.org/whl/cpu": "cpu",
            "https://download.pytorch.org/whl/rocm6.4": "rocm",
            "https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2.1/": "rocm",
            "https://repo.amd.com/rocm/whl/gfx1151/": "rocm",
            "https://mirror.local/rocm-cache/cu128": "cuda",
            "https://mirror.local/gfx-cache/cpu": "cpu",
        }
        for url, expected in cases.items():
            out = sp.run(
                ["sh", str(script), url], capture_output = True, text = True, timeout = 30
            ).stdout.strip()
            assert out == expected, f"{url} classified as {out!r}, expected {expected!r}"


# TEST: CUDA_VISIBLE_DEVICES=""/-1 hides NVIDIA in every usable-GPU helper


_STACK_PATH = PACKAGE_ROOT / "studio" / "install_python_stack.py"
_STACK_SPEC = importlib.util.spec_from_file_location(
    "studio_install_python_stack_followups", _STACK_PATH
)
assert _STACK_SPEC is not None and _STACK_SPEC.loader is not None
stack_mod = importlib.util.module_from_spec(_STACK_SPEC)
sys.modules[_STACK_SPEC.name] = stack_mod
_STACK_SPEC.loader.exec_module(stack_mod)


def _stack_nvidia_usable(cvd):
    """Drive _has_usable_nvidia_gpu with a mocked nvidia-smi that always reports a GPU; cvd=None unsets the env var."""

    def fake_run(cmd, *args, **kwargs):
        result = MagicMock()
        result.returncode = 0
        result.stdout = "GPU 0: NVIDIA Fake (UUID: GPU-x)\n"
        return result

    env = {} if cvd is None else {"CUDA_VISIBLE_DEVICES": cvd}
    with (
        patch.object(
            stack_mod.shutil,
            "which",
            side_effect = lambda n: "/usr/bin/nvidia-smi" if n == "nvidia-smi" else None,
        ),
        patch.object(stack_mod.subprocess, "run", side_effect = fake_run),
        patch.dict(stack_mod.os.environ, env, clear = False),
    ):
        if cvd is None:
            stack_mod.os.environ.pop("CUDA_VISIBLE_DEVICES", None)
        return stack_mod._has_usable_nvidia_gpu()


class TestHiddenCvdNotUsable:
    """CVD ""/-1 hides every NVIDIA device; all three _has_usable_nvidia_gpu impls must report not-usable so AMD/CPU routes run."""

    def test_python_unset_cvd_is_usable(self):
        assert _stack_nvidia_usable(None) is True

    def test_python_empty_cvd_not_usable(self):
        assert _stack_nvidia_usable("") is False

    def test_python_minus_one_not_usable(self):
        assert _stack_nvidia_usable("-1") is False

    def test_python_padded_minus_one_not_usable(self):
        assert _stack_nvidia_usable(" -1 ") is False

    def test_python_explicit_device_is_usable(self):
        assert _stack_nvidia_usable("0") is True

    def test_python_device_list_is_usable(self):
        assert _stack_nvidia_usable("0,1") is True

    def test_hidden_nvidia_restores_rocm_detection(self):
        """Mixed host, NVIDIA hidden via CVD=-1: _has_rocm_gpu must pass the NVIDIA guard and return True (pre-fix it ignored CVD)."""

        def fake_run(cmd, *args, **kwargs):
            result = MagicMock()
            result.returncode = 0
            exe = str(cmd[0])
            if exe.endswith("rocminfo"):
                result.stdout = "  Name:                    gfx1100\n"
            else:
                result.stdout = "GPU 0: NVIDIA Fake (UUID: GPU-x)\n"
            return result

        which_map = {
            "rocminfo": "/usr/bin/rocminfo",
            "nvidia-smi": "/usr/bin/nvidia-smi",
        }
        with (
            patch.object(stack_mod.shutil, "which", side_effect = which_map.get),
            patch.object(stack_mod.subprocess, "run", side_effect = fake_run),
            patch.dict(stack_mod.os.environ, {"CUDA_VISIBLE_DEVICES": "-1"}, clear = False),
        ):
            assert stack_mod._has_rocm_gpu() is True

    @staticmethod
    def _run_sh_helper(tmp_path, src: str, fn_names: list, cvd):
        """Extract shell functions, run the usable-GPU one against a fake nvidia-smi; return "usable"/"not_usable"."""
        import os as _os
        import subprocess as sp

        blocks = []
        for name in fn_names:
            start = src.find(f"{name}() {{")
            assert start >= 0, f"{name} missing"
            end = src.find("\n}", start) + 2
            blocks.append(src[start:end])
        fake_bin = tmp_path / "bin"
        fake_bin.mkdir(exist_ok = True)
        smi = fake_bin / "nvidia-smi"
        smi.write_text("#!/bin/sh\necho 'GPU 0: NVIDIA Fake (UUID: GPU-x)'\n")
        smi.chmod(0o755)
        script = tmp_path / "probe.sh"
        script.write_text(
            "#!/bin/sh\n" + "\n".join(blocks) + "\n"
            f"if {fn_names[-1]}; then echo usable; else echo not_usable; fi\n"
        )
        env = dict(_os.environ)
        env["PATH"] = f"{fake_bin}:{env['PATH']}"
        if cvd is None:
            env.pop("CUDA_VISIBLE_DEVICES", None)
        else:
            env["CUDA_VISIBLE_DEVICES"] = cvd
        return sp.run(
            ["sh", str(script)], capture_output = True, text = True, timeout = 30, env = env
        ).stdout.strip()

    @pytest.mark.parametrize(
        "cvd, expected",
        [(None, "usable"), ("", "not_usable"), ("-1", "not_usable"), ("0", "usable")],
    )
    def test_install_sh_helper_cvd(self, tmp_path, cvd, expected):
        src = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        out = self._run_sh_helper(
            tmp_path,
            src,
            ["_run_bounded", "_cvd_hides_nvidia", "_has_usable_nvidia_gpu"],
            cvd,
        )
        assert out == expected

    @pytest.mark.parametrize(
        "cvd, expected",
        [(None, "usable"), ("", "not_usable"), ("-1", "not_usable"), ("0", "usable")],
    )
    def test_setup_sh_helper_cvd(self, tmp_path, cvd, expected):
        src = SETUP_SH.read_text(encoding = "utf-8")
        out = self._run_sh_helper(
            tmp_path,
            src,
            ["_setup_run_smi", "_setup_cvd_hides_nvidia", "_setup_has_usable_nvidia_gpu"],
            cvd,
        )
        assert out == expected