chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
def get_soc_name():
|
||||
return "Ascend910B"
|
||||
|
||||
|
||||
class rt:
|
||||
@staticmethod
|
||||
def get_device_count():
|
||||
return (4, 0)
|
||||
@@ -0,0 +1,16 @@
|
||||
class SyclContext:
|
||||
def __init__(self, info):
|
||||
pass
|
||||
|
||||
@property
|
||||
def device_count(self):
|
||||
return 6
|
||||
|
||||
|
||||
class SyclDevice:
|
||||
def __init__(self, info):
|
||||
pass
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "Intel(R) Data Center GPU Max 1550"
|
||||
@@ -0,0 +1,16 @@
|
||||
class SyclContext:
|
||||
def __init__(self, info):
|
||||
pass
|
||||
|
||||
@property
|
||||
def device_count(self):
|
||||
return 4
|
||||
|
||||
|
||||
class SyclDevice:
|
||||
def __init__(self, info):
|
||||
pass
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return "Intel(R) Data Center GPU Max 1100"
|
||||
@@ -0,0 +1,42 @@
|
||||
class _MockArch:
|
||||
"""Mock for the PyO3-generated Arch enum."""
|
||||
|
||||
def __init__(self, name: str = "Rngd"):
|
||||
self._name = name
|
||||
|
||||
def __str__(self) -> str:
|
||||
# PyO3 unit enums commonly stringify like "Arch.Rngd".
|
||||
return f"Arch.{self._name}"
|
||||
|
||||
|
||||
class _MockDeviceInfo:
|
||||
def __init__(self, arch_name: str = "Rngd", index: int = 0):
|
||||
self._arch_name = arch_name
|
||||
self._index = index
|
||||
|
||||
def arch(self) -> _MockArch:
|
||||
return _MockArch(self._arch_name)
|
||||
|
||||
def name(self) -> str:
|
||||
return f"npu{self._index}"
|
||||
|
||||
def index(self) -> int:
|
||||
return self._index
|
||||
|
||||
|
||||
class _MockDevice:
|
||||
def __init__(self, idx: int, arch_name: str = "Rngd"):
|
||||
self._idx = idx
|
||||
self._arch_name = arch_name
|
||||
|
||||
def device_info(self) -> _MockDeviceInfo:
|
||||
return _MockDeviceInfo(arch_name=self._arch_name, index=self._idx)
|
||||
|
||||
|
||||
def init():
|
||||
"""Mock for ``furiosa_smi_py.init``. Real SMI library requires init() before use."""
|
||||
return None
|
||||
|
||||
|
||||
def list_devices():
|
||||
return [_MockDevice(i) for i in range(8)]
|
||||
@@ -0,0 +1,101 @@
|
||||
from typing import List
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray._private.thirdparty.pynvml as pynvml
|
||||
|
||||
|
||||
class DeviceHandleMock(dict):
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
uuid: str,
|
||||
mig_devices: List["DeviceHandleMock"] = None,
|
||||
**kwargs
|
||||
):
|
||||
super().__init__()
|
||||
self["name"] = name
|
||||
self["uuid"] = uuid
|
||||
if mig_devices is not None:
|
||||
self["mig_devices"] = mig_devices
|
||||
self.update(kwargs)
|
||||
|
||||
|
||||
# pnvml mock for gpu resources
|
||||
class PyNVMLMock:
|
||||
def __init__(self, mock_data, driver_version="535.104.12"):
|
||||
self._mock_data = mock_data
|
||||
self.driver_version = driver_version
|
||||
|
||||
def nvmlInit(self):
|
||||
return
|
||||
|
||||
def nvmlShutdown(self):
|
||||
return
|
||||
|
||||
def nvmlSystemGetDriverVersion(self):
|
||||
return self.driver_version
|
||||
|
||||
def nvmlDeviceGetCount(self):
|
||||
return len(self._mock_data)
|
||||
|
||||
def nvmlDeviceGetHandleByIndex(self, index):
|
||||
return self._mock_data[index]
|
||||
|
||||
def nvmlDeviceGetName(self, handle):
|
||||
return handle.get("name", "")
|
||||
|
||||
def nvmlDeviceGetMaxMigDeviceCount(self, handle):
|
||||
if "mig_devices" in handle:
|
||||
return max(7, len(handle["mig_devices"]))
|
||||
else:
|
||||
raise pynvml.NVMLError_NotSupported
|
||||
|
||||
def nvmlDeviceGetMigDeviceHandleByIndex(self, handle, mig_index):
|
||||
try:
|
||||
return handle["mig_devices"][mig_index]
|
||||
except IndexError:
|
||||
raise pynvml.NVMLError_NotFound
|
||||
|
||||
def nvmlDeviceGetUUID(self, handle):
|
||||
return handle.get("uuid", "")
|
||||
|
||||
def nvmlDeviceGetComputeInstanceId(self, mig_handle):
|
||||
return mig_handle["ci_id"]
|
||||
|
||||
def nvmlDeviceGetGpuInstanceId(self, mig_handle):
|
||||
return mig_handle["gi_id"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patch_mock_pynvml(mock_nvml):
|
||||
with patch("ray._private.thirdparty.pynvml.nvmlInit", mock_nvml.nvmlInit), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlShutdown", mock_nvml.nvmlShutdown
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlSystemGetDriverVersion",
|
||||
mock_nvml.nvmlSystemGetDriverVersion,
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetCount",
|
||||
mock_nvml.nvmlDeviceGetCount,
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetHandleByIndex",
|
||||
mock_nvml.nvmlDeviceGetHandleByIndex,
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetName", mock_nvml.nvmlDeviceGetName
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetMaxMigDeviceCount",
|
||||
mock_nvml.nvmlDeviceGetMaxMigDeviceCount,
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetMigDeviceHandleByIndex",
|
||||
mock_nvml.nvmlDeviceGetMigDeviceHandleByIndex,
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetUUID", mock_nvml.nvmlDeviceGetUUID
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetComputeInstanceId",
|
||||
mock_nvml.nvmlDeviceGetComputeInstanceId,
|
||||
), patch(
|
||||
"ray._private.thirdparty.pynvml.nvmlDeviceGetGpuInstanceId",
|
||||
mock_nvml.nvmlDeviceGetGpuInstanceId,
|
||||
):
|
||||
yield
|
||||
@@ -0,0 +1,6 @@
|
||||
def device_count():
|
||||
return 4
|
||||
|
||||
|
||||
def get_npu_name():
|
||||
return "RBLN-CA02"
|
||||
@@ -0,0 +1,22 @@
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from ray.util import accelerators
|
||||
from ray.util.annotations import RayDeprecationWarning
|
||||
|
||||
|
||||
def test_accelerators():
|
||||
assert accelerators.NVIDIA_TESLA_K80 == "K80"
|
||||
assert accelerators.NVIDIA_A100 == "A100"
|
||||
with pytest.raises(
|
||||
AttributeError,
|
||||
match="module 'ray.util.accelerators' has no attribute 'NVIDIA_INVALID'",
|
||||
):
|
||||
_ = accelerators.NVIDIA_INVALID
|
||||
with pytest.warns(RayDeprecationWarning):
|
||||
assert accelerators.NVIDIA_TESLA_A100 == "A100"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,92 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._private.accelerators import (
|
||||
AMDGPUAcceleratorManager,
|
||||
get_accelerator_manager_for_resource,
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.AMDGPUAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
|
||||
return_value=4,
|
||||
)
|
||||
def test_visible_amd_gpu_ids(mock_get_num_accelerators, monkeypatch, shutdown_only):
|
||||
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,1,2")
|
||||
# Delete the cache so it can be re-populated the next time
|
||||
# we call get_accelerator_manager_for_resource
|
||||
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
|
||||
ray.init()
|
||||
_ = mock_get_num_accelerators.called
|
||||
assert ray.available_resources()["GPU"] == 3
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.AMDGPUAcceleratorManager._get_amd_device_ids",
|
||||
return_value=["0x74a1", "0x74a1", "0x74a1", "0x74a1"],
|
||||
)
|
||||
def test_visible_amd_gpu_type(mock_get_amd_device_ids, shutdown_only):
|
||||
ray.init()
|
||||
_ = mock_get_amd_device_ids.called
|
||||
assert (
|
||||
AMDGPUAcceleratorManager.get_current_node_accelerator_type()
|
||||
== "AMD-Instinct-MI300X-OAM"
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.AMDGPUAcceleratorManager._get_amd_device_ids",
|
||||
return_value=["0x640f", "0x640f", "0x640f", "0x640f"],
|
||||
)
|
||||
def test_visible_amd_gpu_type_bad_device_id(mock_get_num_accelerators, shutdown_only):
|
||||
ray.init()
|
||||
_ = mock_get_num_accelerators.called
|
||||
assert AMDGPUAcceleratorManager.get_current_node_accelerator_type() is None
|
||||
|
||||
|
||||
def test_get_current_process_visible_accelerator_ids(monkeypatch):
|
||||
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,1,2")
|
||||
assert AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
|
||||
"0",
|
||||
"1",
|
||||
"2",
|
||||
]
|
||||
|
||||
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,2,7")
|
||||
assert AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
|
||||
"0",
|
||||
"2",
|
||||
"7",
|
||||
]
|
||||
|
||||
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "")
|
||||
assert AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == []
|
||||
|
||||
del os.environ["HIP_VISIBLE_DEVICES"]
|
||||
assert (
|
||||
AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() is None
|
||||
)
|
||||
|
||||
|
||||
def test_set_current_process_visible_accelerator_ids():
|
||||
AMDGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0"])
|
||||
env_var = AMDGPUAcceleratorManager.get_visible_accelerator_ids_env_var()
|
||||
assert os.environ[env_var] == "0"
|
||||
|
||||
AMDGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
|
||||
assert os.environ[env_var] == "0,1"
|
||||
|
||||
AMDGPUAcceleratorManager.set_current_process_visible_accelerator_ids(
|
||||
["0", "1", "7"]
|
||||
)
|
||||
assert os.environ[env_var] == "0,1,7"
|
||||
|
||||
del os.environ[env_var]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,159 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from ray._private.accelerators.furiosa import (
|
||||
FURIOSA_VISIBLE_DEVICES_ENV_VAR,
|
||||
NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR,
|
||||
FuriosaAcceleratorManager,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_furiosa_smi_py(monkeypatch):
|
||||
from ray.tests.accelerators import mock_furiosa_smi_py
|
||||
|
||||
monkeypatch.setitem(sys.modules, "furiosa_smi_py", mock_furiosa_smi_py)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clear_furiosa_environment():
|
||||
original_env = os.environ.get(FURIOSA_VISIBLE_DEVICES_ENV_VAR)
|
||||
original_no_set_env = os.environ.get(NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR)
|
||||
|
||||
os.environ.pop(FURIOSA_VISIBLE_DEVICES_ENV_VAR, None)
|
||||
os.environ.pop(NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR, None)
|
||||
|
||||
yield
|
||||
|
||||
if original_env is not None:
|
||||
os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] = original_env
|
||||
if original_no_set_env is not None:
|
||||
os.environ[NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR] = original_no_set_env
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("clear_furiosa_environment")
|
||||
class TestFuriosaAcceleratorManager:
|
||||
def test_get_resource_name(self):
|
||||
assert FuriosaAcceleratorManager.get_resource_name() == "FURIOSA"
|
||||
|
||||
def test_get_visible_accelerator_ids_env_var(self):
|
||||
assert (
|
||||
FuriosaAcceleratorManager.get_visible_accelerator_ids_env_var()
|
||||
== FURIOSA_VISIBLE_DEVICES_ENV_VAR
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"env_value,expected",
|
||||
[
|
||||
# furiosa-llm --devices form (preferred).
|
||||
("npu:0,npu:1,npu:2,npu:3", ["0", "1", "2", "3"]),
|
||||
# Bare integer form is also accepted for convenience.
|
||||
("0,1,2,3", ["0", "1", "2", "3"]),
|
||||
# Core range notation: only the device index is returned.
|
||||
("npu:0:0-3,npu:1:0-3", ["0", "1"]),
|
||||
# Empty string yields an empty list.
|
||||
("", []),
|
||||
# Sentinel ``None`` means the env var is unset.
|
||||
(None, None),
|
||||
],
|
||||
)
|
||||
def test_get_current_process_visible_accelerator_ids(self, env_value, expected):
|
||||
if env_value is None:
|
||||
os.environ.pop(FURIOSA_VISIBLE_DEVICES_ENV_VAR, None)
|
||||
else:
|
||||
os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] = env_value
|
||||
assert (
|
||||
FuriosaAcceleratorManager.get_current_process_visible_accelerator_ids()
|
||||
== expected
|
||||
)
|
||||
|
||||
def test_get_current_node_num_accelerators(self):
|
||||
assert FuriosaAcceleratorManager.get_current_node_num_accelerators() == 8
|
||||
|
||||
def test_get_current_node_accelerator_type(self):
|
||||
assert (
|
||||
FuriosaAcceleratorManager.get_current_node_accelerator_type()
|
||||
== "FURIOSA_RNGD"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arch_name,expected",
|
||||
[
|
||||
# PyO3 enum form (CamelCase).
|
||||
("Rngd", "FURIOSA_RNGD"),
|
||||
("RngdMax", "FURIOSA_RNGDMAX"),
|
||||
("RngdS", "FURIOSA_RNGDS"),
|
||||
("RngdPlus", "FURIOSA_RNGDPLUS"),
|
||||
# ``Arch::ToString`` form is also accepted, and both forms must
|
||||
# resolve to the same label.
|
||||
("rngd-max", "FURIOSA_RNGDMAX"),
|
||||
# ``+`` must NOT be silently stripped, since that would collapse
|
||||
# ``rngd+`` into ``rngd`` and collide with the base RNGD SKU; it
|
||||
# is mapped to ``plus`` so the label matches ``RngdPlus``.
|
||||
("rngd+", "FURIOSA_RNGDPLUS"),
|
||||
],
|
||||
)
|
||||
def test_get_current_node_accelerator_type_dynamic(
|
||||
self, monkeypatch, arch_name, expected
|
||||
):
|
||||
from ray.tests.accelerators import mock_furiosa_smi_py
|
||||
|
||||
def mocked_list_devices():
|
||||
return [mock_furiosa_smi_py._MockDevice(0, arch_name=arch_name)]
|
||||
|
||||
monkeypatch.setattr(mock_furiosa_smi_py, "list_devices", mocked_list_devices)
|
||||
assert FuriosaAcceleratorManager.get_current_node_accelerator_type() == expected
|
||||
|
||||
def test_get_current_node_accelerator_type_no_devices(self, monkeypatch):
|
||||
from ray.tests.accelerators import mock_furiosa_smi_py
|
||||
|
||||
monkeypatch.setattr(mock_furiosa_smi_py, "list_devices", lambda: [])
|
||||
assert FuriosaAcceleratorManager.get_current_node_accelerator_type() is None
|
||||
|
||||
def test_get_current_node_accelerator_type_arch_is_none(self, monkeypatch):
|
||||
"""Regression: arch() returning None must not produce 'FURIOSA_NONE'."""
|
||||
from ray.tests.accelerators import mock_furiosa_smi_py
|
||||
|
||||
class _NullArchDeviceInfo:
|
||||
def arch(self):
|
||||
return None
|
||||
|
||||
class _NullArchDevice:
|
||||
def device_info(self):
|
||||
return _NullArchDeviceInfo()
|
||||
|
||||
monkeypatch.setattr(
|
||||
mock_furiosa_smi_py, "list_devices", lambda: [_NullArchDevice()]
|
||||
)
|
||||
assert FuriosaAcceleratorManager.get_current_node_accelerator_type() is None
|
||||
|
||||
def test_set_current_process_visible_accelerator_ids(self):
|
||||
# Ray's scheduler hands us bare integer IDs; we serialize them in
|
||||
# the ``npu:<id>`` form expected by ``furiosa-llm --devices``.
|
||||
FuriosaAcceleratorManager.set_current_process_visible_accelerator_ids(
|
||||
["0", "1"]
|
||||
)
|
||||
assert os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] == "npu:0,npu:1"
|
||||
|
||||
os.environ[NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR] = "1"
|
||||
FuriosaAcceleratorManager.set_current_process_visible_accelerator_ids(
|
||||
["2", "3"]
|
||||
)
|
||||
assert os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] == "npu:0,npu:1"
|
||||
|
||||
def test_validate_resource_request_quantity(self):
|
||||
valid, _ = FuriosaAcceleratorManager.validate_resource_request_quantity(1)
|
||||
assert valid
|
||||
|
||||
valid, _ = FuriosaAcceleratorManager.validate_resource_request_quantity(2.0)
|
||||
assert valid
|
||||
|
||||
valid, msg = FuriosaAcceleratorManager.validate_resource_request_quantity(0.5)
|
||||
assert not valid
|
||||
assert "whole number" in msg
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,296 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._private.accelerators import HPUAcceleratorManager, hpu
|
||||
from ray.util.placement_group import placement_group
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
|
||||
|
||||
def test_user_configured_more_than_visible(monkeypatch, call_ray_stop_only):
|
||||
# Test more hpus are configured than visible.
|
||||
monkeypatch.setenv("HABANA_VISIBLE_MODULES", "0,1,2")
|
||||
with pytest.raises(ValueError):
|
||||
ray.init(resources={"HPU": 4})
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.HPUAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
|
||||
return_value=4,
|
||||
)
|
||||
def test_auto_detected_more_than_visible(
|
||||
mock_get_num_accelerators, monkeypatch, shutdown_only
|
||||
):
|
||||
# Test more hpus are detected than visible.
|
||||
monkeypatch.setenv("HABANA_VISIBLE_MODULES", "0,1,2")
|
||||
ray.init()
|
||||
_ = mock_get_num_accelerators.called
|
||||
assert ray.available_resources()["HPU"] == 3
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.HPUAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
|
||||
return_value=2,
|
||||
)
|
||||
def test_auto_detect_resources(mock_get_num_accelerators, shutdown_only):
|
||||
# Test that ray node resources are filled with auto detected count.
|
||||
ray.init()
|
||||
_ = mock_get_num_accelerators.called
|
||||
assert ray.available_resources()["HPU"] == 2
|
||||
|
||||
|
||||
def test_get_current_process_visible_accelerator_ids():
|
||||
os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] = "0,1,2"
|
||||
assert HPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
|
||||
"0",
|
||||
"1",
|
||||
"2",
|
||||
] # noqa: E501
|
||||
|
||||
del os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR]
|
||||
assert HPUAcceleratorManager.get_current_process_visible_accelerator_ids() is None
|
||||
|
||||
os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] = ""
|
||||
assert HPUAcceleratorManager.get_current_process_visible_accelerator_ids() == []
|
||||
|
||||
del os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR]
|
||||
|
||||
|
||||
def test_set_current_process_visible_accelerator_ids():
|
||||
HPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0"])
|
||||
assert os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] == "0"
|
||||
|
||||
HPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
|
||||
assert os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] == "0,1"
|
||||
|
||||
HPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1", "2"])
|
||||
assert os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] == "0,1,2"
|
||||
|
||||
del os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_config",
|
||||
[
|
||||
(1, False),
|
||||
(0.5, True),
|
||||
(3, False),
|
||||
],
|
||||
)
|
||||
def test_validate_resource_request_quantity(test_config):
|
||||
num_hpus, expect_error = test_config
|
||||
|
||||
if expect_error:
|
||||
assert (
|
||||
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[0]
|
||||
is False
|
||||
)
|
||||
assert (
|
||||
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[1]
|
||||
is not None
|
||||
)
|
||||
else:
|
||||
assert (
|
||||
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[0]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[1]
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_check_accelerator_info():
|
||||
|
||||
if HPUAcceleratorManager.is_initialized():
|
||||
assert (
|
||||
"Intel-GAUDI" in HPUAcceleratorManager.get_current_node_accelerator_type()
|
||||
)
|
||||
else:
|
||||
assert HPUAcceleratorManager.get_current_node_accelerator_type() is None
|
||||
|
||||
assert HPUAcceleratorManager.get_resource_name() == "HPU"
|
||||
|
||||
|
||||
def test_decorator_args():
|
||||
|
||||
# This is a valid way of using the decorator.
|
||||
@ray.remote(resources={"HPU": 1}) # noqa: F811
|
||||
class Actor: # noqa: F811
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
# This is a valid way of using the decorator.
|
||||
@ray.remote(num_cpus=1, resources={"HPU": 1}) # noqa: F811
|
||||
class Actor: # noqa: F811
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_actor_deletion_with_hpus(shutdown_only):
|
||||
ray.init(num_cpus=1, resources={"HPU": 1})
|
||||
|
||||
# When an actor that uses an HPU exits, make sure that the HPU resources
|
||||
# are released.
|
||||
|
||||
@ray.remote(resources={"HPU": 1})
|
||||
class Actor:
|
||||
def getpid(self):
|
||||
return os.getpid()
|
||||
|
||||
for _ in range(5):
|
||||
# If we can successfully create an actor, that means that enough
|
||||
# HPU resources are available.
|
||||
a = Actor.remote()
|
||||
ray.get(a.getpid.remote())
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
|
||||
def test_actor_hpus(ray_start_cluster):
|
||||
cluster = ray_start_cluster
|
||||
num_nodes = 2
|
||||
num_hpus_per_raylet = 2
|
||||
for i in range(num_nodes):
|
||||
cluster.add_node(num_cpus=10 * 2, resources={"HPU": num_hpus_per_raylet})
|
||||
ray.init(address=cluster.address)
|
||||
|
||||
@ray.remote(resources={"HPU": 1})
|
||||
class Actor1:
|
||||
def __init__(self):
|
||||
resource_ids = ray.get_runtime_context().get_accelerator_ids()
|
||||
self.hpu_ids = resource_ids.get("HPU")
|
||||
|
||||
def get_location_and_ids(self):
|
||||
return (
|
||||
ray.get_runtime_context().get_node_id(),
|
||||
tuple(self.hpu_ids),
|
||||
)
|
||||
|
||||
# Create one actor per HPU.
|
||||
actors = [Actor1.remote() for _ in range(num_nodes * num_hpus_per_raylet)]
|
||||
# Make sure that no two actors are assigned to the same HPU.
|
||||
locations_and_ids = ray.get(
|
||||
[actor.get_location_and_ids.remote() for actor in actors]
|
||||
)
|
||||
node_names = {location for location, hpu_id in locations_and_ids}
|
||||
assert len(node_names) == num_nodes
|
||||
location_actor_combinations = []
|
||||
for node_name in node_names:
|
||||
for hpu_id in range(num_hpus_per_raylet):
|
||||
location_actor_combinations.append((node_name, (f"{hpu_id}",)))
|
||||
|
||||
assert set(locations_and_ids) == set(location_actor_combinations)
|
||||
|
||||
# Creating a new actor should fail because all of the HPUs are being
|
||||
# used.
|
||||
a = Actor1.remote()
|
||||
ready_ids, _ = ray.wait([a.get_location_and_ids.remote()], timeout=0.01)
|
||||
assert ready_ids == []
|
||||
|
||||
|
||||
def test_actor_habana_visible_devices(shutdown_only):
|
||||
"""Test user can overwrite HABANA_VISIBLE_MODULES
|
||||
after the actor is created."""
|
||||
ray.init(resources={"HPU": 1})
|
||||
|
||||
@ray.remote(resources={"HPU": 1})
|
||||
class Actor:
|
||||
def set_habana_visible_devices(self, habana_visible_devices):
|
||||
os.environ["HABANA_VISIBLE_MODULES"] = habana_visible_devices
|
||||
|
||||
def get_habana_visible_devices(self):
|
||||
return os.environ["HABANA_VISIBLE_MODULES"]
|
||||
|
||||
actor = Actor.remote()
|
||||
assert ray.get(actor.get_habana_visible_devices.remote()) == "0"
|
||||
ray.get(actor.set_habana_visible_devices.remote("0,1"))
|
||||
assert ray.get(actor.get_habana_visible_devices.remote()) == "0,1"
|
||||
|
||||
|
||||
def test_hpu_ids(shutdown_only):
|
||||
num_hpus = 3
|
||||
ray.init(num_cpus=num_hpus, resources={"HPU": num_hpus})
|
||||
|
||||
def get_hpu_ids(hpus_per_worker):
|
||||
hpu_ids = ray.get_runtime_context().get_accelerator_ids()["HPU"]
|
||||
assert len(hpu_ids) == hpus_per_worker
|
||||
modules = os.environ.get("HABANA_VISIBLE_MODULES")
|
||||
if modules is not None:
|
||||
assert modules == ",".join([str(i) for i in hpu_ids]) # noqa
|
||||
for hpu_id in hpu_ids:
|
||||
assert hpu_id in [str(i) for i in range(num_hpus)]
|
||||
return hpu_ids
|
||||
|
||||
f0 = ray.remote(resources={"HPU": 0})(lambda: get_hpu_ids(0))
|
||||
f1 = ray.remote(resources={"HPU": 1})(lambda: get_hpu_ids(1))
|
||||
|
||||
list_of_ids = ray.get([f0.remote() for _ in range(10)])
|
||||
assert list_of_ids == 10 * [[]]
|
||||
ray.get([f1.remote() for _ in range(10)])
|
||||
|
||||
# Test that actors have HABANA_VISIBLE_MODULES set properly.
|
||||
|
||||
def _check_hpu_env(expected_num_hpus):
|
||||
hpu_ids = ray.get_runtime_context().get_accelerator_ids()["HPU"]
|
||||
assert len(hpu_ids) == expected_num_hpus
|
||||
if expected_num_hpus > 0:
|
||||
assert os.environ["HABANA_VISIBLE_MODULES"] == ",".join(
|
||||
[str(i) for i in hpu_ids] # noqa
|
||||
)
|
||||
else:
|
||||
assert os.environ.get("HABANA_VISIBLE_MODULES") is None
|
||||
|
||||
@ray.remote
|
||||
class Actor:
|
||||
def __init__(self, num_hpus):
|
||||
self.num_hpus = num_hpus
|
||||
_check_hpu_env(num_hpus)
|
||||
self.x = num_hpus
|
||||
|
||||
def test(self):
|
||||
_check_hpu_env(self.num_hpus)
|
||||
return self.x
|
||||
|
||||
a0 = Actor.remote(0)
|
||||
assert ray.get(a0.test.remote()) == 0
|
||||
|
||||
a1 = Actor.options(resources={"HPU": 1}).remote(1)
|
||||
assert ray.get(a1.test.remote()) == 1
|
||||
|
||||
|
||||
def test_hpu_with_placement_group(shutdown_only):
|
||||
num_hpus = 2
|
||||
ray.init(num_cpus=1, resources={"HPU": num_hpus})
|
||||
|
||||
@ray.remote(resources={"HPU": num_hpus})
|
||||
class HPUActor:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def ready(self):
|
||||
hpu_ids = ray.get_runtime_context().get_accelerator_ids()["HPU"]
|
||||
assert len(hpu_ids) == num_hpus
|
||||
assert os.environ["HABANA_VISIBLE_MODULES"] == ",".join(
|
||||
[str(i) for i in hpu_ids] # noqa
|
||||
)
|
||||
|
||||
# Reserve a placement group of 1 bundle that reserves 1 CPU and 2 HPU.
|
||||
pg = placement_group([{"CPU": 1, "HPU": num_hpus}])
|
||||
|
||||
# Wait until placement group is created.
|
||||
ray.get(pg.ready(), timeout=10)
|
||||
|
||||
actor = HPUActor.options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=pg,
|
||||
)
|
||||
).remote()
|
||||
|
||||
ray.get(actor.ready.remote(), timeout=10)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,117 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._private.accelerators import (
|
||||
IntelGPUAcceleratorManager as Accelerator,
|
||||
get_accelerator_manager_for_resource,
|
||||
)
|
||||
from ray.util.accelerators import INTEL_MAX_1100, INTEL_MAX_1550
|
||||
|
||||
|
||||
def test_visible_intel_gpu_ids(shutdown_only):
|
||||
with patch.object(Accelerator, "get_current_node_num_accelerators", return_value=4):
|
||||
os.environ["ONEAPI_DEVICE_SELECTOR"] = "level_zero:0,1,2"
|
||||
# Delete the cache so it can be re-populated the next time
|
||||
# we call get_accelerator_manager_for_resource
|
||||
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
|
||||
ray.init()
|
||||
manager = get_accelerator_manager_for_resource("GPU")
|
||||
assert manager.get_current_node_num_accelerators() == 4
|
||||
assert manager.__name__ == "IntelGPUAcceleratorManager"
|
||||
assert ray.available_resources()["GPU"] == 3
|
||||
|
||||
|
||||
def test_visible_intel_gpu_type(shutdown_only):
|
||||
with patch.object(
|
||||
Accelerator, "get_current_node_num_accelerators", return_value=4
|
||||
), patch.object(
|
||||
Accelerator, "get_current_node_accelerator_type", return_value=INTEL_MAX_1550
|
||||
):
|
||||
os.environ["ONEAPI_DEVICE_SELECTOR"] = "level_zero:0,1,2"
|
||||
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
|
||||
ray.init()
|
||||
manager = get_accelerator_manager_for_resource("GPU")
|
||||
assert manager.get_current_node_accelerator_type() == INTEL_MAX_1550
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
|
||||
@pytest.mark.skipif(
|
||||
sys.version_info >= (3, 12),
|
||||
reason="Not passing on Python 3.12. Being followed up by external contributors.",
|
||||
)
|
||||
def test_get_current_node_num_accelerators():
|
||||
old_dpctl = None
|
||||
if "dpctl" in sys.modules:
|
||||
old_dpctl = sys.modules["dpctl"]
|
||||
|
||||
sys.modules["dpctl"] = __import__("mock_dpctl_1")
|
||||
assert Accelerator.get_current_node_num_accelerators() == 6
|
||||
|
||||
sys.modules["dpctl"] = __import__("mock_dpctl_2")
|
||||
assert Accelerator.get_current_node_num_accelerators() == 4
|
||||
|
||||
if old_dpctl is not None:
|
||||
sys.modules["dpctl"] = old_dpctl
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
|
||||
@pytest.mark.skipif(
|
||||
sys.version_info >= (3, 12),
|
||||
reason="Not passing on Python 3.12. Being followed up by external contributors.",
|
||||
)
|
||||
def test_get_current_node_accelerator_type():
|
||||
old_dpctl = None
|
||||
if "dpctl" in sys.modules:
|
||||
old_dpctl = sys.modules["dpctl"]
|
||||
|
||||
sys.modules["dpctl"] = __import__("mock_dpctl_1")
|
||||
assert Accelerator.get_current_node_accelerator_type() == INTEL_MAX_1550
|
||||
|
||||
sys.modules["dpctl"] = __import__("mock_dpctl_2")
|
||||
assert Accelerator.get_current_node_accelerator_type() == INTEL_MAX_1100
|
||||
|
||||
if old_dpctl is not None:
|
||||
sys.modules["dpctl"] = old_dpctl
|
||||
|
||||
|
||||
def test_intel_gpu_accelerator_manager_api():
|
||||
assert Accelerator.get_resource_name() == "GPU"
|
||||
assert Accelerator.get_visible_accelerator_ids_env_var() == "ONEAPI_DEVICE_SELECTOR"
|
||||
assert Accelerator.validate_resource_request_quantity(0.1) == (True, None)
|
||||
|
||||
|
||||
def test_get_current_process_visible_accelerator_ids():
|
||||
os.environ["ONEAPI_DEVICE_SELECTOR"] = "level_zero:0,1,2"
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() == ["0", "1", "2"]
|
||||
|
||||
del os.environ["ONEAPI_DEVICE_SELECTOR"]
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() is None
|
||||
|
||||
os.environ["ONEAPI_DEVICE_SELECTOR"] = ""
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() == []
|
||||
|
||||
os.environ["ONEAPI_DEVICE_SELECTOR"] = "NoDevFiles"
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() == []
|
||||
|
||||
del os.environ["ONEAPI_DEVICE_SELECTOR"]
|
||||
|
||||
|
||||
def test_set_current_process_visible_accelerator_ids():
|
||||
Accelerator.set_current_process_visible_accelerator_ids(["0"])
|
||||
assert os.environ["ONEAPI_DEVICE_SELECTOR"] == "level_zero:0"
|
||||
|
||||
Accelerator.set_current_process_visible_accelerator_ids(["0", "1"])
|
||||
assert os.environ["ONEAPI_DEVICE_SELECTOR"] == "level_zero:0,1"
|
||||
|
||||
Accelerator.set_current_process_visible_accelerator_ids(["0", "1", "2"])
|
||||
assert os.environ["ONEAPI_DEVICE_SELECTOR"] == "level_zero:0,1,2"
|
||||
|
||||
del os.environ["ONEAPI_DEVICE_SELECTOR"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,228 @@
|
||||
"""
|
||||
Manual Intel GPU validation tests, not executed in automated runs.
|
||||
|
||||
These tests are basic acceptance tests to validate Intel GPU support in Ray. They
|
||||
require a suitable Intel GPU environment with dpctl installed. They are intended to
|
||||
serve as an approved method to verify Intel GPU-based Ray deployments.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
|
||||
try:
|
||||
import dpctl
|
||||
except ImportError:
|
||||
pytest.skip(
|
||||
"dpctl is not installed, skipping Intel GPU tests.", allow_module_level=True
|
||||
)
|
||||
|
||||
DEFAULT_SCALE_OUT_NODES = 2
|
||||
DEFAULT_SCALE_UP_DEVICES = 2
|
||||
|
||||
USE_GPU = bool(os.environ.get("RAY_PYTEST_USE_GPU", 0))
|
||||
|
||||
if not USE_GPU:
|
||||
pytest.skip("Skipping, these tests require GPUs.", allow_module_level=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ray_gpu_session():
|
||||
"""Start a Ray session with caller-provided init kwargs."""
|
||||
|
||||
def _start_session(**init_kwargs):
|
||||
if ray.is_initialized():
|
||||
ray.shutdown()
|
||||
|
||||
ray.init(**init_kwargs)
|
||||
|
||||
try:
|
||||
yield _start_session
|
||||
finally:
|
||||
if ray.is_initialized():
|
||||
ray.shutdown()
|
||||
|
||||
|
||||
def _is_cluster_configured(address: str = "auto") -> bool:
|
||||
try:
|
||||
ray.init(
|
||||
address=address,
|
||||
)
|
||||
return True
|
||||
except (ray.exceptions.RaySystemError, ConnectionError, TimeoutError):
|
||||
return False
|
||||
finally:
|
||||
if ray.is_initialized():
|
||||
ray.shutdown()
|
||||
|
||||
|
||||
def _detect_available_gpu_count() -> int:
|
||||
"""Return the number of GPU devices detected via dpctl."""
|
||||
try:
|
||||
return dpctl.SyclContext("level_zero:gpu").device_count
|
||||
except Exception:
|
||||
# If dpctl cannot enumerate devices, assume no additional GPUs.
|
||||
return 0
|
||||
|
||||
|
||||
def _require_min_gpus(required: int, context: str) -> None:
|
||||
available = _detect_available_gpu_count()
|
||||
if available < required:
|
||||
pytest.skip(
|
||||
f"Skipping {context}: requires {required} GPUs, detected {available} via dpctl."
|
||||
)
|
||||
|
||||
|
||||
def _require_min_cluster_nodes(required_nodes: int, context: str) -> None:
|
||||
alive_nodes = [node for node in ray.nodes() if node.get("Alive")]
|
||||
unique_node_ids = {node.get("NodeID") for node in alive_nodes if node.get("NodeID")}
|
||||
|
||||
if len(unique_node_ids) < required_nodes:
|
||||
pytest.skip(
|
||||
f"Skipping {context}: requires {required_nodes} alive Ray nodes, detected {len(unique_node_ids)}."
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def gpu_task() -> Dict[str, Any]:
|
||||
context = ray.get_runtime_context()
|
||||
gpu_ids = context.get_accelerator_ids().get("GPU", [])
|
||||
return {
|
||||
"gpu_ids": gpu_ids,
|
||||
"pid": os.getpid(),
|
||||
"oneapi_selector": os.environ.get("ONEAPI_DEVICE_SELECTOR"),
|
||||
}
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def cluster_probe_task() -> Dict[str, Any]:
|
||||
context = ray.get_runtime_context()
|
||||
return {
|
||||
"node_id": context.get_node_id(),
|
||||
"node_ip": ray.util.get_node_ip_address(),
|
||||
"worker_id": context.get_worker_id(),
|
||||
"gpu_ids": context.get_accelerator_ids().get("GPU", []),
|
||||
"selector": os.environ.get("ONEAPI_DEVICE_SELECTOR"),
|
||||
}
|
||||
|
||||
|
||||
def assert_valid_gpu_binding(result: Dict[str, Any], label: str) -> None:
|
||||
primary_gpu_id = _validate_gpu_binding_common(result, label)
|
||||
assert (
|
||||
primary_gpu_id >= 0
|
||||
), f"Expected {label} to bind to a valid GPU, got {result.get('gpu_ids')}"
|
||||
|
||||
|
||||
def _validate_gpu_binding_common(
|
||||
result: Dict[str, Any], label: str, selector_key: str = "oneapi_selector"
|
||||
) -> int:
|
||||
"""Validate basic GPU binding properties shared by single- and multi-GPU tests."""
|
||||
|
||||
gpu_ids = result.get("gpu_ids")
|
||||
assert gpu_ids, f"No GPU IDs assigned for {label}."
|
||||
|
||||
primary_gpu_id = int(gpu_ids[0])
|
||||
|
||||
selector = result.get(selector_key)
|
||||
assert selector, f"ONEAPI_DEVICE_SELECTOR not set in environment for {label}."
|
||||
selector_lower = selector.lower()
|
||||
assert (
|
||||
"level_zero:" in selector_lower
|
||||
), f"ONEAPI_DEVICE_SELECTOR should target GPU devices for {label}, got: {selector}."
|
||||
|
||||
selector_gpu_ids = {int(match) for match in re.findall(r"\b\d+\b", selector_lower)}
|
||||
assert (
|
||||
primary_gpu_id in selector_gpu_ids
|
||||
), f"ONEAPI_DEVICE_SELECTOR does not reference bound GPU id for {label}: {selector}."
|
||||
|
||||
return primary_gpu_id
|
||||
|
||||
|
||||
def assert_valid_multi_gpu_binding(
|
||||
results: List[Dict[str, Any]], num_gpus: int, label: str
|
||||
) -> None:
|
||||
"""Assert that multiple GPU tasks bind to different GPUs correctly."""
|
||||
assert (
|
||||
len(results) == num_gpus
|
||||
), f"Expected {num_gpus} results for {label}, got {len(results)}."
|
||||
|
||||
gpu_ids = []
|
||||
for i, result in enumerate(results):
|
||||
primary_gpu_id = _validate_gpu_binding_common(result, f"{label} instance {i}")
|
||||
gpu_ids.append(primary_gpu_id)
|
||||
assert (
|
||||
len(set(gpu_ids)) == num_gpus
|
||||
), f"Expected {label} to bind to {num_gpus} distinct GPUs, got bindings to GPU IDs: {gpu_ids}."
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_is_cluster_configured(),
|
||||
reason="Environment setup for scale-out, skipping single-node test.",
|
||||
)
|
||||
def test_gpu_task_binding(ray_gpu_session) -> None:
|
||||
_require_min_gpus(1, "single GPU task binding test")
|
||||
ray_gpu_session(num_gpus=1)
|
||||
|
||||
task_result = ray.get(gpu_task.remote())
|
||||
assert_valid_gpu_binding(task_result, "GPU task")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_is_cluster_configured(),
|
||||
reason="Environment setup for scale-out, skipping single-node test.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_gpus", [DEFAULT_SCALE_UP_DEVICES]
|
||||
) # To be extended to required configurations
|
||||
def test_multi_gpu_task_binding(ray_gpu_session, num_gpus) -> None:
|
||||
"""Test that multiple GPU tasks bind to different GPUs correctly."""
|
||||
_require_min_gpus(num_gpus, "multi-GPU task binding test")
|
||||
|
||||
ray_gpu_session(num_gpus=num_gpus)
|
||||
|
||||
task_futures = [gpu_task.remote() for _ in range(num_gpus)]
|
||||
task_results = ray.get(task_futures)
|
||||
|
||||
assert_valid_multi_gpu_binding(task_results, num_gpus, f"GPU tasks (n={num_gpus})")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _is_cluster_configured(), reason="Environment not setup for scale-out test."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_nodes", [DEFAULT_SCALE_OUT_NODES]
|
||||
) # To be extended to required configurations
|
||||
def test_scale_out_task_distribution(ray_gpu_session, num_nodes) -> None:
|
||||
"""Ensure tasks can be scheduled across multiple nodes in the cluster."""
|
||||
ray_gpu_session(address="auto")
|
||||
_require_min_cluster_nodes(num_nodes, "scale-out task distribution test")
|
||||
|
||||
probe_handles = [
|
||||
cluster_probe_task.options(scheduling_strategy="SPREAD").remote()
|
||||
for _ in range(num_nodes)
|
||||
]
|
||||
probe_results = ray.get(probe_handles)
|
||||
|
||||
node_ids = {
|
||||
result.get("node_id") for result in probe_results if result.get("node_id")
|
||||
}
|
||||
node_ips = {
|
||||
result.get("node_ip") for result in probe_results if result.get("node_ip")
|
||||
}
|
||||
|
||||
for result in probe_results:
|
||||
_validate_gpu_binding_common(result, "scale-out probe task", "selector")
|
||||
|
||||
assert len(node_ids) == num_nodes or len(node_ips) == num_nodes, (
|
||||
f"Expected probe tasks to execute on {num_nodes} distinct nodes, "
|
||||
f"got node_ids={node_ids} node_ips={node_ips}."
|
||||
)
|
||||
|
||||
gpu_capable_results = [result for result in probe_results if result.get("gpu_ids")]
|
||||
assert (
|
||||
len(gpu_capable_results) == num_nodes
|
||||
), "Not all probe tasks reported GPU accelerator bindings in the cluster."
|
||||
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._private.accelerators import (
|
||||
MetaxGPUAcceleratorManager,
|
||||
get_accelerator_manager_for_resource,
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.MetaxGPUAcceleratorManager.get_current_node_num_accelerators",
|
||||
return_value=4,
|
||||
)
|
||||
def test_visible_metax_gpu_ids(mock_get_num_accelerators, monkeypatch, shutdown_only):
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1,2")
|
||||
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
|
||||
ray.init()
|
||||
assert mock_get_num_accelerators.called
|
||||
assert ray.available_resources()["GPU"] == 3
|
||||
|
||||
|
||||
def test_get_current_process_visible_accelerator_ids(monkeypatch):
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
|
||||
assert MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
|
||||
"0"
|
||||
]
|
||||
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,4,7")
|
||||
assert MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
|
||||
"0",
|
||||
"4",
|
||||
"7",
|
||||
]
|
||||
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
|
||||
assert (
|
||||
MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == []
|
||||
)
|
||||
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES")
|
||||
assert (
|
||||
MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() is None
|
||||
)
|
||||
|
||||
|
||||
def test_set_current_process_visible_accelerator_ids():
|
||||
MetaxGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0"])
|
||||
assert os.environ["CUDA_VISIBLE_DEVICES"] == "0"
|
||||
|
||||
MetaxGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
|
||||
assert os.environ["CUDA_VISIBLE_DEVICES"] == "0,1"
|
||||
|
||||
MetaxGPUAcceleratorManager.set_current_process_visible_accelerator_ids(
|
||||
["0", "1", "7"]
|
||||
)
|
||||
assert os.environ["CUDA_VISIBLE_DEVICES"] == "0,1,7"
|
||||
|
||||
del os.environ["CUDA_VISIBLE_DEVICES"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if os.environ.get("PARALLEL_CI"):
|
||||
sys.exit(pytest.main(["-n", "auto", "--boxed", "-vs", __file__]))
|
||||
else:
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,104 @@
|
||||
import subprocess
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._private.accelerators import NeuronAcceleratorManager
|
||||
|
||||
|
||||
def test_user_configured_more_than_visible(monkeypatch, call_ray_stop_only):
|
||||
# Test more neuron_cores are configured than visible.
|
||||
monkeypatch.setenv("NEURON_RT_VISIBLE_CORES", "0,1,2")
|
||||
with pytest.raises(ValueError):
|
||||
ray.init(resources={"neuron_cores": 4})
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.NeuronAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
|
||||
return_value=4,
|
||||
)
|
||||
def test_auto_detected_more_than_visible(
|
||||
mock_get_num_accelerators, monkeypatch, shutdown_only
|
||||
):
|
||||
# Test more neuron_cores are detected than visible.
|
||||
monkeypatch.setenv("NEURON_RT_VISIBLE_CORES", "0,1,2")
|
||||
ray.init()
|
||||
_ = mock_get_num_accelerators.called
|
||||
assert ray.available_resources()["neuron_cores"] == 3
|
||||
|
||||
|
||||
@patch(
|
||||
"ray._private.accelerators.NeuronAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
|
||||
return_value=2,
|
||||
)
|
||||
def test_auto_detect_resources(mock_get_num_accelerators, shutdown_only):
|
||||
# Test that ray node resources are filled with auto detected count.
|
||||
ray.init()
|
||||
_ = mock_get_num_accelerators.called
|
||||
assert ray.available_resources()["neuron_cores"] == 2
|
||||
|
||||
|
||||
@patch(
|
||||
"subprocess.run",
|
||||
return_value=subprocess.CompletedProcess(
|
||||
args=[],
|
||||
returncode=0,
|
||||
stdout=(
|
||||
b'[{"neuron_device":0,"bdf":"00:1e.0",'
|
||||
b'"connected_to":null,"nc_count":2,'
|
||||
b'"memory_size":34359738368,"neuron_processes":[]}]'
|
||||
),
|
||||
),
|
||||
)
|
||||
@patch("os.path.isdir", return_value=True)
|
||||
@patch("sys.platform", "linux")
|
||||
def test_get_neuron_core_count_single_device(mock_isdir, mock_subprocess):
|
||||
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 2
|
||||
|
||||
|
||||
@patch(
|
||||
"subprocess.run",
|
||||
return_value=subprocess.CompletedProcess(
|
||||
args=[],
|
||||
returncode=0,
|
||||
stdout=(
|
||||
b'[{"neuron_device":0,"bdf":"00:1e.0",'
|
||||
b'"connected_to":null,"nc_count":2,'
|
||||
b'"memory_size":34359738368,"neuron_processes":[]},'
|
||||
b'{"neuron_device":1,"bdf":"00:1f.0","connected_to":null,'
|
||||
b'"nc_count":2,"memory_size":34359738368,"neuron_processes":[]}]'
|
||||
),
|
||||
),
|
||||
)
|
||||
@patch("os.path.isdir", return_value=True)
|
||||
@patch("sys.platform", "linux")
|
||||
def test_get_neuron_core_count_multiple_devices(mock_isdir, mock_subprocess):
|
||||
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 4
|
||||
|
||||
|
||||
@patch(
|
||||
"subprocess.run",
|
||||
return_value=subprocess.CompletedProcess(
|
||||
args=[], returncode=1, stdout=b"AccessDenied"
|
||||
),
|
||||
)
|
||||
@patch("os.path.isdir", return_value=True)
|
||||
@patch("sys.platform", "linux")
|
||||
def test_get_neuron_core_count_failure_with_error(mock_isdir, mock_subprocess):
|
||||
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 0
|
||||
|
||||
|
||||
@patch(
|
||||
"subprocess.run",
|
||||
return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout=b"[{}]"),
|
||||
)
|
||||
@patch("os.path.isdir", return_value=True)
|
||||
@patch("sys.platform", "linux")
|
||||
def test_get_neuron_core_count_failure_with_empty_results(mock_isdir, mock_subprocess):
|
||||
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,131 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
from ray._private.accelerators import NPUAcceleratorManager as Accelerator
|
||||
|
||||
|
||||
@patch("glob.glob")
|
||||
def test_autodetect_num_npus(mock_glob):
|
||||
with patch.dict(sys.modules):
|
||||
sys.modules["acl"] = None
|
||||
mock_glob.return_value = [f"/dev/davinci{i}" for i in range(64)]
|
||||
assert Accelerator.get_current_node_num_accelerators() == 64
|
||||
|
||||
|
||||
@patch("glob.glob")
|
||||
def test_autodetect_num_npus_without_devices(mock_glob):
|
||||
with patch.dict(sys.modules):
|
||||
sys.modules["acl"] = None
|
||||
mock_glob.side_effect = Exception
|
||||
assert Accelerator.get_current_node_num_accelerators() == 0
|
||||
|
||||
|
||||
def test_ascend_npu_accelerator_manager_api():
|
||||
assert Accelerator.get_resource_name() == "NPU"
|
||||
assert (
|
||||
Accelerator.get_visible_accelerator_ids_env_var() == "ASCEND_RT_VISIBLE_DEVICES"
|
||||
)
|
||||
assert Accelerator.validate_resource_request_quantity(0.5) == (True, None)
|
||||
assert Accelerator.validate_resource_request_quantity(1) == (True, None)
|
||||
|
||||
|
||||
def test_visible_ascend_npu_type(monkeypatch, shutdown_only):
|
||||
with patch.object(
|
||||
Accelerator, "get_current_node_num_accelerators", return_value=4
|
||||
), patch.object(
|
||||
Accelerator, "get_current_node_accelerator_type", return_value="Ascend910B"
|
||||
):
|
||||
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
|
||||
manager = ray._private.accelerators.get_accelerator_manager_for_resource("NPU")
|
||||
assert manager.get_current_node_accelerator_type() == "Ascend910B"
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
|
||||
@pytest.mark.skipif(
|
||||
sys.version_info >= (3, 12),
|
||||
reason="Not passing on Python 3.12. Being followed up by external contributors.",
|
||||
)
|
||||
def test_visible_ascend_npu_ids(monkeypatch, shutdown_only):
|
||||
with patch.dict(sys.modules):
|
||||
sys.modules["acl"] = __import__("mock_acl")
|
||||
|
||||
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
|
||||
with patch.object(
|
||||
Accelerator, "get_current_node_num_accelerators", return_value=4
|
||||
):
|
||||
|
||||
ray.init()
|
||||
manager = ray._private.accelerators.get_accelerator_manager_for_resource(
|
||||
"NPU"
|
||||
)
|
||||
assert manager.get_current_node_num_accelerators() == 4
|
||||
assert manager.__name__ == "NPUAcceleratorManager"
|
||||
assert ray.available_resources()["NPU"] == 3
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
|
||||
@pytest.mark.skipif(
|
||||
sys.version_info >= (3, 12),
|
||||
reason="Not passing on Python 3.12. Being followed up by external contributors.",
|
||||
)
|
||||
def test_acl_api_function(shutdown_only):
|
||||
with patch.dict(sys.modules):
|
||||
sys.modules["acl"] = __import__("mock_acl")
|
||||
|
||||
ray.init()
|
||||
manager = ray._private.accelerators.get_accelerator_manager_for_resource("NPU")
|
||||
assert manager.get_current_node_num_accelerators() == 4
|
||||
assert manager.__name__ == "NPUAcceleratorManager"
|
||||
assert manager.get_current_node_accelerator_type() == "Ascend910B"
|
||||
|
||||
|
||||
def test_get_current_process_visible_accelerator_ids(monkeypatch, shutdown_only):
|
||||
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() == ["0", "1", "2"]
|
||||
|
||||
monkeypatch.delenv("ASCEND_RT_VISIBLE_DEVICES")
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() is None
|
||||
|
||||
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "")
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() == []
|
||||
|
||||
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "NoDevFiles")
|
||||
assert Accelerator.get_current_process_visible_accelerator_ids() == []
|
||||
|
||||
|
||||
def test_set_current_process_visible_accelerator_ids(shutdown_only):
|
||||
Accelerator.set_current_process_visible_accelerator_ids(["0"])
|
||||
assert os.environ["ASCEND_RT_VISIBLE_DEVICES"] == "0"
|
||||
|
||||
Accelerator.set_current_process_visible_accelerator_ids(["0", "1"])
|
||||
assert os.environ["ASCEND_RT_VISIBLE_DEVICES"] == "0,1"
|
||||
|
||||
Accelerator.set_current_process_visible_accelerator_ids(["0", "1", "2"])
|
||||
assert os.environ["ASCEND_RT_VISIBLE_DEVICES"] == "0,1,2"
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
|
||||
@pytest.mark.skipif(
|
||||
sys.version_info >= (3, 12),
|
||||
reason="Not passing on Python 3.12. Being followed up by external contributors.",
|
||||
)
|
||||
def test_auto_detected_more_than_visible(monkeypatch, shutdown_only):
|
||||
with patch.dict(sys.modules):
|
||||
sys.modules["acl"] = __import__("mock_acl")
|
||||
|
||||
with patch.object(
|
||||
Accelerator, "get_current_node_num_accelerators", return_value=4
|
||||
):
|
||||
# If more NPUs are detected than visible.
|
||||
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
|
||||
|
||||
ray.init()
|
||||
assert ray.available_resources()["NPU"] == 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,103 @@
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from ray._private.accelerators import NvidiaGPUAcceleratorManager
|
||||
from ray.tests.accelerators.mock_pynvml import (
|
||||
DeviceHandleMock,
|
||||
PyNVMLMock,
|
||||
patch_mock_pynvml,
|
||||
)
|
||||
|
||||
GPU_MOCK_DATA = [
|
||||
DeviceHandleMock(
|
||||
"Ampere A100-SXM4-40GB",
|
||||
"GPU-8eaaebb8-bb64-8489-fda2-62256e821983",
|
||||
mig_devices=[
|
||||
DeviceHandleMock(
|
||||
"Ampere A100-SXM4-40GB MIG 1g.5gb",
|
||||
"MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f",
|
||||
gi_id=0,
|
||||
ci_instance=0,
|
||||
),
|
||||
DeviceHandleMock(
|
||||
"Ampere A100-SXM4-40GB MIG 1g.5gb",
|
||||
"MIG-0c757cd7-e942-5726-a0b8-0e8fb7067135",
|
||||
gi_id=1,
|
||||
ci_instance=0,
|
||||
),
|
||||
],
|
||||
),
|
||||
DeviceHandleMock(
|
||||
"Ampere A100-SXM4-40GB",
|
||||
"GPU-8eaaebb8-bb64-8489-fda2-62256e821983",
|
||||
mig_devices=[
|
||||
DeviceHandleMock(
|
||||
"Ampere A100-SXM4-40GB MIG 1g.5gb",
|
||||
"MIG-a28ad590-3fda-56dd-84fc-0a0b96edc58d",
|
||||
gi_id=0,
|
||||
ci_instance=0,
|
||||
)
|
||||
],
|
||||
),
|
||||
DeviceHandleMock(
|
||||
"Tesla V100-SXM2-16GB", "GPU-8eaaebb8-bb64-8489-fda2-62256e821983"
|
||||
),
|
||||
]
|
||||
|
||||
mock_nvml = PyNVMLMock(GPU_MOCK_DATA)
|
||||
|
||||
patch_mock_pynvml = patch_mock_pynvml # avoid format error
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mock_nvml", [mock_nvml])
|
||||
def test_num_gpus_parsing(patch_mock_pynvml):
|
||||
# without mig instance
|
||||
assert NvidiaGPUAcceleratorManager.get_current_node_num_accelerators() == len(
|
||||
GPU_MOCK_DATA
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mock_nvml", [mock_nvml])
|
||||
def test_gpu_info_parsing(patch_mock_pynvml):
|
||||
assert NvidiaGPUAcceleratorManager.get_current_node_accelerator_type() == "A100"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,expected",
|
||||
[
|
||||
# Legacy datacenter GPU names: keep labels produced by the previous
|
||||
# parser stable.
|
||||
("Tesla V100-SXM2-16GB", "V100"),
|
||||
("Tesla P100-PCIE-16GB", "P100"),
|
||||
("Tesla T4", "T4"),
|
||||
("Tesla P4", "P4"),
|
||||
("Tesla K80", "K80"),
|
||||
("NVIDIA A10G", "A10G"),
|
||||
("NVIDIA L4", "L4"),
|
||||
("NVIDIA L40S", "L40S"),
|
||||
("NVIDIA A100-SXM4-40GB", "A100"),
|
||||
("NVIDIA H100 80GB HBM3", "H100"),
|
||||
("NVIDIA H200", "H200"),
|
||||
("NVIDIA H20", "H20"),
|
||||
("NVIDIA B200", "B200"),
|
||||
("NVIDIA B300", "B300"),
|
||||
# Consumer GPUs: the regex does not match the mixed-case product line,
|
||||
# so we fall back to a hyphen-joined product name.
|
||||
("NVIDIA GeForce RTX 5090", "GeForce-RTX-5090"),
|
||||
("NVIDIA GeForce RTX 4090", "GeForce-RTX-4090"),
|
||||
# RTX PRO cards: "RTX" alone is just a brand prefix, so the model is
|
||||
# captured through the first digit-containing token instead of
|
||||
# collapsing to the ambiguous "RTX".
|
||||
("NVIDIA RTX PRO 6000 Blackwell Server Edition", "RTX-PRO-6000"),
|
||||
# Edge cases.
|
||||
(None, None),
|
||||
("", None),
|
||||
],
|
||||
)
|
||||
def test_gpu_name_to_accelerator_type(name, expected):
|
||||
assert NvidiaGPUAcceleratorManager._gpu_name_to_accelerator_type(name) == expected
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,82 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from ray._private.accelerators.rbln import (
|
||||
NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR,
|
||||
RBLN_RT_VISIBLE_DEVICES_ENV_VAR,
|
||||
RBLNAcceleratorManager,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_rebel_module(monkeypatch):
|
||||
from ray.tests.accelerators import mock_rebel
|
||||
|
||||
monkeypatch.setitem(sys.modules, "rebel", mock_rebel)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clear_rbln_environment():
|
||||
original_env = os.environ.get(RBLN_RT_VISIBLE_DEVICES_ENV_VAR)
|
||||
original_no_set_env = os.environ.get(NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR)
|
||||
|
||||
os.environ.pop(RBLN_RT_VISIBLE_DEVICES_ENV_VAR, None)
|
||||
os.environ.pop(NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR, None)
|
||||
|
||||
yield
|
||||
|
||||
if original_env is not None:
|
||||
os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = original_env
|
||||
if original_no_set_env is not None:
|
||||
os.environ[NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = original_no_set_env
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("clear_rbln_environment")
|
||||
class TestRBLNAcceleratorManager:
|
||||
def test_get_resource_name(self):
|
||||
assert RBLNAcceleratorManager.get_resource_name() == "RBLN"
|
||||
|
||||
def test_get_visible_accelerator_ids_env_var(self):
|
||||
assert (
|
||||
RBLNAcceleratorManager.get_visible_accelerator_ids_env_var()
|
||||
== RBLN_RT_VISIBLE_DEVICES_ENV_VAR
|
||||
)
|
||||
|
||||
def test_get_current_process_visible_accelerator_ids(self):
|
||||
os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = "0,1,2,3"
|
||||
assert RBLNAcceleratorManager.get_current_process_visible_accelerator_ids() == [
|
||||
"0",
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
]
|
||||
|
||||
os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = ""
|
||||
assert (
|
||||
RBLNAcceleratorManager.get_current_process_visible_accelerator_ids() == []
|
||||
)
|
||||
|
||||
os.environ.pop(RBLN_RT_VISIBLE_DEVICES_ENV_VAR)
|
||||
assert (
|
||||
RBLNAcceleratorManager.get_current_process_visible_accelerator_ids() is None
|
||||
)
|
||||
|
||||
def test_get_current_node_num_accelerators(self):
|
||||
assert RBLNAcceleratorManager.get_current_node_num_accelerators() == 4
|
||||
|
||||
def test_get_current_node_accelerator_type(self):
|
||||
assert RBLNAcceleratorManager.get_current_node_accelerator_type() == "RBLN-CA02"
|
||||
|
||||
def test_set_current_process_visible_accelerator_ids(self):
|
||||
RBLNAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
|
||||
assert os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] == "0,1"
|
||||
|
||||
os.environ[NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = "1"
|
||||
RBLNAcceleratorManager.set_current_process_visible_accelerator_ids(["2", "3"])
|
||||
assert os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] == "0,1"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
@@ -0,0 +1,346 @@
|
||||
import os
|
||||
import sys
|
||||
from unittest import mock
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from ray._private.accelerators import TPUAcceleratorManager, tpu
|
||||
|
||||
|
||||
@patch("glob.glob")
|
||||
def test_autodetect_num_tpus_accel(mock_glob):
|
||||
mock_glob.return_value = [
|
||||
"/dev/accel0",
|
||||
"/dev/accel1",
|
||||
"/dev/accel2",
|
||||
"/dev/accel3",
|
||||
]
|
||||
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
|
||||
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 4
|
||||
|
||||
|
||||
@patch("os.path.isdir")
|
||||
@patch("glob.glob")
|
||||
@patch("os.listdir")
|
||||
def test_autodetect_num_tpus_accel_ignores_blackwell_directory(
|
||||
mock_list, mock_glob, mock_isdir
|
||||
):
|
||||
# NVIDIA drivers 570.x (Blackwell-class GPUs, e.g. RTX 5090) create
|
||||
# /dev/accel as a directory containing /dev/accel/accel0. The non-recursive
|
||||
# glob matches the directory entry; filtering directories out keeps real
|
||||
# TPU chips (character devices at /dev/accel0..N) while rejecting the
|
||||
# NVIDIA false positive.
|
||||
mock_glob.return_value = ["/dev/accel"]
|
||||
mock_isdir.side_effect = lambda p: p == "/dev/accel"
|
||||
mock_list.side_effect = FileNotFoundError
|
||||
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
|
||||
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 0
|
||||
|
||||
|
||||
@patch("glob.glob")
|
||||
@patch("os.listdir")
|
||||
def test_autodetect_num_tpus_vfio(mock_list, mock_glob):
|
||||
mock_glob.return_value = []
|
||||
mock_list.return_value = [f"{i}" for i in range(4)]
|
||||
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
|
||||
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 4
|
||||
|
||||
|
||||
@patch("glob.glob")
|
||||
@patch("os.listdir")
|
||||
def test_autodetect_num_tpus_without_devices(mock_list, mock_glob):
|
||||
mock_list.side_effect = FileNotFoundError
|
||||
mock_glob.return_value = []
|
||||
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
|
||||
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"accelerator_type_version_tuple",
|
||||
[
|
||||
("gce", "v2-8", "TPU-V2"),
|
||||
("gce", "v2-32", "TPU-V2"),
|
||||
("gce", "v3-8", "TPU-V3"),
|
||||
("gce", "v3-128", "TPU-V3"),
|
||||
("gce", "v4-8", "TPU-V4"),
|
||||
("gce", "v4-2048", "TPU-V4"),
|
||||
("gce", "v5p-8", "TPU-V5P"),
|
||||
("gce", "v5litepod-8", "TPU-V5LITEPOD"),
|
||||
("gce", "v6e-8", "TPU-V6E"),
|
||||
("gke", "v2-8", "TPU-V2"),
|
||||
("gke", "v2-32", "TPU-V2"),
|
||||
("gke", "v3-8", "TPU-V3"),
|
||||
("gke", "v3-128", "TPU-V3"),
|
||||
("gke", "v4-8", "TPU-V4"),
|
||||
("gke", "v4-2048", "TPU-V4"),
|
||||
("gke", "v5p-8", "TPU-V5P"),
|
||||
("gke", "v5litepod-8", "TPU-V5LITEPOD"),
|
||||
("gke", "v6e-8", "TPU-V6E"),
|
||||
("gke", "tpu7x-16", "TPU-V7X"),
|
||||
],
|
||||
)
|
||||
@patch("requests.get")
|
||||
@patch("os.getenv")
|
||||
def test_autodetect_tpu_accelerator_type(
|
||||
mock_os, mock_request, accelerator_type_version_tuple
|
||||
):
|
||||
gce_or_gke, accelerator_type, expected_version = accelerator_type_version_tuple
|
||||
if gce_or_gke == "gce":
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = accelerator_type
|
||||
mock_request.return_value = mock_response
|
||||
mock_os.return_value = None
|
||||
else:
|
||||
mock_os.return_value = accelerator_type
|
||||
assert TPUAcceleratorManager.get_current_node_accelerator_type() == expected_version
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_case",
|
||||
[
|
||||
("gce", "0", 0),
|
||||
("gke", "0", 0),
|
||||
],
|
||||
)
|
||||
@patch("requests.get")
|
||||
@patch("os.getenv")
|
||||
def test_get_current_node_tpu_worker_id(mock_os, mock_request, test_case):
|
||||
gce_or_gke, worker_id, expected_value = test_case
|
||||
if gce_or_gke == "gce":
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = worker_id
|
||||
mock_request.return_value = mock_response
|
||||
mock_os.return_value = None
|
||||
else:
|
||||
mock_os.return_value = worker_id
|
||||
assert TPUAcceleratorManager.get_current_node_tpu_worker_id() == expected_value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_case",
|
||||
[
|
||||
("gce", "my-tpu"),
|
||||
("gke", "my-tpu"),
|
||||
],
|
||||
)
|
||||
@patch("requests.get")
|
||||
@patch("os.getenv")
|
||||
def test_get_tpu_unique_id(mock_os, mock_request, test_case):
|
||||
gce_or_gke, worker_id = test_case
|
||||
if gce_or_gke == "gce":
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = worker_id
|
||||
mock_request.return_value = mock_response
|
||||
mock_os.return_value = None
|
||||
else:
|
||||
mock_os.return_value = worker_id
|
||||
assert TPUAcceleratorManager.get_current_node_tpu_name() == worker_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_case",
|
||||
[
|
||||
("gce", "not-a-valid-version"),
|
||||
("gce", "vNOTVALID-8"),
|
||||
("gce", "230498230948230948"),
|
||||
# From issue #39913
|
||||
("gce", ""),
|
||||
("gke", "not-a-valid-version"),
|
||||
("gke", "vNOTVALID-8"),
|
||||
("gke", "230498230948230948"),
|
||||
],
|
||||
)
|
||||
@patch("requests.get")
|
||||
@patch("os.getenv")
|
||||
def test_autodetect_invalid_type(mock_os, mock_request, test_case):
|
||||
gce_or_gke, accelerator_type = test_case
|
||||
if gce_or_gke == "gce":
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = accelerator_type
|
||||
mock_request.return_value = mock_response
|
||||
mock_os.return_value = None
|
||||
else:
|
||||
mock_os.return_value = accelerator_type
|
||||
assert TPUAcceleratorManager.get_current_node_accelerator_type() is None
|
||||
|
||||
|
||||
def test_autodetect_tpu_accelerator_type_fails_gracefully():
|
||||
with patch("requests.get") as mock_get:
|
||||
mock_get.side_effect = requests.exceptions.RequestException
|
||||
assert TPUAcceleratorManager.get_current_node_accelerator_type() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_config",
|
||||
[
|
||||
(1, False),
|
||||
(0.5, True),
|
||||
(3, True),
|
||||
],
|
||||
)
|
||||
def test_validate_resource_request_quantity(test_config):
|
||||
num_tpus, expect_error = test_config
|
||||
|
||||
if expect_error:
|
||||
assert (
|
||||
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[0]
|
||||
is False
|
||||
)
|
||||
assert (
|
||||
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[1]
|
||||
is not None
|
||||
)
|
||||
else:
|
||||
assert (
|
||||
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[0]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[1]
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_case",
|
||||
[
|
||||
(4, ["0"]),
|
||||
(4, ["0", "1"]),
|
||||
(4, ["0", "1", "2", "3"]),
|
||||
(8, ["0", "1", "2", "3", "4", "5", "6", "7"]),
|
||||
],
|
||||
)
|
||||
@patch("glob.glob")
|
||||
def test_set_tpu_visible_ids_and_bounds(mock_glob, test_case):
|
||||
num_devices, tpu_chips = test_case
|
||||
mock_glob.return_value = ["/dev/accel" + str(x) for x in range(num_devices)]
|
||||
with patch.dict("os.environ", {}, clear=True):
|
||||
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
|
||||
TPUAcceleratorManager.set_current_process_visible_accelerator_ids(tpu_chips)
|
||||
if len(tpu_chips) == 1:
|
||||
assert (
|
||||
os.environ[tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR]
|
||||
== tpu.TPU_CHIPS_PER_HOST_BOUNDS_1_CHIP_CONFIG
|
||||
)
|
||||
assert os.environ[tpu.TPU_HOST_BOUNDS_ENV_VAR] == tpu.TPU_SINGLE_HOST_BOUNDS
|
||||
assert os.environ[tpu.TPU_VISIBLE_CHIPS_ENV_VAR] == ",".join(tpu_chips)
|
||||
elif len(tpu_chips) == 2:
|
||||
assert (
|
||||
os.environ[tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR]
|
||||
== tpu.TPU_CHIPS_PER_HOST_BOUNDS_2_CHIP_CONFIG
|
||||
)
|
||||
assert os.environ[tpu.TPU_HOST_BOUNDS_ENV_VAR] == tpu.TPU_SINGLE_HOST_BOUNDS
|
||||
assert os.environ[tpu.TPU_VISIBLE_CHIPS_ENV_VAR] == ",".join(tpu_chips)
|
||||
elif len(tpu_chips) == 4:
|
||||
# Check that nothing is set, let the ML framework use the defaults.
|
||||
assert os.environ.get(tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR, None) is None
|
||||
assert os.environ.get(tpu.TPU_SINGLE_HOST_BOUNDS, None) is None
|
||||
assert os.environ.get(tpu.TPU_VISIBLE_CHIPS_ENV_VAR, None) is None
|
||||
else: # len(tpu_chips) == 8
|
||||
assert os.environ.get(tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR, None) is None
|
||||
assert os.environ.get(tpu.TPU_SINGLE_HOST_BOUNDS, None) is None
|
||||
assert os.environ.get(tpu.TPU_VISIBLE_CHIPS_ENV_VAR, None) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"test_config",
|
||||
[
|
||||
(0, "v4-16", {"TPU-v4-16-head": 1, "my-tpu": 1}),
|
||||
(1, "v4-16", {"my-tpu": 1}),
|
||||
(0, "tpu7x-16", {"TPU-v7x-16-head": 1, "my-tpu": 1}),
|
||||
],
|
||||
)
|
||||
def test_tpu_pod_detect_and_configure_worker(test_config):
|
||||
worker_id, pod_type, expected_value = test_config
|
||||
final_resources = {}
|
||||
with patch(
|
||||
"ray._private.accelerators.tpu.TPUAcceleratorManager.get_current_node_tpu_name",
|
||||
return_value="my-tpu",
|
||||
):
|
||||
with patch(
|
||||
"ray._private.accelerators.tpu.TPUAcceleratorManager.get_current_node_tpu_worker_id",
|
||||
return_value=worker_id,
|
||||
):
|
||||
with patch.dict(os.environ, {"TPU_ACCELERATOR_TYPE": pod_type}):
|
||||
final_resources = (
|
||||
TPUAcceleratorManager.get_current_node_additional_resources()
|
||||
)
|
||||
|
||||
assert final_resources == expected_value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"accelerator_type, expected",
|
||||
[
|
||||
("v2-8", True),
|
||||
("v3-32", True),
|
||||
("v4-8", True),
|
||||
("v5p-8", True),
|
||||
("v5litepod-8", True),
|
||||
("v6e-8", True),
|
||||
("tpu7x-16", True),
|
||||
("v7x-16", True),
|
||||
("v-8", False),
|
||||
("8", False),
|
||||
("tpu-8", False),
|
||||
("v2", False),
|
||||
("v2-", False),
|
||||
("random-string", False),
|
||||
],
|
||||
)
|
||||
def test_is_valid_tpu_accelerator_type(accelerator_type, expected):
|
||||
assert (
|
||||
TPUAcceleratorManager.is_valid_tpu_accelerator_type(accelerator_type)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def test_get_total_chips_from_accelerator_type():
|
||||
assert tpu.get_total_chips_from_accelerator_type("v6e-16") == 16
|
||||
assert tpu.get_total_chips_from_accelerator_type("v6e-8") == 8
|
||||
assert (
|
||||
tpu.get_total_chips_from_accelerator_type("v7x-16") == 8
|
||||
) # v7x has 2 cores per chip
|
||||
assert (
|
||||
tpu.get_total_chips_from_accelerator_type("v4-8") == 4
|
||||
) # v4 has 2 cores per chip
|
||||
|
||||
# Test invalid cases
|
||||
with pytest.raises(ValueError, match="Accelerator type must include size"):
|
||||
tpu.get_total_chips_from_accelerator_type("v6e")
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid accelerator type"):
|
||||
tpu.get_total_chips_from_accelerator_type("invalid-8")
|
||||
|
||||
|
||||
def test_get_num_tpu_visible_chips_per_host():
|
||||
# v6e multi-host (4 chips per VM)
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v6e-16") == 4
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v6e-32") == 4
|
||||
|
||||
# v6e single-host/sub-host (exact chip count)
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v6e-8") == 8
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v6e-4") == 4
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v6e-1") == 1
|
||||
|
||||
# v5litepod multi-host defaults to 4, single-host is 8 chips
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-16") == 4
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-8") == 8
|
||||
|
||||
# v5litepod sub-host
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-4") == 4
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-1") == 1
|
||||
|
||||
# Other TPU generations default to 4
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v4-8") == 4
|
||||
assert tpu.get_num_tpu_visible_chips_per_host("v5p-8") == 4
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
Reference in New Issue
Block a user