chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
@@ -0,0 +1,8 @@
def get_soc_name():
return "Ascend910B"
class rt:
@staticmethod
def get_device_count():
return (4, 0)
@@ -0,0 +1,16 @@
class SyclContext:
def __init__(self, info):
pass
@property
def device_count(self):
return 6
class SyclDevice:
def __init__(self, info):
pass
@property
def name(self):
return "Intel(R) Data Center GPU Max 1550"
@@ -0,0 +1,16 @@
class SyclContext:
def __init__(self, info):
pass
@property
def device_count(self):
return 4
class SyclDevice:
def __init__(self, info):
pass
@property
def name(self):
return "Intel(R) Data Center GPU Max 1100"
@@ -0,0 +1,42 @@
class _MockArch:
"""Mock for the PyO3-generated Arch enum."""
def __init__(self, name: str = "Rngd"):
self._name = name
def __str__(self) -> str:
# PyO3 unit enums commonly stringify like "Arch.Rngd".
return f"Arch.{self._name}"
class _MockDeviceInfo:
def __init__(self, arch_name: str = "Rngd", index: int = 0):
self._arch_name = arch_name
self._index = index
def arch(self) -> _MockArch:
return _MockArch(self._arch_name)
def name(self) -> str:
return f"npu{self._index}"
def index(self) -> int:
return self._index
class _MockDevice:
def __init__(self, idx: int, arch_name: str = "Rngd"):
self._idx = idx
self._arch_name = arch_name
def device_info(self) -> _MockDeviceInfo:
return _MockDeviceInfo(arch_name=self._arch_name, index=self._idx)
def init():
"""Mock for ``furiosa_smi_py.init``. Real SMI library requires init() before use."""
return None
def list_devices():
return [_MockDevice(i) for i in range(8)]
@@ -0,0 +1,101 @@
from typing import List
from unittest.mock import patch
import pytest
import ray._private.thirdparty.pynvml as pynvml
class DeviceHandleMock(dict):
def __init__(
self,
name: str,
uuid: str,
mig_devices: List["DeviceHandleMock"] = None,
**kwargs
):
super().__init__()
self["name"] = name
self["uuid"] = uuid
if mig_devices is not None:
self["mig_devices"] = mig_devices
self.update(kwargs)
# pnvml mock for gpu resources
class PyNVMLMock:
def __init__(self, mock_data, driver_version="535.104.12"):
self._mock_data = mock_data
self.driver_version = driver_version
def nvmlInit(self):
return
def nvmlShutdown(self):
return
def nvmlSystemGetDriverVersion(self):
return self.driver_version
def nvmlDeviceGetCount(self):
return len(self._mock_data)
def nvmlDeviceGetHandleByIndex(self, index):
return self._mock_data[index]
def nvmlDeviceGetName(self, handle):
return handle.get("name", "")
def nvmlDeviceGetMaxMigDeviceCount(self, handle):
if "mig_devices" in handle:
return max(7, len(handle["mig_devices"]))
else:
raise pynvml.NVMLError_NotSupported
def nvmlDeviceGetMigDeviceHandleByIndex(self, handle, mig_index):
try:
return handle["mig_devices"][mig_index]
except IndexError:
raise pynvml.NVMLError_NotFound
def nvmlDeviceGetUUID(self, handle):
return handle.get("uuid", "")
def nvmlDeviceGetComputeInstanceId(self, mig_handle):
return mig_handle["ci_id"]
def nvmlDeviceGetGpuInstanceId(self, mig_handle):
return mig_handle["gi_id"]
@pytest.fixture
def patch_mock_pynvml(mock_nvml):
with patch("ray._private.thirdparty.pynvml.nvmlInit", mock_nvml.nvmlInit), patch(
"ray._private.thirdparty.pynvml.nvmlShutdown", mock_nvml.nvmlShutdown
), patch(
"ray._private.thirdparty.pynvml.nvmlSystemGetDriverVersion",
mock_nvml.nvmlSystemGetDriverVersion,
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetCount",
mock_nvml.nvmlDeviceGetCount,
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetHandleByIndex",
mock_nvml.nvmlDeviceGetHandleByIndex,
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetName", mock_nvml.nvmlDeviceGetName
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetMaxMigDeviceCount",
mock_nvml.nvmlDeviceGetMaxMigDeviceCount,
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetMigDeviceHandleByIndex",
mock_nvml.nvmlDeviceGetMigDeviceHandleByIndex,
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetUUID", mock_nvml.nvmlDeviceGetUUID
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetComputeInstanceId",
mock_nvml.nvmlDeviceGetComputeInstanceId,
), patch(
"ray._private.thirdparty.pynvml.nvmlDeviceGetGpuInstanceId",
mock_nvml.nvmlDeviceGetGpuInstanceId,
):
yield
@@ -0,0 +1,6 @@
def device_count():
return 4
def get_npu_name():
return "RBLN-CA02"
@@ -0,0 +1,22 @@
import sys
import pytest
from ray.util import accelerators
from ray.util.annotations import RayDeprecationWarning
def test_accelerators():
assert accelerators.NVIDIA_TESLA_K80 == "K80"
assert accelerators.NVIDIA_A100 == "A100"
with pytest.raises(
AttributeError,
match="module 'ray.util.accelerators' has no attribute 'NVIDIA_INVALID'",
):
_ = accelerators.NVIDIA_INVALID
with pytest.warns(RayDeprecationWarning):
assert accelerators.NVIDIA_TESLA_A100 == "A100"
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
@@ -0,0 +1,92 @@
import os
import sys
from unittest.mock import patch
import pytest
import ray
from ray._private.accelerators import (
AMDGPUAcceleratorManager,
get_accelerator_manager_for_resource,
)
@patch(
"ray._private.accelerators.AMDGPUAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
return_value=4,
)
def test_visible_amd_gpu_ids(mock_get_num_accelerators, monkeypatch, shutdown_only):
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,1,2")
# Delete the cache so it can be re-populated the next time
# we call get_accelerator_manager_for_resource
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
ray.init()
_ = mock_get_num_accelerators.called
assert ray.available_resources()["GPU"] == 3
@patch(
"ray._private.accelerators.AMDGPUAcceleratorManager._get_amd_device_ids",
return_value=["0x74a1", "0x74a1", "0x74a1", "0x74a1"],
)
def test_visible_amd_gpu_type(mock_get_amd_device_ids, shutdown_only):
ray.init()
_ = mock_get_amd_device_ids.called
assert (
AMDGPUAcceleratorManager.get_current_node_accelerator_type()
== "AMD-Instinct-MI300X-OAM"
)
@patch(
"ray._private.accelerators.AMDGPUAcceleratorManager._get_amd_device_ids",
return_value=["0x640f", "0x640f", "0x640f", "0x640f"],
)
def test_visible_amd_gpu_type_bad_device_id(mock_get_num_accelerators, shutdown_only):
ray.init()
_ = mock_get_num_accelerators.called
assert AMDGPUAcceleratorManager.get_current_node_accelerator_type() is None
def test_get_current_process_visible_accelerator_ids(monkeypatch):
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,1,2")
assert AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
"0",
"1",
"2",
]
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0,2,7")
assert AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
"0",
"2",
"7",
]
monkeypatch.setenv("HIP_VISIBLE_DEVICES", "")
assert AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == []
del os.environ["HIP_VISIBLE_DEVICES"]
assert (
AMDGPUAcceleratorManager.get_current_process_visible_accelerator_ids() is None
)
def test_set_current_process_visible_accelerator_ids():
AMDGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0"])
env_var = AMDGPUAcceleratorManager.get_visible_accelerator_ids_env_var()
assert os.environ[env_var] == "0"
AMDGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
assert os.environ[env_var] == "0,1"
AMDGPUAcceleratorManager.set_current_process_visible_accelerator_ids(
["0", "1", "7"]
)
assert os.environ[env_var] == "0,1,7"
del os.environ[env_var]
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
@@ -0,0 +1,159 @@
import os
import sys
import pytest
from ray._private.accelerators.furiosa import (
FURIOSA_VISIBLE_DEVICES_ENV_VAR,
NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR,
FuriosaAcceleratorManager,
)
@pytest.fixture(autouse=True)
def mock_furiosa_smi_py(monkeypatch):
from ray.tests.accelerators import mock_furiosa_smi_py
monkeypatch.setitem(sys.modules, "furiosa_smi_py", mock_furiosa_smi_py)
@pytest.fixture
def clear_furiosa_environment():
original_env = os.environ.get(FURIOSA_VISIBLE_DEVICES_ENV_VAR)
original_no_set_env = os.environ.get(NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR)
os.environ.pop(FURIOSA_VISIBLE_DEVICES_ENV_VAR, None)
os.environ.pop(NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR, None)
yield
if original_env is not None:
os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] = original_env
if original_no_set_env is not None:
os.environ[NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR] = original_no_set_env
@pytest.mark.usefixtures("clear_furiosa_environment")
class TestFuriosaAcceleratorManager:
def test_get_resource_name(self):
assert FuriosaAcceleratorManager.get_resource_name() == "FURIOSA"
def test_get_visible_accelerator_ids_env_var(self):
assert (
FuriosaAcceleratorManager.get_visible_accelerator_ids_env_var()
== FURIOSA_VISIBLE_DEVICES_ENV_VAR
)
@pytest.mark.parametrize(
"env_value,expected",
[
# furiosa-llm --devices form (preferred).
("npu:0,npu:1,npu:2,npu:3", ["0", "1", "2", "3"]),
# Bare integer form is also accepted for convenience.
("0,1,2,3", ["0", "1", "2", "3"]),
# Core range notation: only the device index is returned.
("npu:0:0-3,npu:1:0-3", ["0", "1"]),
# Empty string yields an empty list.
("", []),
# Sentinel ``None`` means the env var is unset.
(None, None),
],
)
def test_get_current_process_visible_accelerator_ids(self, env_value, expected):
if env_value is None:
os.environ.pop(FURIOSA_VISIBLE_DEVICES_ENV_VAR, None)
else:
os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] = env_value
assert (
FuriosaAcceleratorManager.get_current_process_visible_accelerator_ids()
== expected
)
def test_get_current_node_num_accelerators(self):
assert FuriosaAcceleratorManager.get_current_node_num_accelerators() == 8
def test_get_current_node_accelerator_type(self):
assert (
FuriosaAcceleratorManager.get_current_node_accelerator_type()
== "FURIOSA_RNGD"
)
@pytest.mark.parametrize(
"arch_name,expected",
[
# PyO3 enum form (CamelCase).
("Rngd", "FURIOSA_RNGD"),
("RngdMax", "FURIOSA_RNGDMAX"),
("RngdS", "FURIOSA_RNGDS"),
("RngdPlus", "FURIOSA_RNGDPLUS"),
# ``Arch::ToString`` form is also accepted, and both forms must
# resolve to the same label.
("rngd-max", "FURIOSA_RNGDMAX"),
# ``+`` must NOT be silently stripped, since that would collapse
# ``rngd+`` into ``rngd`` and collide with the base RNGD SKU; it
# is mapped to ``plus`` so the label matches ``RngdPlus``.
("rngd+", "FURIOSA_RNGDPLUS"),
],
)
def test_get_current_node_accelerator_type_dynamic(
self, monkeypatch, arch_name, expected
):
from ray.tests.accelerators import mock_furiosa_smi_py
def mocked_list_devices():
return [mock_furiosa_smi_py._MockDevice(0, arch_name=arch_name)]
monkeypatch.setattr(mock_furiosa_smi_py, "list_devices", mocked_list_devices)
assert FuriosaAcceleratorManager.get_current_node_accelerator_type() == expected
def test_get_current_node_accelerator_type_no_devices(self, monkeypatch):
from ray.tests.accelerators import mock_furiosa_smi_py
monkeypatch.setattr(mock_furiosa_smi_py, "list_devices", lambda: [])
assert FuriosaAcceleratorManager.get_current_node_accelerator_type() is None
def test_get_current_node_accelerator_type_arch_is_none(self, monkeypatch):
"""Regression: arch() returning None must not produce 'FURIOSA_NONE'."""
from ray.tests.accelerators import mock_furiosa_smi_py
class _NullArchDeviceInfo:
def arch(self):
return None
class _NullArchDevice:
def device_info(self):
return _NullArchDeviceInfo()
monkeypatch.setattr(
mock_furiosa_smi_py, "list_devices", lambda: [_NullArchDevice()]
)
assert FuriosaAcceleratorManager.get_current_node_accelerator_type() is None
def test_set_current_process_visible_accelerator_ids(self):
# Ray's scheduler hands us bare integer IDs; we serialize them in
# the ``npu:<id>`` form expected by ``furiosa-llm --devices``.
FuriosaAcceleratorManager.set_current_process_visible_accelerator_ids(
["0", "1"]
)
assert os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] == "npu:0,npu:1"
os.environ[NOSET_FURIOSA_VISIBLE_DEVICES_ENV_VAR] = "1"
FuriosaAcceleratorManager.set_current_process_visible_accelerator_ids(
["2", "3"]
)
assert os.environ[FURIOSA_VISIBLE_DEVICES_ENV_VAR] == "npu:0,npu:1"
def test_validate_resource_request_quantity(self):
valid, _ = FuriosaAcceleratorManager.validate_resource_request_quantity(1)
assert valid
valid, _ = FuriosaAcceleratorManager.validate_resource_request_quantity(2.0)
assert valid
valid, msg = FuriosaAcceleratorManager.validate_resource_request_quantity(0.5)
assert not valid
assert "whole number" in msg
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
+296
View File
@@ -0,0 +1,296 @@
import os
import sys
from unittest.mock import patch
import pytest
import ray
from ray._private.accelerators import HPUAcceleratorManager, hpu
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
def test_user_configured_more_than_visible(monkeypatch, call_ray_stop_only):
# Test more hpus are configured than visible.
monkeypatch.setenv("HABANA_VISIBLE_MODULES", "0,1,2")
with pytest.raises(ValueError):
ray.init(resources={"HPU": 4})
@patch(
"ray._private.accelerators.HPUAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
return_value=4,
)
def test_auto_detected_more_than_visible(
mock_get_num_accelerators, monkeypatch, shutdown_only
):
# Test more hpus are detected than visible.
monkeypatch.setenv("HABANA_VISIBLE_MODULES", "0,1,2")
ray.init()
_ = mock_get_num_accelerators.called
assert ray.available_resources()["HPU"] == 3
@patch(
"ray._private.accelerators.HPUAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
return_value=2,
)
def test_auto_detect_resources(mock_get_num_accelerators, shutdown_only):
# Test that ray node resources are filled with auto detected count.
ray.init()
_ = mock_get_num_accelerators.called
assert ray.available_resources()["HPU"] == 2
def test_get_current_process_visible_accelerator_ids():
os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] = "0,1,2"
assert HPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
"0",
"1",
"2",
] # noqa: E501
del os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR]
assert HPUAcceleratorManager.get_current_process_visible_accelerator_ids() is None
os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] = ""
assert HPUAcceleratorManager.get_current_process_visible_accelerator_ids() == []
del os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR]
def test_set_current_process_visible_accelerator_ids():
HPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0"])
assert os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] == "0"
HPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
assert os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] == "0,1"
HPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1", "2"])
assert os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR] == "0,1,2"
del os.environ[hpu.HABANA_VISIBLE_DEVICES_ENV_VAR]
@pytest.mark.parametrize(
"test_config",
[
(1, False),
(0.5, True),
(3, False),
],
)
def test_validate_resource_request_quantity(test_config):
num_hpus, expect_error = test_config
if expect_error:
assert (
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[0]
is False
)
assert (
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[1]
is not None
)
else:
assert (
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[0]
is True
)
assert (
HPUAcceleratorManager.validate_resource_request_quantity(num_hpus)[1]
is None
)
def test_check_accelerator_info():
if HPUAcceleratorManager.is_initialized():
assert (
"Intel-GAUDI" in HPUAcceleratorManager.get_current_node_accelerator_type()
)
else:
assert HPUAcceleratorManager.get_current_node_accelerator_type() is None
assert HPUAcceleratorManager.get_resource_name() == "HPU"
def test_decorator_args():
# This is a valid way of using the decorator.
@ray.remote(resources={"HPU": 1}) # noqa: F811
class Actor: # noqa: F811
def __init__(self):
pass
# This is a valid way of using the decorator.
@ray.remote(num_cpus=1, resources={"HPU": 1}) # noqa: F811
class Actor: # noqa: F811
def __init__(self):
pass
def test_actor_deletion_with_hpus(shutdown_only):
ray.init(num_cpus=1, resources={"HPU": 1})
# When an actor that uses an HPU exits, make sure that the HPU resources
# are released.
@ray.remote(resources={"HPU": 1})
class Actor:
def getpid(self):
return os.getpid()
for _ in range(5):
# If we can successfully create an actor, that means that enough
# HPU resources are available.
a = Actor.remote()
ray.get(a.getpid.remote())
@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
def test_actor_hpus(ray_start_cluster):
cluster = ray_start_cluster
num_nodes = 2
num_hpus_per_raylet = 2
for i in range(num_nodes):
cluster.add_node(num_cpus=10 * 2, resources={"HPU": num_hpus_per_raylet})
ray.init(address=cluster.address)
@ray.remote(resources={"HPU": 1})
class Actor1:
def __init__(self):
resource_ids = ray.get_runtime_context().get_accelerator_ids()
self.hpu_ids = resource_ids.get("HPU")
def get_location_and_ids(self):
return (
ray.get_runtime_context().get_node_id(),
tuple(self.hpu_ids),
)
# Create one actor per HPU.
actors = [Actor1.remote() for _ in range(num_nodes * num_hpus_per_raylet)]
# Make sure that no two actors are assigned to the same HPU.
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors]
)
node_names = {location for location, hpu_id in locations_and_ids}
assert len(node_names) == num_nodes
location_actor_combinations = []
for node_name in node_names:
for hpu_id in range(num_hpus_per_raylet):
location_actor_combinations.append((node_name, (f"{hpu_id}",)))
assert set(locations_and_ids) == set(location_actor_combinations)
# Creating a new actor should fail because all of the HPUs are being
# used.
a = Actor1.remote()
ready_ids, _ = ray.wait([a.get_location_and_ids.remote()], timeout=0.01)
assert ready_ids == []
def test_actor_habana_visible_devices(shutdown_only):
"""Test user can overwrite HABANA_VISIBLE_MODULES
after the actor is created."""
ray.init(resources={"HPU": 1})
@ray.remote(resources={"HPU": 1})
class Actor:
def set_habana_visible_devices(self, habana_visible_devices):
os.environ["HABANA_VISIBLE_MODULES"] = habana_visible_devices
def get_habana_visible_devices(self):
return os.environ["HABANA_VISIBLE_MODULES"]
actor = Actor.remote()
assert ray.get(actor.get_habana_visible_devices.remote()) == "0"
ray.get(actor.set_habana_visible_devices.remote("0,1"))
assert ray.get(actor.get_habana_visible_devices.remote()) == "0,1"
def test_hpu_ids(shutdown_only):
num_hpus = 3
ray.init(num_cpus=num_hpus, resources={"HPU": num_hpus})
def get_hpu_ids(hpus_per_worker):
hpu_ids = ray.get_runtime_context().get_accelerator_ids()["HPU"]
assert len(hpu_ids) == hpus_per_worker
modules = os.environ.get("HABANA_VISIBLE_MODULES")
if modules is not None:
assert modules == ",".join([str(i) for i in hpu_ids]) # noqa
for hpu_id in hpu_ids:
assert hpu_id in [str(i) for i in range(num_hpus)]
return hpu_ids
f0 = ray.remote(resources={"HPU": 0})(lambda: get_hpu_ids(0))
f1 = ray.remote(resources={"HPU": 1})(lambda: get_hpu_ids(1))
list_of_ids = ray.get([f0.remote() for _ in range(10)])
assert list_of_ids == 10 * [[]]
ray.get([f1.remote() for _ in range(10)])
# Test that actors have HABANA_VISIBLE_MODULES set properly.
def _check_hpu_env(expected_num_hpus):
hpu_ids = ray.get_runtime_context().get_accelerator_ids()["HPU"]
assert len(hpu_ids) == expected_num_hpus
if expected_num_hpus > 0:
assert os.environ["HABANA_VISIBLE_MODULES"] == ",".join(
[str(i) for i in hpu_ids] # noqa
)
else:
assert os.environ.get("HABANA_VISIBLE_MODULES") is None
@ray.remote
class Actor:
def __init__(self, num_hpus):
self.num_hpus = num_hpus
_check_hpu_env(num_hpus)
self.x = num_hpus
def test(self):
_check_hpu_env(self.num_hpus)
return self.x
a0 = Actor.remote(0)
assert ray.get(a0.test.remote()) == 0
a1 = Actor.options(resources={"HPU": 1}).remote(1)
assert ray.get(a1.test.remote()) == 1
def test_hpu_with_placement_group(shutdown_only):
num_hpus = 2
ray.init(num_cpus=1, resources={"HPU": num_hpus})
@ray.remote(resources={"HPU": num_hpus})
class HPUActor:
def __init__(self):
pass
def ready(self):
hpu_ids = ray.get_runtime_context().get_accelerator_ids()["HPU"]
assert len(hpu_ids) == num_hpus
assert os.environ["HABANA_VISIBLE_MODULES"] == ",".join(
[str(i) for i in hpu_ids] # noqa
)
# Reserve a placement group of 1 bundle that reserves 1 CPU and 2 HPU.
pg = placement_group([{"CPU": 1, "HPU": num_hpus}])
# Wait until placement group is created.
ray.get(pg.ready(), timeout=10)
actor = HPUActor.options(
scheduling_strategy=PlacementGroupSchedulingStrategy(
placement_group=pg,
)
).remote()
ray.get(actor.ready.remote(), timeout=10)
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
@@ -0,0 +1,117 @@
import os
import sys
from unittest.mock import patch
import pytest
import ray
from ray._private.accelerators import (
IntelGPUAcceleratorManager as Accelerator,
get_accelerator_manager_for_resource,
)
from ray.util.accelerators import INTEL_MAX_1100, INTEL_MAX_1550
def test_visible_intel_gpu_ids(shutdown_only):
with patch.object(Accelerator, "get_current_node_num_accelerators", return_value=4):
os.environ["ONEAPI_DEVICE_SELECTOR"] = "level_zero:0,1,2"
# Delete the cache so it can be re-populated the next time
# we call get_accelerator_manager_for_resource
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
ray.init()
manager = get_accelerator_manager_for_resource("GPU")
assert manager.get_current_node_num_accelerators() == 4
assert manager.__name__ == "IntelGPUAcceleratorManager"
assert ray.available_resources()["GPU"] == 3
def test_visible_intel_gpu_type(shutdown_only):
with patch.object(
Accelerator, "get_current_node_num_accelerators", return_value=4
), patch.object(
Accelerator, "get_current_node_accelerator_type", return_value=INTEL_MAX_1550
):
os.environ["ONEAPI_DEVICE_SELECTOR"] = "level_zero:0,1,2"
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
ray.init()
manager = get_accelerator_manager_for_resource("GPU")
assert manager.get_current_node_accelerator_type() == INTEL_MAX_1550
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
@pytest.mark.skipif(
sys.version_info >= (3, 12),
reason="Not passing on Python 3.12. Being followed up by external contributors.",
)
def test_get_current_node_num_accelerators():
old_dpctl = None
if "dpctl" in sys.modules:
old_dpctl = sys.modules["dpctl"]
sys.modules["dpctl"] = __import__("mock_dpctl_1")
assert Accelerator.get_current_node_num_accelerators() == 6
sys.modules["dpctl"] = __import__("mock_dpctl_2")
assert Accelerator.get_current_node_num_accelerators() == 4
if old_dpctl is not None:
sys.modules["dpctl"] = old_dpctl
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
@pytest.mark.skipif(
sys.version_info >= (3, 12),
reason="Not passing on Python 3.12. Being followed up by external contributors.",
)
def test_get_current_node_accelerator_type():
old_dpctl = None
if "dpctl" in sys.modules:
old_dpctl = sys.modules["dpctl"]
sys.modules["dpctl"] = __import__("mock_dpctl_1")
assert Accelerator.get_current_node_accelerator_type() == INTEL_MAX_1550
sys.modules["dpctl"] = __import__("mock_dpctl_2")
assert Accelerator.get_current_node_accelerator_type() == INTEL_MAX_1100
if old_dpctl is not None:
sys.modules["dpctl"] = old_dpctl
def test_intel_gpu_accelerator_manager_api():
assert Accelerator.get_resource_name() == "GPU"
assert Accelerator.get_visible_accelerator_ids_env_var() == "ONEAPI_DEVICE_SELECTOR"
assert Accelerator.validate_resource_request_quantity(0.1) == (True, None)
def test_get_current_process_visible_accelerator_ids():
os.environ["ONEAPI_DEVICE_SELECTOR"] = "level_zero:0,1,2"
assert Accelerator.get_current_process_visible_accelerator_ids() == ["0", "1", "2"]
del os.environ["ONEAPI_DEVICE_SELECTOR"]
assert Accelerator.get_current_process_visible_accelerator_ids() is None
os.environ["ONEAPI_DEVICE_SELECTOR"] = ""
assert Accelerator.get_current_process_visible_accelerator_ids() == []
os.environ["ONEAPI_DEVICE_SELECTOR"] = "NoDevFiles"
assert Accelerator.get_current_process_visible_accelerator_ids() == []
del os.environ["ONEAPI_DEVICE_SELECTOR"]
def test_set_current_process_visible_accelerator_ids():
Accelerator.set_current_process_visible_accelerator_ids(["0"])
assert os.environ["ONEAPI_DEVICE_SELECTOR"] == "level_zero:0"
Accelerator.set_current_process_visible_accelerator_ids(["0", "1"])
assert os.environ["ONEAPI_DEVICE_SELECTOR"] == "level_zero:0,1"
Accelerator.set_current_process_visible_accelerator_ids(["0", "1", "2"])
assert os.environ["ONEAPI_DEVICE_SELECTOR"] == "level_zero:0,1,2"
del os.environ["ONEAPI_DEVICE_SELECTOR"]
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
@@ -0,0 +1,228 @@
"""
Manual Intel GPU validation tests, not executed in automated runs.
These tests are basic acceptance tests to validate Intel GPU support in Ray. They
require a suitable Intel GPU environment with dpctl installed. They are intended to
serve as an approved method to verify Intel GPU-based Ray deployments.
"""
import os
import re
from typing import Any, Dict, List
import pytest
import ray
try:
import dpctl
except ImportError:
pytest.skip(
"dpctl is not installed, skipping Intel GPU tests.", allow_module_level=True
)
DEFAULT_SCALE_OUT_NODES = 2
DEFAULT_SCALE_UP_DEVICES = 2
USE_GPU = bool(os.environ.get("RAY_PYTEST_USE_GPU", 0))
if not USE_GPU:
pytest.skip("Skipping, these tests require GPUs.", allow_module_level=True)
@pytest.fixture
def ray_gpu_session():
"""Start a Ray session with caller-provided init kwargs."""
def _start_session(**init_kwargs):
if ray.is_initialized():
ray.shutdown()
ray.init(**init_kwargs)
try:
yield _start_session
finally:
if ray.is_initialized():
ray.shutdown()
def _is_cluster_configured(address: str = "auto") -> bool:
try:
ray.init(
address=address,
)
return True
except (ray.exceptions.RaySystemError, ConnectionError, TimeoutError):
return False
finally:
if ray.is_initialized():
ray.shutdown()
def _detect_available_gpu_count() -> int:
"""Return the number of GPU devices detected via dpctl."""
try:
return dpctl.SyclContext("level_zero:gpu").device_count
except Exception:
# If dpctl cannot enumerate devices, assume no additional GPUs.
return 0
def _require_min_gpus(required: int, context: str) -> None:
available = _detect_available_gpu_count()
if available < required:
pytest.skip(
f"Skipping {context}: requires {required} GPUs, detected {available} via dpctl."
)
def _require_min_cluster_nodes(required_nodes: int, context: str) -> None:
alive_nodes = [node for node in ray.nodes() if node.get("Alive")]
unique_node_ids = {node.get("NodeID") for node in alive_nodes if node.get("NodeID")}
if len(unique_node_ids) < required_nodes:
pytest.skip(
f"Skipping {context}: requires {required_nodes} alive Ray nodes, detected {len(unique_node_ids)}."
)
@ray.remote(num_gpus=1)
def gpu_task() -> Dict[str, Any]:
context = ray.get_runtime_context()
gpu_ids = context.get_accelerator_ids().get("GPU", [])
return {
"gpu_ids": gpu_ids,
"pid": os.getpid(),
"oneapi_selector": os.environ.get("ONEAPI_DEVICE_SELECTOR"),
}
@ray.remote(num_gpus=1)
def cluster_probe_task() -> Dict[str, Any]:
context = ray.get_runtime_context()
return {
"node_id": context.get_node_id(),
"node_ip": ray.util.get_node_ip_address(),
"worker_id": context.get_worker_id(),
"gpu_ids": context.get_accelerator_ids().get("GPU", []),
"selector": os.environ.get("ONEAPI_DEVICE_SELECTOR"),
}
def assert_valid_gpu_binding(result: Dict[str, Any], label: str) -> None:
primary_gpu_id = _validate_gpu_binding_common(result, label)
assert (
primary_gpu_id >= 0
), f"Expected {label} to bind to a valid GPU, got {result.get('gpu_ids')}"
def _validate_gpu_binding_common(
result: Dict[str, Any], label: str, selector_key: str = "oneapi_selector"
) -> int:
"""Validate basic GPU binding properties shared by single- and multi-GPU tests."""
gpu_ids = result.get("gpu_ids")
assert gpu_ids, f"No GPU IDs assigned for {label}."
primary_gpu_id = int(gpu_ids[0])
selector = result.get(selector_key)
assert selector, f"ONEAPI_DEVICE_SELECTOR not set in environment for {label}."
selector_lower = selector.lower()
assert (
"level_zero:" in selector_lower
), f"ONEAPI_DEVICE_SELECTOR should target GPU devices for {label}, got: {selector}."
selector_gpu_ids = {int(match) for match in re.findall(r"\b\d+\b", selector_lower)}
assert (
primary_gpu_id in selector_gpu_ids
), f"ONEAPI_DEVICE_SELECTOR does not reference bound GPU id for {label}: {selector}."
return primary_gpu_id
def assert_valid_multi_gpu_binding(
results: List[Dict[str, Any]], num_gpus: int, label: str
) -> None:
"""Assert that multiple GPU tasks bind to different GPUs correctly."""
assert (
len(results) == num_gpus
), f"Expected {num_gpus} results for {label}, got {len(results)}."
gpu_ids = []
for i, result in enumerate(results):
primary_gpu_id = _validate_gpu_binding_common(result, f"{label} instance {i}")
gpu_ids.append(primary_gpu_id)
assert (
len(set(gpu_ids)) == num_gpus
), f"Expected {label} to bind to {num_gpus} distinct GPUs, got bindings to GPU IDs: {gpu_ids}."
@pytest.mark.skipif(
_is_cluster_configured(),
reason="Environment setup for scale-out, skipping single-node test.",
)
def test_gpu_task_binding(ray_gpu_session) -> None:
_require_min_gpus(1, "single GPU task binding test")
ray_gpu_session(num_gpus=1)
task_result = ray.get(gpu_task.remote())
assert_valid_gpu_binding(task_result, "GPU task")
@pytest.mark.skipif(
_is_cluster_configured(),
reason="Environment setup for scale-out, skipping single-node test.",
)
@pytest.mark.parametrize(
"num_gpus", [DEFAULT_SCALE_UP_DEVICES]
) # To be extended to required configurations
def test_multi_gpu_task_binding(ray_gpu_session, num_gpus) -> None:
"""Test that multiple GPU tasks bind to different GPUs correctly."""
_require_min_gpus(num_gpus, "multi-GPU task binding test")
ray_gpu_session(num_gpus=num_gpus)
task_futures = [gpu_task.remote() for _ in range(num_gpus)]
task_results = ray.get(task_futures)
assert_valid_multi_gpu_binding(task_results, num_gpus, f"GPU tasks (n={num_gpus})")
@pytest.mark.skipif(
not _is_cluster_configured(), reason="Environment not setup for scale-out test."
)
@pytest.mark.parametrize(
"num_nodes", [DEFAULT_SCALE_OUT_NODES]
) # To be extended to required configurations
def test_scale_out_task_distribution(ray_gpu_session, num_nodes) -> None:
"""Ensure tasks can be scheduled across multiple nodes in the cluster."""
ray_gpu_session(address="auto")
_require_min_cluster_nodes(num_nodes, "scale-out task distribution test")
probe_handles = [
cluster_probe_task.options(scheduling_strategy="SPREAD").remote()
for _ in range(num_nodes)
]
probe_results = ray.get(probe_handles)
node_ids = {
result.get("node_id") for result in probe_results if result.get("node_id")
}
node_ips = {
result.get("node_ip") for result in probe_results if result.get("node_ip")
}
for result in probe_results:
_validate_gpu_binding_common(result, "scale-out probe task", "selector")
assert len(node_ids) == num_nodes or len(node_ips) == num_nodes, (
f"Expected probe tasks to execute on {num_nodes} distinct nodes, "
f"got node_ids={node_ids} node_ips={node_ips}."
)
gpu_capable_results = [result for result in probe_results if result.get("gpu_ids")]
assert (
len(gpu_capable_results) == num_nodes
), "Not all probe tasks reported GPU accelerator bindings in the cluster."
@@ -0,0 +1,69 @@
import os
import sys
from unittest.mock import patch
import pytest
import ray
from ray._private.accelerators import (
MetaxGPUAcceleratorManager,
get_accelerator_manager_for_resource,
)
@patch(
"ray._private.accelerators.MetaxGPUAcceleratorManager.get_current_node_num_accelerators",
return_value=4,
)
def test_visible_metax_gpu_ids(mock_get_num_accelerators, monkeypatch, shutdown_only):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1,2")
del get_accelerator_manager_for_resource._resource_name_to_accelerator_manager
ray.init()
assert mock_get_num_accelerators.called
assert ray.available_resources()["GPU"] == 3
def test_get_current_process_visible_accelerator_ids(monkeypatch):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
assert MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
"0"
]
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,4,7")
assert MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == [
"0",
"4",
"7",
]
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
assert (
MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() == []
)
monkeypatch.delenv("CUDA_VISIBLE_DEVICES")
assert (
MetaxGPUAcceleratorManager.get_current_process_visible_accelerator_ids() is None
)
def test_set_current_process_visible_accelerator_ids():
MetaxGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0"])
assert os.environ["CUDA_VISIBLE_DEVICES"] == "0"
MetaxGPUAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
assert os.environ["CUDA_VISIBLE_DEVICES"] == "0,1"
MetaxGPUAcceleratorManager.set_current_process_visible_accelerator_ids(
["0", "1", "7"]
)
assert os.environ["CUDA_VISIBLE_DEVICES"] == "0,1,7"
del os.environ["CUDA_VISIBLE_DEVICES"]
if __name__ == "__main__":
if os.environ.get("PARALLEL_CI"):
sys.exit(pytest.main(["-n", "auto", "--boxed", "-vs", __file__]))
else:
sys.exit(pytest.main(["-sv", __file__]))
@@ -0,0 +1,104 @@
import subprocess
import sys
from unittest.mock import patch
import pytest
import ray
from ray._private.accelerators import NeuronAcceleratorManager
def test_user_configured_more_than_visible(monkeypatch, call_ray_stop_only):
# Test more neuron_cores are configured than visible.
monkeypatch.setenv("NEURON_RT_VISIBLE_CORES", "0,1,2")
with pytest.raises(ValueError):
ray.init(resources={"neuron_cores": 4})
@patch(
"ray._private.accelerators.NeuronAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
return_value=4,
)
def test_auto_detected_more_than_visible(
mock_get_num_accelerators, monkeypatch, shutdown_only
):
# Test more neuron_cores are detected than visible.
monkeypatch.setenv("NEURON_RT_VISIBLE_CORES", "0,1,2")
ray.init()
_ = mock_get_num_accelerators.called
assert ray.available_resources()["neuron_cores"] == 3
@patch(
"ray._private.accelerators.NeuronAcceleratorManager.get_current_node_num_accelerators", # noqa: E501
return_value=2,
)
def test_auto_detect_resources(mock_get_num_accelerators, shutdown_only):
# Test that ray node resources are filled with auto detected count.
ray.init()
_ = mock_get_num_accelerators.called
assert ray.available_resources()["neuron_cores"] == 2
@patch(
"subprocess.run",
return_value=subprocess.CompletedProcess(
args=[],
returncode=0,
stdout=(
b'[{"neuron_device":0,"bdf":"00:1e.0",'
b'"connected_to":null,"nc_count":2,'
b'"memory_size":34359738368,"neuron_processes":[]}]'
),
),
)
@patch("os.path.isdir", return_value=True)
@patch("sys.platform", "linux")
def test_get_neuron_core_count_single_device(mock_isdir, mock_subprocess):
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 2
@patch(
"subprocess.run",
return_value=subprocess.CompletedProcess(
args=[],
returncode=0,
stdout=(
b'[{"neuron_device":0,"bdf":"00:1e.0",'
b'"connected_to":null,"nc_count":2,'
b'"memory_size":34359738368,"neuron_processes":[]},'
b'{"neuron_device":1,"bdf":"00:1f.0","connected_to":null,'
b'"nc_count":2,"memory_size":34359738368,"neuron_processes":[]}]'
),
),
)
@patch("os.path.isdir", return_value=True)
@patch("sys.platform", "linux")
def test_get_neuron_core_count_multiple_devices(mock_isdir, mock_subprocess):
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 4
@patch(
"subprocess.run",
return_value=subprocess.CompletedProcess(
args=[], returncode=1, stdout=b"AccessDenied"
),
)
@patch("os.path.isdir", return_value=True)
@patch("sys.platform", "linux")
def test_get_neuron_core_count_failure_with_error(mock_isdir, mock_subprocess):
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 0
@patch(
"subprocess.run",
return_value=subprocess.CompletedProcess(args=[], returncode=0, stdout=b"[{}]"),
)
@patch("os.path.isdir", return_value=True)
@patch("sys.platform", "linux")
def test_get_neuron_core_count_failure_with_empty_results(mock_isdir, mock_subprocess):
assert NeuronAcceleratorManager.get_current_node_num_accelerators() == 0
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
+131
View File
@@ -0,0 +1,131 @@
import os
import sys
from unittest.mock import patch
import pytest
import ray
from ray._private.accelerators import NPUAcceleratorManager as Accelerator
@patch("glob.glob")
def test_autodetect_num_npus(mock_glob):
with patch.dict(sys.modules):
sys.modules["acl"] = None
mock_glob.return_value = [f"/dev/davinci{i}" for i in range(64)]
assert Accelerator.get_current_node_num_accelerators() == 64
@patch("glob.glob")
def test_autodetect_num_npus_without_devices(mock_glob):
with patch.dict(sys.modules):
sys.modules["acl"] = None
mock_glob.side_effect = Exception
assert Accelerator.get_current_node_num_accelerators() == 0
def test_ascend_npu_accelerator_manager_api():
assert Accelerator.get_resource_name() == "NPU"
assert (
Accelerator.get_visible_accelerator_ids_env_var() == "ASCEND_RT_VISIBLE_DEVICES"
)
assert Accelerator.validate_resource_request_quantity(0.5) == (True, None)
assert Accelerator.validate_resource_request_quantity(1) == (True, None)
def test_visible_ascend_npu_type(monkeypatch, shutdown_only):
with patch.object(
Accelerator, "get_current_node_num_accelerators", return_value=4
), patch.object(
Accelerator, "get_current_node_accelerator_type", return_value="Ascend910B"
):
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
manager = ray._private.accelerators.get_accelerator_manager_for_resource("NPU")
assert manager.get_current_node_accelerator_type() == "Ascend910B"
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
@pytest.mark.skipif(
sys.version_info >= (3, 12),
reason="Not passing on Python 3.12. Being followed up by external contributors.",
)
def test_visible_ascend_npu_ids(monkeypatch, shutdown_only):
with patch.dict(sys.modules):
sys.modules["acl"] = __import__("mock_acl")
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
with patch.object(
Accelerator, "get_current_node_num_accelerators", return_value=4
):
ray.init()
manager = ray._private.accelerators.get_accelerator_manager_for_resource(
"NPU"
)
assert manager.get_current_node_num_accelerators() == 4
assert manager.__name__ == "NPUAcceleratorManager"
assert ray.available_resources()["NPU"] == 3
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
@pytest.mark.skipif(
sys.version_info >= (3, 12),
reason="Not passing on Python 3.12. Being followed up by external contributors.",
)
def test_acl_api_function(shutdown_only):
with patch.dict(sys.modules):
sys.modules["acl"] = __import__("mock_acl")
ray.init()
manager = ray._private.accelerators.get_accelerator_manager_for_resource("NPU")
assert manager.get_current_node_num_accelerators() == 4
assert manager.__name__ == "NPUAcceleratorManager"
assert manager.get_current_node_accelerator_type() == "Ascend910B"
def test_get_current_process_visible_accelerator_ids(monkeypatch, shutdown_only):
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
assert Accelerator.get_current_process_visible_accelerator_ids() == ["0", "1", "2"]
monkeypatch.delenv("ASCEND_RT_VISIBLE_DEVICES")
assert Accelerator.get_current_process_visible_accelerator_ids() is None
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "")
assert Accelerator.get_current_process_visible_accelerator_ids() == []
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "NoDevFiles")
assert Accelerator.get_current_process_visible_accelerator_ids() == []
def test_set_current_process_visible_accelerator_ids(shutdown_only):
Accelerator.set_current_process_visible_accelerator_ids(["0"])
assert os.environ["ASCEND_RT_VISIBLE_DEVICES"] == "0"
Accelerator.set_current_process_visible_accelerator_ids(["0", "1"])
assert os.environ["ASCEND_RT_VISIBLE_DEVICES"] == "0,1"
Accelerator.set_current_process_visible_accelerator_ids(["0", "1", "2"])
assert os.environ["ASCEND_RT_VISIBLE_DEVICES"] == "0,1,2"
@pytest.mark.skipif(sys.platform == "win32", reason="Not supported mock on Windows")
@pytest.mark.skipif(
sys.version_info >= (3, 12),
reason="Not passing on Python 3.12. Being followed up by external contributors.",
)
def test_auto_detected_more_than_visible(monkeypatch, shutdown_only):
with patch.dict(sys.modules):
sys.modules["acl"] = __import__("mock_acl")
with patch.object(
Accelerator, "get_current_node_num_accelerators", return_value=4
):
# If more NPUs are detected than visible.
monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "0,1,2")
ray.init()
assert ray.available_resources()["NPU"] == 3
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
@@ -0,0 +1,103 @@
import sys
import pytest
from ray._private.accelerators import NvidiaGPUAcceleratorManager
from ray.tests.accelerators.mock_pynvml import (
DeviceHandleMock,
PyNVMLMock,
patch_mock_pynvml,
)
GPU_MOCK_DATA = [
DeviceHandleMock(
"Ampere A100-SXM4-40GB",
"GPU-8eaaebb8-bb64-8489-fda2-62256e821983",
mig_devices=[
DeviceHandleMock(
"Ampere A100-SXM4-40GB MIG 1g.5gb",
"MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f",
gi_id=0,
ci_instance=0,
),
DeviceHandleMock(
"Ampere A100-SXM4-40GB MIG 1g.5gb",
"MIG-0c757cd7-e942-5726-a0b8-0e8fb7067135",
gi_id=1,
ci_instance=0,
),
],
),
DeviceHandleMock(
"Ampere A100-SXM4-40GB",
"GPU-8eaaebb8-bb64-8489-fda2-62256e821983",
mig_devices=[
DeviceHandleMock(
"Ampere A100-SXM4-40GB MIG 1g.5gb",
"MIG-a28ad590-3fda-56dd-84fc-0a0b96edc58d",
gi_id=0,
ci_instance=0,
)
],
),
DeviceHandleMock(
"Tesla V100-SXM2-16GB", "GPU-8eaaebb8-bb64-8489-fda2-62256e821983"
),
]
mock_nvml = PyNVMLMock(GPU_MOCK_DATA)
patch_mock_pynvml = patch_mock_pynvml # avoid format error
@pytest.mark.parametrize("mock_nvml", [mock_nvml])
def test_num_gpus_parsing(patch_mock_pynvml):
# without mig instance
assert NvidiaGPUAcceleratorManager.get_current_node_num_accelerators() == len(
GPU_MOCK_DATA
)
@pytest.mark.parametrize("mock_nvml", [mock_nvml])
def test_gpu_info_parsing(patch_mock_pynvml):
assert NvidiaGPUAcceleratorManager.get_current_node_accelerator_type() == "A100"
@pytest.mark.parametrize(
"name,expected",
[
# Legacy datacenter GPU names: keep labels produced by the previous
# parser stable.
("Tesla V100-SXM2-16GB", "V100"),
("Tesla P100-PCIE-16GB", "P100"),
("Tesla T4", "T4"),
("Tesla P4", "P4"),
("Tesla K80", "K80"),
("NVIDIA A10G", "A10G"),
("NVIDIA L4", "L4"),
("NVIDIA L40S", "L40S"),
("NVIDIA A100-SXM4-40GB", "A100"),
("NVIDIA H100 80GB HBM3", "H100"),
("NVIDIA H200", "H200"),
("NVIDIA H20", "H20"),
("NVIDIA B200", "B200"),
("NVIDIA B300", "B300"),
# Consumer GPUs: the regex does not match the mixed-case product line,
# so we fall back to a hyphen-joined product name.
("NVIDIA GeForce RTX 5090", "GeForce-RTX-5090"),
("NVIDIA GeForce RTX 4090", "GeForce-RTX-4090"),
# RTX PRO cards: "RTX" alone is just a brand prefix, so the model is
# captured through the first digit-containing token instead of
# collapsing to the ambiguous "RTX".
("NVIDIA RTX PRO 6000 Blackwell Server Edition", "RTX-PRO-6000"),
# Edge cases.
(None, None),
("", None),
],
)
def test_gpu_name_to_accelerator_type(name, expected):
assert NvidiaGPUAcceleratorManager._gpu_name_to_accelerator_type(name) == expected
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
@@ -0,0 +1,82 @@
import os
import sys
import pytest
from ray._private.accelerators.rbln import (
NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR,
RBLN_RT_VISIBLE_DEVICES_ENV_VAR,
RBLNAcceleratorManager,
)
@pytest.fixture(autouse=True)
def mock_rebel_module(monkeypatch):
from ray.tests.accelerators import mock_rebel
monkeypatch.setitem(sys.modules, "rebel", mock_rebel)
@pytest.fixture
def clear_rbln_environment():
original_env = os.environ.get(RBLN_RT_VISIBLE_DEVICES_ENV_VAR)
original_no_set_env = os.environ.get(NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR)
os.environ.pop(RBLN_RT_VISIBLE_DEVICES_ENV_VAR, None)
os.environ.pop(NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR, None)
yield
if original_env is not None:
os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = original_env
if original_no_set_env is not None:
os.environ[NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = original_no_set_env
@pytest.mark.usefixtures("clear_rbln_environment")
class TestRBLNAcceleratorManager:
def test_get_resource_name(self):
assert RBLNAcceleratorManager.get_resource_name() == "RBLN"
def test_get_visible_accelerator_ids_env_var(self):
assert (
RBLNAcceleratorManager.get_visible_accelerator_ids_env_var()
== RBLN_RT_VISIBLE_DEVICES_ENV_VAR
)
def test_get_current_process_visible_accelerator_ids(self):
os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = "0,1,2,3"
assert RBLNAcceleratorManager.get_current_process_visible_accelerator_ids() == [
"0",
"1",
"2",
"3",
]
os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = ""
assert (
RBLNAcceleratorManager.get_current_process_visible_accelerator_ids() == []
)
os.environ.pop(RBLN_RT_VISIBLE_DEVICES_ENV_VAR)
assert (
RBLNAcceleratorManager.get_current_process_visible_accelerator_ids() is None
)
def test_get_current_node_num_accelerators(self):
assert RBLNAcceleratorManager.get_current_node_num_accelerators() == 4
def test_get_current_node_accelerator_type(self):
assert RBLNAcceleratorManager.get_current_node_accelerator_type() == "RBLN-CA02"
def test_set_current_process_visible_accelerator_ids(self):
RBLNAcceleratorManager.set_current_process_visible_accelerator_ids(["0", "1"])
assert os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] == "0,1"
os.environ[NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR] = "1"
RBLNAcceleratorManager.set_current_process_visible_accelerator_ids(["2", "3"])
assert os.environ[RBLN_RT_VISIBLE_DEVICES_ENV_VAR] == "0,1"
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))
+346
View File
@@ -0,0 +1,346 @@
import os
import sys
from unittest import mock
from unittest.mock import patch
import pytest
import requests
from ray._private.accelerators import TPUAcceleratorManager, tpu
@patch("glob.glob")
def test_autodetect_num_tpus_accel(mock_glob):
mock_glob.return_value = [
"/dev/accel0",
"/dev/accel1",
"/dev/accel2",
"/dev/accel3",
]
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 4
@patch("os.path.isdir")
@patch("glob.glob")
@patch("os.listdir")
def test_autodetect_num_tpus_accel_ignores_blackwell_directory(
mock_list, mock_glob, mock_isdir
):
# NVIDIA drivers 570.x (Blackwell-class GPUs, e.g. RTX 5090) create
# /dev/accel as a directory containing /dev/accel/accel0. The non-recursive
# glob matches the directory entry; filtering directories out keeps real
# TPU chips (character devices at /dev/accel0..N) while rejecting the
# NVIDIA false positive.
mock_glob.return_value = ["/dev/accel"]
mock_isdir.side_effect = lambda p: p == "/dev/accel"
mock_list.side_effect = FileNotFoundError
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 0
@patch("glob.glob")
@patch("os.listdir")
def test_autodetect_num_tpus_vfio(mock_list, mock_glob):
mock_glob.return_value = []
mock_list.return_value = [f"{i}" for i in range(4)]
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 4
@patch("glob.glob")
@patch("os.listdir")
def test_autodetect_num_tpus_without_devices(mock_list, mock_glob):
mock_list.side_effect = FileNotFoundError
mock_glob.return_value = []
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
assert TPUAcceleratorManager.get_current_node_num_accelerators() == 0
@pytest.mark.parametrize(
"accelerator_type_version_tuple",
[
("gce", "v2-8", "TPU-V2"),
("gce", "v2-32", "TPU-V2"),
("gce", "v3-8", "TPU-V3"),
("gce", "v3-128", "TPU-V3"),
("gce", "v4-8", "TPU-V4"),
("gce", "v4-2048", "TPU-V4"),
("gce", "v5p-8", "TPU-V5P"),
("gce", "v5litepod-8", "TPU-V5LITEPOD"),
("gce", "v6e-8", "TPU-V6E"),
("gke", "v2-8", "TPU-V2"),
("gke", "v2-32", "TPU-V2"),
("gke", "v3-8", "TPU-V3"),
("gke", "v3-128", "TPU-V3"),
("gke", "v4-8", "TPU-V4"),
("gke", "v4-2048", "TPU-V4"),
("gke", "v5p-8", "TPU-V5P"),
("gke", "v5litepod-8", "TPU-V5LITEPOD"),
("gke", "v6e-8", "TPU-V6E"),
("gke", "tpu7x-16", "TPU-V7X"),
],
)
@patch("requests.get")
@patch("os.getenv")
def test_autodetect_tpu_accelerator_type(
mock_os, mock_request, accelerator_type_version_tuple
):
gce_or_gke, accelerator_type, expected_version = accelerator_type_version_tuple
if gce_or_gke == "gce":
mock_response = mock.MagicMock()
mock_response.status_code = 200
mock_response.text = accelerator_type
mock_request.return_value = mock_response
mock_os.return_value = None
else:
mock_os.return_value = accelerator_type
assert TPUAcceleratorManager.get_current_node_accelerator_type() == expected_version
@pytest.mark.parametrize(
"test_case",
[
("gce", "0", 0),
("gke", "0", 0),
],
)
@patch("requests.get")
@patch("os.getenv")
def test_get_current_node_tpu_worker_id(mock_os, mock_request, test_case):
gce_or_gke, worker_id, expected_value = test_case
if gce_or_gke == "gce":
mock_response = mock.MagicMock()
mock_response.status_code = 200
mock_response.text = worker_id
mock_request.return_value = mock_response
mock_os.return_value = None
else:
mock_os.return_value = worker_id
assert TPUAcceleratorManager.get_current_node_tpu_worker_id() == expected_value
@pytest.mark.parametrize(
"test_case",
[
("gce", "my-tpu"),
("gke", "my-tpu"),
],
)
@patch("requests.get")
@patch("os.getenv")
def test_get_tpu_unique_id(mock_os, mock_request, test_case):
gce_or_gke, worker_id = test_case
if gce_or_gke == "gce":
mock_response = mock.MagicMock()
mock_response.status_code = 200
mock_response.text = worker_id
mock_request.return_value = mock_response
mock_os.return_value = None
else:
mock_os.return_value = worker_id
assert TPUAcceleratorManager.get_current_node_tpu_name() == worker_id
@pytest.mark.parametrize(
"test_case",
[
("gce", "not-a-valid-version"),
("gce", "vNOTVALID-8"),
("gce", "230498230948230948"),
# From issue #39913
("gce", ""),
("gke", "not-a-valid-version"),
("gke", "vNOTVALID-8"),
("gke", "230498230948230948"),
],
)
@patch("requests.get")
@patch("os.getenv")
def test_autodetect_invalid_type(mock_os, mock_request, test_case):
gce_or_gke, accelerator_type = test_case
if gce_or_gke == "gce":
mock_response = mock.MagicMock()
mock_response.status_code = 200
mock_response.text = accelerator_type
mock_request.return_value = mock_response
mock_os.return_value = None
else:
mock_os.return_value = accelerator_type
assert TPUAcceleratorManager.get_current_node_accelerator_type() is None
def test_autodetect_tpu_accelerator_type_fails_gracefully():
with patch("requests.get") as mock_get:
mock_get.side_effect = requests.exceptions.RequestException
assert TPUAcceleratorManager.get_current_node_accelerator_type() is None
@pytest.mark.parametrize(
"test_config",
[
(1, False),
(0.5, True),
(3, True),
],
)
def test_validate_resource_request_quantity(test_config):
num_tpus, expect_error = test_config
if expect_error:
assert (
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[0]
is False
)
assert (
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[1]
is not None
)
else:
assert (
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[0]
is True
)
assert (
TPUAcceleratorManager.validate_resource_request_quantity(num_tpus)[1]
is None
)
@pytest.mark.parametrize(
"test_case",
[
(4, ["0"]),
(4, ["0", "1"]),
(4, ["0", "1", "2", "3"]),
(8, ["0", "1", "2", "3", "4", "5", "6", "7"]),
],
)
@patch("glob.glob")
def test_set_tpu_visible_ids_and_bounds(mock_glob, test_case):
num_devices, tpu_chips = test_case
mock_glob.return_value = ["/dev/accel" + str(x) for x in range(num_devices)]
with patch.dict("os.environ", {}, clear=True):
TPUAcceleratorManager.get_current_node_num_accelerators.cache_clear()
TPUAcceleratorManager.set_current_process_visible_accelerator_ids(tpu_chips)
if len(tpu_chips) == 1:
assert (
os.environ[tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR]
== tpu.TPU_CHIPS_PER_HOST_BOUNDS_1_CHIP_CONFIG
)
assert os.environ[tpu.TPU_HOST_BOUNDS_ENV_VAR] == tpu.TPU_SINGLE_HOST_BOUNDS
assert os.environ[tpu.TPU_VISIBLE_CHIPS_ENV_VAR] == ",".join(tpu_chips)
elif len(tpu_chips) == 2:
assert (
os.environ[tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR]
== tpu.TPU_CHIPS_PER_HOST_BOUNDS_2_CHIP_CONFIG
)
assert os.environ[tpu.TPU_HOST_BOUNDS_ENV_VAR] == tpu.TPU_SINGLE_HOST_BOUNDS
assert os.environ[tpu.TPU_VISIBLE_CHIPS_ENV_VAR] == ",".join(tpu_chips)
elif len(tpu_chips) == 4:
# Check that nothing is set, let the ML framework use the defaults.
assert os.environ.get(tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR, None) is None
assert os.environ.get(tpu.TPU_SINGLE_HOST_BOUNDS, None) is None
assert os.environ.get(tpu.TPU_VISIBLE_CHIPS_ENV_VAR, None) is None
else: # len(tpu_chips) == 8
assert os.environ.get(tpu.TPU_CHIPS_PER_HOST_BOUNDS_ENV_VAR, None) is None
assert os.environ.get(tpu.TPU_SINGLE_HOST_BOUNDS, None) is None
assert os.environ.get(tpu.TPU_VISIBLE_CHIPS_ENV_VAR, None) is None
@pytest.mark.parametrize(
"test_config",
[
(0, "v4-16", {"TPU-v4-16-head": 1, "my-tpu": 1}),
(1, "v4-16", {"my-tpu": 1}),
(0, "tpu7x-16", {"TPU-v7x-16-head": 1, "my-tpu": 1}),
],
)
def test_tpu_pod_detect_and_configure_worker(test_config):
worker_id, pod_type, expected_value = test_config
final_resources = {}
with patch(
"ray._private.accelerators.tpu.TPUAcceleratorManager.get_current_node_tpu_name",
return_value="my-tpu",
):
with patch(
"ray._private.accelerators.tpu.TPUAcceleratorManager.get_current_node_tpu_worker_id",
return_value=worker_id,
):
with patch.dict(os.environ, {"TPU_ACCELERATOR_TYPE": pod_type}):
final_resources = (
TPUAcceleratorManager.get_current_node_additional_resources()
)
assert final_resources == expected_value
@pytest.mark.parametrize(
"accelerator_type, expected",
[
("v2-8", True),
("v3-32", True),
("v4-8", True),
("v5p-8", True),
("v5litepod-8", True),
("v6e-8", True),
("tpu7x-16", True),
("v7x-16", True),
("v-8", False),
("8", False),
("tpu-8", False),
("v2", False),
("v2-", False),
("random-string", False),
],
)
def test_is_valid_tpu_accelerator_type(accelerator_type, expected):
assert (
TPUAcceleratorManager.is_valid_tpu_accelerator_type(accelerator_type)
== expected
)
def test_get_total_chips_from_accelerator_type():
assert tpu.get_total_chips_from_accelerator_type("v6e-16") == 16
assert tpu.get_total_chips_from_accelerator_type("v6e-8") == 8
assert (
tpu.get_total_chips_from_accelerator_type("v7x-16") == 8
) # v7x has 2 cores per chip
assert (
tpu.get_total_chips_from_accelerator_type("v4-8") == 4
) # v4 has 2 cores per chip
# Test invalid cases
with pytest.raises(ValueError, match="Accelerator type must include size"):
tpu.get_total_chips_from_accelerator_type("v6e")
with pytest.raises(ValueError, match="Invalid accelerator type"):
tpu.get_total_chips_from_accelerator_type("invalid-8")
def test_get_num_tpu_visible_chips_per_host():
# v6e multi-host (4 chips per VM)
assert tpu.get_num_tpu_visible_chips_per_host("v6e-16") == 4
assert tpu.get_num_tpu_visible_chips_per_host("v6e-32") == 4
# v6e single-host/sub-host (exact chip count)
assert tpu.get_num_tpu_visible_chips_per_host("v6e-8") == 8
assert tpu.get_num_tpu_visible_chips_per_host("v6e-4") == 4
assert tpu.get_num_tpu_visible_chips_per_host("v6e-1") == 1
# v5litepod multi-host defaults to 4, single-host is 8 chips
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-16") == 4
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-8") == 8
# v5litepod sub-host
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-4") == 4
assert tpu.get_num_tpu_visible_chips_per_host("v5litepod-1") == 1
# Other TPU generations default to 4
assert tpu.get_num_tpu_visible_chips_per_host("v4-8") == 4
assert tpu.get_num_tpu_visible_chips_per_host("v5p-8") == 4
if __name__ == "__main__":
sys.exit(pytest.main(["-sv", __file__]))