chore: import upstream snapshot with attribution

python/ray/dashboard/modules/reporter/gpu_profile_manager.py

@@ -0,0 +1,326 @@
+import asyncio
+import functools
+import logging
+import os
+import shutil
+import subprocess
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Tuple
+
+from ray.dashboard.modules.reporter.profile_manager import (
+    _format_failed_profiler_command,
+)
+
+import psutil
+
+logger = logging.getLogger(__name__)
+
+
+class GpuProfilingManager:
+    """GPU profiling manager for Ray Dashboard.
+
+    NOTE: The current implementation is based on the `dynolog` OSS project,
+    but these are mostly implementation details that can be changed in the future.
+    `dynolog` needs to be installed on the nodes where profiling is being done.
+
+    This only supports Torch training scripts with KINETO_USE_DAEMON=1 set.
+    It is not supported for other frameworks.
+    """
+
+    # Port for the monitoring daemon.
+    # This port was chosen arbitrarily to a value to avoid conflicts.
+    _DYNOLOG_PORT = 65406
+
+    # Default timeout for the profiling operation.
+    _DEFAULT_TIMEOUT_S = 5 * 60
+
+    _NO_PROCESSES_MATCHED_ERROR_MESSAGE_PREFIX = "No processes were matched"
+
+    _DISABLED_ERROR_MESSAGE = (
+        "GPU profiling is not enabled on node {ip_address}. "
+        "This is the case if no GPUs are detected on the node or if "
+        "the profiling dependency `dynolog` is not installed on the node.\n"
+        "Please ensure that GPUs are available on the node and that "
+        "`dynolog` is installed."
+    )
+    _NO_PROCESSES_MATCHED_ERROR_MESSAGE = (
+        "The profiling command failed for pid={pid} on node {ip_address}. "
+        "There are a few potential reasons for this:\n"
+        "1. The `KINETO_USE_DAEMON=1 KINETO_DAEMON_INIT_DELAY_S=5` environment variables "
+        "are not set for the training worker processes.\n"
+        "2. The process requested for profiling is not running a "
+        "PyTorch training script. GPU profiling is only supported for "
+        "PyTorch training scripts, typically launched via "
+        "`ray.train.torch.TorchTrainer`."
+    )
+    _DEAD_PROCESS_ERROR_MESSAGE = (
+        "The requested process to profile with pid={pid} on node "
+        "{ip_address} is no longer running. "
+        "GPU profiling is not available for this process."
+    )
+
+    def __init__(self, profile_dir_path: str, *, ip_address: str):
+        # Dump trace files to: /tmp/ray/session_latest/logs/profiles/
+        self._root_log_dir = Path(profile_dir_path)
+        self._profile_dir_path = self._root_log_dir / "profiles"
+        self._daemon_log_file_path = (
+            self._profile_dir_path / f"dynolog_daemon_{os.getpid()}.log"
+        )
+        self._ip_address = ip_address
+
+        self._dynolog_bin = shutil.which("dynolog")
+        self._dyno_bin = shutil.which("dyno")
+
+        self._dynolog_daemon_process: Optional[subprocess.Popen] = None
+
+        if not self._dynolog_bin or not self._dyno_bin:
+            logger.warning(
+                "[GpuProfilingManager] `dynolog` is not installed, GPU profiling will not be available."
+            )
+        elif not self.node_has_gpus():
+            logger.warning(
+                "[GpuProfilingManager] No GPUs found on this node, GPU profiling will not be setup."
+            )
+
+        self._profile_dir_path.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def enabled(self) -> bool:
+        return (
+            self._dynolog_bin is not None
+            and self._dyno_bin is not None
+            and self.node_has_gpus()
+        )
+
+    @property
+    def is_monitoring_daemon_running(self) -> bool:
+        return (
+            self._dynolog_daemon_process is not None
+            and self._dynolog_daemon_process.poll() is None
+        )
+
+    @classmethod
+    @functools.cache
+    def node_has_gpus(cls) -> bool:
+        try:
+            subprocess.check_output(
+                ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"],
+                stderr=subprocess.DEVNULL,
+                timeout=10,
+            )
+            return True
+        except subprocess.TimeoutExpired:
+            logger.warning(
+                "[GpuProfilingManager] `nvidia-smi` command timed out after 10s. "
+                "GPU profiling may not function correctly."
+            )
+            return False
+        except Exception:
+            return False
+
+    @classmethod
+    def is_pid_alive(cls, pid: int) -> bool:
+        try:
+            return psutil.pid_exists(pid) and psutil.Process(pid).is_running()
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            return False
+
+    def start_monitoring_daemon(self):
+        """Start the GPU profiling monitoring daemon if it's possible.
+        This must be called before profiling.
+        """
+
+        if not self.enabled:
+            logger.warning(
+                "[GpuProfilingManager] GPU profiling is disabled, skipping daemon setup."
+            )
+            return
+
+        if self.is_monitoring_daemon_running:
+            logger.warning(
+                "[GpuProfilingManager] GPU profiling monitoring daemon is already running."
+            )
+            return
+
+        try:
+            with open(self._daemon_log_file_path, "ab") as log_file:
+                daemon = subprocess.Popen(
+                    [
+                        self._dynolog_bin,
+                        "--enable_ipc_monitor",
+                        "--port",
+                        str(self._DYNOLOG_PORT),
+                    ],
+                    stdout=log_file,
+                    stderr=log_file,
+                    stdin=subprocess.DEVNULL,
+                    start_new_session=True,
+                )
+        except (FileNotFoundError, PermissionError, OSError) as e:
+            logger.error(
+                f"[GpuProfilingManager] Failed to launch GPU profiling monitoring daemon: {e}\n"
+                f"Check error log for more details: {self._daemon_log_file_path}"
+            )
+            return
+
+        logger.info(
+            "[GpuProfilingManager] Launched GPU profiling monitoring daemon "
+            f"(pid={daemon.pid}, port={self._DYNOLOG_PORT})\n"
+            f"Redirecting logs to: {self._daemon_log_file_path}"
+        )
+        self._dynolog_daemon_process = daemon
+
+    def _get_trace_filename(self) -> str:
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        return f"gputrace_{self._ip_address}_{timestamp}.json"
+
+    async def gpu_profile(
+        self, pid: int, num_iterations: int, _timeout_s: int = _DEFAULT_TIMEOUT_S
+    ) -> Tuple[bool, str]:
+        """
+        Perform GPU profiling on a specified process.
+
+        Args:
+            pid: The process ID (PID) of the target process to be profiled.
+            num_iterations: The number of iterations to profile.
+            _timeout_s: Maximum time in seconds to wait for profiling to complete.
+                This is an advanced parameter that catches edge cases where the
+                profiling request never completes and hangs indefinitely.
+
+        Returns:
+            Tuple[bool, str]: A tuple containing a boolean indicating the success
+                of the profiling operation and a string with the
+                filepath of the trace file relative to the root log directory,
+                or an error message.
+        """
+        if not self.enabled:
+            return False, self._DISABLED_ERROR_MESSAGE.format(
+                ip_address=self._ip_address
+            )
+
+        if not self._dynolog_daemon_process:
+            raise RuntimeError("Must call `start_monitoring_daemon` before profiling.")
+
+        if not self.is_monitoring_daemon_running:
+            error_msg = (
+                f"GPU monitoring daemon (pid={self._dynolog_daemon_process.pid}) "
+                f"is not running on node {self._ip_address}. "
+                f"See log for more details: {self._daemon_log_file_path}"
+            )
+            logger.error(f"[GpuProfilingManager] {error_msg}")
+            return False, error_msg
+
+        if not self.is_pid_alive(pid):
+            error_msg = self._DEAD_PROCESS_ERROR_MESSAGE.format(
+                pid=pid, ip_address=self._ip_address
+            )
+            logger.error(f"[GpuProfilingManager] {error_msg}")
+            return False, error_msg
+
+        trace_file_name = self._get_trace_filename()
+        trace_file_path = self._profile_dir_path / trace_file_name
+
+        cmd = [
+            self._dyno_bin,
+            "--port",
+            str(self._DYNOLOG_PORT),
+            "gputrace",
+            "--pids",
+            str(pid),
+            "--log-file",
+            str(trace_file_path),
+            "--process-limit",
+            str(1),
+            "--iterations",
+            str(num_iterations),
+        ]
+
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        if process.returncode != 0:
+            return False, _format_failed_profiler_command(cmd, "dyno", stdout, stderr)
+
+        stdout_str = stdout.decode("utf-8")
+        logger.info(f"[GpuProfilingManager] Launched profiling: {stdout_str}")
+
+        # The initial launch command returns immediately,
+        # so wait for the profiling to actually finish before returning.
+        # The indicator of the profiling finishing is the creation of the trace file,
+        # when the completed trace is moved from <prefix>.tmp.json -> <prefix>.json
+        # If the profiling request is invalid (e.g. "No processes were matched"),
+        # the trace file will not be created and this will hang indefinitely,
+        # up until the timeout is reached.
+
+        # TODO(ml-team): This logic is brittle, we should find a better way to do this.
+        if self._NO_PROCESSES_MATCHED_ERROR_MESSAGE_PREFIX in stdout_str:
+            error_msg = self._NO_PROCESSES_MATCHED_ERROR_MESSAGE.format(
+                pid=pid, ip_address=self._ip_address
+            )
+            logger.error(f"[GpuProfilingManager] {error_msg}")
+            return False, error_msg
+
+        # The actual trace file gets dumped with a suffix of `_{pid}.json
+        trace_file_name_pattern = trace_file_name.replace(".json", "*.json")
+
+        return await self._wait_for_trace_file(pid, trace_file_name_pattern, _timeout_s)
+
+    async def _wait_for_trace_file(
+        self,
+        pid: int,
+        trace_file_name_pattern: str,
+        timeout_s: int,
+        sleep_interval_s: float = 0.25,
+    ) -> Tuple[bool, str]:
+        """Wait for the trace file to be created.
+
+        Args:
+            pid: The target process to be profiled.
+            trace_file_name_pattern: The pattern of the trace file to be created
+                within the `<log_dir>/profiles` directory.
+            timeout_s: Maximum time in seconds to wait for profiling to complete.
+            sleep_interval_s: Time in seconds to sleep between checking for the trace file.
+
+        Returns:
+            Tuple[bool, str]: (success, trace file path relative to the *root* log directory)
+        """
+        remaining_timeout_s = timeout_s
+
+        logger.info(
+            "[GpuProfilingManager] Waiting for trace file to be created "
+            f"with the pattern: {trace_file_name_pattern}"
+        )
+
+        while True:
+            dumped_trace_file_path = next(
+                self._profile_dir_path.glob(trace_file_name_pattern), None
+            )
+            if dumped_trace_file_path is not None:
+                break
+
+            await asyncio.sleep(sleep_interval_s)
+
+            remaining_timeout_s -= sleep_interval_s
+            if remaining_timeout_s <= 0:
+                return (
+                    False,
+                    f"GPU profiling timed out after {timeout_s} seconds, please try again.",
+                )
+
+            # If the process has already exited, return an error.
+            if not self.is_pid_alive(pid):
+                return (
+                    False,
+                    self._DEAD_PROCESS_ERROR_MESSAGE.format(
+                        pid=pid, ip_address=self._ip_address
+                    ),
+                )
+
+        logger.info(
+            f"[GpuProfilingManager] GPU profiling finished, trace file: {dumped_trace_file_path}"
+        )
+        return True, str(dumped_trace_file_path.relative_to(self._root_log_dir))

python/ray/dashboard/modules/reporter/gpu_providers.py

@@ -0,0 +1,615 @@
+"""GPU providers for monitoring GPU usage in Ray dashboard.
+
+This module provides an object-oriented interface for different GPU providers
+(NVIDIA, AMD) to collect GPU utilization information.
+"""
+
+import abc
+import enum
+import logging
+import subprocess
+import time
+from typing import Dict, List, Optional, TypedDict, Union
+
+try:
+    from typing import NotRequired
+except ImportError:
+    from typing_extensions import NotRequired
+
+from ray.util.debug import log_once
+
+logger = logging.getLogger(__name__)
+
+# Constants
+MB = 1024 * 1024
+
+# Types
+Percentage = int
+Megabytes = int
+Bytes = int
+
+
+class GpuProviderType(enum.Enum):
+    """Enum for GPU provider types."""
+
+    NVIDIA = "nvidia"
+    AMD = "amd"
+
+
+class ProcessGPUInfo(TypedDict):
+    """Information about GPU usage for a single process."""
+
+    pid: int
+    gpu_memory_usage: Megabytes
+    gpu_utilization: Optional[Percentage]
+
+
+class GpuUtilizationInfo(TypedDict):
+    """GPU utilization information for a single GPU device."""
+
+    index: int
+    name: str
+    uuid: str
+    utilization_gpu: Optional[Percentage]
+    memory_used: Megabytes
+    memory_total: Megabytes
+    processes_pids: Optional[Dict[int, ProcessGPUInfo]]
+    # Optional: power in milliwatts, temperature in Celsius (e.g. from NVIDIA/AMD)
+    power_mw: NotRequired[Optional[int]]
+    temperature_c: NotRequired[Optional[int]]
+
+
+# tpu utilization for google tpu
+class TpuUtilizationInfo(TypedDict):
+    index: int
+    name: str
+    tpu_type: str
+    tpu_topology: str
+    tensorcore_utilization: Percentage
+    hbm_utilization: Percentage
+    duty_cycle: Percentage
+    memory_used: Bytes
+    memory_total: Bytes
+
+
+class GpuProvider(abc.ABC):
+    """Abstract base class for GPU providers."""
+
+    def __init__(self):
+        self._initialized = False
+
+    @abc.abstractmethod
+    def get_provider_name(self) -> GpuProviderType:
+        """Return the type of the GPU provider."""
+        pass
+
+    @abc.abstractmethod
+    def is_available(self) -> bool:
+        """Check if the GPU provider is available on this system."""
+        pass
+
+    @abc.abstractmethod
+    def _initialize(self) -> bool:
+        """Initialize the GPU provider. Returns True if successful."""
+        pass
+
+    @abc.abstractmethod
+    def _shutdown(self):
+        """Shutdown the GPU provider and clean up resources."""
+        pass
+
+    @abc.abstractmethod
+    def get_gpu_utilization(self) -> List[GpuUtilizationInfo]:
+        """Get GPU utilization information for all available GPUs."""
+        pass
+
+    @staticmethod
+    def _decode(b: Union[str, bytes]) -> str:
+        """Decode bytes to string for Python 3 compatibility."""
+        if isinstance(b, bytes):
+            return b.decode("utf-8")
+        return b
+
+
+class NvidiaGpuProvider(GpuProvider):
+    """NVIDIA GPU provider using pynvml."""
+
+    def __init__(self):
+        super().__init__()
+        self._pynvml = None
+        # Maintain per-GPU sampling timestamps when using process utilization API
+        self._gpu_process_last_sample_ts: Dict[int, int] = {}
+
+    def get_provider_name(self) -> GpuProviderType:
+        return GpuProviderType.NVIDIA
+
+    def is_available(self) -> bool:
+        """Check if NVIDIA GPUs are available."""
+        try:
+            import ray._private.thirdparty.pynvml as pynvml
+
+            pynvml.nvmlInit()
+            pynvml.nvmlShutdown()
+            return True
+        except Exception as e:
+            logger.debug(f"NVIDIA GPU not available: {e}")
+            return False
+
+    def _initialize(self) -> bool:
+        """Initialize the NVIDIA GPU provider."""
+        if self._initialized:
+            return True
+
+        try:
+            import ray._private.thirdparty.pynvml as pynvml
+
+            self._pynvml = pynvml
+            self._pynvml.nvmlInit()
+            self._initialized = True
+            return True
+        except Exception as e:
+            logger.debug(f"Failed to initialize NVIDIA GPU provider: {e}")
+            return False
+
+    def _shutdown(self):
+        """Shutdown the NVIDIA GPU provider."""
+        if self._initialized and self._pynvml:
+            try:
+                self._pynvml.nvmlShutdown()
+            except Exception as e:
+                logger.debug(f"Error shutting down NVIDIA GPU provider: {e}")
+            finally:
+                self._initialized = False
+
+    def get_gpu_utilization(self) -> List[GpuUtilizationInfo]:
+        """Get GPU utilization information for all NVIDIA GPUs and MIG devices."""
+
+        return self._get_pynvml_gpu_usage()
+
+    def _get_pynvml_gpu_usage(self) -> List[GpuUtilizationInfo]:
+        if not self._initialized:
+            if not self._initialize():
+                return []
+
+        gpu_utilizations = []
+
+        try:
+            num_gpus = self._pynvml.nvmlDeviceGetCount()
+
+            for i in range(num_gpus):
+                gpu_handle = self._pynvml.nvmlDeviceGetHandleByIndex(i)
+
+                # Check if MIG mode is enabled on this GPU
+                try:
+                    mig_mode = self._pynvml.nvmlDeviceGetMigMode(gpu_handle)
+                    if mig_mode[0]:  # MIG mode is enabled
+                        # Get MIG device instances
+                        mig_devices = self._get_mig_devices(gpu_handle, i)
+                        gpu_utilizations.extend(mig_devices)
+                        continue
+                except (self._pynvml.NVMLError, AttributeError):
+                    # MIG not supported or not enabled, continue with regular GPU
+                    pass
+
+                # Process regular GPU (non-MIG)
+                gpu_info = self._get_gpu_info(gpu_handle, i)
+                if gpu_info:
+                    gpu_utilizations.append(gpu_info)
+
+        except Exception as e:
+            logger.warning(f"Error getting NVIDIA GPU utilization: {e}")
+        finally:
+            self._shutdown()
+
+        return gpu_utilizations
+
+    def _get_mig_devices(self, gpu_handle, gpu_index: int) -> List[GpuUtilizationInfo]:
+        """Get MIG device information for a GPU with MIG enabled."""
+        mig_devices = []
+
+        try:
+            # Get all MIG device instances
+            mig_count = self._pynvml.nvmlDeviceGetMaxMigDeviceCount(gpu_handle)
+
+            for mig_idx in range(mig_count):
+                try:
+                    # Get MIG device handle
+                    mig_handle = self._pynvml.nvmlDeviceGetMigDeviceHandleByIndex(
+                        gpu_handle, mig_idx
+                    )
+
+                    # Get MIG device info
+                    mig_info = self._get_mig_device_info(mig_handle, gpu_index, mig_idx)
+                    if mig_info:
+                        mig_devices.append(mig_info)
+
+                except self._pynvml.NVMLError:
+                    # MIG device not available at this index
+                    continue
+
+        except (self._pynvml.NVMLError, AttributeError) as e:
+            logger.debug(f"Error getting MIG devices: {e}")
+
+        return mig_devices
+
+    def _get_mig_device_info(
+        self, mig_handle, gpu_index: int, mig_index: int
+    ) -> Optional[GpuUtilizationInfo]:
+        """Get utilization info for a single MIG device."""
+        try:
+            memory_info = self._pynvml.nvmlDeviceGetMemoryInfo(mig_handle)
+
+            # Get MIG device utilization
+            utilization = -1
+            try:
+                utilization_info = self._pynvml.nvmlDeviceGetUtilizationRates(
+                    mig_handle
+                )
+                utilization = int(utilization_info.gpu)
+            except self._pynvml.NVMLError as e:
+                logger.debug(f"Failed to retrieve MIG device utilization: {e}")
+
+            # Get running processes on MIG device
+            processes_pids = {}
+            try:
+                nv_comp_processes = self._pynvml.nvmlDeviceGetComputeRunningProcesses(
+                    mig_handle
+                )
+                nv_graphics_processes = (
+                    self._pynvml.nvmlDeviceGetGraphicsRunningProcesses(mig_handle)
+                )
+
+                for nv_process in nv_comp_processes + nv_graphics_processes:
+                    processes_pids[int(nv_process.pid)] = ProcessGPUInfo(
+                        pid=int(nv_process.pid),
+                        gpu_memory_usage=(
+                            int(nv_process.usedGpuMemory) // MB
+                            if nv_process.usedGpuMemory
+                            else 0
+                        ),
+                        # NOTE: According to nvml, this is not currently available in MIG mode
+                        gpu_utilization=None,
+                    )
+            except self._pynvml.NVMLError as e:
+                logger.debug(f"Failed to retrieve MIG device processes: {e}")
+
+            # Get MIG device UUID and name
+            try:
+                mig_uuid = self._decode(self._pynvml.nvmlDeviceGetUUID(mig_handle))
+                mig_name = self._decode(self._pynvml.nvmlDeviceGetName(mig_handle))
+            except self._pynvml.NVMLError:
+                # Fallback for older drivers
+                try:
+                    parent_name = self._decode(
+                        self._pynvml.nvmlDeviceGetName(
+                            self._pynvml.nvmlDeviceGetHandleByIndex(gpu_index)
+                        )
+                    )
+                    mig_name = f"{parent_name} MIG {mig_index}"
+                    mig_uuid = f"MIG-GPU-{gpu_index}-{mig_index}"
+                except Exception:
+                    mig_name = f"NVIDIA MIG Device {gpu_index}.{mig_index}"
+                    mig_uuid = f"MIG-{gpu_index}-{mig_index}"
+
+            return GpuUtilizationInfo(
+                index=gpu_index * 1000 + mig_index,  # Unique index for MIG devices
+                name=mig_name,
+                uuid=mig_uuid,
+                utilization_gpu=utilization,
+                memory_used=int(memory_info.used) // MB,
+                memory_total=int(memory_info.total) // MB,
+                processes_pids=processes_pids,
+                power_mw=None,  # MIG devices don't expose per-slice power in NVML
+                temperature_c=None,
+            )
+
+        except Exception as e:
+            logger.debug(f"Error getting MIG device info: {e}")
+            return None
+
+    def _get_gpu_info(self, gpu_handle, gpu_index: int) -> Optional[GpuUtilizationInfo]:
+        """Get utilization info for a regular (non-MIG) GPU."""
+        try:
+            memory_info = self._pynvml.nvmlDeviceGetMemoryInfo(gpu_handle)
+
+            # Get GPU utilization
+            utilization = -1
+            try:
+                utilization_info = self._pynvml.nvmlDeviceGetUtilizationRates(
+                    gpu_handle
+                )
+                utilization = int(utilization_info.gpu)
+            except self._pynvml.NVMLError as e:
+                if log_once("gpu_utilization"):
+                    logger.info(
+                        f"Failed to retrieve GPU utilization via `nvmlDeviceGetUtilizationRates`: {e}"
+                    )
+
+            # Get running processes
+            processes_pids = {}
+
+            # Get per-process memory usage from the running-processes APIs.
+            try:
+                nv_comp_processes = self._pynvml.nvmlDeviceGetComputeRunningProcesses(
+                    gpu_handle
+                )
+                nv_graphics_processes = (
+                    self._pynvml.nvmlDeviceGetGraphicsRunningProcesses(gpu_handle)
+                )
+                for nv_process in nv_comp_processes + nv_graphics_processes:
+                    pid = int(nv_process.pid)
+                    processes_pids[pid] = ProcessGPUInfo(
+                        pid=pid,
+                        gpu_memory_usage=int(nv_process.usedGpuMemory) // MB
+                        if nv_process.usedGpuMemory
+                        else 0,
+                        gpu_utilization=None,
+                    )
+            except self._pynvml.NVMLError as e:
+                if log_once("gpu_per_process_memory"):
+                    logger.info(
+                        "Failed to retrieve per-process GPU memory via `nvmlDeviceGetComputeRunningProcesses` "
+                        f"and `nvmlDeviceGetGraphicsRunningProcesses` APIs: {e}"
+                    )
+
+            # Use a newer API (driver 550+) to get per-process SM utilization, but the user
+            # may not always have the access to the newest API.
+            try:
+                current_ts_ms = int(time.time() * 1000)
+                last_ts_ms = self._gpu_process_last_sample_ts.get(gpu_index, 0)
+                nv_processes = self._pynvml.nvmlDeviceGetProcessesUtilizationInfo(
+                    gpu_handle, last_ts_ms
+                )
+                self._gpu_process_last_sample_ts[gpu_index] = current_ts_ms
+
+                for nv_process in nv_processes:
+                    pid = int(nv_process.pid)
+                    if pid not in processes_pids:
+                        # Note that it's pretty unlikely that nvmlDeviceGetProcessesUtilizationInfo
+                        # will include a process that nvmlDeviceGetComputeRunningProcesses +
+                        # nvmlDeviceGetGraphicsRunningProcesses didn't find, but doing this just in case.
+                        processes_pids[pid] = ProcessGPUInfo(
+                            pid=pid,
+                            gpu_memory_usage=0,
+                            gpu_utilization=int(nv_process.smUtil),
+                        )
+                    else:
+                        processes_pids[pid]["gpu_utilization"] = int(nv_process.smUtil)
+            except self._pynvml.NVMLError as e:
+                if log_once("gpu_process_sm_utilization"):
+                    logger.info(
+                        f"Failed to retrieve GPU process SM utilization using `nvmlDeviceGetProcessesUtilizationInfo`, error: {e}"
+                    )
+
+            # Optional: power (milliwatts) and temperature (Celsius)
+            power_mw = None
+            temperature_c = None
+            try:
+                power_mw = self._pynvml.nvmlDeviceGetPowerUsage(gpu_handle)
+            except (self._pynvml.NVMLError, AttributeError) as e:
+                if log_once("gpu_power"):
+                    logger.info(f"Failed to retrieve GPU power: {e}")
+            try:
+                # NVML_TEMPERATURE_GPU = 0
+                temperature_c = self._pynvml.nvmlDeviceGetTemperature(
+                    gpu_handle, self._pynvml.NVML_TEMPERATURE_GPU
+                )
+            except (self._pynvml.NVMLError, AttributeError) as e:
+                if log_once("gpu_temperature"):
+                    logger.info(f"Failed to retrieve GPU temperature: {e}")
+
+            return GpuUtilizationInfo(
+                index=gpu_index,
+                name=self._decode(self._pynvml.nvmlDeviceGetName(gpu_handle)),
+                uuid=self._decode(self._pynvml.nvmlDeviceGetUUID(gpu_handle)),
+                utilization_gpu=utilization,
+                memory_used=int(memory_info.used) // MB,
+                memory_total=int(memory_info.total) // MB,
+                processes_pids=processes_pids,
+                power_mw=power_mw,
+                temperature_c=temperature_c,
+            )
+
+        except Exception as e:
+            logger.debug(f"Error getting GPU info: {e}")
+            return None
+
+
+class AmdGpuProvider(GpuProvider):
+    """AMD GPU provider using pyamdsmi."""
+
+    def __init__(self):
+        super().__init__()
+        self._pyamdsmi = None
+
+    def get_provider_name(self) -> GpuProviderType:
+        return GpuProviderType.AMD
+
+    def is_available(self) -> bool:
+        """Check if AMD GPUs are available."""
+        try:
+            import ray._private.thirdparty.pyamdsmi as pyamdsmi
+
+            pyamdsmi.smi_initialize()
+            pyamdsmi.smi_shutdown()
+            return True
+        except Exception as e:
+            logger.debug(f"AMD GPU not available: {e}")
+            return False
+
+    def _initialize(self) -> bool:
+        """Initialize the AMD GPU provider."""
+        if self._initialized:
+            return True
+
+        try:
+            import ray._private.thirdparty.pyamdsmi as pyamdsmi
+
+            self._pyamdsmi = pyamdsmi
+            self._pyamdsmi.smi_initialize()
+            self._initialized = True
+            return True
+        except Exception as e:
+            logger.debug(f"Failed to initialize AMD GPU provider: {e}")
+            return False
+
+    def _shutdown(self):
+        """Shutdown the AMD GPU provider."""
+        if self._initialized and self._pyamdsmi:
+            try:
+                self._pyamdsmi.smi_shutdown()
+            except Exception as e:
+                logger.debug(f"Error shutting down AMD GPU provider: {e}")
+            finally:
+                self._initialized = False
+
+    def get_gpu_utilization(self) -> List[GpuUtilizationInfo]:
+        """Get GPU utilization information for all AMD GPUs."""
+        if not self._initialized:
+            if not self._initialize():
+                return []
+
+        gpu_utilizations = []
+
+        try:
+            num_gpus = self._pyamdsmi.smi_get_device_count()
+            processes = self._pyamdsmi.smi_get_device_compute_process()
+
+            for i in range(num_gpus):
+                utilization = self._pyamdsmi.smi_get_device_utilization(i)
+                if utilization == -1:
+                    utilization = -1
+
+                # Get running processes
+                processes_pids = {}
+                for process in self._pyamdsmi.smi_get_compute_process_info_by_device(
+                    i, processes
+                ):
+                    if process.vram_usage:
+                        processes_pids[int(process.process_id)] = ProcessGPUInfo(
+                            pid=int(process.process_id),
+                            gpu_memory_usage=int(process.vram_usage) // MB,
+                            gpu_utilization=None,
+                        )
+
+                # Optional: power in milliwatts (AMD returns watts)
+                power_mw = None
+                try:
+                    power_watts = self._pyamdsmi.smi_get_device_average_power(i)
+                    if power_watts >= 0:
+                        power_mw = int(power_watts * 1000)
+                except Exception as e:
+                    logger.debug(f"Failed to retrieve AMD GPU power: {e}")
+
+                info = GpuUtilizationInfo(
+                    index=i,
+                    name=self._decode(self._pyamdsmi.smi_get_device_name(i)),
+                    uuid=self._pyamdsmi.smi_get_device_unique_id(i),
+                    utilization_gpu=utilization,
+                    memory_used=int(self._pyamdsmi.smi_get_device_memory_used(i)) // MB,
+                    memory_total=int(self._pyamdsmi.smi_get_device_memory_total(i))
+                    // MB,
+                    processes_pids=processes_pids,
+                    power_mw=power_mw,
+                    temperature_c=None,  # not exposed in vendored pyamdsmi
+                )
+                gpu_utilizations.append(info)
+
+        except Exception as e:
+            logger.warning(f"Error getting AMD GPU utilization: {e}")
+        finally:
+            self._shutdown()
+
+        return gpu_utilizations
+
+
+class GpuMetricProvider:
+    """Provider class for GPU metrics collection."""
+
+    def __init__(self):
+        self._provider: Optional[GpuProvider] = None
+        self._enable_metric_report = True
+        self._providers = [NvidiaGpuProvider(), AmdGpuProvider()]
+        self._initialized = False
+
+    def initialize(self) -> bool:
+        """Initialize the GPU metric provider by detecting available GPU providers."""
+        if self._initialized:
+            return True
+
+        self._provider = self._detect_gpu_provider()
+
+        if self._provider is None:
+            # Check if we should disable GPU check entirely
+            try:
+                # Try NVIDIA first to check for the specific error condition
+                nvidia_provider = NvidiaGpuProvider()
+                nvidia_provider._initialize()
+            except Exception as e:
+                if self._should_disable_gpu_check(e):
+                    self._enable_metric_report = False
+        else:
+            logger.info(f"Using GPU Provider: {type(self._provider).__name__}")
+
+        self._initialized = True
+        return self._provider is not None
+
+    def _detect_gpu_provider(self) -> Optional[GpuProvider]:
+        """Detect and return the first available GPU provider."""
+        for provider in self._providers:
+            if provider.is_available():
+                return provider
+        return None
+
+    def _should_disable_gpu_check(self, nvidia_error: Exception) -> bool:
+        """
+        Check if we should disable GPU usage check based on the error.
+
+        On machines without GPUs, pynvml.nvmlInit() can run subprocesses that
+        spew to stderr. Then with log_to_driver=True, we get log spew from every
+        single raylet. To avoid this, disable the GPU usage check on certain errors.
+
+        See: https://github.com/ray-project/ray/issues/14305
+        """
+        if type(nvidia_error).__name__ != "NVMLError_DriverNotLoaded":
+            return False
+
+        try:
+            result = subprocess.check_output(
+                "cat /sys/module/amdgpu/initstate |grep live",
+                shell=True,
+                stderr=subprocess.DEVNULL,
+            )
+            # If AMD GPU module is not live and NVIDIA driver not loaded,
+            # disable GPU check
+            return len(str(result)) == 0
+        except Exception:
+            return False
+
+    def get_gpu_usage(self) -> List[GpuUtilizationInfo]:
+        """Get GPU usage information from the available provider."""
+        if not self._enable_metric_report:
+            return []
+
+        if not self._initialized:
+            self.initialize()
+
+        if self._provider is None:
+            return []
+
+        try:
+            gpu_info_list = self._provider.get_gpu_utilization()
+            return gpu_info_list  # Return TypedDict instances directly
+        except Exception as e:
+            logger.debug(
+                f"Error getting GPU usage from {self._provider.get_provider_name().value}: {e}"
+            )
+            return []
+
+    def get_provider_name(self) -> Optional[str]:
+        """Get the name of the current GPU provider."""
+        return self._provider.get_provider_name().value if self._provider else None
+
+    def is_metric_report_enabled(self) -> bool:
+        """Check if GPU metric reporting is enabled."""
+        return self._enable_metric_report

python/ray/dashboard/modules/reporter/healthz_agent.py

@@ -0,0 +1,103 @@
+import asyncio
+import logging
+
+from aiohttp.web import Request, Response
+
+import ray.dashboard.optional_utils as optional_utils
+import ray.dashboard.utils as dashboard_utils
+import ray.exceptions
+from ray._raylet import NodeID
+from ray.dashboard.modules.reporter.utils import HealthChecker
+
+routes = optional_utils.DashboardAgentRouteTable
+
+logger = logging.getLogger(__name__)
+
+
+class HealthzAgent(dashboard_utils.DashboardAgentModule):
+    """Health check in the agent.
+
+    This module adds health check related endpoint to the agent to check
+    local components' health.
+    """
+
+    def __init__(self, dashboard_agent):
+        super().__init__(dashboard_agent)
+        node_id = (
+            NodeID.from_hex(dashboard_agent.node_id)
+            if dashboard_agent.node_id
+            else None
+        )
+        self._health_checker = HealthChecker(
+            dashboard_agent.gcs_client,
+            node_id,
+        )
+
+    @routes.get("/api/local_raylet_healthz")
+    async def health_check(self, req: Request) -> Response:
+        try:
+            await self.raylet_health()
+        except Exception as e:
+            return Response(status=503, text=str(e), content_type="application/text")
+
+        return Response(
+            text="success",
+            content_type="application/text",
+        )
+
+    async def raylet_health(self) -> str:
+        try:
+            alive = await self._health_checker.check_local_raylet_liveness()
+            if alive is False:
+                raise Exception("Local Raylet failed")
+        except ray.exceptions.RpcError as e:
+            # We only consider the error other than GCS unreachable as raylet failure
+            # to avoid false positive.
+            # In case of GCS failed, Raylet will crash eventually if GCS is not back
+            # within a given time and the check will fail since agent can't live
+            # without a local raylet.
+            if e.rpc_code not in (
+                ray._raylet.GRPC_STATUS_CODE_UNAVAILABLE,
+                ray._raylet.GRPC_STATUS_CODE_UNKNOWN,
+                ray._raylet.GRPC_STATUS_CODE_DEADLINE_EXCEEDED,
+            ):
+                raise Exception(f"Health check failed due to: {e}")
+        return "success"
+
+    async def local_gcs_health(self) -> str:
+        # If GCS is not local, don't check its health.
+        if not self._dashboard_agent.is_head:
+            return "success (no local gcs)"
+        gcs_alive = await self._health_checker.check_gcs_liveness()
+        if not gcs_alive:
+            raise Exception("GCS health check failed.")
+        return "success"
+
+    @routes.get("/api/healthz")
+    async def unified_health(self, req: Request) -> Response:
+        [raylet_check, gcs_check] = await asyncio.gather(
+            self.raylet_health(),
+            self.local_gcs_health(),
+            return_exceptions=True,
+        )
+        checks = {"raylet": raylet_check, "gcs": gcs_check}
+
+        # Log failures.
+        status = 200
+        for name, result in checks.items():
+            if isinstance(result, Exception):
+                status = 503
+                logger.warning(f"health check {name} failed: {result}")
+
+        return Response(
+            status=status,
+            text="\n".join([f"{name}: {result}" for name, result in checks.items()]),
+            content_type="application/text",
+        )
+
+    async def run(self, server):
+        pass
+
+    @staticmethod
+    def is_minimal_module():
+        return False

python/ray/dashboard/modules/reporter/jax_profile_manager.py

@@ -0,0 +1,101 @@
+import asyncio
+import logging
+from concurrent.futures import Future, ThreadPoolExecutor
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class JaxProfilingManager:
+    """JAX profiling manager for Ray Dashboard.
+
+    It connects to the JAX profiler server running on the worker
+    and captures a trace using TensorFlow's profiler client.
+    """
+
+    def __init__(self, profile_dir_path: str):
+        self._root_log_dir = Path(profile_dir_path)
+        self._profile_dir_path = self._root_log_dir / "profiles"
+        self._profile_dir_path.mkdir(parents=True, exist_ok=True)
+        self._executor = ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix="jax_profiling_executor"
+        )
+        self._inflight: Optional[Future] = None
+
+    async def jax_profile(
+        self, pid: int, port: int, duration_s: int = 5
+    ) -> Tuple[bool, str]:
+        """Perform JAX profiling by connecting to the JAX server.
+
+        Args:
+            pid: The process ID of the target process (for logging/tracking).
+            port: The port where JAX profiler server is listening.
+            duration_s: Duration of the profiling in seconds.
+
+        Returns:
+            Tuple[bool, str]: (success, trace file path relative to root log dir)
+        """
+        if self._inflight is not None and not self._inflight.done():
+            return (
+                False,
+                "Another JAX profiling session is already in progress on this node.",
+            )
+
+        try:
+            from tensorflow.python.profiler import profiler_client
+        except ImportError as e:
+            return (
+                False,
+                "TensorFlow is required to capture JAX profiles from the Dashboard. "
+                f"Please install `tensorflow` on the node. Error: {e}",
+            )
+
+        address = f"grpc://localhost:{port}"
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        capture_dir = self._profile_dir_path / f"{pid}_{timestamp}"
+        logger.info(
+            f"Capturing JAX profile from {address} for pid {pid} "
+            f"for {duration_s} seconds..."
+        )
+
+        def _capture():
+            try:
+                # profiler_client.trace captures the trace and saves it to logdir
+                profiler_client.trace(
+                    address,
+                    logdir=str(capture_dir),
+                    duration_ms=duration_s * 1000,
+                )
+                return True, ""
+            except Exception as e:
+                return False, f"Failed to capture trace: {e}"
+
+        # Run in executor because trace is blocking
+        future = self._executor.submit(_capture)
+        self._inflight = future
+        try:
+            success, error_msg = await asyncio.wait_for(
+                asyncio.wrap_future(future),
+                timeout=duration_s + 20,
+            )
+        except asyncio.TimeoutError:
+            logger.error(
+                f"JAX profiling timed out after {duration_s + 20} seconds. "
+                "The capture may still be finishing in the background."
+            )
+            return (
+                False,
+                f"JAX profiling timed out after {duration_s + 20} seconds. "
+                "The capture may still be finishing, retry shortly.",
+            )
+
+        if not success:
+            logger.error(f"JAX profiling failed: {error_msg}")
+            return False, error_msg
+
+        logger.info(f"JAX profiling finished. Files saved in {capture_dir}")
+
+        # Return the directory path relative to the root log directory
+        return True, str(capture_dir.relative_to(self._root_log_dir))

python/ray/dashboard/modules/reporter/profile_manager.py

@@ -0,0 +1,378 @@
+import asyncio
+import logging
+import os
+import shutil
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Tuple, Union
+
+logger = logging.getLogger(__name__)
+
+DARWIN_SET_CHOWN_CMD = "sudo chown root: `which {profiler}`"
+LINUX_SET_CHOWN_CMD = "sudo chown root:root `which {profiler}`"
+
+PROFILER_PERMISSIONS_ERROR_MESSAGE = """
+Note that this command requires `{profiler}` to be installed with root permissions. You
+can install `{profiler}` and give it root permissions as follows:
+  $ pip install {profiler}
+  $ {set_chown_command}
+  $ sudo chmod u+s `which {profiler}`
+
+Alternatively, you can start Ray with passwordless sudo / root permissions.
+
+"""
+
+
+def decode(string: Union[str, bytes]):
+    if isinstance(string, bytes):
+        return string.decode("utf-8")
+    return string
+
+
+def _format_failed_profiler_command(cmd, profiler, stdout, stderr) -> str:
+    stderr_str = decode(stderr)
+    extra_message = ""
+
+    # If some sort of permission error returned, show a message about how
+    # to set up permissions correctly.
+    if "permission" in stderr_str.lower():
+        set_chown_command = (
+            DARWIN_SET_CHOWN_CMD.format(profiler=profiler)
+            if sys.platform == "darwin"
+            else LINUX_SET_CHOWN_CMD.format(profiler=profiler)
+        )
+        extra_message = PROFILER_PERMISSIONS_ERROR_MESSAGE.format(
+            profiler=profiler, set_chown_command=set_chown_command
+        )
+
+    return f"""Failed to execute `{cmd}`.
+{extra_message}
+=== stderr ===
+{decode(stderr)}
+
+=== stdout ===
+{decode(stdout)}
+"""
+
+
+# If we can sudo, always try that. Otherwise, py-spy will only work if the user has
+# root privileges or has configured setuid on the py-spy script.
+async def _can_passwordless_sudo() -> bool:
+    try:
+        process = await asyncio.create_subprocess_exec(
+            "sudo",
+            "-n",
+            "true",
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except FileNotFoundError:
+        return False
+    else:
+        _, _ = await process.communicate()
+        return process.returncode == 0
+
+
+class CpuProfilingManager:
+    def __init__(self, profile_dir_path: str):
+        self.profile_dir_path = Path(profile_dir_path)
+        self.profile_dir_path.mkdir(exist_ok=True)
+        self.profiler_name = "py-spy"
+
+    async def trace_dump(
+        self, pid: int, native: bool = False, subprocesses: bool = False
+    ) -> Tuple[bool, str]:
+        """Capture and dump a trace for a specified process.
+
+        Args:
+            pid: The process ID (PID) of the target process for trace capture.
+            native: If True, includes native (C/C++) stack frames.
+                Default is False.
+            subprocesses: If True, also dumps stack traces for
+                child processes of the target process. Default is False.
+
+        Returns:
+             A tuple containing a boolean indicating the success of the trace
+                capture operation and a string with the trace data or an error message.
+        """
+        pyspy = shutil.which(self.profiler_name)
+        if pyspy is None:
+            return False, "Failed to execute: py-spy is not installed"
+
+        cmd = [pyspy, "dump", "-p", str(pid)]
+        # We
+        if sys.platform == "linux" and native:
+            cmd.append("--native")
+        if subprocesses:
+            cmd.append("--subprocesses")
+        if await _can_passwordless_sudo():
+            cmd = ["sudo", "-n"] + cmd
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        if process.returncode != 0:
+            return False, _format_failed_profiler_command(
+                cmd, self.profiler_name, stdout, stderr
+            )
+        else:
+            return True, decode(stdout)
+
+    async def cpu_profile(
+        self,
+        pid: int,
+        format: str = "flamegraph",
+        duration: float = 5,
+        native: bool = False,
+        idle: bool = False,
+        subprocesses: bool = False,
+    ) -> Tuple[bool, str]:
+        """Perform CPU profiling on a specified process.
+
+        Args:
+            pid: The process ID (PID) of the target process to be profiled.
+            format: The format of the CPU profile output. Default is "flamegraph".
+            duration: The duration of the profiling session in seconds.
+                Default is 5 seconds.
+            native: If True, includes native (C/C++) stack frames. Default is False.
+            idle: If True, includes off-CPU / sleeping threads (e.g. threads
+                blocked on locks, I/O, or CUDA syncs). Default is False.
+            subprocesses: If True, also profiles child
+                processes of the target process (e.g. data loader or
+                multiprocess inference workers). Default is False.
+
+        Returns:
+            A tuple containing a boolean indicating the success of the profiling
+                operation and a string with the profile data or an error message.
+        """
+        pyspy = shutil.which(self.profiler_name)
+        if pyspy is None:
+            return False, "Failed to execute: py-spy is not installed"
+
+        if format not in ("flamegraph", "raw", "speedscope"):
+            return (
+                False,
+                f"Failed to execute: Invalid format {format}, "
+                + "must be [flamegraph, raw, speedscope]",
+            )
+
+        if format == "flamegraph":
+            extension = "svg"
+        else:
+            extension = "txt"
+        profile_file_path = (
+            self.profile_dir_path / f"{format}_{pid}_cpu_profiling.{extension}"
+        )
+        cmd = [
+            pyspy,
+            "record",
+            "-o",
+            profile_file_path,
+            "-p",
+            str(pid),
+            "-d",
+            str(duration),
+            "-f",
+            format,
+        ]
+        if sys.platform == "linux" and native:
+            cmd.append("--native")
+        if idle:
+            cmd.append("--idle")
+        if subprocesses:
+            cmd.append("--subprocesses")
+        if await _can_passwordless_sudo():
+            cmd = ["sudo", "-n"] + cmd
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        if process.returncode != 0:
+            return False, _format_failed_profiler_command(
+                cmd, self.profiler_name, stdout, stderr
+            )
+        else:
+            return True, open(profile_file_path, "rb").read()
+
+
+class MemoryProfilingManager:
+    def __init__(self, profile_dir_path: str):
+        self.profile_dir_path = Path(profile_dir_path) / "memray"
+        self.profile_dir_path.mkdir(exist_ok=True)
+        self.profiler_name = "memray"
+
+    async def get_profile_result(
+        self,
+        pid: int,
+        profiler_filename: str,
+        format: str = "flamegraph",
+        leaks: bool = False,
+    ) -> Tuple[bool, str]:
+        """Convert the Memray profile result to specified format.
+
+        Args:
+            pid: The process ID (PID) associated with the profiling operation.
+            profiler_filename: The filename of the profiler output to be processed.
+            format: The format of the profile result. Default is "flamegraph".
+            leaks: If True, include memory leak information in the profile result.
+
+        Returns:
+            A tuple containing a boolean indicating the success of the operation
+                and a string with the processed profile result or an error message.
+        """
+        memray = shutil.which(self.profiler_name)
+        if memray is None:
+            return False, "Failed to execute: memray is not installed"
+
+        profile_file_path = self.profile_dir_path / profiler_filename
+        if not Path(profile_file_path).is_file():
+            return False, f"Failed to execute: process {pid} has not been profiled"
+
+        profiler_name, _ = os.path.splitext(profiler_filename)
+        profile_visualize_path = self.profile_dir_path / f"{profiler_name}.html"
+        if format == "flamegraph":
+            visualize_cmd = [
+                memray,
+                "flamegraph",
+                "-o",
+                profile_visualize_path,
+                "-f",
+            ]
+        elif format == "table":
+            visualize_cmd = [
+                memray,
+                "table",
+                "-o",
+                profile_visualize_path,
+                "-f",
+            ]
+        else:
+            return (
+                False,
+                f"Failed to execute: Report with format: {format} is not supported",
+            )
+
+        if leaks:
+            visualize_cmd.append("--leaks")
+        visualize_cmd.append(profile_file_path)
+
+        process = await asyncio.create_subprocess_exec(
+            *visualize_cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        if process.returncode != 0:
+            return False, _format_failed_profiler_command(
+                visualize_cmd, self.profiler_name, stdout, stderr
+            )
+
+        return True, open(profile_visualize_path, "rb").read()
+
+    async def attach_profiler(
+        self,
+        pid: int,
+        native: bool = False,
+        trace_python_allocators: bool = False,
+        verbose: bool = False,
+    ) -> Tuple[bool, str]:
+        """Attach a Memray profiler to a specified process.
+
+        Args:
+            pid: The process ID (PID) of the target process which
+                the profiler attached to.
+            native: If True, includes native (C/C++) stack frames. Default is False.
+            trace_python_allocators: If True, includes Python stack frames.
+                Default is False.
+            verbose: If True, enables verbose output. Default is False.
+
+        Returns:
+            A tuple containing a boolean indicating the success of the operation
+                and a string of a success message or an error message.
+        """
+        memray = shutil.which(self.profiler_name)
+        if memray is None:
+            return False, None, "Failed to execute: memray is not installed"
+
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        profiler_filename = f"{pid}_memory_profiling_{timestamp}.bin"
+        profile_file_path = self.profile_dir_path / profiler_filename
+        cmd = [memray, "attach", str(pid), "-o", profile_file_path]
+
+        if native:
+            cmd.append("--native")
+        if trace_python_allocators:
+            cmd.append("--trace-python-allocators")
+        if verbose:
+            cmd.append("--verbose")
+        if await _can_passwordless_sudo():
+            cmd = ["sudo", "-n"] + cmd
+
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        stdout, stderr = await process.communicate()
+        if process.returncode != 0:
+            return (
+                False,
+                None,
+                _format_failed_profiler_command(
+                    cmd, self.profiler_name, stdout, stderr
+                ),
+            )
+        else:
+            return (
+                True,
+                profiler_filename,
+                f"Success attaching memray to process {pid}",
+            )
+
+    async def detach_profiler(
+        self,
+        pid: int,
+        verbose: bool = False,
+    ) -> Tuple[bool, str]:
+        """Detach a profiler from a specified process.
+
+        Args:
+            pid: The process ID (PID) of the target process the
+                profiler detached from.
+            verbose: If True, enables verbose output. Default is False.
+
+        Returns:
+            A tuple containing a boolean indicating the success of the operation
+                and a string of a success message or an error message.
+        """
+        memray = shutil.which(self.profiler_name)
+        if memray is None:
+            return False, "Failed to execute: memray is not installed"
+
+        cmd = [memray, "detach"]
+        if verbose:
+            cmd.append("--verbose")
+        cmd.append(str(pid))
+
+        if await _can_passwordless_sudo():
+            cmd = ["sudo", "-n"] + cmd
+
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        stdout, stderr = await process.communicate()
+        if process.returncode != 0:
+            return False, _format_failed_profiler_command(
+                cmd, self.profiler_name, stdout, stderr
+            )
+        else:
+            return True, f"Success detaching memray from process {pid}"

python/ray/dashboard/modules/reporter/reporter_consts.py

@@ -0,0 +1,7 @@
+import ray._private.ray_constants as ray_constants
+
+REPORTER_PREFIX = "RAY_REPORTER:"
+# The reporter will report its statistics this often (milliseconds).
+REPORTER_UPDATE_INTERVAL_MS = ray_constants.env_integer(
+    "REPORTER_UPDATE_INTERVAL_MS", 5000
+)

python/ray/dashboard/modules/reporter/reporter_models.py

@@ -0,0 +1,198 @@
+from typing import Dict, List, Optional, Tuple
+
+from ray._common.pydantic_compat import PYDANTIC_INSTALLED, BaseModel
+
+if PYDANTIC_INSTALLED:
+    from pydantic import field_validator
+
+    # TODO(aguo): Use these pydantic models in the dashboard API as well.
+    class ProcessGPUInfo(BaseModel):
+        """
+        Information about GPU usage for a single process.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        pid: int
+        gpuMemoryUsage: int  # in MB
+        gpuUtilization: Optional[int] = None  # percentage
+
+    class GpuUtilizationInfo(BaseModel):
+        """
+        GPU utilization information for a single GPU device.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        index: int
+        name: str
+        uuid: str
+        utilizationGpu: Optional[int] = None  # percentage
+        memoryUsed: int  # in MB
+        memoryTotal: int  # in MB
+        processesPids: Optional[
+            List[ProcessGPUInfo]
+        ] = None  # converted to list in _compose_stats_payload
+        powerMw: Optional[int] = None  # current power draw in milliwatts
+        temperatureC: Optional[int] = None  # temperature in Celsius
+
+    class TpuUtilizationInfo(BaseModel):
+        """
+        TPU utilization information for a single TPU device.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        index: int
+        name: str
+        tpuType: str
+        tpuTopology: str
+        tensorcoreUtilization: float  # percentage
+        hbmUtilization: float  # percentage
+        dutyCycle: float  # percentage
+        memoryUsed: int  # in bytes
+        memoryTotal: int  # in bytes
+
+    class CpuTimes(BaseModel):
+        """
+        CPU times information based on psutil.scputimes.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        user: float
+        system: float
+        childrenUser: float
+        childrenSystem: float
+
+    class MemoryInfo(BaseModel):
+        """
+        Memory information based on psutil.svmem.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        rss: float
+        vms: float
+        pfaults: Optional[float] = None
+        pageins: Optional[float] = None
+
+    class MemoryFullInfo(MemoryInfo):
+        """
+        Memory full information based on psutil.smem.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        uss: float
+
+    class ProcessInfo(BaseModel):
+        """
+        Process information from psutil.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        pid: int
+        createTime: float
+        cpuPercent: float
+        cpuTimes: Optional[CpuTimes] = None  # psutil._pslinux.scputimes object
+        cmdline: List[str]
+        memoryInfo: Optional[MemoryInfo] = None  # psutil._pslinux.svmem object
+        memoryFullInfo: Optional[MemoryFullInfo] = None  # psutil._pslinux.smem object
+        numFds: Optional[int] = None  # Not available on Windows
+        gpuMemoryUsage: Optional[int] = None  # in MB, added by _get_workers
+        gpuUtilization: Optional[int] = None  # percentage, added by _get_workers
+
+        @field_validator("cmdline", mode="before")
+        @classmethod
+        def _normalize_cmdline(cls, value):
+            return [] if value is None else value
+
+    # Note: The actual data structure uses tuples for some fields, not structured objects
+    # These are type aliases to document the tuple structure
+    MemoryUsage = Tuple[
+        int, int, float, int
+    ]  # (total, available, percent, used) in bytes
+    LoadAverage = Tuple[
+        Tuple[float, float, float], Optional[Tuple[float, float, float]]
+    ]  # (load, perCpuLoad)
+    NetworkStats = Tuple[int, int]  # (sent, received) in bytes
+    DiskIOStats = Tuple[
+        int, int, int, int
+    ]  # (readBytes, writeBytes, readCount, writeCount)
+    DiskIOSpeed = Tuple[
+        float, float, float, float
+    ]  # (readSpeed, writeSpeed, readIops, writeIops)
+    NetworkSpeed = Tuple[float, float]  # (sendSpeed, receiveSpeed) in bytes/sec
+
+    class DiskUsage(BaseModel):
+        """
+        Disk usage information based on psutil.diskusage.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        total: int
+        used: int
+        free: int
+        percent: float
+
+    class StatsPayload(BaseModel):
+        """
+        Main stats payload returned by _compose_stats_payload.
+        NOTE: Backwards compatibility for this model must be maintained.
+        If broken, the downstream dashboard API and UI code will break.
+        If you must make a backwards-incompatible change, you must make sure
+        to update the relevant code in the dashboard API and UI as well.
+        """
+
+        now: float  # POSIX timestamp
+        hostname: str
+        ip: str
+        cpu: float  # CPU usage percentage
+        cpus: Tuple[int, int]  # (logicalCpuCount, physicalCpuCount)
+        mem: MemoryUsage  # (total, available, percent, used) in bytes
+        hostMem: Tuple[int, int]  # host physical memory (used, totall) in bytes
+        cgroupMem: Optional[
+            Tuple[int, int]
+        ] = None  # (used, total) from cgroup, or None
+        shm: Optional[int] = None  # shared memory in bytes, None if not available
+        workers: List[ProcessInfo]
+        raylet: Optional[ProcessInfo] = None
+        agent: Optional[ProcessInfo] = None
+        bootTime: float  # POSIX timestamp
+        loadAvg: LoadAverage  # (load, perCpuLoad) where load is (1min, 5min, 15min)
+        disk: Dict[str, DiskUsage]  # mount point -> psutil disk usage object
+        diskIo: DiskIOStats  # (readBytes, writeBytes, readCount, writeCount)
+        diskIoSpeed: DiskIOSpeed  # (readSpeed, writeSpeed, readIops, writeIops)
+        gpus: List[GpuUtilizationInfo]
+        tpus: List[TpuUtilizationInfo]
+        network: NetworkStats  # (sent, received) in bytes
+        networkSpeed: NetworkSpeed  # (sendSpeed, receiveSpeed) in bytes/sec
+        cmdline: List[str]  # deprecated field from raylet
+        gcs: Optional[ProcessInfo] = None  # only present on head node
+
+        @field_validator("cmdline", mode="before")
+        @classmethod
+        def _normalize_cmdline(cls, value):
+            return [] if value is None else value
+
+else:
+    StatsPayload = None

python/ray/dashboard/modules/reporter/tests/test_gpu_profiler_manager.py

@@ -0,0 +1,377 @@
+"""Unit tests for the GPU profiler manager.
+
+All GPU and dynolog dependencies are mocked out.
+This test just verifies that commands are launched correctly and that
+validations are correctly performed.
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from ray.dashboard.modules.reporter.gpu_profile_manager import GpuProfilingManager
+
+
+@pytest.fixture
+def mock_node_has_gpus(monkeypatch):
+    monkeypatch.setattr(GpuProfilingManager, "node_has_gpus", lambda cls: True)
+    yield
+
+
+@pytest.fixture
+def mock_dynolog_binaries(monkeypatch):
+    monkeypatch.setattr("shutil.which", lambda cmd: f"/usr/bin/fake_{cmd}")
+    yield
+
+
+@pytest.fixture
+def mock_subprocess_popen(monkeypatch):
+    mock_popen = MagicMock()
+    mock_proc = MagicMock()
+    mock_popen.return_value = mock_proc
+
+    monkeypatch.setattr("subprocess.Popen", mock_popen)
+    yield (mock_popen, mock_proc)
+
+
+LOCALHOST = "127.0.0.1"
+
+
+@pytest.fixture
+def mock_asyncio_create_subprocess_exec(monkeypatch):
+    mock_create_subprocess_exec = AsyncMock()
+    mock_async_proc = mock_create_subprocess_exec.return_value = AsyncMock()
+    mock_async_proc.communicate.return_value = b"mock stdout", b"mock stderr"
+    mock_async_proc.returncode = 0
+    monkeypatch.setattr("asyncio.create_subprocess_exec", mock_create_subprocess_exec)
+    yield (mock_create_subprocess_exec, mock_async_proc)
+
+
+def test_node_has_gpus_uses_query_gpu_flag(tmp_path, monkeypatch):
+    """node_has_gpus() must use --query-gpu to avoid FabricManager stalls."""
+    captured = {}
+
+    def fake_check_output(cmd, **kwargs):
+        captured["cmd"] = list(cmd)
+        return b""
+
+    monkeypatch.setattr("subprocess.check_output", fake_check_output)
+    # Clear the cache so the monkeypatched check_output is actually called.
+    GpuProfilingManager.node_has_gpus.cache_clear()
+    result = GpuProfilingManager.node_has_gpus()
+    GpuProfilingManager.node_has_gpus.cache_clear()
+
+    assert result is True
+    assert captured["cmd"] == [
+        "nvidia-smi",
+        "--query-gpu=name",
+        "--format=csv,noheader",
+    ]
+
+
+def test_enabled_does_not_call_node_has_gpus_when_dynolog_missing(
+    tmp_path, monkeypatch
+):
+    """enabled must short-circuit on missing dynolog bins before node_has_gpus()."""
+    node_has_gpus_called = []
+
+    def spy_node_has_gpus(self_or_cls):
+        node_has_gpus_called.append(True)
+        return True
+
+    monkeypatch.setattr(GpuProfilingManager, "node_has_gpus", spy_node_has_gpus)
+    # No dynolog binaries on PATH → shutil.which returns None for both.
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    assert not gpu_profiler.enabled
+    assert (
+        not node_has_gpus_called
+    ), "node_has_gpus() must not be called when dynolog binaries are absent"
+
+
+def test_enabled(tmp_path, mock_node_has_gpus, mock_dynolog_binaries):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    assert gpu_profiler.enabled
+
+
+def test_disabled_no_gpus(tmp_path, monkeypatch):
+    monkeypatch.setattr(
+        GpuProfilingManager, "node_has_gpus", classmethod(lambda cls: False)
+    )
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    assert not gpu_profiler.enabled
+
+
+def test_disabled_no_dynolog_bin(tmp_path, mock_node_has_gpus):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    assert not gpu_profiler.enabled
+
+
+def test_start_monitoring_daemon(
+    tmp_path, mock_node_has_gpus, mock_dynolog_binaries, mock_subprocess_popen
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+
+    mocked_popen, mocked_proc = mock_subprocess_popen
+    mocked_proc.pid = 123
+    mocked_proc.poll.return_value = None
+
+    gpu_profiler.start_monitoring_daemon()
+    assert gpu_profiler.is_monitoring_daemon_running
+
+    assert mocked_popen.call_count == 1
+    assert mocked_popen.call_args[0][0] == [
+        "/usr/bin/fake_dynolog",
+        "--enable_ipc_monitor",
+        "--port",
+        str(gpu_profiler._DYNOLOG_PORT),
+    ]
+
+    # "Terminate" the daemon
+    mocked_proc.poll.return_value = 0
+    assert not gpu_profiler.is_monitoring_daemon_running
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_disabled(tmp_path):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    assert not gpu_profiler.enabled
+
+    success, output = await gpu_profiler.gpu_profile(pid=123, num_iterations=1)
+    assert not success
+    assert output == gpu_profiler._DISABLED_ERROR_MESSAGE.format(
+        ip_address=gpu_profiler._ip_address
+    )
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_without_starting_daemon(
+    tmp_path, mock_node_has_gpus, mock_dynolog_binaries
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    assert not gpu_profiler.is_monitoring_daemon_running
+
+    with pytest.raises(RuntimeError, match="start_monitoring_daemon"):
+        await gpu_profiler.gpu_profile(pid=123, num_iterations=1)
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_with_dead_daemon(
+    tmp_path, mock_node_has_gpus, mock_dynolog_binaries, mock_subprocess_popen
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    gpu_profiler.start_monitoring_daemon()
+
+    mocked_popen, mocked_proc = mock_subprocess_popen
+    mocked_proc.pid = 123
+    # "Terminate" the daemon
+    mocked_proc.poll.return_value = 0
+    assert not gpu_profiler.is_monitoring_daemon_running
+
+    success, output = await gpu_profiler.gpu_profile(pid=456, num_iterations=1)
+    assert not success
+    print(output)
+    assert "GPU monitoring daemon" in output
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_on_dead_process(
+    tmp_path,
+    monkeypatch,
+    mock_node_has_gpus,
+    mock_dynolog_binaries,
+    mock_subprocess_popen,
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    gpu_profiler.start_monitoring_daemon()
+
+    _, mocked_proc = mock_subprocess_popen
+    mocked_proc.pid = 123
+    mocked_proc.poll.return_value = None
+
+    monkeypatch.setattr(GpuProfilingManager, "is_pid_alive", lambda cls, pid: False)
+
+    success, output = await gpu_profiler.gpu_profile(pid=456, num_iterations=1)
+    assert not success
+    assert output == gpu_profiler._DEAD_PROCESS_ERROR_MESSAGE.format(
+        pid=456, ip_address=gpu_profiler._ip_address
+    )
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_no_matched_processes(
+    tmp_path,
+    monkeypatch,
+    mock_node_has_gpus,
+    mock_dynolog_binaries,
+    mock_subprocess_popen,
+    mock_asyncio_create_subprocess_exec,
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    gpu_profiler.start_monitoring_daemon()
+
+    # Mock the daemon process
+    _, mocked_daemon_proc = mock_subprocess_popen
+    mocked_daemon_proc.pid = 123
+    mocked_daemon_proc.poll.return_value = None
+
+    monkeypatch.setattr(GpuProfilingManager, "is_pid_alive", lambda cls, pid: True)
+
+    # Mock the asyncio.create_subprocess_exec
+    (
+        mocked_create_subprocess_exec,
+        mocked_async_proc,
+    ) = mock_asyncio_create_subprocess_exec
+    mocked_async_proc.communicate.return_value = (
+        f"{gpu_profiler._NO_PROCESSES_MATCHED_ERROR_MESSAGE_PREFIX}".encode(),
+        b"dummy stderr",
+    )
+    process_pid = 456
+    num_iterations = 1
+    success, output = await gpu_profiler.gpu_profile(
+        pid=process_pid, num_iterations=num_iterations
+    )
+
+    assert mocked_create_subprocess_exec.call_count == 1
+
+    assert not success
+    assert output == gpu_profiler._NO_PROCESSES_MATCHED_ERROR_MESSAGE.format(
+        pid=process_pid, ip_address=gpu_profiler._ip_address
+    )
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_timeout(
+    tmp_path,
+    monkeypatch,
+    mock_node_has_gpus,
+    mock_dynolog_binaries,
+    mock_subprocess_popen,
+    mock_asyncio_create_subprocess_exec,
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    gpu_profiler.start_monitoring_daemon()
+
+    # Mock the daemon process
+    _, mocked_daemon_proc = mock_subprocess_popen
+    mocked_daemon_proc.pid = 123
+    mocked_daemon_proc.poll.return_value = None
+
+    monkeypatch.setattr(GpuProfilingManager, "is_pid_alive", lambda cls, pid: True)
+
+    process_pid = 456
+    num_iterations = 1
+    task = asyncio.create_task(
+        gpu_profiler.gpu_profile(
+            pid=process_pid, num_iterations=num_iterations, _timeout_s=0.1
+        )
+    )
+
+    await asyncio.sleep(0.2)
+    success, output = await task
+    assert not success
+    assert "timed out" in output
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_process_dies_during_profiling(
+    tmp_path,
+    monkeypatch,
+    mock_node_has_gpus,
+    mock_dynolog_binaries,
+    mock_subprocess_popen,
+    mock_asyncio_create_subprocess_exec,
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    gpu_profiler.start_monitoring_daemon()
+
+    # Mock the daemon process
+    _, mocked_daemon_proc = mock_subprocess_popen
+    mocked_daemon_proc.pid = 123
+    mocked_daemon_proc.poll.return_value = None
+
+    monkeypatch.setattr(GpuProfilingManager, "is_pid_alive", lambda cls, pid: True)
+
+    process_pid = 456
+    num_iterations = 1
+    task = asyncio.create_task(
+        gpu_profiler.gpu_profile(pid=process_pid, num_iterations=num_iterations)
+    )
+
+    monkeypatch.setattr(GpuProfilingManager, "is_pid_alive", lambda cls, pid: False)
+
+    await asyncio.sleep(0.2)
+
+    success, output = await task
+    assert not success
+    assert output == gpu_profiler._DEAD_PROCESS_ERROR_MESSAGE.format(
+        pid=process_pid, ip_address=gpu_profiler._ip_address
+    )
+
+
+@pytest.mark.asyncio
+async def test_gpu_profile_success(
+    tmp_path,
+    monkeypatch,
+    mock_node_has_gpus,
+    mock_dynolog_binaries,
+    mock_subprocess_popen,
+    mock_asyncio_create_subprocess_exec,
+):
+    gpu_profiler = GpuProfilingManager(tmp_path, ip_address=LOCALHOST)
+    gpu_profiler.start_monitoring_daemon()
+
+    # Mock the daemon process
+    _, mocked_daemon_proc = mock_subprocess_popen
+    mocked_daemon_proc.pid = 123
+    mocked_daemon_proc.poll.return_value = None
+
+    monkeypatch.setattr(GpuProfilingManager, "is_pid_alive", lambda cls, pid: True)
+    monkeypatch.setattr(
+        GpuProfilingManager, "_get_trace_filename", lambda cls: "dummy_trace.json"
+    )
+    dumped_trace_filepath = gpu_profiler._profile_dir_path / "dummy_trace.json"
+    dumped_trace_filepath.touch()
+
+    # Mock the asyncio.create_subprocess_exec
+    (
+        mocked_create_subprocess_exec,
+        mocked_async_proc,
+    ) = mock_asyncio_create_subprocess_exec
+    process_pid = 456
+    num_iterations = 1
+    success, output = await gpu_profiler.gpu_profile(
+        pid=process_pid, num_iterations=num_iterations
+    )
+
+    # Verify the command was launched correctly
+    assert mocked_create_subprocess_exec.call_count == 1
+    profile_launch_args = list(mocked_create_subprocess_exec.call_args[0])
+    assert profile_launch_args[:6] == [
+        "/usr/bin/fake_dyno",
+        "--port",
+        str(gpu_profiler._DYNOLOG_PORT),
+        "gputrace",
+        "--pids",
+        str(process_pid),
+    ]
+
+    assert "--log-file" in profile_launch_args
+    profile_log_file_arg = profile_launch_args[
+        profile_launch_args.index("--log-file") + 1
+    ]
+    assert Path(profile_log_file_arg).is_relative_to(tmp_path)
+
+    assert "--iterations" in profile_launch_args
+    assert profile_launch_args[profile_launch_args.index("--iterations") + 1] == str(
+        num_iterations
+    )
+
+    assert success
+    assert output == str(dumped_trace_filepath.relative_to(tmp_path))
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py

@@ -0,0 +1,629 @@
+"""Unit tests for GPU providers."""
+
+import unittest
+from unittest.mock import Mock, patch
+
+from ray.dashboard.modules.reporter.gpu_providers import (
+    MB,
+    AmdGpuProvider,
+    GpuMetricProvider,
+    GpuProvider,
+    GpuProviderType,
+    GpuUtilizationInfo,
+    NvidiaGpuProvider,
+    ProcessGPUInfo,
+)
+
+
+class TestProcessGPUInfo(unittest.TestCase):
+    """Test ProcessGPUInfo TypedDict."""
+
+    def test_creation(self):
+        """Test ProcessGPUInfo creation."""
+        process_info = ProcessGPUInfo(
+            pid=1234, gpu_memory_usage=256, gpu_utilization=None
+        )
+
+        self.assertEqual(process_info["pid"], 1234)
+        self.assertEqual(process_info["gpu_memory_usage"], 256)
+        self.assertIsNone(process_info["gpu_utilization"])
+
+
+class TestGpuUtilizationInfo(unittest.TestCase):
+    """Test GpuUtilizationInfo TypedDict."""
+
+    def test_creation_with_processes(self):
+        """Test GpuUtilizationInfo with process information."""
+        process1 = ProcessGPUInfo(pid=1234, gpu_memory_usage=256, gpu_utilization=None)
+        process2 = ProcessGPUInfo(pid=5678, gpu_memory_usage=512, gpu_utilization=None)
+
+        gpu_info = GpuUtilizationInfo(
+            index=0,
+            name="NVIDIA GeForce RTX 3080",
+            uuid="GPU-12345678-1234-1234-1234-123456789abc",
+            utilization_gpu=75,
+            memory_used=8192,
+            memory_total=10240,
+            processes_pids={1234: process1, 5678: process2},
+        )
+
+        self.assertEqual(gpu_info["index"], 0)
+        self.assertEqual(gpu_info["name"], "NVIDIA GeForce RTX 3080")
+        self.assertEqual(gpu_info["uuid"], "GPU-12345678-1234-1234-1234-123456789abc")
+        self.assertEqual(gpu_info["utilization_gpu"], 75)
+        self.assertEqual(gpu_info["memory_used"], 8192)
+        self.assertEqual(gpu_info["memory_total"], 10240)
+        self.assertEqual(len(gpu_info["processes_pids"]), 2)
+        self.assertIn(1234, gpu_info["processes_pids"])
+        self.assertIn(5678, gpu_info["processes_pids"])
+        self.assertEqual(gpu_info["processes_pids"][1234]["pid"], 1234)
+        self.assertEqual(gpu_info["processes_pids"][1234]["gpu_memory_usage"], 256)
+        self.assertEqual(gpu_info["processes_pids"][5678]["pid"], 5678)
+        self.assertEqual(gpu_info["processes_pids"][5678]["gpu_memory_usage"], 512)
+
+    def test_creation_without_processes(self):
+        """Test GpuUtilizationInfo without process information."""
+        gpu_info = GpuUtilizationInfo(
+            index=1,
+            name="AMD Radeon RX 6800 XT",
+            uuid="GPU-87654321-4321-4321-4321-ba9876543210",
+            utilization_gpu=None,
+            memory_used=4096,
+            memory_total=16384,
+            processes_pids=None,
+        )
+
+        self.assertEqual(gpu_info["index"], 1)
+        self.assertEqual(gpu_info["name"], "AMD Radeon RX 6800 XT")
+        self.assertEqual(gpu_info["uuid"], "GPU-87654321-4321-4321-4321-ba9876543210")
+        self.assertIsNone(gpu_info["utilization_gpu"])  # Should be None, not -1
+        self.assertEqual(gpu_info["memory_used"], 4096)
+        self.assertEqual(gpu_info["memory_total"], 16384)
+        self.assertIsNone(gpu_info["processes_pids"])  # Should be None, not []
+
+
+class TestGpuProvider(unittest.TestCase):
+    """Test abstract GpuProvider class."""
+
+    def test_decode_bytes(self):
+        """Test _decode method with bytes input."""
+        result = GpuProvider._decode(b"test string")
+        self.assertEqual(result, "test string")
+
+    def test_decode_string(self):
+        """Test _decode method with string input."""
+        result = GpuProvider._decode("test string")
+        self.assertEqual(result, "test string")
+
+    def test_abstract_methods_not_implemented(self):
+        """Test that abstract methods raise NotImplementedError."""
+
+        class IncompleteProvider(GpuProvider):
+            pass
+
+        with self.assertRaises(TypeError):
+            IncompleteProvider()
+
+
+class TestNvidiaGpuProvider(unittest.TestCase):
+    """Test NvidiaGpuProvider class."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.provider = NvidiaGpuProvider()
+
+    def test_get_provider_name(self):
+        """Test provider name."""
+        self.assertEqual(self.provider.get_provider_name(), GpuProviderType.NVIDIA)
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_is_available_success(self, mock_pynvml):
+        """Test is_available when NVIDIA GPU is available."""
+        mock_pynvml.nvmlInit.return_value = None
+        mock_pynvml.nvmlShutdown.return_value = None
+
+        # Mock sys.modules to make the import work
+        import sys
+
+        original_modules = sys.modules.copy()
+        sys.modules["ray._private.thirdparty.pynvml"] = mock_pynvml
+
+        try:
+            self.assertTrue(self.provider.is_available())
+            mock_pynvml.nvmlInit.assert_called_once()
+            mock_pynvml.nvmlShutdown.assert_called_once()
+        finally:
+            # Restore original modules
+            sys.modules.clear()
+            sys.modules.update(original_modules)
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_is_available_failure(self, mock_pynvml):
+        """Test is_available when NVIDIA GPU is not available."""
+        mock_pynvml.nvmlInit.side_effect = Exception("NVIDIA driver not found")
+
+        # Mock sys.modules to make the import work but nvmlInit fail
+        import sys
+
+        original_modules = sys.modules.copy()
+        sys.modules["ray._private.thirdparty.pynvml"] = mock_pynvml
+
+        try:
+            self.assertFalse(self.provider.is_available())
+        finally:
+            # Restore original modules
+            sys.modules.clear()
+            sys.modules.update(original_modules)
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_initialize_success(self, mock_pynvml):
+        """Test successful initialization."""
+        # Ensure provider starts fresh
+        self.provider._initialized = False
+
+        mock_pynvml.nvmlInit.return_value = None
+
+        # Mock sys.modules to make the import work
+        import sys
+
+        original_modules = sys.modules.copy()
+        sys.modules["ray._private.thirdparty.pynvml"] = mock_pynvml
+
+        try:
+            self.assertTrue(self.provider._initialize())
+            self.assertTrue(self.provider._initialized)
+            mock_pynvml.nvmlInit.assert_called_once()
+        finally:
+            # Restore original modules
+            sys.modules.clear()
+            sys.modules.update(original_modules)
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_initialize_failure(self, mock_pynvml):
+        """Test failed initialization."""
+        # Ensure provider starts fresh
+        self.provider._initialized = False
+
+        # Make nvmlInit fail
+        mock_pynvml.nvmlInit.side_effect = Exception("Initialization failed")
+
+        # Mock sys.modules to make the import work but nvmlInit fail
+        import sys
+
+        original_modules = sys.modules.copy()
+        sys.modules["ray._private.thirdparty.pynvml"] = mock_pynvml
+
+        try:
+            self.assertFalse(self.provider._initialize())
+            self.assertFalse(self.provider._initialized)
+        finally:
+            # Restore original modules
+            sys.modules.clear()
+            sys.modules.update(original_modules)
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_initialize_already_initialized(self, mock_pynvml):
+        """Test initialization when already initialized."""
+        self.provider._initialized = True
+
+        self.assertTrue(self.provider._initialize())
+        mock_pynvml.nvmlInit.assert_not_called()
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_shutdown(self, mock_pynvml):
+        """Test shutdown."""
+        self.provider._initialized = True
+        self.provider._pynvml = mock_pynvml
+
+        self.provider._shutdown()
+
+        self.assertFalse(self.provider._initialized)
+        mock_pynvml.nvmlShutdown.assert_called_once()
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_shutdown_not_initialized(self, mock_pynvml):
+        """Test shutdown when not initialized."""
+        self.provider._shutdown()
+        mock_pynvml.nvmlShutdown.assert_not_called()
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_get_gpu_utilization_success(self, mock_pynvml):
+        """Test successful GPU utilization retrieval."""
+        # Mock GPU device
+        mock_handle = Mock()
+        mock_memory_info = Mock()
+        mock_memory_info.used = 8 * MB * 1024  # 8GB used
+        mock_memory_info.total = 12 * MB * 1024  # 12GB total
+
+        mock_utilization_info = Mock()
+        mock_utilization_info.gpu = 75
+
+        mock_process = Mock()
+        mock_process.pid = 1234
+        mock_process.usedGpuMemory = 256 * MB
+
+        # Configure mocks
+        mock_pynvml.nvmlInit.return_value = None
+        mock_pynvml.nvmlDeviceGetCount.return_value = 1
+        mock_pynvml.nvmlDeviceGetHandleByIndex.return_value = mock_handle
+        mock_pynvml.nvmlDeviceGetMemoryInfo.return_value = mock_memory_info
+        mock_pynvml.nvmlDeviceGetUtilizationRates.return_value = mock_utilization_info
+        mock_pynvml.nvmlDeviceGetComputeRunningProcesses.return_value = [mock_process]
+        mock_pynvml.nvmlDeviceGetGraphicsRunningProcesses.return_value = []
+        mock_pynvml.nvmlDeviceGetName.return_value = b"NVIDIA GeForce RTX 3080"
+        mock_pynvml.nvmlDeviceGetUUID.return_value = (
+            b"GPU-12345678-1234-1234-1234-123456789abc"
+        )
+        mock_pynvml.nvmlShutdown.return_value = None
+
+        # Set up provider state
+        self.provider._pynvml = mock_pynvml
+        self.provider._initialized = True
+
+        result = self.provider.get_gpu_utilization()
+
+        self.assertEqual(len(result), 1)
+        gpu_info = result[0]
+
+        self.assertEqual(gpu_info["index"], 0)
+        self.assertEqual(gpu_info["name"], "NVIDIA GeForce RTX 3080")
+        self.assertEqual(gpu_info["uuid"], "GPU-12345678-1234-1234-1234-123456789abc")
+        self.assertEqual(gpu_info["utilization_gpu"], 75)
+        self.assertEqual(gpu_info["memory_used"], 8 * 1024)  # 8GB in MB
+        self.assertEqual(gpu_info["memory_total"], 12 * 1024)  # 12GB in MB
+        self.assertEqual(len(gpu_info["processes_pids"]), 1)
+        self.assertEqual(gpu_info["processes_pids"][1234]["pid"], 1234)
+        self.assertEqual(gpu_info["processes_pids"][1234]["gpu_memory_usage"], 256)
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_get_gpu_utilization_with_errors(self, mock_pynvml):
+        """Test GPU utilization retrieval with partial errors."""
+        mock_handle = Mock()
+        mock_memory_info = Mock()
+        mock_memory_info.used = 4 * MB * 1024
+        mock_memory_info.total = 8 * MB * 1024
+
+        # Create mock NVML error class
+        class MockNVMLError(Exception):
+            pass
+
+        mock_pynvml.NVMLError = MockNVMLError
+
+        # Configure mocks with some failures
+        mock_pynvml.nvmlInit.return_value = None
+        mock_pynvml.nvmlDeviceGetCount.return_value = 1
+        mock_pynvml.nvmlDeviceGetHandleByIndex.return_value = mock_handle
+        mock_pynvml.nvmlDeviceGetMemoryInfo.return_value = mock_memory_info
+        mock_pynvml.nvmlDeviceGetUtilizationRates.side_effect = MockNVMLError(
+            "Utilization not available"
+        )
+        mock_pynvml.nvmlDeviceGetComputeRunningProcesses.side_effect = MockNVMLError(
+            "Process info not available"
+        )
+        mock_pynvml.nvmlDeviceGetGraphicsRunningProcesses.side_effect = MockNVMLError(
+            "Process info not available"
+        )
+        mock_pynvml.nvmlDeviceGetName.return_value = b"NVIDIA Tesla V100"
+        mock_pynvml.nvmlDeviceGetUUID.return_value = (
+            b"GPU-87654321-4321-4321-4321-ba9876543210"
+        )
+        mock_pynvml.nvmlShutdown.return_value = None
+
+        # Set up provider state
+        self.provider._pynvml = mock_pynvml
+        self.provider._initialized = True
+
+        result = self.provider.get_gpu_utilization()
+
+        self.assertEqual(len(result), 1)
+        gpu_info = result[0]
+
+        self.assertEqual(gpu_info["index"], 0)
+        self.assertEqual(gpu_info["name"], "NVIDIA Tesla V100")
+        self.assertEqual(gpu_info["utilization_gpu"], -1)  # Should be -1 due to error
+        self.assertEqual(
+            gpu_info["processes_pids"], {}
+        )  # Should be empty dict due to error
+
+    @patch("ray._private.thirdparty.pynvml", create=True)
+    def test_get_gpu_utilization_with_mig(self, mock_pynvml):
+        """Test GPU utilization retrieval with MIG devices."""
+        # Mock regular GPU handle
+        mock_gpu_handle = Mock()
+        mock_memory_info = Mock()
+        mock_memory_info.used = 4 * MB * 1024
+        mock_memory_info.total = 8 * MB * 1024
+
+        # Mock MIG device handle and info
+        mock_mig_handle = Mock()
+        mock_mig_memory_info = Mock()
+        mock_mig_memory_info.used = 2 * MB * 1024
+        mock_mig_memory_info.total = 4 * MB * 1024
+
+        mock_mig_utilization_info = Mock()
+        mock_mig_utilization_info.gpu = 80
+
+        # Configure mocks for MIG-enabled GPU
+        mock_pynvml.nvmlInit.return_value = None
+        mock_pynvml.nvmlDeviceGetCount.return_value = 1
+        mock_pynvml.nvmlDeviceGetHandleByIndex.return_value = mock_gpu_handle
+
+        # MIG mode enabled
+        mock_pynvml.nvmlDeviceGetMigMode.return_value = (
+            True,
+            True,
+        )  # (current, pending)
+        mock_pynvml.nvmlDeviceGetMaxMigDeviceCount.return_value = 1  # Only 1 MIG device
+        mock_pynvml.nvmlDeviceGetMigDeviceHandleByIndex.return_value = mock_mig_handle
+
+        # MIG device info
+        mock_pynvml.nvmlDeviceGetMemoryInfo.return_value = mock_mig_memory_info
+        mock_pynvml.nvmlDeviceGetUtilizationRates.return_value = (
+            mock_mig_utilization_info
+        )
+        mock_pynvml.nvmlDeviceGetComputeRunningProcesses.return_value = []
+        mock_pynvml.nvmlDeviceGetGraphicsRunningProcesses.return_value = []
+        mock_pynvml.nvmlDeviceGetName.return_value = b"NVIDIA A100-SXM4-40GB MIG 1g.5gb"
+        mock_pynvml.nvmlDeviceGetUUID.return_value = (
+            b"MIG-12345678-1234-1234-1234-123456789abc"
+        )
+        mock_pynvml.nvmlShutdown.return_value = None
+
+        # Set up provider state
+        self.provider._pynvml = mock_pynvml
+        self.provider._initialized = True
+
+        result = self.provider.get_gpu_utilization()
+
+        # Should return MIG device info instead of regular GPU
+        self.assertEqual(
+            len(result), 1
+        )  # Only one MIG device due to exception handling
+        gpu_info = result[0]
+
+        self.assertEqual(gpu_info["index"], 0)  # First MIG device (0 * 1000 + 0)
+        self.assertEqual(gpu_info["name"], "NVIDIA A100-SXM4-40GB MIG 1g.5gb")
+        self.assertEqual(gpu_info["uuid"], "MIG-12345678-1234-1234-1234-123456789abc")
+        self.assertEqual(gpu_info["utilization_gpu"], 80)
+        self.assertEqual(gpu_info["memory_used"], 2 * 1024)  # 2GB in MB
+        self.assertEqual(gpu_info["memory_total"], 4 * 1024)  # 4GB in MB
+        self.assertEqual(gpu_info["processes_pids"], {})
+
+
+class TestAmdGpuProvider(unittest.TestCase):
+    """Test AmdGpuProvider class."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.provider = AmdGpuProvider()
+
+    def test_get_provider_name(self):
+        """Test provider name."""
+        self.assertEqual(self.provider.get_provider_name(), GpuProviderType.AMD)
+
+    @patch("ray._private.thirdparty.pyamdsmi", create=True)
+    def test_is_available_success(self, mock_pyamdsmi):
+        """Test is_available when AMD GPU is available."""
+        mock_pyamdsmi.smi_initialize.return_value = None
+        mock_pyamdsmi.smi_shutdown.return_value = None
+
+        self.assertTrue(self.provider.is_available())
+        mock_pyamdsmi.smi_initialize.assert_called_once()
+        mock_pyamdsmi.smi_shutdown.assert_called_once()
+
+    @patch("ray._private.thirdparty.pyamdsmi", create=True)
+    def test_is_available_failure(self, mock_pyamdsmi):
+        """Test is_available when AMD GPU is not available."""
+        mock_pyamdsmi.smi_initialize.side_effect = Exception("AMD driver not found")
+
+        self.assertFalse(self.provider.is_available())
+
+    @patch("ray._private.thirdparty.pyamdsmi", create=True)
+    def test_initialize_success(self, mock_pyamdsmi):
+        """Test successful initialization."""
+        mock_pyamdsmi.smi_initialize.return_value = None
+
+        self.assertTrue(self.provider._initialize())
+        self.assertTrue(self.provider._initialized)
+        mock_pyamdsmi.smi_initialize.assert_called_once()
+
+    @patch("ray._private.thirdparty.pyamdsmi", create=True)
+    def test_get_gpu_utilization_success(self, mock_pyamdsmi):
+        """Test successful GPU utilization retrieval."""
+        mock_process = Mock()
+        mock_process.process_id = 5678
+        mock_process.vram_usage = 512 * MB
+
+        # Configure mocks
+        mock_pyamdsmi.smi_initialize.return_value = None
+        mock_pyamdsmi.smi_get_device_count.return_value = 1
+        mock_pyamdsmi.smi_get_device_id.return_value = "device_0"
+        mock_pyamdsmi.smi_get_device_utilization.return_value = 85
+        mock_pyamdsmi.smi_get_device_compute_process.return_value = [mock_process]
+        mock_pyamdsmi.smi_get_compute_process_info_by_device.return_value = [
+            mock_process
+        ]
+        mock_pyamdsmi.smi_get_device_name.return_value = b"AMD Radeon RX 6800 XT"
+        mock_pyamdsmi.smi_get_device_unique_id.return_value = (
+            "GPU-13579bdf-9abc-def0-0000-000000000000"
+        )
+        mock_pyamdsmi.smi_get_device_memory_used.return_value = 6 * MB * 1024
+        mock_pyamdsmi.smi_get_device_memory_total.return_value = 16 * MB * 1024
+        mock_pyamdsmi.smi_shutdown.return_value = None
+
+        # Set up provider state
+        self.provider._pyamdsmi = mock_pyamdsmi
+        self.provider._initialized = True
+
+        result = self.provider.get_gpu_utilization()
+
+        self.assertEqual(len(result), 1)
+        gpu_info = result[0]
+
+        self.assertEqual(gpu_info["index"], 0)
+        self.assertEqual(gpu_info["name"], "AMD Radeon RX 6800 XT")
+        self.assertEqual(gpu_info["uuid"], "GPU-13579bdf-9abc-def0-0000-000000000000")
+        self.assertEqual(gpu_info["utilization_gpu"], 85)
+        self.assertEqual(gpu_info["memory_used"], 6 * 1024)  # 6GB in MB
+        self.assertEqual(gpu_info["memory_total"], 16 * 1024)  # 16GB in MB
+        self.assertEqual(len(gpu_info["processes_pids"]), 1)
+        self.assertEqual(gpu_info["processes_pids"][5678]["pid"], 5678)
+        self.assertEqual(gpu_info["processes_pids"][5678]["gpu_memory_usage"], 512)
+
+
+class TestGpuMetricProvider(unittest.TestCase):
+    """Test GpuMetricProvider class."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.provider = GpuMetricProvider()
+
+    def test_init(self):
+        """Test GpuMetricProvider initialization."""
+        self.assertIsNone(self.provider._provider)
+        self.assertTrue(self.provider._enable_metric_report)
+        self.assertEqual(len(self.provider._providers), 2)
+        self.assertFalse(self.provider._initialized)
+
+    @patch.object(NvidiaGpuProvider, "is_available", return_value=True)
+    @patch.object(AmdGpuProvider, "is_available", return_value=False)
+    def test_detect_gpu_provider_nvidia(
+        self, mock_amd_available, mock_nvidia_available
+    ):
+        """Test GPU provider detection when NVIDIA is available."""
+        provider = self.provider._detect_gpu_provider()
+
+        self.assertIsInstance(provider, NvidiaGpuProvider)
+        mock_nvidia_available.assert_called_once()
+
+    @patch.object(NvidiaGpuProvider, "is_available", return_value=False)
+    @patch.object(AmdGpuProvider, "is_available", return_value=True)
+    def test_detect_gpu_provider_amd(self, mock_amd_available, mock_nvidia_available):
+        """Test GPU provider detection when AMD is available."""
+        provider = self.provider._detect_gpu_provider()
+
+        self.assertIsInstance(provider, AmdGpuProvider)
+        mock_nvidia_available.assert_called_once()
+        mock_amd_available.assert_called_once()
+
+    @patch.object(NvidiaGpuProvider, "is_available", return_value=False)
+    @patch.object(AmdGpuProvider, "is_available", return_value=False)
+    def test_detect_gpu_provider_none(self, mock_amd_available, mock_nvidia_available):
+        """Test GPU provider detection when no GPUs are available."""
+        provider = self.provider._detect_gpu_provider()
+
+        self.assertIsNone(provider)
+
+    @patch("subprocess.check_output")
+    def test_should_disable_gpu_check_true(self, mock_subprocess):
+        """Test should_disable_gpu_check returns True for specific conditions."""
+        mock_subprocess.return_value = ""  # Empty result means AMD GPU module not live
+
+        class MockNVMLError(Exception):
+            pass
+
+        MockNVMLError.__name__ = "NVMLError_DriverNotLoaded"
+
+        error = MockNVMLError("NVIDIA driver not loaded")
+
+        result = self.provider._should_disable_gpu_check(error)
+        self.assertTrue(result)
+
+    @patch("subprocess.check_output")
+    def test_should_disable_gpu_check_false_wrong_error(self, mock_subprocess):
+        """Test should_disable_gpu_check returns False for wrong error type."""
+        mock_subprocess.return_value = ""
+
+        error = Exception("Some other error")
+
+        result = self.provider._should_disable_gpu_check(error)
+        self.assertFalse(result)
+
+    @patch("subprocess.check_output")
+    def test_should_disable_gpu_check_false_amd_present(self, mock_subprocess):
+        """Test should_disable_gpu_check returns False when AMD GPU is present."""
+        mock_subprocess.return_value = "live"  # AMD GPU module is live
+
+        class MockNVMLError(Exception):
+            pass
+
+        MockNVMLError.__name__ = "NVMLError_DriverNotLoaded"
+
+        error = MockNVMLError("NVIDIA driver not loaded")
+
+        result = self.provider._should_disable_gpu_check(error)
+        self.assertFalse(result)
+
+    def test_get_gpu_usage_disabled(self):
+        """Test get_gpu_usage when GPU usage check is disabled."""
+        self.provider._enable_metric_report = False
+
+        result = self.provider.get_gpu_usage()
+        self.assertEqual(result, [])
+
+    @patch.object(GpuMetricProvider, "_detect_gpu_provider")
+    def test_get_gpu_usage_no_provider(self, mock_detect):
+        """Test get_gpu_usage when no GPU provider is available."""
+        mock_detect.return_value = None
+
+        with patch.object(
+            NvidiaGpuProvider, "_initialize", side_effect=Exception("No GPU")
+        ):
+            result = self.provider.get_gpu_usage()
+
+        self.assertEqual(result, [])
+        self.provider._initialized = False  # Reset for clean test
+        mock_detect.assert_called_once()
+
+    @patch.object(GpuMetricProvider, "_detect_gpu_provider")
+    def test_get_gpu_usage_success(self, mock_detect):
+        """Test successful get_gpu_usage."""
+        mock_provider = Mock()
+        mock_provider.get_gpu_utilization.return_value = [
+            GpuUtilizationInfo(
+                index=0,
+                name="Test GPU",
+                uuid="test-uuid",
+                utilization_gpu=50,
+                memory_used=1024,
+                memory_total=2048,
+                processes_pids={
+                    1234: ProcessGPUInfo(
+                        pid=1234, gpu_memory_usage=1024, gpu_utilization=None
+                    )
+                },
+            )
+        ]
+        mock_detect.return_value = mock_provider
+
+        result = self.provider.get_gpu_usage()
+
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["index"], 0)
+        self.assertEqual(result[0]["name"], "Test GPU")
+        mock_provider.get_gpu_utilization.assert_called_once()
+
+    def test_get_provider_name_no_provider(self):
+        """Test get_provider_name when no provider is set."""
+        result = self.provider.get_provider_name()
+        self.assertIsNone(result)
+
+    def test_get_provider_name_with_provider(self):
+        """Test get_provider_name when provider is set."""
+        mock_provider = Mock()
+        mock_provider.get_provider_name.return_value = GpuProviderType.NVIDIA
+        self.provider._provider = mock_provider
+
+        result = self.provider.get_provider_name()
+        self.assertEqual(result, "nvidia")
+
+    def test_is_metric_report_enabled(self):
+        """Test is_metric_report_enabled."""
+        self.assertTrue(self.provider.is_metric_report_enabled())
+
+        self.provider._enable_metric_report = False
+        self.assertFalse(self.provider.is_metric_report_enabled())
+
+
+if __name__ == "__main__":
+    unittest.main()

python/ray/dashboard/modules/reporter/tests/test_healthz.py

@@ -0,0 +1,121 @@
+import signal
+import sys
+
+import pytest
+import requests
+
+import ray._private.ray_constants as ray_constants
+from ray._common.network_utils import find_free_port
+from ray._common.test_utils import wait_for_condition
+from ray.tests.conftest import *  # noqa: F401 F403
+
+
+def test_healthz_head(monkeypatch, ray_start_cluster):
+    dashboard_port = find_free_port()
+    h = ray_start_cluster.add_node(dashboard_port=dashboard_port)
+    uri = f"http://localhost:{dashboard_port}/api/gcs_healthz"
+    wait_for_condition(lambda: requests.get(uri).status_code == 200)
+    h.all_processes[ray_constants.PROCESS_TYPE_GCS_SERVER][0].process.kill()
+    # It'll either timeout or just return an error
+    try:
+        wait_for_condition(lambda: requests.get(uri, timeout=1) != 200, timeout=4)
+    except RuntimeError as e:
+        assert "Read timed out" in str(e)
+
+
+def test_healthz_agent_1(monkeypatch, ray_start_cluster):
+    agent_port = find_free_port()
+    h = ray_start_cluster.add_node(dashboard_agent_listen_port=agent_port)
+    uri = f"http://{h.node_ip_address}:{agent_port}/api/local_raylet_healthz"
+
+    wait_for_condition(lambda: requests.get(uri).status_code == 200)
+
+    h.all_processes[ray_constants.PROCESS_TYPE_GCS_SERVER][0].process.kill()
+    # GCS's failure will not lead to healthz failure
+    assert requests.get(uri).status_code == 200
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="SIGSTOP only on posix")
+def test_healthz_agent_2(monkeypatch, ray_start_cluster):
+    monkeypatch.setenv("RAY_health_check_failure_threshold", "3")
+    monkeypatch.setenv("RAY_health_check_timeout_ms", "100")
+    monkeypatch.setenv("RAY_health_check_period_ms", "1000")
+    monkeypatch.setenv("RAY_health_check_initial_delay_ms", "0")
+
+    agent_port = find_free_port()
+    h = ray_start_cluster.add_node(dashboard_agent_listen_port=agent_port)
+    uri = f"http://{h.node_ip_address}:{agent_port}/api/local_raylet_healthz"
+
+    wait_for_condition(lambda: requests.get(uri).status_code == 200)
+
+    h.all_processes[ray_constants.PROCESS_TYPE_RAYLET][0].process.send_signal(
+        signal.SIGSTOP
+    )
+
+    # GCS still think raylet is alive.
+    assert requests.get(uri).status_code == 200
+    # But after heartbeat timeout, it'll think the raylet is down.
+    wait_for_condition(lambda: requests.get(uri).status_code != 200)
+
+
+def test_unified_healthz_head(monkeypatch, ray_start_cluster):
+    agent_port = find_free_port()
+    h = ray_start_cluster.add_node(dashboard_agent_listen_port=agent_port)
+    uri = f"http://{h.node_ip_address}:{agent_port}/api/healthz"
+
+    wait_for_condition(lambda: requests.get(uri).status_code == 200)
+    resp = requests.get(uri)
+    assert "raylet: success" in resp.text
+    assert "gcs: success" in resp.text
+
+    h.all_processes[ray_constants.PROCESS_TYPE_GCS_SERVER][0].process.kill()
+    wait_for_condition(lambda: requests.get(uri).status_code == 503)
+    resp = requests.get(uri)
+    assert "gcs: " in resp.text
+    assert "gcs: success" not in resp.text
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="SIGSTOP only on posix")
+def test_unified_healthz_worker(monkeypatch, ray_start_cluster):
+    monkeypatch.setenv("RAY_health_check_failure_threshold", "3")
+    monkeypatch.setenv("RAY_health_check_timeout_ms", "100")
+    monkeypatch.setenv("RAY_health_check_period_ms", "1000")
+    monkeypatch.setenv("RAY_health_check_initial_delay_ms", "0")
+
+    ray_start_cluster.add_node()
+    agent_port = find_free_port()
+    h = ray_start_cluster.add_node(dashboard_agent_listen_port=agent_port)
+    uri = f"http://{h.node_ip_address}:{agent_port}/api/healthz"
+
+    wait_for_condition(lambda: requests.get(uri).status_code == 200)
+    resp = requests.get(uri)
+    assert "gcs: success (no local gcs)" in resp.text
+
+    # Stop local raylet and verify this makes /healthz fail.
+    h.all_processes[ray_constants.PROCESS_TYPE_RAYLET][0].process.send_signal(
+        signal.SIGSTOP
+    )
+    wait_for_condition(lambda: requests.get(uri).status_code == 503)
+    resp = requests.get(uri)
+    assert "raylet: Local Raylet failed" in resp.text
+
+
+def test_unified_healthz_worker_gcs_down(monkeypatch, ray_start_cluster):
+    h_head = ray_start_cluster.add_node()
+    agent_port = find_free_port()
+    h_worker = ray_start_cluster.add_node(dashboard_agent_listen_port=agent_port)
+    uri = f"http://{h_worker.node_ip_address}:{agent_port}/api/healthz"
+
+    wait_for_condition(lambda: requests.get(uri).status_code == 200)
+    resp = requests.get(uri)
+    assert "gcs: success (no local gcs)" in resp.text
+
+    # Stop the head GCS server.
+    h_head.all_processes[ray_constants.PROCESS_TYPE_GCS_SERVER][0].process.kill()
+
+    # Worker health check should still succeed.
+    assert requests.get(uri).status_code == 200
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/reporter/tests/test_jax_profiler_manager.py

@@ -0,0 +1,106 @@
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from ray.dashboard.modules.reporter.jax_profile_manager import JaxProfilingManager
+from ray.util.tpu import init_jax_profiler
+
+
+@pytest.fixture
+def mock_profiler_client():
+    mock_client = MagicMock()
+    mock_profiler_module = MagicMock()
+    mock_profiler_module.profiler_client = mock_client
+
+    modules_to_patch = {
+        "tensorflow": MagicMock(),
+        "tensorflow.python": MagicMock(),
+        "tensorflow.python.profiler": mock_profiler_module,
+    }
+
+    with patch.dict("sys.modules", modules_to_patch):
+        yield mock_client
+
+
+@pytest.mark.asyncio
+async def test_jax_profile_success(tmp_path, mock_profiler_client):
+    manager = JaxProfilingManager(tmp_path)
+
+    # Mock success
+    mock_profiler_client.trace.return_value = None
+
+    success, output = await manager.jax_profile(pid=123, port=6000, duration_s=2)
+
+    assert success
+    assert output.startswith("profiles")
+    assert "123_" in output
+
+    mock_profiler_client.trace.assert_called_once()
+    call = mock_profiler_client.trace.call_args
+    assert call.args[0] == "grpc://localhost:6000"
+    assert call.kwargs["logdir"].startswith(str(tmp_path / "profiles"))
+    assert call.kwargs["duration_ms"] == 2000
+
+
+@pytest.mark.asyncio
+async def test_jax_profile_failure(tmp_path, mock_profiler_client):
+    manager = JaxProfilingManager(tmp_path)
+
+    # Mock failure
+    mock_profiler_client.trace.side_effect = Exception("Connection failed")
+
+    success, output = await manager.jax_profile(pid=123, port=6000, duration_s=2)
+
+    assert not success
+    assert "Failed to capture trace: Connection failed" in output
+
+
+@pytest.mark.asyncio
+async def test_jax_profile_no_tensorflow(tmp_path):
+    manager = JaxProfilingManager(tmp_path)
+
+    # Force ImportError on tensorflow
+    with patch.dict("sys.modules", {"tensorflow": None}):
+        success, output = await manager.jax_profile(pid=123, port=6000, duration_s=2)
+
+        assert not success
+        assert "TensorFlow is required" in output
+
+
+@patch("ray.util.tpu.os.getpid")
+@patch("ray.util.tpu.os.getenv")
+def test_setup_jax_profiler_success(mock_getenv, mock_getpid):
+    mock_getenv.return_value = "9999"
+    mock_getpid.return_value = 12345
+
+    mock_jax = MagicMock()
+    mock_worker = MagicMock()
+    mock_worker.node.node_id = "mock_node_id_hex"
+
+    with (
+        patch.dict("sys.modules", {"jax": mock_jax}),
+        patch("ray._private.worker.global_worker", mock_worker),
+        patch("ray.experimental.internal_kv._internal_kv_put") as mock_kv_put,
+    ):
+
+        init_jax_profiler()
+
+        mock_jax.profiler.start_server.assert_called_once_with(9999)
+        import ray
+
+        mock_kv_put.assert_called_once_with(
+            "jax_profiler_port:mock_node_id_hex:12345",
+            b"9999",
+            namespace=ray._private.ray_constants.KV_NAMESPACE_DASHBOARD,
+        )
+
+
+def test_setup_jax_profiler_no_jax():
+    with patch.dict("sys.modules", {"jax": None}):
+        # Should skip starting profiler and not raise error
+        init_jax_profiler()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/reporter/tests/test_profile_manager.py

@@ -0,0 +1,270 @@
+import os
+import sys
+import tempfile
+import time
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+import ray
+from ray.dashboard.modules.reporter.profile_manager import (
+    CpuProfilingManager,
+    MemoryProfilingManager,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+
+@pytest.fixture
+def setup_memory_profiler():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        memory_profiler = MemoryProfilingManager(tmpdir)
+
+        @ray.remote
+        class Actor:
+            def getpid(self):
+                return os.getpid()
+
+            def long_run(self):
+                print("Long-running task began.")
+                time.sleep(1000)
+                print("Long-running task completed.")
+
+        actor = Actor.remote()
+
+        yield actor, memory_profiler
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    os.environ.get("RAY_MINIMAL") == "1",
+    reason="This test is not supposed to work for minimal installation.",
+)
+@pytest.mark.skipif(sys.platform == "win32", reason="No memray on Windows.")
+@pytest.mark.skipif(
+    sys.platform == "darwin",
+    reason="Fails on OSX, requires memray & lldb installed in osx image",
+)
+class TestMemoryProfiling:
+    async def test_basic_attach_profiler(self, setup_memory_profiler, shutdown_only):
+        # test basic attach profiler to running process
+        actor, memory_profiler = setup_memory_profiler
+        pid = ray.get(actor.getpid.remote())
+        actor.long_run.remote()
+        success, profiler_filename, message = await memory_profiler.attach_profiler(
+            pid, verbose=True
+        )
+
+        assert success, message
+        assert f"Success attaching memray to process {pid}" in message
+        assert profiler_filename in os.listdir(memory_profiler.profile_dir_path)
+
+    async def test_profiler_multiple_attach(self, setup_memory_profiler, shutdown_only):
+        # test multiple attaches
+        actor, memory_profiler = setup_memory_profiler
+        pid = ray.get(actor.getpid.remote())
+        actor.long_run.remote()
+        success, profiler_filename, message = await memory_profiler.attach_profiler(
+            pid, verbose=True
+        )
+
+        assert success, message
+        assert f"Success attaching memray to process {pid}" in message
+        assert profiler_filename in os.listdir(memory_profiler.profile_dir_path)
+
+        success, _, message = await memory_profiler.attach_profiler(pid)
+        assert success, message
+        assert f"Success attaching memray to process {pid}" in message
+
+    async def test_detach_profiler_successful(
+        self, setup_memory_profiler, shutdown_only
+    ):
+        # test basic detach profiler
+        actor, memory_profiler = setup_memory_profiler
+        pid = ray.get(actor.getpid.remote())
+        actor.long_run.remote()
+        success, _, message = await memory_profiler.attach_profiler(pid, verbose=True)
+        assert success, message
+
+        success, message = await memory_profiler.detach_profiler(pid, verbose=True)
+        assert success, message
+        assert f"Success detaching memray from process {pid}" in message
+
+    async def test_detach_profiler_without_attach(
+        self, setup_memory_profiler, shutdown_only
+    ):
+        # test detach profiler from unattached process
+        actor, memory_profiler = setup_memory_profiler
+        pid = ray.get(actor.getpid.remote())
+
+        success, message = await memory_profiler.detach_profiler(pid)
+        assert not success, message
+        assert "Failed to execute" in message
+        assert "no previous `memray attach`" in message
+
+    async def test_profiler_memray_not_installed(
+        self, setup_memory_profiler, shutdown_only
+    ):
+        # test profiler when memray is not installed
+        actor, memory_profiler = setup_memory_profiler
+        pid = ray.get(actor.getpid.remote())
+
+        with patch("shutil.which", return_value=None):
+            success, _, message = await memory_profiler.attach_profiler(pid)
+            assert not success
+            assert "memray is not installed" in message
+
+    async def test_profiler_attach_process_not_found(
+        self, setup_memory_profiler, shutdown_only
+    ):
+        # test basic attach profiler to non-existing process
+        _, memory_profiler = setup_memory_profiler
+        pid = 123456
+        success, _, message = await memory_profiler.attach_profiler(pid)
+        assert not success, message
+        assert "Failed to execute" in message
+        assert "The given process ID does not exist" in message
+
+    async def test_profiler_get_profiler_result(
+        self, setup_memory_profiler, shutdown_only
+    ):
+        # test get profiler result from running process
+        actor, memory_profiler = setup_memory_profiler
+        pid = ray.get(actor.getpid.remote())
+        actor.long_run.remote()
+        success, profiler_filename, message = await memory_profiler.attach_profiler(
+            pid, verbose=True
+        )
+        assert success, message
+        assert f"Success attaching memray to process {pid}" in message
+
+        # get profiler result in flamegraph and table format
+        supported_formats = ["flamegraph", "table"]
+        unsupported_formats = ["json"]
+        for format in supported_formats + unsupported_formats:
+            success, message = await memory_profiler.get_profile_result(
+                pid, profiler_filename=profiler_filename, format=format
+            )
+            if format in supported_formats:
+                assert success, message
+                assert f"{format} report" in message.decode("utf-8")
+            else:
+                assert not success, message
+                assert f"{format} is not supported" in message
+
+    async def test_profiler_result_not_exist(
+        self, setup_memory_profiler, shutdown_only
+    ):
+        # test get profiler result from unexisting process
+        _, memory_profiler = setup_memory_profiler
+        pid = 123456
+        profiler_filename = "non-existing-file"
+
+        success, message = await memory_profiler.get_profile_result(
+            pid, profiler_filename=profiler_filename, format=format
+        )
+        assert not success, message
+        assert f"process {pid} has not been profiled" in message
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(sys.platform == "win32", reason="No py-spy on Windows.")
+class TestCpuProfiling:
+    async def _capture_pyspy_cmd(self, **cpu_profile_kwargs):
+        """Run cpu_profile with subprocess execution mocked out and return the
+        py-spy command that would have been executed.
+
+        We patch ``asyncio.create_subprocess_exec`` (the same primitive the
+        manager uses) and have the fake process exit non-zero so that
+        ``cpu_profile`` short-circuits before attempting to read the (never
+        created) output file. The command is fully constructed before the
+        subprocess is spawned, so the captured args are valid regardless.
+        """
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cpu_profiler = CpuProfilingManager(tmpdir)
+
+            fake_process = AsyncMock()
+            fake_process.communicate.return_value = (b"", b"boom")
+            fake_process.returncode = 1
+
+            with patch(
+                "ray.dashboard.modules.reporter.profile_manager.shutil.which",
+                return_value="/fake/py-spy",
+            ), patch(
+                "ray.dashboard.modules.reporter.profile_manager."
+                "_can_passwordless_sudo",
+                new=AsyncMock(return_value=False),
+            ), patch(
+                "asyncio.create_subprocess_exec",
+                new=AsyncMock(return_value=fake_process),
+            ) as mock_exec:
+                await cpu_profiler.cpu_profile(pid=12345, **cpu_profile_kwargs)
+
+            assert mock_exec.call_count == 1
+            # create_subprocess_exec(*cmd, ...) -> positional args are the cmd.
+            return list(mock_exec.call_args.args)
+
+    async def test_cpu_profile_idle_flag_added(self):
+        # idle=True should append `--idle` to the py-spy command.
+        cmd = await self._capture_pyspy_cmd(idle=True)
+        assert "--idle" in cmd
+
+    async def test_cpu_profile_idle_not_added_by_default(self):
+        # By default (idle=False) the `--idle` flag should be absent.
+        cmd = await self._capture_pyspy_cmd()
+        assert "--idle" not in cmd
+
+    async def test_cpu_profile_subprocesses_flag_added(self):
+        # subprocesses=True should append `--subprocesses` to the py-spy command.
+        cmd = await self._capture_pyspy_cmd(subprocesses=True)
+        assert "--subprocesses" in cmd
+
+    async def test_cpu_profile_subprocesses_not_added_by_default(self):
+        # By default the `--subprocesses` flag should be absent.
+        cmd = await self._capture_pyspy_cmd()
+        assert "--subprocesses" not in cmd
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(sys.platform == "win32", reason="No py-spy on Windows.")
+class TestTraceDump:
+    async def _capture_pyspy_cmd(self, **trace_dump_kwargs):
+        """Run trace_dump with subprocess execution mocked out and return the
+        py-spy command that would have been executed.
+        """
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cpu_profiler = CpuProfilingManager(tmpdir)
+
+            fake_process = AsyncMock()
+            fake_process.communicate.return_value = (b"", b"boom")
+            fake_process.returncode = 1
+
+            with patch(
+                "ray.dashboard.modules.reporter.profile_manager.shutil.which",
+                return_value="/fake/py-spy",
+            ), patch(
+                "ray.dashboard.modules.reporter.profile_manager."
+                "_can_passwordless_sudo",
+                new=AsyncMock(return_value=False),
+            ), patch(
+                "asyncio.create_subprocess_exec",
+                new=AsyncMock(return_value=fake_process),
+            ) as mock_exec:
+                await cpu_profiler.trace_dump(pid=12345, **trace_dump_kwargs)
+
+            assert mock_exec.call_count == 1
+            return list(mock_exec.call_args.args)
+
+    async def test_trace_dump_subprocesses_flag_added(self):
+        # subprocesses=True should append `--subprocesses` to the py-spy dump command.
+        cmd = await self._capture_pyspy_cmd(subprocesses=True)
+        assert "dump" in cmd
+        assert "--subprocesses" in cmd
+
+    async def test_trace_dump_subprocesses_not_added_by_default(self):
+        # By default the `--subprocesses` flag should be absent.
+        cmd = await self._capture_pyspy_cmd()
+        assert "--subprocesses" not in cmd
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/reporter/utils.py

@@ -0,0 +1,19 @@
+from typing import Optional
+
+from ray._raylet import GcsClient, NodeID
+
+
+class HealthChecker:
+    def __init__(self, gcs_client: GcsClient, local_node_id: Optional[NodeID] = None):
+        self._gcs_client = gcs_client
+        self._local_node_id = local_node_id
+
+    async def check_local_raylet_liveness(self) -> bool:
+        if self._local_node_id is None:
+            return False
+        liveness = await self._gcs_client.async_check_alive([self._local_node_id], 0.1)
+        return liveness[0]
+
+    async def check_gcs_liveness(self) -> bool:
+        await self._gcs_client.async_check_alive([], 0.1)
+        return True