chore: import upstream snapshot with attribution

python/ray/dashboard/modules/aggregator/aggregator_agent.py

@@ -0,0 +1,230 @@
+import asyncio
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+import ray
+import ray.dashboard.utils as dashboard_utils
+from ray._private import ray_constants
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.core.generated import (
+    events_event_aggregator_service_pb2,
+    events_event_aggregator_service_pb2_grpc,
+)
+from ray.dashboard.modules.aggregator.constants import AGGREGATOR_AGENT_METRIC_PREFIX
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+from ray.dashboard.modules.aggregator.publisher.async_publisher_client import (
+    AsyncGCSTaskEventsPublisherClient,
+    AsyncHttpPublisherClient,
+)
+from ray.dashboard.modules.aggregator.publisher.ray_event_publisher import (
+    NoopPublisher,
+    RayEventPublisher,
+)
+from ray.dashboard.modules.aggregator.task_events_metadata_buffer import (
+    TaskEventsMetadataBuffer,
+)
+
+logger = logging.getLogger(__name__)
+
+# Max number of threads for the thread pool executor handling CPU intensive tasks
+THREAD_POOL_EXECUTOR_MAX_WORKERS = ray_constants.env_integer(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_THREAD_POOL_EXECUTOR_MAX_WORKERS", 1
+)
+# Interval to check the main thread liveness
+CHECK_MAIN_THREAD_LIVENESS_INTERVAL_SECONDS = ray_constants.env_float(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_CHECK_MAIN_THREAD_LIVENESS_INTERVAL_SECONDS", 0.1
+)
+# Maximum size of the event buffer in the aggregator agent
+# The default value was 1,000,000 but was reduced to 100,000 now to avoid being OOM Killed.
+# We observed that the previous 1,000,000 could take up to 20 GB of memory.
+# TODO (rueian): Find a better way for the event buffer to store events while avoiding being OOM Killed. For example:
+# 1. Store bytes instead of python objects and count the size in bytes.
+# 2. Compress the bytes before storing them in the buffer? (This will increase the CPU usage)
+# 3. Don't be fixed at 10,0000 but adjust the buffer size based on the available memory on startup.
+MAX_EVENT_BUFFER_SIZE = ray_constants.env_integer(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_BUFFER_SIZE", 100000
+)
+# Maximum number of events to send in a single batch to the destination
+MAX_EVENT_SEND_BATCH_SIZE = ray_constants.env_integer(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_SEND_BATCH_SIZE", 1000
+)
+# Address of the external service to send events with format of "http://<ip>:<port>"
+EVENTS_EXPORT_ADDR = os.environ.get(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR", ""
+)
+# flag to enable publishing events to the external HTTP service
+PUBLISH_EVENTS_TO_EXTERNAL_HTTP_SERVICE = ray_constants.env_bool(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISH_EVENTS_TO_EXTERNAL_HTTP_SERVICE", True
+)
+# flag to enable publishing events to GCS
+PUBLISH_EVENTS_TO_GCS = ray_constants.env_bool(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISH_EVENTS_TO_GCS", False
+)
+# flag to control whether preserve the proto field name when converting the events to
+# JSON. If True, the proto field name will be preserved. If False, the proto field name
+# will be converted to camel case.
+PRESERVE_PROTO_FIELD_NAME = ray_constants.env_bool(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_PRESERVE_PROTO_FIELD_NAME", False
+)
+
+
+class AggregatorAgent(
+    dashboard_utils.DashboardAgentModule,
+    events_event_aggregator_service_pb2_grpc.EventAggregatorServiceServicer,
+):
+    """
+    AggregatorAgent is a dashboard agent module that collects events sent with
+    gRPC from other components, buffers them, and periodically sends them to GCS and
+    an external service with HTTP POST requests for further processing or storage
+    """
+
+    def __init__(self, dashboard_agent) -> None:
+        super().__init__(dashboard_agent)
+        self._ip = dashboard_agent.ip
+        self._pid = os.getpid()
+
+        # common prometheus labels for aggregator-owned metrics
+        self._common_tags = {
+            "ip": self._ip,
+            "pid": str(self._pid),
+            "Version": ray.__version__,
+            "Component": "aggregator_agent",
+            "SessionName": self.session_name,
+        }
+
+        self._event_buffer = MultiConsumerEventBuffer(
+            max_size=MAX_EVENT_BUFFER_SIZE,
+            max_batch_size=MAX_EVENT_SEND_BATCH_SIZE,
+            common_metric_tags=self._common_tags,
+        )
+        self._executor = ThreadPoolExecutor(
+            max_workers=THREAD_POOL_EXECUTOR_MAX_WORKERS,
+            thread_name_prefix="aggregator_agent_executor",
+        )
+
+        # Task metadata buffer accumulates dropped task attempts for GCS publishing
+        self._task_metadata_buffer = TaskEventsMetadataBuffer(
+            common_metric_tags=self._common_tags
+        )
+
+        self._events_export_addr = (
+            dashboard_agent.events_export_addr or EVENTS_EXPORT_ADDR
+        )
+
+        self._event_processing_enabled = False
+        if PUBLISH_EVENTS_TO_EXTERNAL_HTTP_SERVICE and self._events_export_addr:
+            logger.info(
+                f"Publishing events to external HTTP service is enabled. events_export_addr: {self._events_export_addr}"
+            )
+            self._event_processing_enabled = True
+            self._http_endpoint_publisher = RayEventPublisher(
+                name="http_service",
+                publish_client=AsyncHttpPublisherClient(
+                    endpoint=self._events_export_addr,
+                    executor=self._executor,
+                    preserve_proto_field_name=PRESERVE_PROTO_FIELD_NAME,
+                ),
+                event_buffer=self._event_buffer,
+                common_metric_tags=self._common_tags,
+            )
+        else:
+            logger.info(
+                f"Event HTTP target is not enabled or publishing events to external HTTP service is disabled. Skipping sending events to external HTTP service. events_export_addr: {self._events_export_addr}"
+            )
+            self._http_endpoint_publisher = NoopPublisher()
+
+        if PUBLISH_EVENTS_TO_GCS:
+            logger.info("Publishing events to GCS is enabled")
+            self._event_processing_enabled = True
+            self._gcs_publisher = RayEventPublisher(
+                name="ray_gcs",
+                publish_client=AsyncGCSTaskEventsPublisherClient(
+                    gcs_client=self._dashboard_agent.gcs_client,
+                    executor=self._executor,
+                ),
+                event_buffer=self._event_buffer,
+                common_metric_tags=self._common_tags,
+                task_metadata_buffer=self._task_metadata_buffer,
+            )
+        else:
+            logger.info("Publishing events to GCS is disabled")
+            self._gcs_publisher = NoopPublisher()
+
+        # Metrics
+        self._open_telemetry_metric_recorder = OpenTelemetryMetricRecorder()
+
+        # Register counter metrics
+        self._events_received_metric_name = (
+            f"{AGGREGATOR_AGENT_METRIC_PREFIX}_events_received_total"
+        )
+        self._open_telemetry_metric_recorder.register_counter_metric(
+            self._events_received_metric_name,
+            "Total number of events received via AddEvents gRPC.",
+        )
+
+        self._events_failed_to_add_metric_name = (
+            f"{AGGREGATOR_AGENT_METRIC_PREFIX}_events_buffer_add_failures_total"
+        )
+        self._open_telemetry_metric_recorder.register_counter_metric(
+            self._events_failed_to_add_metric_name,
+            "Total number of events that failed to be added to the event buffer.",
+        )
+
+    async def AddEvents(self, request, context) -> None:
+        """
+        gRPC handler for adding events to the event aggregator. Receives events from the
+        request and adds them to the event buffer.
+        """
+        if not self._event_processing_enabled:
+            return events_event_aggregator_service_pb2.AddEventsReply()
+
+        received_count = len(request.events_data.events)
+        failed_count = 0
+        events_data = request.events_data
+
+        if PUBLISH_EVENTS_TO_GCS:
+            self._task_metadata_buffer.merge(events_data.task_events_metadata)
+
+        for event in events_data.events:
+            try:
+                await self._event_buffer.add_event(event)
+            except Exception as e:
+                failed_count += 1
+                logger.error(
+                    f"Failed to add event with id={event.event_id.decode()} to buffer. "
+                    "Error: %s",
+                    e,
+                )
+
+        if received_count > 0:
+            self._open_telemetry_metric_recorder.set_metric_value(
+                self._events_received_metric_name, self._common_tags, received_count
+            )
+        if failed_count > 0:
+            self._open_telemetry_metric_recorder.set_metric_value(
+                self._events_failed_to_add_metric_name, self._common_tags, failed_count
+            )
+
+        return events_event_aggregator_service_pb2.AddEventsReply()
+
+    async def run(self, server) -> None:
+        if server:
+            events_event_aggregator_service_pb2_grpc.add_EventAggregatorServiceServicer_to_server(
+                self, server
+            )
+        try:
+            await asyncio.gather(
+                self._http_endpoint_publisher.run_forever(),
+                self._gcs_publisher.run_forever(),
+            )
+        finally:
+            self._executor.shutdown()
+
+    @staticmethod
+    def is_minimal_module() -> bool:
+        return False

python/ray/dashboard/modules/aggregator/constants.py

@@ -0,0 +1,2 @@
+AGGREGATOR_AGENT_METRIC_PREFIX = "aggregator_agent"
+CONSUMER_TAG_KEY = "consumer"

python/ray/dashboard/modules/aggregator/multi_consumer_event_buffer.py

@@ -0,0 +1,194 @@
+import asyncio
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.core.generated import (
+    events_base_event_pb2,
+)
+from ray.core.generated.events_base_event_pb2 import RayEvent
+from ray.dashboard.modules.aggregator.constants import (
+    AGGREGATOR_AGENT_METRIC_PREFIX,
+    CONSUMER_TAG_KEY,
+)
+
+
+@dataclass
+class _ConsumerState:
+    # Index of the next event to be consumed by this consumer
+    cursor_index: int
+
+
+class MultiConsumerEventBuffer:
+    """A buffer which allows adding one event at a time and consuming events in batches.
+    Supports multiple consumers, each with their own cursor index. Tracks the number of events evicted for each consumer.
+
+    Buffer is not thread-safe but is asyncio-friendly. All operations must be called from within the same event loop.
+
+    Arguments:
+        max_size: Maximum number of events to store in the buffer.
+        max_batch_size: Maximum number of events to return in a batch when calling wait_for_batch.
+        common_metric_tags: Tags to add to all metrics.
+    """
+
+    def __init__(
+        self,
+        max_size: int,
+        max_batch_size: int,
+        common_metric_tags: Optional[Dict[str, str]] = None,
+    ):
+        self._buffer = deque(maxlen=max_size)
+        self._max_size = max_size
+        self._lock = asyncio.Lock()
+        self._has_new_events_to_consume = asyncio.Condition(self._lock)
+        self._consumers: Dict[str, _ConsumerState] = {}
+
+        self._max_batch_size = max_batch_size
+
+        self._common_metrics_tags = common_metric_tags or {}
+        self._metric_recorder = OpenTelemetryMetricRecorder()
+        self.evicted_events_metric_name = (
+            f"{AGGREGATOR_AGENT_METRIC_PREFIX}_queue_dropped_events"
+        )
+        self._metric_recorder.register_counter_metric(
+            self.evicted_events_metric_name,
+            "Total number of events dropped because the publish/buffer queue was full.",
+        )
+
+    async def add_event(self, event: events_base_event_pb2.RayEvent) -> None:
+        """Add an event to the buffer.
+
+        If the buffer is full, the oldest event is dropped.
+        """
+        async with self._lock:
+            dropped_event = None
+            if len(self._buffer) >= self._max_size:
+                dropped_event = self._buffer.popleft()
+            self._buffer.append(event)
+
+            if dropped_event is not None:
+                for consumer_name, consumer_state in self._consumers.items():
+                    # Update consumer cursor index and evicted events metric if an event was dropped
+                    if consumer_state.cursor_index == 0:
+                        # The dropped event was the next event this consumer would have consumed, publish eviction metric
+                        self._metric_recorder.set_metric_value(
+                            self.evicted_events_metric_name,
+                            {
+                                **self._common_metrics_tags,
+                                CONSUMER_TAG_KEY: consumer_name,
+                                "event_type": RayEvent.EventType.Name(
+                                    dropped_event.event_type
+                                ),
+                            },
+                            1,
+                        )
+                    else:
+                        # The dropped event was already consumed by the consumer, so we need to adjust the cursor
+                        consumer_state.cursor_index -= 1
+
+            # Signal the consumers that there are new events to consume
+            self._has_new_events_to_consume.notify_all()
+
+    def _evict_old_events(self) -> None:
+        """Clean the buffer by removing events from the buffer who have index lower than
+        all the cursor indexes of all consumers and updating the cursor index of all
+        consumers.
+        """
+        if not self._consumers:
+            return
+
+        min_cursor_index = min(
+            consumer_state.cursor_index for consumer_state in self._consumers.values()
+        )
+        for _ in range(min_cursor_index):
+            self._buffer.popleft()
+
+        # update the cursor index of all consumers
+        for consumer_state in self._consumers.values():
+            consumer_state.cursor_index -= min_cursor_index
+
+    async def wait_for_batch(
+        self, consumer_name: str, timeout_seconds: float = 1.0
+    ) -> List[events_base_event_pb2.RayEvent]:
+        """Wait for batch respecting self.max_batch_size and timeout_seconds.
+
+        Returns a batch of up to self.max_batch_size items. Waits for up to
+        timeout_seconds after receiving the first event that will be in
+        the next batch. After the timeout, returns as many items as are ready.
+
+        Always returns a batch with at least one item - will block
+        indefinitely until an item comes in.
+
+        Arguments:
+            consumer_name: name of the consumer consuming the batch
+            timeout_seconds: maximum time to wait for a batch
+
+        Returns:
+            A list of up to max_batch_size events ready for consumption.
+            The list always contains at least one event.
+        """
+        max_batch = self._max_batch_size
+        batch = []
+        async with self._has_new_events_to_consume:
+            consumer_state = self._consumers.get(consumer_name)
+            if consumer_state is None:
+                raise KeyError(f"unknown consumer '{consumer_name}'")
+
+            # Phase 1: read the first event, wait indefinitely until there is at least one event to consume
+            while consumer_state.cursor_index >= len(self._buffer):
+                await self._has_new_events_to_consume.wait()
+
+            # Add the first event to the batch
+            event = self._buffer[consumer_state.cursor_index]
+            consumer_state.cursor_index += 1
+            batch.append(event)
+
+            # Phase 2: add items to the batch up to timeout or until full
+            deadline = time.monotonic() + max(0.0, float(timeout_seconds))
+            while len(batch) < max_batch:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    break
+
+                # Drain whatever is available
+                while len(batch) < max_batch and consumer_state.cursor_index < len(
+                    self._buffer
+                ):
+                    batch.append(self._buffer[consumer_state.cursor_index])
+                    consumer_state.cursor_index += 1
+
+                if len(batch) >= max_batch:
+                    break
+
+                # There is still room in the batch, but no new events to consume; wait until notified or timeout
+                try:
+                    await asyncio.wait_for(
+                        self._has_new_events_to_consume.wait(), remaining
+                    )
+                except asyncio.TimeoutError:
+                    # Timeout, return the current batch
+                    break
+
+            self._evict_old_events()
+        return batch
+
+    async def register_consumer(self, consumer_name: str) -> None:
+        """Register a new consumer with a name.
+
+        Arguments:
+            consumer_name: A unique name for the consumer.
+
+        """
+        async with self._lock:
+            if self._consumers.get(consumer_name) is not None:
+                raise ValueError(f"consumer '{consumer_name}' already registered")
+
+            self._consumers[consumer_name] = _ConsumerState(cursor_index=0)
+
+    async def size(self) -> int:
+        """Get total number of events in the buffer. Does not take consumer cursors into account."""
+        return len(self._buffer)

python/ray/dashboard/modules/aggregator/publisher/async_publisher_client.py

@@ -0,0 +1,291 @@
+import json
+import logging
+from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import List, Optional
+
+import aiohttp
+
+import ray.dashboard.utils as dashboard_utils
+from ray._common.utils import get_or_create_event_loop
+from ray._private.protobuf_compat import message_to_json
+from ray._raylet import GcsClient
+from ray.core.generated import (
+    events_base_event_pb2,
+    events_event_aggregator_service_pb2,
+)
+from ray.dashboard.modules.aggregator.publisher.configs import (
+    GCS_EXPOSABLE_EVENT_TYPES,
+    HTTP_EXPOSABLE_EVENT_TYPES,
+    PUBLISHER_TIMEOUT_SECONDS,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PublishStats:
+    """Data class that represents stats of publishing a batch of events."""
+
+    # Whether the publish was successful
+    is_publish_successful: bool
+    # Number of events published
+    num_events_published: int
+    # Number of events filtered out
+    num_events_filtered_out: int
+
+
+@dataclass
+class PublishBatch:
+    """Data class that represents a batch of events to publish."""
+
+    # The list of events to publish
+    events: list[events_base_event_pb2.RayEvent]
+    # dropped task events metadata
+    task_events_metadata: Optional[
+        events_event_aggregator_service_pb2.TaskEventsMetadata
+    ] = None
+
+
+class PublisherClientInterface(ABC):
+    """Abstract interface for publishing Ray event batches to external destinations.
+
+    Implementations should handle the actual publishing logic, filtering,
+    and format conversion appropriate for their specific destination type.
+    """
+
+    def __init__(self):
+        self._exposable_event_types_list: List[str] = []
+        self._allow_all_event_types: bool = False
+
+    def count_num_events_in_batch(self, batch: PublishBatch) -> int:
+        """Count the number of events in a given PublishBatch."""
+        return len(batch.events)
+
+    def _can_expose_event(self, event) -> bool:
+        """
+        Check if an event should be allowed to be published.
+        """
+        if self._allow_all_event_types:
+            return True
+        if not self._exposable_event_types_list:
+            return False
+
+        event_type_name = events_base_event_pb2.RayEvent.EventType.Name(
+            event.event_type
+        )
+        return event_type_name in self._exposable_event_types_list
+
+    @abstractmethod
+    async def publish(self, batch: PublishBatch) -> PublishStats:
+        """Publish a batch of events to the destination."""
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Clean up any resources used by this client. Should be called when the publisherClient is no longer required"""
+        pass
+
+
+class AsyncHttpPublisherClient(PublisherClientInterface):
+    """Client for publishing ray event batches to an external HTTP service."""
+
+    def __init__(
+        self,
+        endpoint: str,
+        executor: ThreadPoolExecutor,
+        timeout: float = PUBLISHER_TIMEOUT_SECONDS,
+        preserve_proto_field_name: bool = False,
+    ) -> None:
+        super().__init__()
+        self._endpoint = endpoint
+        self._executor = executor
+        self._timeout = aiohttp.ClientTimeout(total=timeout)
+        self._session = None
+        self._preserve_proto_field_name = preserve_proto_field_name
+
+        if HTTP_EXPOSABLE_EVENT_TYPES.strip().upper() == "ALL":
+            self._allow_all_event_types = True
+            self._exposable_event_types_list = []
+        else:
+            self._exposable_event_types_list = [
+                event_type.strip()
+                for event_type in HTTP_EXPOSABLE_EVENT_TYPES.split(",")
+                if event_type.strip()
+            ]
+
+    async def publish(self, batch: PublishBatch) -> PublishStats:
+        events_batch: list[events_base_event_pb2.RayEvent] = batch.events
+        if not events_batch:
+            # Nothing to publish -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+        filtered = [e for e in events_batch if self._can_expose_event(e)]
+        num_filtered_out = len(events_batch) - len(filtered)
+        if not filtered:
+            # All filtered out -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=num_filtered_out,
+            )
+
+        # Convert protobuf objects to python dictionaries for HTTP POST. Run in executor to avoid blocking the event loop.
+        filtered_json = await get_or_create_event_loop().run_in_executor(
+            self._executor,
+            lambda: [
+                json.loads(
+                    message_to_json(
+                        e,
+                        always_print_fields_with_no_presence=True,
+                        preserving_proto_field_name=self._preserve_proto_field_name,
+                    )
+                )
+                for e in filtered
+            ],
+        )
+
+        try:
+            # Create session on first use (lazy initialization)
+            if not self._session:
+                self._session = aiohttp.ClientSession(timeout=self._timeout)
+
+            return await self._send_http_request(filtered_json, num_filtered_out)
+        except Exception as e:
+            logger.error("Failed to send events to external service. Error: %r", e)
+            return PublishStats(
+                is_publish_successful=False,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+
+    async def _send_http_request(self, json_data, num_filtered_out) -> PublishStats:
+        async with self._session.post(
+            self._endpoint,
+            json=json_data,
+        ) as resp:
+            resp.raise_for_status()
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=len(json_data),
+                num_events_filtered_out=num_filtered_out,
+            )
+
+    async def close(self) -> None:
+        """Closes the http session if one was created. Should be called when the publisherClient is no longer required"""
+        if self._session:
+            await self._session.close()
+            self._session = None
+
+    def set_session(self, session) -> None:
+        """Inject an HTTP client session.
+
+        If a session is set explicitly, it will be used and managed by close().
+        """
+        self._session = session
+
+
+class AsyncGCSTaskEventsPublisherClient(PublisherClientInterface):
+    """Client for publishing ray event batches to GCS."""
+
+    def __init__(
+        self,
+        gcs_client: GcsClient,
+        executor: ThreadPoolExecutor,
+        timeout_s: float = PUBLISHER_TIMEOUT_SECONDS,
+    ) -> None:
+        super().__init__()
+        self._gcs_client = gcs_client
+        self._executor = executor
+        self._timeout_s = timeout_s
+
+        self._exposable_event_types_list = GCS_EXPOSABLE_EVENT_TYPES
+
+    async def publish(
+        self,
+        batch: PublishBatch,
+    ) -> PublishStats:
+        events = batch.events
+        task_events_metadata = batch.task_events_metadata
+        has_dropped_task_attempts = (
+            task_events_metadata and task_events_metadata.dropped_task_attempts
+        )
+        if not events and not has_dropped_task_attempts:
+            # Nothing to publish -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+
+        # Filter events based on exposable event types
+        filtered_events = [e for e in events if self._can_expose_event(e)]
+        num_filtered_out = len(events) - len(filtered_events)
+
+        if not filtered_events and not has_dropped_task_attempts:
+            # all events filtered out and no task events metadata -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=num_filtered_out,
+            )
+
+        try:
+            events_data = self._create_ray_events_data(
+                filtered_events, task_events_metadata
+            )
+            request = events_event_aggregator_service_pb2.AddEventsRequest(
+                events_data=events_data
+            )
+            serialized_request = await get_or_create_event_loop().run_in_executor(
+                self._executor,
+                lambda: request.SerializeToString(),
+            )
+            status_code = await self._gcs_client.async_add_events(
+                serialized_request, self._timeout_s, self._executor
+            )
+
+            if status_code != dashboard_utils.HTTPStatusCode.OK:
+                logger.error(f"GCS AddEvents failed: {status_code}")
+                return PublishStats(
+                    is_publish_successful=False,
+                    num_events_published=0,
+                    num_events_filtered_out=0,
+                )
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=len(filtered_events),
+                num_events_filtered_out=num_filtered_out,
+            )
+        except Exception as e:
+            logger.error(f"Failed to send events to GCS: {e}")
+            return PublishStats(
+                is_publish_successful=False,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+
+    def _create_ray_events_data(
+        self,
+        event_batch: List[events_base_event_pb2.RayEvent],
+        task_events_metadata: Optional[
+            events_event_aggregator_service_pb2.TaskEventsMetadata
+        ] = None,
+    ) -> events_event_aggregator_service_pb2.RayEventsData:
+        """
+        Helper method to create RayEventsData from event batch and metadata.
+        """
+        events_data = events_event_aggregator_service_pb2.RayEventsData()
+        events_data.events.extend(event_batch)
+
+        if task_events_metadata:
+            events_data.task_events_metadata.CopyFrom(task_events_metadata)
+
+        return events_data
+
+    async def close(self) -> None:
+        pass

python/ray/dashboard/modules/aggregator/publisher/configs.py

@@ -0,0 +1,54 @@
+# Environment variables for the aggregator agent publisher component.
+import os
+
+from ray._private import ray_constants
+
+env_var_prefix = "RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER"
+# Timeout for the publisher to publish events to the destination
+PUBLISHER_TIMEOUT_SECONDS = ray_constants.env_integer(
+    f"{env_var_prefix}_TIMEOUT_SECONDS", 3
+)
+# Maximum number of retries for publishing events to the destination, if less than 0, will retry indefinitely
+PUBLISHER_MAX_RETRIES = ray_constants.env_integer(f"{env_var_prefix}_MAX_RETRIES", -1)
+# Initial backoff time for publishing events to the destination
+PUBLISHER_INITIAL_BACKOFF_SECONDS = ray_constants.env_float(
+    f"{env_var_prefix}_INITIAL_BACKOFF_SECONDS", 0.01
+)
+# Maximum backoff time for publishing events to the destination
+PUBLISHER_MAX_BACKOFF_SECONDS = ray_constants.env_float(
+    f"{env_var_prefix}_MAX_BACKOFF_SECONDS", 5.0
+)
+# Jitter ratio for publishing events to the destination
+PUBLISHER_JITTER_RATIO = ray_constants.env_float(f"{env_var_prefix}_JITTER_RATIO", 0.1)
+# Maximum sleep time between sending batches of events to the destination, should be greater than 0.0 to avoid busy looping
+PUBLISHER_MAX_BUFFER_SEND_INTERVAL_SECONDS = ray_constants.env_float(
+    f"{env_var_prefix}_MAX_BUFFER_SEND_INTERVAL_SECONDS", 0.1
+)
+
+# HTTP Publisher specific configurations
+# Comma-separated list of event types that are allowed to be exposed to external HTTP services
+# Valid values: TASK_DEFINITION_EVENT, TASK_LIFECYCLE_EVENT, ACTOR_TASK_DEFINITION_EVENT, etc.
+# Set to "ALL" to allow all event types.
+# The list of all supported event types can be found in src/ray/protobuf/public/events_base_event.proto (EventType enum)
+# By default TASK_PROFILE_EVENT is not exposed to external services
+DEFAULT_HTTP_EXPOSABLE_EVENT_TYPES = (
+    "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT,"
+    "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT,"
+    "ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,"
+    "NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT,"
+    "PLATFORM_EVENT,"
+)
+HTTP_EXPOSABLE_EVENT_TYPES = os.environ.get(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES",
+    DEFAULT_HTTP_EXPOSABLE_EVENT_TYPES,
+)
+
+# GCS Publisher specific configurations
+# List of event types that are allowed to be exposed to GCS, not overridden by environment variable
+# as GCS only supports Task event types
+GCS_EXPOSABLE_EVENT_TYPES = [
+    "TASK_DEFINITION_EVENT",
+    "TASK_LIFECYCLE_EVENT",
+    "TASK_PROFILE_EVENT",
+    "ACTOR_TASK_DEFINITION_EVENT",
+]

python/ray/dashboard/modules/aggregator/publisher/metrics.py

@@ -0,0 +1,53 @@
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.dashboard.modules.aggregator.constants import (
+    AGGREGATOR_AGENT_METRIC_PREFIX,
+)
+
+# OpenTelemetry metrics setup (registered once at import time)
+metric_recorder = OpenTelemetryMetricRecorder()
+
+# Counter metrics
+published_counter_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_published_events"
+metric_recorder.register_counter_metric(
+    published_counter_name,
+    "Total number of events successfully published to the destination.",
+)
+
+filtered_counter_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_filtered_events"
+metric_recorder.register_counter_metric(
+    filtered_counter_name,
+    "Total number of events filtered out before publishing to the destination.",
+)
+
+failed_counter_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_publish_failures"
+metric_recorder.register_counter_metric(
+    failed_counter_name,
+    "Total number of events that failed to publish after retries.",
+)
+
+# Histogram metric
+publish_latency_hist_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_publish_latency_seconds"
+metric_recorder.register_histogram_metric(
+    publish_latency_hist_name,
+    "Duration of publish calls in seconds.",
+    [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5],
+)
+
+# Gauge metrics
+consecutive_failures_gauge_name = (
+    f"{AGGREGATOR_AGENT_METRIC_PREFIX}_consecutive_failures_since_last_success"
+)
+metric_recorder.register_gauge_metric(
+    consecutive_failures_gauge_name,
+    "Number of consecutive failed publish attempts since the last success.",
+)
+
+time_since_last_success_gauge_name = (
+    f"{AGGREGATOR_AGENT_METRIC_PREFIX}_time_since_last_success_seconds"
+)
+metric_recorder.register_gauge_metric(
+    time_since_last_success_gauge_name,
+    "Seconds since the last successful publish to the destination.",
+)

python/ray/dashboard/modules/aggregator/publisher/ray_event_publisher.py

@@ -0,0 +1,289 @@
+import asyncio
+import logging
+import random
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+from ray.dashboard.modules.aggregator.constants import (
+    CONSUMER_TAG_KEY,
+)
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+from ray.dashboard.modules.aggregator.publisher.async_publisher_client import (
+    PublishBatch,
+    PublisherClientInterface,
+)
+from ray.dashboard.modules.aggregator.publisher.configs import (
+    PUBLISHER_INITIAL_BACKOFF_SECONDS,
+    PUBLISHER_JITTER_RATIO,
+    PUBLISHER_MAX_BACKOFF_SECONDS,
+    PUBLISHER_MAX_BUFFER_SEND_INTERVAL_SECONDS,
+    PUBLISHER_MAX_RETRIES,
+)
+from ray.dashboard.modules.aggregator.publisher.metrics import (
+    consecutive_failures_gauge_name,
+    failed_counter_name,
+    filtered_counter_name,
+    metric_recorder,
+    publish_latency_hist_name,
+    published_counter_name,
+    time_since_last_success_gauge_name,
+)
+from ray.dashboard.modules.aggregator.task_events_metadata_buffer import (
+    TaskEventsMetadataBuffer,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class RayEventPublisherInterface(ABC):
+    """Abstract interface for publishing Ray event batches to external destinations."""
+
+    @abstractmethod
+    async def run_forever(self) -> None:
+        """Run the publisher forever until cancellation or process death."""
+        pass
+
+    @abstractmethod
+    async def wait_until_running(self, timeout: Optional[float] = None) -> bool:
+        """Wait until the publisher has started."""
+        pass
+
+
+class RayEventPublisher(RayEventPublisherInterface):
+    """RayEvents publisher that publishes batches of events to a destination by running a worker loop.
+
+    The worker loop continuously pulls batches from the event buffer and publishes them to the destination.
+    """
+
+    # Cap the exponent to avoid computing unnecessarily large intermediate
+    _MAX_BACKOFF_EXPONENT = 30
+
+    def __init__(
+        self,
+        name: str,
+        publish_client: PublisherClientInterface,
+        event_buffer: MultiConsumerEventBuffer,
+        common_metric_tags: Optional[Dict[str, str]] = None,
+        task_metadata_buffer: Optional[TaskEventsMetadataBuffer] = None,
+        max_retries: int = PUBLISHER_MAX_RETRIES,
+        initial_backoff: float = PUBLISHER_INITIAL_BACKOFF_SECONDS,
+        max_backoff: float = PUBLISHER_MAX_BACKOFF_SECONDS,
+        jitter_ratio: float = PUBLISHER_JITTER_RATIO,
+    ) -> None:
+        """Initialize a RayEventsPublisher.
+
+        Args:
+            name: Name identifier for this publisher instance
+            publish_client: Client for publishing events to the destination
+            event_buffer: Buffer for reading batches of events
+            common_metric_tags: Common labels for all prometheus metrics
+            task_metadata_buffer: Buffer for reading a batch of droppedtask metadata
+            max_retries: Maximum number of retries for failed publishes
+            initial_backoff: Initial backoff time between retries in seconds
+            max_backoff: Maximum backoff time between retries in seconds
+            jitter_ratio: Random jitter ratio to add to backoff times
+        """
+        self._name = name
+        self._common_metric_tags = dict(common_metric_tags or {})
+        self._common_metric_tags[CONSUMER_TAG_KEY] = name
+        self._max_retries = int(max_retries)
+        self._initial_backoff = float(initial_backoff)
+        self._max_backoff = float(max_backoff)
+        self._jitter_ratio = float(jitter_ratio)
+        self._publish_client = publish_client
+        self._event_buffer = event_buffer
+        self._task_metadata_buffer = task_metadata_buffer
+
+        # Event set once the publisher has registered as a consumer and is ready to publish events
+        self._started_event: asyncio.Event = asyncio.Event()
+
+    async def run_forever(self) -> None:
+        """Run the publisher forever until cancellation or process death.
+
+        Registers as a consumer, starts the worker loop, and handles cleanup on cancellation.
+        """
+        await self._event_buffer.register_consumer(self._name)
+
+        # Signal that the publisher is ready to publish events
+        self._started_event.set()
+
+        try:
+            logger.info(f"Starting publisher {self._name}")
+            while True:
+                events_batch = await self._event_buffer.wait_for_batch(
+                    self._name,
+                    PUBLISHER_MAX_BUFFER_SEND_INTERVAL_SECONDS,
+                )
+                publish_batch = PublishBatch(events=events_batch)
+
+                if self._task_metadata_buffer is not None:
+                    task_metadata_batch = self._task_metadata_buffer.get()
+                    publish_batch.task_events_metadata = task_metadata_batch
+
+                await self._async_publish_with_retries(publish_batch)
+        except asyncio.CancelledError:
+            logger.info(f"Publisher {self._name} cancelled, shutting down gracefully")
+            raise
+        except Exception as e:
+            logger.error(f"Publisher {self._name} encountered error: {e}")
+            raise
+        finally:
+            self._started_event.clear()
+            await self._publish_client.close()
+
+    async def wait_until_running(self, timeout: Optional[float] = None) -> bool:
+        """Wait until the publisher has started.
+
+        Args:
+            timeout: Maximum time to wait in seconds. If None, waits indefinitely.
+
+        Returns:
+            True if the publisher started before the timeout, False otherwise.
+            If timeout is None, waits indefinitely.
+        """
+        if timeout is None:
+            await self._started_event.wait()
+            return True
+        try:
+            await asyncio.wait_for(self._started_event.wait(), timeout)
+            return True
+        except asyncio.TimeoutError:
+            return False
+
+    async def _async_publish_with_retries(self, batch) -> None:
+        """Attempts to publish a batch with retries.
+
+        Will retry failed publishes up to max_retries times with increasing delays.
+        """
+        num_events_in_batch = self._publish_client.count_num_events_in_batch(batch)
+        failed_attempts_since_last_success = 0
+        while True:
+            start = asyncio.get_running_loop().time()
+            result = await self._publish_client.publish(batch)
+            duration = asyncio.get_running_loop().time() - start
+
+            if result.is_publish_successful:
+                await self._record_success(
+                    num_published=int(result.num_events_published),
+                    num_filtered=int(result.num_events_filtered_out),
+                    duration=float(duration),
+                )
+                failed_attempts_since_last_success = 0
+                return
+
+            # Failed attempt
+            # case 1: if max retries are exhausted mark as failed and break out, retry indefinitely if max_retries is less than 0
+            if (
+                self._max_retries >= 0
+                and failed_attempts_since_last_success >= self._max_retries
+            ):
+                await self._record_final_failure(
+                    num_failed_events=int(num_events_in_batch),
+                    duration=float(duration),
+                )
+                return
+
+            # case 2: max retries not exhausted, increment failed attempts counter and add latency to failure list, retry publishing batch with backoff
+            failed_attempts_since_last_success += 1
+            await self._record_retry_failure(
+                duration=float(duration),
+                failed_attempts=int(failed_attempts_since_last_success),
+            )
+
+            await self._async_sleep_with_backoff(failed_attempts_since_last_success)
+
+    async def _async_sleep_with_backoff(self, attempt: int) -> None:
+        """Sleep with exponential backoff and optional jitter.
+
+        Args:
+            attempt: The current attempt number (0-based)
+        """
+        capped_attempt = min(attempt, self._MAX_BACKOFF_EXPONENT)
+        delay = min(
+            self._max_backoff,
+            self._initial_backoff * (2**capped_attempt),
+        )
+        if self._jitter_ratio > 0:
+            jitter = delay * self._jitter_ratio
+            delay = max(0.0, random.uniform(delay - jitter, delay + jitter))
+        await asyncio.sleep(delay)
+
+    async def _record_success(
+        self, num_published: int, num_filtered: int, duration: float
+    ) -> None:
+        """Update in-memory stats and Prometheus metrics for a successful publish."""
+        if num_published > 0:
+            metric_recorder.set_metric_value(
+                published_counter_name,
+                self._common_metric_tags,
+                int(num_published),
+            )
+        if num_filtered > 0:
+            metric_recorder.set_metric_value(
+                filtered_counter_name, self._common_metric_tags, int(num_filtered)
+            )
+        metric_recorder.set_metric_value(
+            consecutive_failures_gauge_name, self._common_metric_tags, 0
+        )
+        metric_recorder.set_metric_value(
+            time_since_last_success_gauge_name, self._common_metric_tags, 0
+        )
+        metric_recorder.set_metric_value(
+            publish_latency_hist_name,
+            {**self._common_metric_tags, "Outcome": "success"},
+            float(duration),
+        )
+
+    async def _record_retry_failure(
+        self, duration: float, failed_attempts: int
+    ) -> None:
+        """Update Prometheus metrics for a retryable failure attempt."""
+        metric_recorder.set_metric_value(
+            consecutive_failures_gauge_name,
+            self._common_metric_tags,
+            int(failed_attempts),
+        )
+        metric_recorder.set_metric_value(
+            publish_latency_hist_name,
+            {**self._common_metric_tags, "Outcome": "failure"},
+            float(duration),
+        )
+
+    async def _record_final_failure(
+        self, num_failed_events: int, duration: float
+    ) -> None:
+        """Update in-memory stats and Prometheus metrics for a final (non-retryable) failure."""
+        if num_failed_events > 0:
+            metric_recorder.set_metric_value(
+                failed_counter_name,
+                self._common_metric_tags,
+                int(num_failed_events),
+            )
+        metric_recorder.set_metric_value(
+            consecutive_failures_gauge_name, self._common_metric_tags, 0
+        )
+        metric_recorder.set_metric_value(
+            publish_latency_hist_name,
+            {**self._common_metric_tags, "Outcome": "failure"},
+            float(duration),
+        )
+
+
+class NoopPublisher(RayEventPublisherInterface):
+    """A no-op publisher that adheres to the minimal interface used by AggregatorAgent.
+
+    Used when a destination is disabled. It runs forever but does nothing.
+    """
+
+    async def run_forever(self) -> None:
+        """Run forever doing nothing until cancellation."""
+        try:
+            await asyncio.Event().wait()
+        except asyncio.CancelledError:
+            logger.info("NoopPublisher cancelled")
+            raise
+
+    async def wait_until_running(self, timeout: Optional[float] = None) -> bool:
+        return True

python/ray/dashboard/modules/aggregator/task_events_metadata_buffer.py

@@ -0,0 +1,85 @@
+from collections import deque
+from typing import Dict, Optional
+
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.core.generated import events_event_aggregator_service_pb2
+from ray.dashboard.modules.aggregator.constants import AGGREGATOR_AGENT_METRIC_PREFIX
+
+
+class TaskEventsMetadataBuffer:
+    """Buffer for accumulating task event metadata and batching it into a bounded queue.
+
+    This buffer is used to construct TaskEventsMetadata protobuf messages (defined in events_event_aggregator_service.proto).
+    """
+
+    def __init__(
+        self,
+        max_buffer_size: int = 1000,
+        max_dropped_attempts_per_metadata_entry: int = 100,
+        common_metric_tags: Optional[Dict[str, str]] = None,
+    ):
+        self._buffer_maxlen = max(
+            max_buffer_size - 1, 1
+        )  # -1 to account for the current batch
+        self._buffer = deque(maxlen=self._buffer_maxlen)
+        self._current_metadata_batch = (
+            events_event_aggregator_service_pb2.TaskEventsMetadata()
+        )
+        self._max_dropped_attempts = max_dropped_attempts_per_metadata_entry
+
+        self._common_metric_tags = common_metric_tags or {}
+        self._metric_recorder = OpenTelemetryMetricRecorder()
+        self._dropped_metadata_count_metric_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_task_metadata_buffer_dropped_attempts_total"
+        self._metric_recorder.register_counter_metric(
+            self._dropped_metadata_count_metric_name,
+            "Total number of dropped task attempt metadata entries which were dropped due to buffer being full",
+        )
+
+    def merge(
+        self,
+        new_metadata: Optional[events_event_aggregator_service_pb2.TaskEventsMetadata],
+    ) -> None:
+        """Merge new task event metadata into the current entry, enqueuing when limits are reached."""
+        if new_metadata is None:
+            return
+
+        for new_attempt in new_metadata.dropped_task_attempts:
+            if (
+                len(self._current_metadata_batch.dropped_task_attempts)
+                >= self._max_dropped_attempts
+            ):
+                # Add current metadata to buffer, if buffer is full, drop the oldest entry
+                if len(self._buffer) >= self._buffer_maxlen:
+                    # Record the number of dropped attempts
+                    oldest_entry = self._buffer.popleft()
+                    self._metric_recorder.set_metric_value(
+                        self._dropped_metadata_count_metric_name,
+                        self._common_metric_tags,
+                        len(oldest_entry.dropped_task_attempts),
+                    )
+
+                # Enqueue current metadata batch and start a new batch
+                metadata_copy = events_event_aggregator_service_pb2.TaskEventsMetadata()
+                metadata_copy.CopyFrom(self._current_metadata_batch)
+                self._buffer.append(metadata_copy)
+                self._current_metadata_batch.Clear()
+
+            # Now add the new attempt
+            new_entry = self._current_metadata_batch.dropped_task_attempts.add()
+            new_entry.CopyFrom(new_attempt)
+
+    def get(self) -> events_event_aggregator_service_pb2.TaskEventsMetadata:
+        """Return the next buffered metadata entry or a snapshot of the current one and reset state."""
+        if len(self._buffer) == 0:
+            # create a copy of the current metadata and return it
+            current_metadata = events_event_aggregator_service_pb2.TaskEventsMetadata()
+            current_metadata.CopyFrom(self._current_metadata_batch)
+
+            # Reset the current metadata and start merging afresh
+            self._current_metadata_batch.Clear()
+
+            return current_metadata
+
+        return self._buffer.popleft()

python/ray/dashboard/modules/aggregator/tests/test_multi_consumer_event_buffer.py

@@ -0,0 +1,270 @@
+import asyncio
+import random
+import sys
+
+import pytest
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from ray.core.generated.events_base_event_pb2 import RayEvent
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+
+
+def _create_test_event(
+    event_id: bytes = b"test",
+    event_type_enum=RayEvent.EventType.TASK_DEFINITION_EVENT,
+    message: str = "test message",
+):
+    """Helper function to create a test RayEvent."""
+    event = RayEvent()
+    event.event_id = event_id
+    event.source_type = RayEvent.SourceType.CORE_WORKER
+    event.event_type = event_type_enum
+    event.severity = RayEvent.Severity.INFO
+    event.message = message
+    event.session_name = "test_session"
+
+    # Set timestamp
+    timestamp = Timestamp()
+    timestamp.GetCurrentTime()
+    event.timestamp.CopyFrom(timestamp)
+
+    return event
+
+
+class TestMultiConsumerEventBuffer:
+    @pytest.mark.asyncio
+    async def test_add_and_consume_event_basic(self):
+        """Test basic event addition."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+        assert await buffer.size() == 0
+
+        event = _create_test_event(b"event1")
+        await buffer.add_event(event)
+
+        assert await buffer.size() == 1
+
+        batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0)
+        assert len(batch) == 1
+        assert batch[0] == event
+
+    @pytest.mark.asyncio
+    async def test_add_event_buffer_overflow(self):
+        """Test buffer overflow behavior and eviction logic."""
+        buffer = MultiConsumerEventBuffer(max_size=3, max_batch_size=2)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        # Add events to fill buffer
+        events = []
+        event_types = [
+            RayEvent.EventType.TASK_DEFINITION_EVENT,
+            RayEvent.EventType.TASK_LIFECYCLE_EVENT,
+            RayEvent.EventType.ACTOR_TASK_DEFINITION_EVENT,
+        ]
+        for i in range(3):
+            event = _create_test_event(f"event{i}".encode(), event_types[i])
+            events.append(event)
+            await buffer.add_event(event)
+
+        assert await buffer.size() == 3
+
+        # Add one more event to trigger eviction
+        overflow_event = _create_test_event(
+            b"overflow", RayEvent.EventType.TASK_PROFILE_EVENT
+        )
+        await buffer.add_event(overflow_event)
+
+        assert await buffer.size() == 3  # Still max size
+
+    @pytest.mark.asyncio
+    async def test_wait_for_batch_multiple_events(self):
+        """Test waiting for batch when multiple events are immediately available and when when not all events are available."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=3)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        # Add multiple events
+        events = []
+        for i in range(5):
+            event = _create_test_event(f"event{i}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        # Should get max_batch_size events immediately
+        batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0.1)
+        assert len(batch) == 3  # max_batch_size
+        assert batch == events[:3]
+        # should now get the leftover events (< max_batch_size)
+        batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0.1)
+        assert len(batch) == 2
+        assert batch == events[3:]
+
+    @pytest.mark.asyncio
+    async def test_wait_for_batch_unknown_consumer(self):
+        """Test error handling for unknown consumer."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+
+        with pytest.raises(KeyError, match="unknown consumer"):
+            await buffer.wait_for_batch("nonexistent_consumer", timeout_seconds=0)
+
+    @pytest.mark.asyncio
+    async def test_register_consumer_duplicate(self):
+        """Test error handling for duplicate consumer registration."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+        with pytest.raises(
+            ValueError, match="consumer 'test_consumer' already registered"
+        ):
+            await buffer.register_consumer(consumer_name)
+
+    @pytest.mark.asyncio
+    async def test_multiple_consumers_independent_cursors(self):
+        """Test that multiple consumers have independent cursors."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=2)
+        consumer_name_1 = "test_consumer_1"
+        consumer_name_2 = "test_consumer_2"
+        await buffer.register_consumer(consumer_name_1)
+        await buffer.register_consumer(consumer_name_2)
+
+        # Add events
+        events = []
+        for i in range(10):
+            event = _create_test_event(f"event{i}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        # Consumer 1 reads first batch
+        batch1 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch1 == events[:2]
+
+        # Consumer 2 reads from beginning
+        batch2 = await buffer.wait_for_batch(consumer_name_2, timeout_seconds=0.1)
+        assert batch2 == events[:2]
+
+        # consumer 1 reads another batch
+        batch3 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch3 == events[2:4]
+
+        # more events are added leading to events not consumed by consumer 2 getting evicted
+        # 4 events get evicted, consumer 1 has processed all 4 evicted events previously
+        # but consumer 2 has only processed 2 out of the 4 evicted events
+        for i in range(4):
+            event = _create_test_event(f"event{i + 10}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        # Just ensure buffer remains at max size
+        assert await buffer.size() == 10
+
+        # consumer 1 will read the next 2 events, not affected by the evictions
+        # consumer 1's cursor is adjusted internally to account for the evicted events
+        batch4 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch4 == events[4:6]
+
+        # consumer 2 will read 2 events, skipping the evicted events
+        batch5 = await buffer.wait_for_batch(consumer_name_2, timeout_seconds=0.1)
+        assert batch5 == events[4:6]  # events[2:4] are lost
+
+    @pytest.mark.asyncio
+    async def test_wait_for_batch_blocks_until_event_available(self):
+        """Test that wait_for_batch blocks until at least one event is available."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        # Start waiting for batch (should block)
+        async def wait_for_batch():
+            return await buffer.wait_for_batch(consumer_name, timeout_seconds=2.0)
+
+        wait_task = asyncio.create_task(wait_for_batch())
+
+        # Wait a bit to ensure the task is waiting
+        await asyncio.sleep(4.0)
+        assert not wait_task.done()
+
+        # Add an event
+        event = _create_test_event(b"event1")
+        await buffer.add_event(event)
+
+        # Now the task should complete
+        batch = await wait_task
+        assert len(batch) == 1
+        assert batch[0] == event
+
+    @pytest.mark.asyncio
+    async def test_concurrent_producer_consumer_random_sleeps_with_overall_timeout(
+        self,
+    ):
+        """Producer with random sleeps and consumer reading until all events are received.
+
+        Uses an overall asyncio timeout to ensure the test fails if it hangs
+        before consuming all events.
+        """
+        total_events = 40
+        max_batch_size = 2
+        buffer = MultiConsumerEventBuffer(max_size=100, max_batch_size=max_batch_size)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        produced_events = []
+        consumed_events = []
+
+        random.seed(0)
+
+        async def producer():
+            for i in range(total_events):
+                event = _create_test_event(f"e{i}".encode())
+                produced_events.append(event)
+                await buffer.add_event(event)
+                await asyncio.sleep(random.uniform(0.0, 0.02))
+
+        async def consumer():
+            while len(consumed_events) < total_events:
+                batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0.1)
+                consumed_events.extend(batch)
+
+        # The test should fail if this times out before all events are consumed
+        await asyncio.wait_for(asyncio.gather(producer(), consumer()), timeout=5.0)
+
+        assert len(consumed_events) == total_events
+        assert consumed_events == produced_events
+
+    @pytest.mark.asyncio
+    async def test_events_are_evicted_once_consumed_by_all_consumers(self):
+        """Test events are evicted from the buffer once they are consumed by all consumers"""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=2)
+        consumer_name_1 = "test_consumer_1"
+        consumer_name_2 = "test_consumer_2"
+        await buffer.register_consumer(consumer_name_1)
+        await buffer.register_consumer(consumer_name_2)
+
+        # Add events
+        events = []
+        for i in range(10):
+            event = _create_test_event(f"event{i}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        assert await buffer.size() == 10
+        # Consumer 1 reads first batch
+        batch1 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch1 == events[:2]
+
+        # buffer size does not change as consumer 2 is yet to consume these events
+        assert await buffer.size() == 10
+
+        # Consumer 2 reads from beginning
+        batch2 = await buffer.wait_for_batch(consumer_name_2, timeout_seconds=0.1)
+        assert batch2 == events[:2]
+
+        # size reduces by 2 as both consumers have consumed 2 events
+        assert await buffer.size() == 8
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_actor_events.py

@@ -0,0 +1,125 @@
+import base64
+import json
+import sys
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_ACTOR_EVENT_PORT = 12346
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _ACTOR_EVENT_PORT)
+
+
+def test_ray_actor_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_ACTOR_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    head_node_id = cluster.head_node.node_id
+    all_nodes_ids = [node.node_id for node in cluster.list_all_nodes()]
+
+    class A:
+        def ping(self):
+            return "pong"
+
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Create an actor to trigger definition + lifecycle events
+    a = ray.remote(A).options(name="actor-test").remote()
+    ray.get(a.ping.remote())
+
+    # Check that an actor definition and a lifecycle event are published.
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 1)
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+    # We expect batched events containing definition then lifecycle
+    assert len(req_json) >= 2
+    # Verify event types and IDs exist
+    assert (
+        base64.b64decode(req_json[0]["actorDefinitionEvent"]["actorId"]).hex()
+        == a._actor_id.hex()
+    )
+    assert base64.b64decode(req_json[0]["nodeId"]).hex() == head_node_id
+    # Verify ActorId and state for ActorLifecycleEvents
+    has_alive_state = False
+    for actorLifeCycleEvent in req_json[1:]:
+        assert base64.b64decode(actorLifeCycleEvent["nodeId"]).hex() == head_node_id
+        assert (
+            base64.b64decode(
+                actorLifeCycleEvent["actorLifecycleEvent"]["actorId"]
+            ).hex()
+            == a._actor_id.hex()
+        )
+        for stateTransition in actorLifeCycleEvent["actorLifecycleEvent"][
+            "stateTransitions"
+        ]:
+            assert stateTransition["state"] in [
+                "DEPENDENCIES_UNREADY",
+                "PENDING_CREATION",
+                "ALIVE",
+                "RESTARTING",
+                "DEAD",
+            ]
+            if stateTransition["state"] == "ALIVE":
+                has_alive_state = True
+                assert (
+                    base64.b64decode(stateTransition["nodeId"]).hex() in all_nodes_ids
+                )
+                assert base64.b64decode(stateTransition["workerId"]).hex() != ""
+    assert has_alive_state
+
+    # Kill the actor and verify we get a DEAD state with death cause
+    ray.kill(a)
+
+    # Wait for the death event to be published
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 2)
+
+    has_dead_state = False
+    for death_req, _ in httpserver.log:
+        death_req_json = json.loads(death_req.data)
+
+        for actorLifeCycleEvent in death_req_json:
+            if "actorLifecycleEvent" in actorLifeCycleEvent:
+                assert (
+                    base64.b64decode(
+                        actorLifeCycleEvent["actorLifecycleEvent"]["actorId"]
+                    ).hex()
+                    == a._actor_id.hex()
+                )
+
+                for stateTransition in actorLifeCycleEvent["actorLifecycleEvent"][
+                    "stateTransitions"
+                ]:
+                    if stateTransition["state"] == "DEAD":
+                        has_dead_state = True
+                        assert (
+                            stateTransition["deathCause"]["actorDiedErrorContext"][
+                                "reason"
+                            ]
+                            == "RAY_KILL"
+                        )
+
+    assert has_dead_state
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_event_publisher.py

@@ -0,0 +1,164 @@
+import asyncio
+import sys
+import uuid
+
+import pytest
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from ray._common.test_utils import async_wait_for_condition
+from ray.core.generated import events_base_event_pb2
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+from ray.dashboard.modules.aggregator.publisher.async_publisher_client import (
+    PublisherClientInterface,
+    PublishStats,
+)
+from ray.dashboard.modules.aggregator.publisher.ray_event_publisher import (
+    NoopPublisher,
+    RayEventPublisher,
+)
+
+
+class MockPublisherClient(PublisherClientInterface):
+    """Test implementation of PublisherClientInterface."""
+
+    def __init__(
+        self,
+        batch_size: int = 1,
+        side_effect=lambda batch: PublishStats(True, 1, 0),
+    ):
+        self.batch_size = batch_size
+        self.publish_calls = []
+        self._side_effect = side_effect
+
+    async def publish(self, batch) -> PublishStats:
+        self.publish_calls.append(batch)
+        return self._side_effect(batch)
+
+    def count_num_events_in_batch(self, batch) -> int:
+        return self.batch_size
+
+    async def close(self) -> None:
+        pass
+
+
+@pytest.fixture
+def base_kwargs():
+    """Common kwargs for publisher initialization."""
+    return {
+        "name": "test",
+        "max_retries": 2,
+        "initial_backoff": 0,
+        "max_backoff": 0,
+        "jitter_ratio": 0,
+        "enable_publisher_stats": True,
+    }
+
+
+class TestRayEventPublisher:
+    """Test the main RayEventsPublisher functionality."""
+
+    @pytest.mark.asyncio
+    async def test_publish_with_retries_failure_then_success(self, base_kwargs):
+        """Test publish that fails then succeeds."""
+        call_count = {"count": 0}
+
+        # fail the first publish call but succeed on retry
+        def side_effect(batch):
+            call_count["count"] += 1
+            if call_count["count"] == 1:
+                return PublishStats(False, 0, 0)
+            return PublishStats(True, 1, 0)
+
+        client = MockPublisherClient(side_effect=side_effect)
+        event_buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=10)
+        publisher = RayEventPublisher(
+            name=base_kwargs["name"] + str(uuid.uuid4()),
+            publish_client=client,
+            event_buffer=event_buffer,
+            max_retries=base_kwargs["max_retries"],
+            initial_backoff=base_kwargs["initial_backoff"],
+            max_backoff=base_kwargs["max_backoff"],
+            jitter_ratio=base_kwargs["jitter_ratio"],
+        )
+
+        task = asyncio.create_task(publisher.run_forever())
+        try:
+            # ensure consumer is registered
+            assert await publisher.wait_until_running(2.0)
+            # Enqueue one event into buffer
+            e = events_base_event_pb2.RayEvent(
+                event_id=b"1",
+                source_type=events_base_event_pb2.RayEvent.SourceType.CORE_WORKER,
+                event_type=events_base_event_pb2.RayEvent.EventType.TASK_DEFINITION_EVENT,
+                timestamp=Timestamp(seconds=123, nanos=0),
+                severity=events_base_event_pb2.RayEvent.Severity.INFO,
+                message="hello",
+            )
+            await event_buffer.add_event(e)
+
+            # wait for two publish attempts (failure then success)
+            await async_wait_for_condition(lambda: len(client.publish_calls) == 2)
+        finally:
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+
+    @pytest.mark.asyncio
+    async def test_publish_with_retries_max_retries_exceeded(self, base_kwargs):
+        """Test publish that fails all retries and records failed events."""
+        client = MockPublisherClient(
+            side_effect=lambda batch: PublishStats(False, 0, 0)
+        )
+        event_buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=10)
+        publisher = RayEventPublisher(
+            name=base_kwargs["name"] + str(uuid.uuid4()),
+            publish_client=client,
+            event_buffer=event_buffer,
+            max_retries=2,  # override to finite retries
+            initial_backoff=0,
+            max_backoff=0,
+            jitter_ratio=0,
+        )
+
+        task = asyncio.create_task(publisher.run_forever())
+        try:
+            # ensure consumer is registered
+            assert await publisher.wait_until_running(2.0)
+            e = events_base_event_pb2.RayEvent(
+                event_id=b"1",
+                source_type=events_base_event_pb2.RayEvent.SourceType.CORE_WORKER,
+                event_type=events_base_event_pb2.RayEvent.EventType.TASK_DEFINITION_EVENT,
+                timestamp=Timestamp(seconds=123, nanos=0),
+                severity=events_base_event_pb2.RayEvent.Severity.INFO,
+                message="hello",
+            )
+            await event_buffer.add_event(e)
+
+            # wait for publish attempts (initial + 2 retries)
+            await async_wait_for_condition(lambda: len(client.publish_calls) == 3)
+            assert len(client.publish_calls) == 3
+        finally:
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+
+
+class TestNoopPublisher:
+    """Test no-op publisher implementation."""
+
+    @pytest.mark.asyncio
+    async def test_all_methods_noop(self):
+        """Test that run_forever can be cancelled and metrics return expected values."""
+        publisher = NoopPublisher()
+
+        # Start and cancel run_forever
+        task = asyncio.create_task(publisher.run_forever())
+        task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_job_events.py

@@ -0,0 +1,58 @@
+import base64
+import json
+import sys
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_RAY_EVENT_PORT = 12345
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _RAY_EVENT_PORT)
+
+
+def test_ray_job_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_RAY_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Submit a ray job
+    @ray.remote
+    def f():
+        return 1
+
+    ray.get(f.remote())
+
+    # Check that a driver job event with the correct job id is published.
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 1)
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+    head_node_id = cluster.head_node.node_id
+    assert base64.b64decode(req_json[0]["nodeId"]).hex() == head_node_id
+    assert (
+        base64.b64decode(req_json[0]["driverJobDefinitionEvent"]["jobId"]).hex()
+        == ray.get_runtime_context().get_job_id()
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_node_events.py

@@ -0,0 +1,75 @@
+import base64
+import json
+import os
+import socket
+import sys
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_RAY_EVENT_PORT = 12345
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _RAY_EVENT_PORT)
+
+
+def test_ray_node_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        node_name="test-head-node",
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_RAY_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    head_node_id = cluster.head_node.node_id
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Check that a node definition and a node lifecycle event are published.
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 1)
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+    assert len(req_json) == 2
+    assert base64.b64decode(req_json[0]["nodeId"]).hex() == head_node_id
+    assert (
+        base64.b64decode(req_json[0]["nodeDefinitionEvent"]["nodeId"]).hex()
+        == cluster.head_node.node_id
+    )
+
+    node_def_event = req_json[0]["nodeDefinitionEvent"]
+    assert node_def_event["hostname"] == socket.gethostname()
+    assert node_def_event["nodeName"] == "test-head-node"
+    # instanceId and instanceTypeName are set via env vars by cloud providers.
+    # In local/CI environments these are typically empty.
+    assert node_def_event["instanceId"] == os.environ.get("RAY_CLOUD_INSTANCE_ID", "")
+    assert node_def_event["instanceTypeName"] == os.environ.get(
+        "RAY_CLOUD_INSTANCE_TYPE_NAME", ""
+    )
+    assert base64.b64decode(req_json[1]["nodeId"]).hex() == head_node_id
+    assert (
+        base64.b64decode(req_json[1]["nodeLifecycleEvent"]["nodeId"]).hex()
+        == cluster.head_node.node_id
+    )
+    assert req_json[1]["nodeLifecycleEvent"]["stateTransitions"][0]["state"] == "ALIVE"
+    assert (
+        req_json[1]["nodeLifecycleEvent"]["stateTransitions"][0]["aliveSubState"]
+        == "UNSPECIFIED"
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_platform_events.py

@@ -0,0 +1,117 @@
+import base64
+import json
+import sys
+import time
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_PLATFORM_EVENT_PORT = 12348
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _PLATFORM_EVENT_PORT)
+
+
+def test_ray_platform_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_PLATFORM_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "PLATFORM_EVENT",
+            "RAY_ENABLE_PYTHON_RAY_EVENT_TYPES": "PLATFORM_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    head_node_id = cluster.head_node.node_id
+
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Define a task that explicitly initializes and emits a platform event via EventRecorder
+    @ray.remote
+    def emit_test_platform_event(aggregator_port, node_ip, node_id):
+        from ray._common.observability.platform_events import PlatformEventBuilder
+        from ray._raylet import EventRecorder
+        from ray.core.generated.events_base_event_pb2 import RayEvent
+        from ray.core.generated.platform_event_pb2 import Source
+
+        EventRecorder.initialize(
+            aggregator_port=aggregator_port,
+            node_ip=node_ip,
+            node_id_hex=node_id,
+            max_buffer_size=1000,
+            metric_source="platform_events",
+        )
+
+        builder = PlatformEventBuilder(
+            event_uid="uid-test-platform-e2e",
+            platform=Source.Platform.KUBERNETES,
+            object_kind="Pod",
+            object_name="test-pod-name",
+            reason="OOMKilled",
+            message="Container exited with code 137",
+            severity=RayEvent.Severity.WARNING,
+            component="kubelet",
+        )
+        cython_event = builder.build(
+            event_id=b"uid-test-platform-e2e",
+            timestamp_ns=int(time.time() * 1e9),
+        )
+        EventRecorder.emit(cython_event)
+        EventRecorder.shutdown()
+        return True
+
+    # Expect the POST request on the HTTP server
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+
+    # Fetch the aggregator agent's address from GCS
+    from ray._private.test_utils import GcsClient, get_dashboard_agent_address
+
+    gcs_client = GcsClient(address=cluster.address)
+    agent_address = get_dashboard_agent_address(gcs_client, head_node_id)
+    ip, port_str = agent_address.split(":")
+    aggregator_port = int(port_str)
+
+    # Execute the remote task to emit the event on the node
+    ray.get(emit_test_platform_event.remote(aggregator_port, ip, head_node_id))
+
+    # Wait for the HTTP log collector to receive the batched payload
+    wait_for_condition(lambda: len(httpserver.log) >= 1, timeout=20)
+
+    # Validate the captured POST payload
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+
+    assert len(req_json) >= 1
+    platform_event_entry = None
+    for entry in req_json:
+        if "platformEvent" in entry:
+            platform_event_entry = entry
+            break
+
+    assert platform_event_entry is not None
+    assert platform_event_entry["eventType"] == "PLATFORM_EVENT"
+    assert base64.b64decode(platform_event_entry["nodeId"]).hex() == head_node_id
+
+    pe_data = platform_event_entry["platformEvent"]
+    assert pe_data["objectKind"] == "Pod"
+    assert pe_data["objectName"] == "test-pod-name"
+    assert pe_data["reason"] == "OOMKilled"
+    assert pe_data["message"] == "Container exited with code 137"
+    assert pe_data["source"]["platform"] == "KUBERNETES"
+    assert pe_data["source"]["component"] == "kubelet"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_task_events_metadata_buffer.py

@@ -0,0 +1,115 @@
+import sys
+
+import pytest
+
+from ray.core.generated.events_event_aggregator_service_pb2 import TaskEventsMetadata
+from ray.dashboard.modules.aggregator.task_events_metadata_buffer import (
+    TaskEventsMetadataBuffer,
+)
+
+
+def _create_test_metadata(dropped_task_ids: list = None, attempt_number=1):
+    """Helper function to create test metadata"""
+    metadata = TaskEventsMetadata()
+    if dropped_task_ids:
+        for task_id in dropped_task_ids:
+            attempt = metadata.dropped_task_attempts.add()
+            attempt.task_id = task_id.encode()
+            attempt.attempt_number = attempt_number
+    return metadata
+
+
+def _result_to_attempts_list(result):
+    """Normalize return value from buffer.get() to a python list of attempts."""
+    if hasattr(result, "dropped_task_attempts"):
+        attempts = result.dropped_task_attempts
+    else:
+        attempts = result
+    return list(attempts)
+
+
+def _drain_all_attempts(buffer: TaskEventsMetadataBuffer):
+    """Drain the buffer completely via public API and return list of bytes task_ids.
+
+    Continues calling get() until it returns an empty set of attempts.
+    """
+    collected_ids = []
+    num_metadata_entries = 0
+    while True:
+        result = buffer.get()
+        attempts = _result_to_attempts_list(result)
+        if len(attempts) == 0:
+            break
+
+        num_metadata_entries += 1
+        collected_ids.extend([a.task_id for a in attempts])
+    return collected_ids, num_metadata_entries
+
+
+class TestTaskMetadataBuffer:
+    """tests for TaskMetadataBuffer class"""
+
+    def test_merge_and_get(self):
+        """Test merging multiple metadata objects and verify task attempts are combined."""
+        buffer = TaskEventsMetadataBuffer(
+            max_buffer_size=100, max_dropped_attempts_per_metadata_entry=10
+        )
+
+        # Create two separate metadata objects with different task IDs
+        metadata1 = _create_test_metadata(["task_1", "task_2"])
+        metadata2 = _create_test_metadata(["task_3", "task_4"])
+
+        # Merge both metadata objects
+        buffer.merge(metadata1)
+        buffer.merge(metadata2)
+
+        # Get the merged results
+        result = buffer.get()
+        attempts = _result_to_attempts_list(result)
+
+        # Verify we have all 4 task attempts
+        assert len(attempts) == 4
+
+        # Verify all expected task IDs are present
+        task_ids = [attempt.task_id for attempt in attempts]
+        assert sorted(task_ids) == [b"task_1", b"task_2", b"task_3", b"task_4"]
+
+    @pytest.mark.parametrize(
+        "max_attempts_per_metadata_entry,num_tasks,max_buffer_size,expected_drop_attempts,expected_num_metadata_entries",
+        [
+            # No overflow, two metadata entries should be created
+            (2, 3, 100, 0, 2),
+            # No overflow, three metadata entries should be created
+            (5, 15, 100, 0, 3),
+            # Overflow scenario: buffer too small, ensure drop count is tracked.
+            (1, 4, 2, 2, 2),
+        ],
+    )
+    def test_buffer_merge_and_overflow(
+        self,
+        max_attempts_per_metadata_entry,
+        num_tasks,
+        max_buffer_size,
+        expected_drop_attempts,
+        expected_num_metadata_entries,
+    ):
+        buffer = TaskEventsMetadataBuffer(
+            max_buffer_size=max_buffer_size,
+            max_dropped_attempts_per_metadata_entry=max_attempts_per_metadata_entry,
+        )
+
+        for i in range(num_tasks):
+            test_metadata = _create_test_metadata([f"task_{i}"])
+            buffer.merge(test_metadata)
+
+        # Drain everything and verify number of attempts in buffer is as expected
+        drained_ids, num_metadata_entries = _drain_all_attempts(buffer)
+        assert len(drained_ids) == num_tasks - expected_drop_attempts
+        assert num_metadata_entries == expected_num_metadata_entries
+
+        # Buffer should now be empty
+        assert len(_result_to_attempts_list(buffer.get())) == 0
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/dashboard_sdk.py

@@ -0,0 +1,477 @@
+import dataclasses
+import importlib
+import json
+import logging
+import os
+import ssl
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+import packaging.version
+import yaml
+
+import ray
+from ray._private.authentication.http_token_authentication import (
+    format_authentication_http_error,
+    get_auth_headers_if_auth_enabled,
+)
+from ray._private.runtime_env.packaging import (
+    create_package,
+    get_uri_for_directory,
+    get_uri_for_package,
+)
+from ray._private.runtime_env.py_modules import upload_py_modules_if_needed
+from ray._private.runtime_env.working_dir import upload_working_dir_if_needed
+from ray._private.utils import split_address
+from ray.autoscaler._private.cli_logger import cli_logger
+from ray.dashboard.modules.job.common import uri_to_http_components
+from ray.exceptions import AuthenticationError
+from ray.util.annotations import DeveloperAPI, PublicAPI
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# By default, connect to local cluster.
+DEFAULT_DASHBOARD_ADDRESS = "http://localhost:8265"
+
+
+def parse_runtime_env_args(
+    runtime_env: Optional[str] = None,
+    runtime_env_json: Optional[str] = None,
+    working_dir: Optional[str] = None,
+):
+    """
+    Generates a runtime_env dictionary using `runtime_env`, `runtime_env_json`,
+    and `working_dir` CLI options. Only one of `runtime_env` or
+    `runtime_env_json` may be defined. `working_dir` overwrites the
+    `working_dir` from any other option.
+    """
+
+    final_runtime_env = {}
+    if runtime_env is not None:
+        if runtime_env_json is not None:
+            raise ValueError(
+                "Only one of --runtime_env and --runtime-env-json can be provided."
+            )
+        with open(runtime_env, "r") as f:
+            final_runtime_env = yaml.safe_load(f)
+
+    elif runtime_env_json is not None:
+        final_runtime_env = json.loads(runtime_env_json)
+
+    if working_dir is not None:
+        if "working_dir" in final_runtime_env:
+            cli_logger.warning(
+                "Overriding runtime_env working_dir with --working-dir option"
+            )
+
+        final_runtime_env["working_dir"] = working_dir
+
+    return final_runtime_env
+
+
+@dataclasses.dataclass
+class ClusterInfo:
+    address: str
+    cookies: Optional[Dict[str, Any]] = None
+    metadata: Optional[Dict[str, Any]] = None
+    headers: Optional[Dict[str, Any]] = None
+
+
+# TODO (shrekris-anyscale): renaming breaks compatibility, do NOT rename
+def get_job_submission_client_cluster_info(
+    address: str,
+    # For backwards compatibility
+    *,
+    # only used in importlib case in parse_cluster_info, but needed
+    # in function signature.
+    create_cluster_if_needed: Optional[bool] = False,
+    cookies: Optional[Dict[str, Any]] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+    headers: Optional[Dict[str, Any]] = None,
+    _use_tls: Optional[bool] = False,
+    **kwargs,
+) -> ClusterInfo:
+    """Get address, cookies, and metadata used for SubmissionClient.
+
+    If no port is specified in `address`, the Ray dashboard default will be
+    inserted.
+
+    Args:
+        address: Address without the module prefix that is passed
+            to SubmissionClient.
+        create_cluster_if_needed: Indicates whether the cluster
+            of the address returned needs to be running. Ray doesn't
+            start a cluster before interacting with jobs, but other
+            implementations may do so.
+        cookies: Optional cookies forwarded to ``SubmissionClient``.
+        metadata: Optional metadata forwarded to ``SubmissionClient``.
+        headers: Optional HTTP headers forwarded to ``SubmissionClient``.
+        _use_tls: When True, use ``https`` instead of ``http`` for the
+            constructed address.
+        **kwargs: Reserved for forward-compatibility with other client
+            implementations; unused here.
+
+    Returns:
+        ClusterInfo object consisting of address, cookies, and metadata
+        for SubmissionClient to use.
+    """
+
+    scheme = "https" if _use_tls else "http"
+    return ClusterInfo(
+        address=f"{scheme}://{address}",
+        cookies=cookies,
+        metadata=metadata,
+        headers=headers,
+    )
+
+
+def parse_cluster_info(
+    address: Optional[str] = None,
+    create_cluster_if_needed: bool = False,
+    cookies: Optional[Dict[str, Any]] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+    headers: Optional[Dict[str, Any]] = None,
+    **kwargs,
+) -> ClusterInfo:
+    """Create a cluster if needed and return its address, cookies, and metadata."""
+    if address is None:
+        if (
+            ray.is_initialized()
+            and ray._private.worker.global_worker.node.address_info["webui_url"]
+            is not None
+        ):
+            address = (
+                "http://"
+                f"{ray._private.worker.global_worker.node.address_info['webui_url']}"
+            )
+            logger.info(
+                f"No address provided but Ray is running; using address {address}."
+            )
+        else:
+            logger.info(
+                f"No address provided, defaulting to {DEFAULT_DASHBOARD_ADDRESS}."
+            )
+            address = DEFAULT_DASHBOARD_ADDRESS
+
+    if address == "auto":
+        raise ValueError("Internal error: unexpected address 'auto'.")
+
+    if "://" not in address:
+        # Default to HTTP.
+        logger.info(
+            "No scheme (e.g. 'http://') or module string (e.g. 'ray://') "
+            f"provided in address {address}, defaulting to HTTP."
+        )
+        address = f"http://{address}"
+
+    module_string, inner_address = split_address(address)
+
+    if module_string == "ray":
+        raise ValueError(f"Internal error: unexpected Ray Client address {address}.")
+    # If user passes http(s)://, go through normal parsing.
+    if module_string in {"http", "https"}:
+        return get_job_submission_client_cluster_info(
+            inner_address,
+            create_cluster_if_needed=create_cluster_if_needed,
+            cookies=cookies,
+            metadata=metadata,
+            headers=headers,
+            _use_tls=(module_string == "https"),
+            **kwargs,
+        )
+    # Try to dynamically import the function to get cluster info.
+    else:
+        try:
+            module = importlib.import_module(module_string)
+        except Exception:
+            raise RuntimeError(
+                f"Module: {module_string} does not exist.\n"
+                f"This module was parsed from address: {address}"
+            ) from None
+        assert "get_job_submission_client_cluster_info" in dir(module), (
+            f"Module: {module_string} does "
+            "not have `get_job_submission_client_cluster_info`.\n"
+            f"This module was parsed from address: {address}"
+        )
+
+        return module.get_job_submission_client_cluster_info(
+            inner_address,
+            create_cluster_if_needed=create_cluster_if_needed,
+            cookies=cookies,
+            metadata=metadata,
+            headers=headers,
+            **kwargs,
+        )
+
+
+class SubmissionClient:
+    def __init__(
+        self,
+        address: Optional[str] = None,
+        create_cluster_if_needed: bool = False,
+        cookies: Optional[Dict[str, Any]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, Any]] = None,
+        verify: Optional[Union[str, bool]] = True,
+        **kwargs,
+    ):
+        # Remove any trailing slashes
+        if address is not None and address.endswith("/"):
+            address = address.rstrip("/")
+            logger.debug(
+                "The submission address cannot contain trailing slashes. Removing "
+                f'them from the requested submission address of "{address}".'
+            )
+
+        cluster_info = parse_cluster_info(
+            address, create_cluster_if_needed, cookies, metadata, headers, **kwargs
+        )
+        self._address = cluster_info.address
+        self._cookies = cluster_info.cookies
+        self._default_metadata = cluster_info.metadata or {}
+        # Headers used for all requests sent to job server, optional and only
+        # needed for cases like authentication to remote cluster.
+        self._headers = cluster_info.headers or {}
+        self._headers.update(**get_auth_headers_if_auth_enabled(self._headers))
+
+        # Set SSL verify parameter for the requests library and create an ssl_context
+        # object when needed for the aiohttp library.
+        self._verify = verify
+        if isinstance(self._verify, str):
+            if os.path.isdir(self._verify):
+                cafile, capath = None, self._verify
+            elif os.path.isfile(self._verify):
+                cafile, capath = self._verify, None
+            else:
+                raise FileNotFoundError(
+                    f"Path to CA certificates: '{self._verify}', does not exist."
+                )
+            self._ssl_context = ssl.create_default_context(cafile=cafile, capath=capath)
+        else:
+            if self._verify is False:
+                self._ssl_context = False
+            else:
+                self._ssl_context = None
+
+        self._server_ray_version: Optional[str] = None
+
+    def _check_connection_and_version(
+        self, min_version: str = "1.9", version_error_message: str = None
+    ):
+        self._check_connection_and_version_with_url(min_version, version_error_message)
+
+    def _check_connection_and_version_with_url(
+        self,
+        min_version: str = "1.9",
+        version_error_message: str = None,
+        url: str = "/api/version",
+    ):
+        if version_error_message is None:
+            version_error_message = (
+                f"Please ensure the cluster is running Ray {min_version} or higher."
+            )
+
+        try:
+            r = self._do_request("GET", url)
+            if r.status_code == 404:
+                raise RuntimeError(
+                    "Version check returned 404. " + version_error_message
+                )
+            r.raise_for_status()
+
+            running_ray_version = r.json()["ray_version"]
+            self._server_ray_version = running_ray_version
+            if packaging.version.parse(running_ray_version) < packaging.version.parse(
+                min_version
+            ):
+                raise RuntimeError(
+                    f"Ray version {running_ray_version} is running on the cluster. "
+                    + version_error_message
+                )
+        except requests.exceptions.ConnectionError:
+            raise ConnectionError(
+                f"Failed to connect to Ray at address: {self._address}."
+            )
+
+    def _raise_error(self, r: "requests.Response"):
+        raise RuntimeError(
+            f"Request failed with status code {r.status_code}: {r.text}."
+        )
+
+    def _do_request(
+        self,
+        method: str,
+        endpoint: str,
+        *,
+        data: Optional[bytes] = None,
+        json_data: Optional[dict] = None,
+        **kwargs,
+    ) -> "requests.Response":
+        """Perform the actual HTTP request with authentication error handling.
+
+        Keyword arguments other than "cookies", "headers" are forwarded to the
+        `requests.request()`.
+        """
+        url = self._address + endpoint
+        logger.debug(f"Sending request to {url} with json data: {json_data or {}}.")
+
+        response = requests.request(
+            method,
+            url,
+            cookies=self._cookies,
+            data=data,
+            json=json_data,
+            headers=self._headers,
+            verify=self._verify,
+            **kwargs,
+        )
+
+        # Check for authentication errors and provide helpful messages
+        formatted_error = format_authentication_http_error(
+            response.status_code, response.text
+        )
+        if formatted_error:
+            raise AuthenticationError(formatted_error)
+
+        return response
+
+    def _package_exists(
+        self,
+        package_uri: str,
+    ) -> bool:
+        protocol, package_name = uri_to_http_components(package_uri)
+        r = self._do_request("GET", f"/api/packages/{protocol}/{package_name}")
+
+        if r.status_code == 200:
+            logger.debug(f"Package {package_uri} already exists.")
+            return True
+        elif r.status_code == 404:
+            logger.debug(f"Package {package_uri} does not exist.")
+            return False
+        else:
+            self._raise_error(r)
+
+    def _upload_package(
+        self,
+        package_uri: str,
+        package_path: str,
+        include_gitignore: bool,
+        include_parent_dir: Optional[bool] = False,
+        excludes: Optional[List[str]] = None,
+        is_file: bool = False,
+    ) -> bool:
+        logger.info(f"Uploading package {package_uri}.")
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            protocol, package_name = uri_to_http_components(package_uri)
+            if is_file:
+                package_file = Path(package_path)
+            else:
+                package_file = Path(tmp_dir) / package_name
+                create_package(
+                    package_path,
+                    package_file,
+                    include_gitignore=include_gitignore,
+                    include_parent_dir=include_parent_dir,
+                    excludes=excludes,
+                )
+            try:
+                r = self._do_request(
+                    "PUT",
+                    f"/api/packages/{protocol}/{package_name}",
+                    data=package_file.read_bytes(),
+                )
+                if r.status_code != 200:
+                    self._raise_error(r)
+            finally:
+                # If the package is a user's existing file, don't delete it.
+                if not is_file:
+                    package_file.unlink()
+
+    def _upload_package_if_needed(
+        self,
+        package_path: str,
+        include_gitignore: bool,
+        include_parent_dir: bool = False,
+        excludes: Optional[List[str]] = None,
+        is_file: bool = False,
+    ) -> str:
+        if is_file:
+            package_uri = get_uri_for_package(Path(package_path))
+        else:
+            package_uri = get_uri_for_directory(
+                package_path, include_gitignore, excludes=excludes
+            )
+
+        if not self._package_exists(package_uri):
+            self._upload_package(
+                package_uri,
+                package_path,
+                include_gitignore=include_gitignore,
+                include_parent_dir=include_parent_dir,
+                excludes=excludes,
+                is_file=is_file,
+            )
+        else:
+            logger.info(f"Package {package_uri} already exists, skipping upload.")
+
+        return package_uri
+
+    def _upload_working_dir_if_needed(self, runtime_env: Dict[str, Any]):
+        from ray._private.ray_constants import RAY_RUNTIME_ENV_IGNORE_GITIGNORE
+
+        # Determine whether to respect .gitignore files based on environment variable
+        # Default is True (respect .gitignore). Set to False if env var is "1".
+        include_gitignore = os.environ.get(RAY_RUNTIME_ENV_IGNORE_GITIGNORE, "0") != "1"
+
+        def _upload_fn(working_dir, excludes, is_file=False):
+            self._upload_package_if_needed(
+                working_dir,
+                include_gitignore=include_gitignore,
+                include_parent_dir=False,
+                excludes=excludes,
+                is_file=is_file,
+            )
+
+        upload_working_dir_if_needed(
+            runtime_env, include_gitignore=include_gitignore, upload_fn=_upload_fn
+        )
+
+    def _upload_py_modules_if_needed(self, runtime_env: Dict[str, Any]):
+        from ray._private.ray_constants import RAY_RUNTIME_ENV_IGNORE_GITIGNORE
+
+        # Determine whether to respect .gitignore files based on environment variable
+        # Default is True (respect .gitignore). Set to False if env var is "1".
+        include_gitignore = os.environ.get(RAY_RUNTIME_ENV_IGNORE_GITIGNORE, "0") != "1"
+
+        def _upload_fn(module_path, excludes, is_file=False):
+            self._upload_package_if_needed(
+                module_path,
+                include_gitignore=include_gitignore,
+                include_parent_dir=True,
+                excludes=excludes,
+                is_file=is_file,
+            )
+
+        upload_py_modules_if_needed(
+            runtime_env, include_gitignore=include_gitignore, upload_fn=_upload_fn
+        )
+
+    @PublicAPI(stability="beta")
+    def get_version(self) -> str:
+        r = self._do_request("GET", "/api/version")
+        if r.status_code == 200:
+            return r.json().get("version")
+        else:
+            self._raise_error(r)
+
+    @DeveloperAPI
+    def get_address(self) -> str:
+        return self._address

python/ray/dashboard/modules/data/data_head.py

@@ -0,0 +1,161 @@
+import json
+import logging
+import os
+from enum import Enum
+from urllib.parse import quote
+
+import aiohttp
+from aiohttp.web import Request, Response
+
+import ray.dashboard.optional_utils as optional_utils
+from ray.dashboard.modules.metrics.metrics_head import (
+    DEFAULT_PROMETHEUS_HEADERS,
+    DEFAULT_PROMETHEUS_HOST,
+    PROMETHEUS_HEADERS_ENV_VAR,
+    PROMETHEUS_HOST_ENV_VAR,
+    PrometheusQueryError,
+    parse_prom_headers,
+)
+from ray.dashboard.subprocesses.module import SubprocessModule
+from ray.dashboard.subprocesses.routes import SubprocessRouteTable as routes
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+# Window and sampling rate used for certain Prometheus queries.
+# Datapoints up until `MAX_TIME_WINDOW` ago are queried at `SAMPLE_RATE` intervals.
+MAX_TIME_WINDOW = "1h"
+SAMPLE_RATE = "1s"
+
+
+class PrometheusQuery(Enum):
+    """Enum to store types of Prometheus queries for a given metric and grouping."""
+
+    VALUE = ("value", "sum({}{{SessionName='{}'}}) by ({})")
+    MAX = (
+        "max",
+        "max_over_time(sum({}{{SessionName='{}'}}) by ({})["
+        + f"{MAX_TIME_WINDOW}:{SAMPLE_RATE}])",
+    )
+
+
+DATASET_METRICS = {
+    "ray_data_output_rows": (PrometheusQuery.MAX,),
+    "ray_data_spilled_bytes": (PrometheusQuery.MAX,),
+    "ray_data_current_bytes": (PrometheusQuery.VALUE, PrometheusQuery.MAX),
+    "ray_data_cpu_usage_cores": (PrometheusQuery.VALUE, PrometheusQuery.MAX),
+    "ray_data_gpu_usage_cores": (PrometheusQuery.VALUE, PrometheusQuery.MAX),
+}
+
+
+class DataHead(SubprocessModule):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.prometheus_host = os.environ.get(
+            PROMETHEUS_HOST_ENV_VAR, DEFAULT_PROMETHEUS_HOST
+        )
+        self.prometheus_headers = parse_prom_headers(
+            os.environ.get(
+                PROMETHEUS_HEADERS_ENV_VAR,
+                DEFAULT_PROMETHEUS_HEADERS,
+            )
+        )
+
+    @routes.get("/api/data/datasets/{job_id}")
+    @optional_utils.init_ray_and_catch_exceptions()
+    async def get_datasets(self, req: Request) -> Response:
+        job_id = req.match_info["job_id"]
+
+        try:
+            from ray.data._internal.stats import get_or_create_stats_actor
+
+            _stats_actor = get_or_create_stats_actor()
+            datasets = await _stats_actor.get_datasets.remote(job_id)
+            # Initializes dataset metric values
+            for dataset in datasets:
+                for metric, queries in DATASET_METRICS.items():
+                    datasets[dataset][metric] = {query.value[0]: 0 for query in queries}
+                    for operator in datasets[dataset]["operators"]:
+                        datasets[dataset]["operators"][operator][metric] = {
+                            query.value[0]: 0 for query in queries
+                        }
+            # Query dataset metric values from prometheus
+            try:
+                # TODO (Zandew): store results of completed datasets in stats actor.
+                for metric, queries in DATASET_METRICS.items():
+                    for query in queries:
+                        query_name, prom_query = query.value
+                        # Dataset level
+                        dataset_result = await self._query_prometheus(
+                            prom_query.format(metric, self.session_name, "dataset")
+                        )
+                        for res in dataset_result["data"]["result"]:
+                            dataset, value = res["metric"]["dataset"], res["value"][1]
+                            if dataset in datasets:
+                                datasets[dataset][metric][query_name] = value
+
+                        # Operator level
+                        operator_result = await self._query_prometheus(
+                            prom_query.format(
+                                metric, self.session_name, "dataset, operator"
+                            )
+                        )
+                        for res in operator_result["data"]["result"]:
+                            dataset, operator, value = (
+                                res["metric"]["dataset"],
+                                res["metric"]["operator"],
+                                res["value"][1],
+                            )
+                            # Check if dataset/operator is in current _StatsActor scope.
+                            # Prometheus server may contain metrics from previous
+                            # cluster if not reset.
+                            if (
+                                dataset in datasets
+                                and operator in datasets[dataset]["operators"]
+                            ):
+                                datasets[dataset]["operators"][operator][metric][
+                                    query_name
+                                ] = value
+            except aiohttp.client_exceptions.ClientConnectorError:
+                # Prometheus server may not be running,
+                # leave these values blank and return other data
+                logging.exception(
+                    "Exception occurred while querying Prometheus. "
+                    "The Prometheus server may not be running."
+                )
+            # Flatten response
+            for dataset in datasets:
+                datasets[dataset]["operators"] = list(
+                    map(
+                        lambda item: {"operator": item[0], **item[1]},
+                        datasets[dataset]["operators"].items(),
+                    )
+                )
+            datasets = list(
+                map(lambda item: {"dataset": item[0], **item[1]}, datasets.items())
+            )
+            # Sort by descending start time
+            datasets = sorted(datasets, key=lambda x: x["start_time"], reverse=True)
+            return Response(
+                text=json.dumps({"datasets": datasets}),
+                content_type="application/json",
+            )
+        except Exception as e:
+            logging.exception("Exception occurred while getting datasets.")
+            return Response(
+                status=503,
+                text=str(e),
+            )
+
+    async def _query_prometheus(self, query):
+        async with self.http_session.get(
+            f"{self.prometheus_host}/api/v1/query?query={quote(query)}",
+            headers=self.prometheus_headers,
+        ) as resp:
+            if resp.status == 200:
+                prom_data = await resp.json()
+                return prom_data
+
+            message = await resp.text()
+            raise PrometheusQueryError(resp.status, message)

python/ray/dashboard/modules/data/tests/test_data_head.py

@@ -0,0 +1,141 @@
+import os
+import sys
+
+import pytest
+import requests
+
+import ray
+from ray.job_submission import JobSubmissionClient
+from ray.tests.conftest import *  # noqa
+
+# For local testing on a Macbook, set `export TEST_ON_DARWIN=1`.
+TEST_ON_DARWIN = os.environ.get("TEST_ON_DARWIN", "0") == "1"
+
+DATA_HEAD_URLS = {"GET": "http://localhost:8265/api/data/datasets/{job_id}"}
+
+DATA_SCHEMA = [
+    "state",
+    "progress",
+    "total",
+    "total_rows",
+    "ray_data_output_rows",
+    "ray_data_spilled_bytes",
+    "ray_data_current_bytes",
+    "ray_data_cpu_usage_cores",
+    "ray_data_gpu_usage_cores",
+]
+
+RESPONSE_SCHEMA = [
+    "dataset",
+    "job_id",
+    "start_time",
+    "end_time",
+    "operators",
+] + DATA_SCHEMA
+
+OPERATOR_SCHEMA = [
+    "name",
+    "operator",
+    "queued_blocks",
+] + DATA_SCHEMA
+
+
+@pytest.mark.skipif(
+    sys.platform == "darwin" and not TEST_ON_DARWIN, reason="Flaky on OSX."
+)
+def test_unique_operator_id(ray_start_regular_shared):
+    # This regression test addresses a bug caused by using a non-unique operator ID
+    # format. Specifically, the third operator's name is limit11 with the ID limit112,
+    # while the thirteenth operator's name is limit1 with the same ID limit112, leading
+    # to a collision.
+    ds = ray.data.range(100, override_num_blocks=20).limit(11)  # 3 operators
+    for i in range(11):  # 11 more operators
+        ds = ds.limit(1)
+    ds._set_name("unique_operator_id_test")
+    ds.materialize()
+
+    client = JobSubmissionClient()
+    jobs = client.list_jobs()
+    assert len(jobs) == 1, jobs
+    job_id = jobs[0].job_id
+
+    data = requests.get(DATA_HEAD_URLS["GET"].format(job_id=job_id)).json()
+    datasets = [
+        dataset
+        for dataset in data["datasets"]
+        if dataset["dataset"].startswith("unique_operator_id_test")
+    ]
+    assert len(datasets) == 1
+    dataset = datasets[0]
+
+    operators = dataset["operators"]
+    assert len(operators) == 3  # Should be 3 because of limiter operator fusion.
+
+
+@pytest.mark.skipif(
+    sys.platform == "darwin" and not TEST_ON_DARWIN, reason="Flaky on OSX."
+)
+def test_get_datasets(ray_start_regular_shared):
+    ds = ray.data.range(100, override_num_blocks=20).map_batches(lambda x: x)
+    ds.set_name("data_head_test")
+    ds.materialize()
+
+    client = JobSubmissionClient()
+    jobs = client.list_jobs()
+    assert len(jobs) == 1, jobs
+    job_id = jobs[0].job_id
+
+    data = requests.get(DATA_HEAD_URLS["GET"].format(job_id=job_id)).json()
+    datasets = [
+        dataset
+        for dataset in data["datasets"]
+        if dataset["dataset"].startswith("data_head_test")
+    ]
+
+    assert len(datasets) == 1
+    assert sorted(datasets[0].keys()) == sorted(RESPONSE_SCHEMA)
+
+    dataset = datasets[0]
+    assert dataset["dataset"].startswith("data_head_test")
+    assert dataset["job_id"] == job_id
+    assert dataset["state"] == "FINISHED"
+    assert dataset["end_time"] is not None
+
+    operators = dataset["operators"]
+    assert len(operators) == 2
+    op0 = operators[0]
+    op1 = operators[1]
+    assert sorted(op0.keys()) == sorted(OPERATOR_SCHEMA)
+    assert sorted(op1.keys()) == sorted(OPERATOR_SCHEMA)
+    assert {
+        "operator": "Input_0",
+        "name": "Input",
+        "state": "FINISHED",
+        "progress": 20,
+        "total": 20,
+    }.items() <= op0.items()
+    assert {
+        "operator": "ReadRange->MapBatches(<lambda>)_1",
+        "name": "ReadRange->MapBatches(<lambda>)",
+        "state": "FINISHED",
+        "progress": 20,
+        "total": 20,
+    }.items() <= op1.items()
+
+    ds._set_name("another_data_head_test")
+    ds.map_batches(lambda x: x).materialize()
+    data = requests.get(DATA_HEAD_URLS["GET"].format(job_id=job_id)).json()
+
+    dataset = [
+        dataset
+        for dataset in data["datasets"]
+        if dataset["dataset"].startswith("another_data_head_test")
+    ][0]
+    assert dataset["dataset"].startswith("another_data_head_test")
+    assert dataset["job_id"] == job_id
+    assert dataset["state"] == "FINISHED"
+    assert dataset["end_time"] is not None
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-vv", __file__]))

python/ray/dashboard/modules/event/event_agent.py

@@ -0,0 +1,138 @@
+import asyncio
+import logging
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Union
+
+import ray._private.ray_constants as ray_constants
+import ray.dashboard.consts as dashboard_consts
+import ray.dashboard.utils as dashboard_utils
+from ray._private.authentication.http_token_authentication import (
+    get_auth_headers_if_auth_enabled,
+)
+from ray.dashboard.modules.event import event_consts
+from ray.dashboard.modules.event.event_utils import monitor_events
+from ray.dashboard.utils import async_loop_forever, create_task
+
+logger = logging.getLogger(__name__)
+
+
+# NOTE: Executor in this head is intentionally constrained to just 1 thread by
+#       default to limit its concurrency, therefore reducing potential for
+#       GIL contention
+RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS = ray_constants.env_integer(
+    "RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS", 1
+)
+
+
+class EventAgent(dashboard_utils.DashboardAgentModule):
+    def __init__(self, dashboard_agent):
+        super().__init__(dashboard_agent)
+        self._event_dir = os.path.join(self._dashboard_agent.log_dir, "events")
+        os.makedirs(self._event_dir, exist_ok=True)
+        self._monitor: Union[asyncio.Task, None] = None
+        # Lazy initialized on first use. Once initialized, it will not be
+        # changed.
+        self._dashboard_http_address = None
+        self._cached_events = asyncio.Queue(event_consts.EVENT_AGENT_CACHE_SIZE)
+        self._gcs_client = dashboard_agent.gcs_client
+        # Total number of event created from this agent.
+        self.total_event_reported = 0
+        # Total number of event report request sent.
+        self.total_request_sent = 0
+        self.module_started = time.monotonic()
+
+        self._executor = ThreadPoolExecutor(
+            max_workers=RAY_DASHBOARD_EVENT_AGENT_TPE_MAX_WORKERS,
+            thread_name_prefix="event_agent_executor",
+        )
+
+        logger.info("Event agent cache buffer size: %s", self._cached_events.maxsize)
+
+    async def _get_dashboard_http_address(self):
+        """
+        Lazily get the dashboard http address from InternalKV. If it's not set, sleep
+        and retry forever.
+        """
+        while True:
+            if self._dashboard_http_address:
+                return self._dashboard_http_address
+            try:
+                dashboard_http_address = await self._gcs_client.async_internal_kv_get(
+                    ray_constants.DASHBOARD_ADDRESS.encode(),
+                    namespace=ray_constants.KV_NAMESPACE_DASHBOARD,
+                    timeout=dashboard_consts.GCS_RPC_TIMEOUT_SECONDS,
+                )
+                if not dashboard_http_address:
+                    raise ValueError("Dashboard http address not found in InternalKV.")
+                address = dashboard_http_address.decode()
+                if not address.startswith(("http://", "https://")):
+                    address = f"http://{address}"
+                self._dashboard_http_address = address
+                return self._dashboard_http_address
+            except Exception:
+                logger.exception("Get dashboard http address failed.")
+            await asyncio.sleep(1)
+
+    @async_loop_forever(event_consts.EVENT_AGENT_REPORT_INTERVAL_SECONDS)
+    async def report_events(self):
+        """Report events from cached events queue. Reconnect to dashboard if
+        report failed. Log error after retry EVENT_AGENT_RETRY_TIMES.
+
+        This method will never returns.
+        """
+        dashboard_http_address = await self._get_dashboard_http_address()
+        data = await self._cached_events.get()
+        self.total_event_reported += len(data)
+        last_exception = None
+        for _ in range(event_consts.EVENT_AGENT_RETRY_TIMES):
+            try:
+                logger.debug("Report %s events.", len(data))
+                async with self._dashboard_agent.http_session.post(
+                    f"{dashboard_http_address}/report_events",
+                    json=data,
+                    headers=get_auth_headers_if_auth_enabled({}),
+                ) as response:
+                    response.raise_for_status()
+                self.total_request_sent += 1
+                break
+            except Exception as e:
+                logger.warning(f"Report event failed, retrying... {e}")
+                last_exception = e
+        else:
+            data_str = str(data)
+            limit = event_consts.LOG_ERROR_EVENT_STRING_LENGTH_LIMIT
+            logger.error(
+                "Report event failed: %s",
+                data_str[:limit] + (data_str[limit:] and "..."),
+                exc_info=last_exception,
+            )
+
+    async def get_internal_states(self):
+        if self.total_event_reported <= 0 or self.total_request_sent <= 0:
+            return
+
+        elapsed = time.monotonic() - self.module_started
+        return {
+            "total_events_reported": self.total_event_reported,
+            "Total_report_request": self.total_request_sent,
+            "queue_size": self._cached_events.qsize(),
+            "total_uptime": elapsed,
+        }
+
+    async def run(self, server):
+        # Start monitor task.
+        self._monitor = monitor_events(
+            self._event_dir,
+            lambda data: create_task(self._cached_events.put(data)),
+            self._executor,
+        )
+
+        await asyncio.gather(
+            self.report_events(),
+        )
+
+    @staticmethod
+    def is_minimal_module():
+        return False

python/ray/dashboard/modules/event/event_consts.py

@@ -0,0 +1,20 @@
+from ray._private.ray_constants import env_float, env_integer
+from ray.core.generated import event_pb2
+
+LOG_ERROR_EVENT_STRING_LENGTH_LIMIT = 1000
+# Monitor events
+SCAN_EVENT_DIR_INTERVAL_SECONDS = env_integer("SCAN_EVENT_DIR_INTERVAL_SECONDS", 2)
+SCAN_EVENT_START_OFFSET_SECONDS = -30 * 60
+CONCURRENT_READ_LIMIT = 50
+EVENT_READ_LINE_COUNT_LIMIT = 200
+EVENT_READ_LINE_LENGTH_LIMIT = env_integer(
+    "EVENT_READ_LINE_LENGTH_LIMIT", 2 * 1024 * 1024
+)  # 2MB
+# Report events
+EVENT_AGENT_REPORT_INTERVAL_SECONDS = env_float(
+    "EVENT_AGENT_REPORT_INTERVAL_SECONDS", 0.1
+)
+EVENT_AGENT_RETRY_TIMES = 10
+EVENT_AGENT_CACHE_SIZE = 10240
+# Event sources
+EVENT_SOURCE_ALL = event_pb2.Event.SourceType.keys()

python/ray/dashboard/modules/event/event_head.py

@@ -0,0 +1,237 @@
+import asyncio
+import logging
+import os
+import time
+from collections import OrderedDict, defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from itertools import islice
+from typing import Dict, List, Union
+
+import aiohttp.web
+
+import ray
+import ray.dashboard.optional_utils as dashboard_optional_utils
+import ray.dashboard.utils as dashboard_utils
+from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag
+from ray._common.utils import get_or_create_event_loop
+from ray._private.ray_constants import env_integer
+from ray.dashboard.consts import (
+    RAY_STATE_SERVER_MAX_HTTP_REQUEST,
+    RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED,
+    RAY_STATE_SERVER_MAX_HTTP_REQUEST_ENV_NAME,
+)
+from ray.dashboard.modules.event.event_utils import monitor_events, parse_event_strings
+from ray.dashboard.state_api_utils import do_filter, handle_list_api
+from ray.dashboard.subprocesses.module import SubprocessModule
+from ray.dashboard.subprocesses.routes import SubprocessRouteTable as routes
+from ray.util.state.common import ClusterEventState, ListApiOptions, ListApiResponse
+
+logger = logging.getLogger(__name__)
+
+JobEvents = OrderedDict
+dashboard_utils._json_compatible_types.add(JobEvents)
+
+MAX_EVENTS_TO_CACHE = int(os.environ.get("RAY_DASHBOARD_MAX_EVENTS_TO_CACHE", 10000))
+
+# NOTE: Executor in this head is intentionally constrained to just 1 thread by
+#       default to limit its concurrency, therefore reducing potential for
+#       GIL contention
+RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS = env_integer(
+    "RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS", 1
+)
+
+
+async def _list_cluster_events_impl(
+    *,
+    all_events: Dict[str, JobEvents],
+    executor: ThreadPoolExecutor,
+    option: ListApiOptions,
+) -> ListApiResponse:
+    """List all cluster events from the cluster. Made a free function to allow unit tests.
+
+    Args:
+        all_events: Mapping of ``job_id`` to per-job event dictionaries.
+        executor: Executor used to run the (CPU-bound) transform off the event loop.
+        option: Query options (filters, limit, detail flag).
+
+    Returns:
+        A list of cluster events in the cluster.
+        The schema of returned "dict" is equivalent to the
+        `ClusterEventState` protobuf message.
+    """
+
+    def transform(all_events) -> ListApiResponse:
+        result = []
+        for _, events in all_events.items():
+            for _, event in events.items():
+                event["time"] = str(datetime.fromtimestamp(int(event["timestamp"])))
+                result.append(event)
+
+        num_after_truncation = len(result)
+        result.sort(key=lambda entry: entry["timestamp"])
+        total = len(result)
+        result = do_filter(result, option.filters, ClusterEventState, option.detail)
+        num_filtered = len(result)
+        # Sort to make the output deterministic.
+        result = list(islice(result, option.limit))
+        return ListApiResponse(
+            result=result,
+            total=total,
+            num_after_truncation=num_after_truncation,
+            num_filtered=num_filtered,
+        )
+
+    return await get_or_create_event_loop().run_in_executor(
+        executor, transform, all_events
+    )
+
+
+class EventHead(
+    SubprocessModule,
+    dashboard_utils.RateLimitedModule,
+):
+    def __init__(self, *args, **kwargs):
+        SubprocessModule.__init__(self, *args, **kwargs)
+        dashboard_utils.RateLimitedModule.__init__(
+            self,
+            min(
+                RAY_STATE_SERVER_MAX_HTTP_REQUEST,
+                RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED,
+            ),
+        )
+        self._event_dir = os.path.join(self.log_dir, "events")
+        os.makedirs(self._event_dir, exist_ok=True)
+        self._monitor: Union[asyncio.Task, None] = None
+        self.total_report_events_count = 0
+        self.total_events_received = 0
+        self.module_started = time.monotonic()
+        # {job_id hex(str): {event_id (str): event (dict)}}
+        self.events: Dict[str, JobEvents] = defaultdict(JobEvents)
+
+        self._executor = ThreadPoolExecutor(
+            max_workers=RAY_DASHBOARD_EVENT_HEAD_TPE_MAX_WORKERS,
+            thread_name_prefix="event_head_executor",
+        )
+
+        # To init gcs_client in internal_kv for record_extra_usage_tag.
+        assert self.gcs_client is not None
+        assert ray.experimental.internal_kv._internal_kv_initialized()
+
+    async def limit_handler_(self):
+        return dashboard_optional_utils.rest_response(
+            status_code=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR,
+            error_message=(
+                "Max number of in-progress requests="
+                f"{self.max_num_call_} reached. "
+                "To set a higher limit, set environment variable: "
+                f"export {RAY_STATE_SERVER_MAX_HTTP_REQUEST_ENV_NAME}='xxx'. "
+                f"Max allowed = {RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED}"
+            ),
+            result=None,
+        )
+
+    def _update_events(self, event_list):
+        # {job_id: {event_id: event}}
+        all_job_events = defaultdict(JobEvents)
+        for event in event_list:
+            event_id = event["event_id"]
+            custom_fields = event.get("custom_fields")
+            system_event = False
+            if custom_fields:
+                job_id = custom_fields.get("job_id", "global") or "global"
+            else:
+                job_id = "global"
+            if system_event is False:
+                all_job_events[job_id][event_id] = event
+
+        for job_id, new_job_events in all_job_events.items():
+            job_events = self.events[job_id]
+            job_events.update(new_job_events)
+
+            # Limit the # of events cached if it exceeds the threshold.
+            if len(job_events) > MAX_EVENTS_TO_CACHE * 1.1:
+                while len(job_events) > MAX_EVENTS_TO_CACHE:
+                    job_events.popitem(last=False)
+
+    @routes.post("/report_events")
+    async def report_events(self, request):
+        """
+        Report events to the dashboard.
+        The request body is a JSON array of event strings in type string.
+        Response should contain {"success": true}.
+        """
+        try:
+            request_body: List[str] = await request.json()
+        except Exception as e:
+            logger.warning(f"Failed to parse request body: {request=}, {e=}")
+            raise aiohttp.web.HTTPBadRequest()
+        if not isinstance(request_body, list):
+            logger.warning(f"Request body is not a list, {request_body=}")
+            raise aiohttp.web.HTTPBadRequest()
+        events = parse_event_strings(request_body)
+        logger.debug("Received %d events", len(events))
+        self._update_events(events)
+        self.total_report_events_count += 1
+        self.total_events_received += len(events)
+        return dashboard_optional_utils.rest_response(
+            success=True,
+            message="",
+            status_code=dashboard_utils.HTTPStatusCode.OK,
+        )
+
+    async def _periodic_state_print(self):
+        if self.total_events_received <= 0 or self.total_report_events_count <= 0:
+            return
+
+        elapsed = time.monotonic() - self.module_started
+        return {
+            "total_events_received": self.total_events_received,
+            "Total_requests_received": self.total_report_events_count,
+            "total_uptime": elapsed,
+        }
+
+    @routes.get("/events")
+    @dashboard_optional_utils.aiohttp_cache
+    async def get_event(self, req) -> aiohttp.web.Response:
+        job_id = req.query.get("job_id")
+        if job_id is None:
+            all_events = {
+                job_id: list(job_events.values())
+                for job_id, job_events in self.events.items()
+            }
+            return dashboard_optional_utils.rest_response(
+                status_code=dashboard_utils.HTTPStatusCode.OK,
+                message="All events fetched.",
+                events=all_events,
+            )
+
+        job_events = self.events[job_id]
+        return dashboard_optional_utils.rest_response(
+            status_code=dashboard_utils.HTTPStatusCode.OK,
+            message="Job events fetched.",
+            job_id=job_id,
+            events=list(job_events.values()),
+        )
+
+    @routes.get("/api/v0/cluster_events")
+    @dashboard_utils.RateLimitedModule.enforce_max_concurrent_calls
+    async def list_cluster_events(
+        self, req: aiohttp.web.Request
+    ) -> aiohttp.web.Response:
+        record_extra_usage_tag(TagKey.CORE_STATE_API_LIST_CLUSTER_EVENTS, "1")
+
+        async def list_api_fn(option: ListApiOptions):
+            return await _list_cluster_events_impl(
+                all_events=self.events, executor=self._executor, option=option
+            )
+
+        return await handle_list_api(list_api_fn, req)
+
+    async def run(self):
+        await super().run()
+        self._monitor = monitor_events(
+            self._event_dir,
+            lambda data: self._update_events(parse_event_strings(data)),
+            self._executor,
+        )

python/ray/dashboard/modules/event/event_utils.py

@@ -0,0 +1,210 @@
+import asyncio
+import collections
+import fnmatch
+import itertools
+import json
+import logging.handlers
+import mmap
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable, Dict, List, Optional
+
+from ray._common.utils import get_or_create_event_loop, run_background_task
+from ray.dashboard.modules.event import event_consts
+from ray.dashboard.utils import async_loop_forever
+
+logger = logging.getLogger(__name__)
+
+
+def _get_source_files(event_dir, source_types=None, event_file_filter=None):
+    event_log_names = os.listdir(event_dir)
+    source_files = {}
+    all_source_types = set(event_consts.EVENT_SOURCE_ALL)
+    for source_type in source_types or event_consts.EVENT_SOURCE_ALL:
+        assert source_type in all_source_types, f"Invalid source type: {source_type}"
+        files = []
+        for n in event_log_names:
+            if fnmatch.fnmatch(n, f"*{source_type}*.log"):
+                f = os.path.join(event_dir, n)
+                if event_file_filter is not None and not event_file_filter(f):
+                    continue
+                files.append(f)
+        if files:
+            source_files[source_type] = files
+    return source_files
+
+
+def _restore_newline(event_dict):
+    try:
+        event_dict["message"] = (
+            event_dict["message"].replace("\\n", "\n").replace("\\r", "\n")
+        )
+    except Exception:
+        logger.exception("Restore newline for event failed: %s", event_dict)
+    return event_dict
+
+
+def _parse_line(event_str):
+    return _restore_newline(json.loads(event_str))
+
+
+def parse_event_strings(event_string_list):
+    events = []
+    for data in event_string_list:
+        if not data:
+            continue
+        try:
+            event = _parse_line(data)
+            events.append(event)
+        except Exception:
+            logger.exception("Parse event line failed: %s", repr(data))
+    return events
+
+
+ReadFileResult = collections.namedtuple(
+    "ReadFileResult", ["fid", "size", "mtime", "position", "lines"]
+)
+
+
+def _read_file(
+    file, pos, n_lines=event_consts.EVENT_READ_LINE_COUNT_LIMIT, closefd=True
+):
+    with open(file, "rb", closefd=closefd) as f:
+        # The ino may be 0 on Windows.
+        stat = os.stat(f.fileno())
+        fid = stat.st_ino or file
+        lines = []
+        with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+            start = pos
+            for _ in range(n_lines):
+                sep = mm.find(b"\n", start)
+                if sep == -1:
+                    break
+                if sep - start <= event_consts.EVENT_READ_LINE_LENGTH_LIMIT:
+                    lines.append(mm[start:sep].decode("utf-8"))
+                else:
+                    truncated_size = min(100, event_consts.EVENT_READ_LINE_LENGTH_LIMIT)
+                    logger.warning(
+                        "Ignored long string: %s...(%s chars)",
+                        mm[start : start + truncated_size].decode("utf-8"),
+                        sep - start,
+                    )
+                start = sep + 1
+        return ReadFileResult(fid, stat.st_size, stat.st_mtime, start, lines)
+
+
+def monitor_events(
+    event_dir: str,
+    callback: Callable[[List[str]], None],
+    monitor_thread_pool_executor: ThreadPoolExecutor,
+    scan_interval_seconds: float = event_consts.SCAN_EVENT_DIR_INTERVAL_SECONDS,
+    start_mtime: float = time.time() + event_consts.SCAN_EVENT_START_OFFSET_SECONDS,
+    monitor_files: Optional[Dict[int, tuple]] = None,
+    source_types: Optional[List[str]] = None,
+) -> asyncio.Task:
+    """Monitor events in directory. New events will be read and passed to the
+    callback.
+
+    Args:
+        event_dir: The event log directory.
+        callback: A callback that accepts a list of event strings.
+        monitor_thread_pool_executor: A thread pool exector to monitor/update
+            events. None means it will use the default execturo which uses
+            num_cpus of the machine * 5 threads (before python 3.8) or
+            min(32, num_cpus + 5) (from Python 3.8).
+        scan_interval_seconds: An interval seconds between two scans.
+        start_mtime: Only the event log files whose last modification
+            time is greater than start_mtime are monitored.
+        monitor_files: The map from event log file id to MonitorFile object.
+            Monitor all files start from the beginning if the value is None.
+        source_types: A list of source type name from
+            event_pb2.Event.SourceType.keys(). Monitor all source types if the
+            value is None.
+
+    Returns:
+        The background ``asyncio.Task`` driving the periodic directory scan.
+    """
+    loop = get_or_create_event_loop()
+    if monitor_files is None:
+        monitor_files = {}
+
+    logger.info(
+        "Monitor events logs modified after %s on %s, the source types are %s.",
+        start_mtime,
+        event_dir,
+        "all" if source_types is None else source_types,
+    )
+
+    MonitorFile = collections.namedtuple("MonitorFile", ["size", "mtime", "position"])
+
+    def _source_file_filter(source_file):
+        stat = os.stat(source_file)
+        return stat.st_mtime > start_mtime
+
+    def _read_monitor_file(file, pos):
+        assert isinstance(
+            file, str
+        ), f"File should be a str, but a {type(file)}({file}) found"
+        fd = os.open(file, os.O_RDONLY)
+        try:
+            stat = os.stat(fd)
+            # Check the file size to avoid raising the exception
+            # ValueError: cannot mmap an empty file
+            if stat.st_size <= 0:
+                return []
+            fid = stat.st_ino or file
+            monitor_file = monitor_files.get(fid)
+            if monitor_file:
+                if (
+                    monitor_file.position == monitor_file.size
+                    and monitor_file.size == stat.st_size
+                    and monitor_file.mtime == stat.st_mtime
+                ):
+                    logger.debug(
+                        "Skip reading the file because there is no change: %s", file
+                    )
+                    return []
+                position = monitor_file.position
+            else:
+                logger.info("Found new event log file: %s", file)
+                position = pos
+            # Close the fd in finally.
+            r = _read_file(fd, position, closefd=False)
+            # It should be fine to update the dict in executor thread.
+            monitor_files[r.fid] = MonitorFile(r.size, r.mtime, r.position)
+            loop.call_soon_threadsafe(callback, r.lines)
+        except Exception as e:
+            raise Exception(f"Read event file failed: {file}") from e
+        finally:
+            os.close(fd)
+
+    @async_loop_forever(scan_interval_seconds, cancellable=True)
+    async def _scan_event_log_files():
+        # Scan event files.
+        source_files = await loop.run_in_executor(
+            monitor_thread_pool_executor,
+            _get_source_files,
+            event_dir,
+            source_types,
+            _source_file_filter,
+        )
+
+        # Limit concurrent read to avoid fd exhaustion.
+        semaphore = asyncio.Semaphore(event_consts.CONCURRENT_READ_LIMIT)
+
+        async def _concurrent_coro(filename):
+            async with semaphore:
+                return await loop.run_in_executor(
+                    monitor_thread_pool_executor, _read_monitor_file, filename, 0
+                )
+
+        # Read files.
+        await asyncio.gather(
+            *[
+                _concurrent_coro(filename)
+                for filename in list(itertools.chain(*source_files.values()))
+            ]
+        )
+
+    return run_background_task(_scan_event_log_files())

python/ray/dashboard/modules/event/tests/test_event.py

@@ -0,0 +1,780 @@
+import asyncio
+import copy
+import json
+import logging
+import os
+import random
+import socket
+import sys
+import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime
+from pprint import pprint
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+import requests
+
+import ray
+from ray._common.test_utils import wait_for_condition
+from ray._common.utils import binary_to_hex
+from ray._private.event.event_logger import (
+    EventLoggerAdapter,
+    filter_event_by_level,
+    get_event_id,
+    get_event_logger,
+)
+from ray._private.event.export_event_logger import (
+    EventLogType,
+    ExportEventLoggerAdapter,
+    get_export_event_logger,
+)
+from ray._private.protobuf_compat import message_to_dict
+from ray._private.state_api_test_utils import create_api_options, verify_schema
+from ray._private.test_utils import (
+    format_web_url,
+    wait_until_server_available,
+)
+from ray.cluster_utils import AutoscalingCluster
+from ray.core.generated import (
+    event_pb2,
+    export_submission_job_event_pb2,
+)
+from ray.dashboard.modules.event import event_consts
+from ray.dashboard.modules.event.event_head import _list_cluster_events_impl
+from ray.dashboard.modules.event.event_utils import monitor_events
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.job_submission import JobSubmissionClient
+from ray.util.state import list_cluster_events
+from ray.util.state.common import ClusterEventState
+
+logger = logging.getLogger(__name__)
+
+
+def _get_event(msg="empty message", job_id=None, source_type=None):
+    return {
+        "event_id": binary_to_hex(np.random.bytes(18)),
+        "source_type": (
+            random.choice(event_pb2.Event.SourceType.keys())
+            if source_type is None
+            else source_type
+        ),
+        "host_name": "po-dev.inc.alipay.net",
+        "pid": random.randint(1, 65536),
+        "label": "",
+        "message": msg,
+        "timestamp": time.time(),
+        "severity": "INFO",
+        "custom_fields": {
+            "job_id": (
+                ray.JobID.from_int(random.randint(1, 100)).hex()
+                if job_id is None
+                else job_id
+            ),
+            "node_id": "",
+            "task_id": "",
+        },
+    }
+
+
+def _test_logger(name, log_file, max_bytes, backup_count):
+    handler = logging.handlers.RotatingFileHandler(
+        log_file, maxBytes=max_bytes, backupCount=backup_count
+    )
+    formatter = logging.Formatter("%(message)s")
+    handler.setFormatter(formatter)
+
+    logger = logging.getLogger(name)
+    logger.propagate = False
+    logger.setLevel(logging.INFO)
+    logger.addHandler(handler)
+
+    return logger
+
+
+def test_python_global_event_logger(tmp_path):
+    logger = get_event_logger(event_pb2.Event.SourceType.GCS, str(tmp_path))
+    logger.set_global_context({"test_meta": "1"})
+    logger.info("message", a="a", b="b")
+    logger.error("message", a="a", b="b")
+    logger.warning("message", a="a", b="b")
+    logger.fatal("message", a="a", b="b")
+    event_dir = tmp_path / "events"
+    assert event_dir.exists()
+    event_file = event_dir / "event_GCS.log"
+    assert event_file.exists()
+
+    line_severities = ["INFO", "ERROR", "WARNING", "FATAL"]
+
+    with event_file.open() as f:
+        for line, severity in zip(f.readlines(), line_severities):
+            data = json.loads(line)
+            assert data["severity"] == severity
+            assert data["label"] == ""
+            assert "timestamp" in data
+            assert len(data["event_id"]) == 36
+            assert data["message"] == "message"
+            assert data["source_type"] == "GCS"
+            assert data["source_hostname"] == socket.gethostname()
+            assert data["source_pid"] == os.getpid()
+            assert data["custom_fields"]["a"] == "a"
+            assert data["custom_fields"]["b"] == "b"
+
+
+def test_event_basic(disable_aiohttp_cache, ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
+    webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
+    session_dir = ray_start_with_dashboard["session_dir"]
+    event_dir = os.path.join(session_dir, "logs", "events")
+    job_id = ray.JobID.from_int(100).hex()
+
+    source_type_gcs = event_pb2.Event.SourceType.Name(event_pb2.Event.GCS)
+    source_type_raylet = event_pb2.Event.SourceType.Name(event_pb2.Event.RAYLET)
+    test_count = 20
+
+    for source_type in [source_type_gcs, source_type_raylet]:
+        test_log_file = os.path.join(event_dir, f"event_{source_type}.log")
+        test_logger = _test_logger(
+            __name__ + str(random.random()),
+            test_log_file,
+            max_bytes=2000,
+            backup_count=0,
+        )
+        for i in range(test_count):
+            sample_event = _get_event(str(i), job_id=job_id, source_type=source_type)
+            test_logger.info("%s", json.dumps(sample_event))
+
+    def _check_events():
+        try:
+            resp = requests.get(f"{webui_url}/events")
+            resp.raise_for_status()
+            result = resp.json()
+            all_events = result["data"]["events"]
+            job_events = all_events[job_id]
+            assert len(job_events) >= test_count * 2
+            source_messages = {}
+            for e in job_events:
+                source_type = e["sourceType"]
+                message = e["message"]
+                source_messages.setdefault(source_type, set()).add(message)
+            assert len(source_messages[source_type_gcs]) >= test_count
+            assert len(source_messages[source_type_raylet]) >= test_count
+            data = {str(i) for i in range(test_count)}
+            assert data & source_messages[source_type_gcs] == data
+            assert data & source_messages[source_type_raylet] == data
+            return True
+        except Exception as ex:
+            logger.exception(ex)
+            return False
+
+    wait_for_condition(_check_events, timeout=15)
+
+
+def test_event_message_limit(
+    small_event_line_limit, disable_aiohttp_cache, ray_start_with_dashboard
+):
+    event_read_line_length_limit = small_event_line_limit
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
+    webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
+    session_dir = ray_start_with_dashboard["session_dir"]
+    event_dir = os.path.join(session_dir, "logs", "events")
+    job_id = ray.JobID.from_int(100).hex()
+    events = []
+    # Sample event equals with limit.
+    sample_event = _get_event("", job_id=job_id)
+    message_len = event_read_line_length_limit - len(json.dumps(sample_event))
+    for i in range(10):
+        sample_event = copy.deepcopy(sample_event)
+        sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
+        sample_event["message"] = str(i) * message_len
+        assert len(json.dumps(sample_event)) == event_read_line_length_limit
+        events.append(sample_event)
+    # Sample event longer than limit.
+    sample_event = copy.deepcopy(sample_event)
+    sample_event["event_id"] = binary_to_hex(np.random.bytes(18))
+    sample_event["message"] = "2" * (message_len + 1)
+    assert len(json.dumps(sample_event)) > event_read_line_length_limit
+    events.append(sample_event)
+
+    for i in range(event_consts.EVENT_READ_LINE_COUNT_LIMIT):
+        events.append(_get_event(str(i), job_id=job_id))
+
+    with open(os.path.join(event_dir, "tmp.log"), "w") as f:
+        f.writelines([(json.dumps(e) + "\n") for e in events])
+
+    try:
+        os.remove(os.path.join(event_dir, "event_GCS.log"))
+    except Exception:
+        pass
+    os.rename(
+        os.path.join(event_dir, "tmp.log"), os.path.join(event_dir, "event_GCS.log")
+    )
+
+    def _check_events():
+        try:
+            resp = requests.get(f"{webui_url}/events")
+            resp.raise_for_status()
+            result = resp.json()
+            all_events = result["data"]["events"]
+            assert (
+                len(all_events[job_id]) >= event_consts.EVENT_READ_LINE_COUNT_LIMIT + 10
+            )
+            messages = [e["message"] for e in all_events[job_id]]
+            for i in range(10):
+                assert str(i) * message_len in messages
+            assert "2" * (message_len + 1) not in messages
+            assert str(event_consts.EVENT_READ_LINE_COUNT_LIMIT - 1) in messages
+            return True
+        except Exception as ex:
+            logger.exception(ex)
+            return False
+
+    wait_for_condition(_check_events, timeout=15)
+
+
+def test_report_events(ray_start_with_dashboard):
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
+    webui_url = format_web_url(ray_start_with_dashboard["webui_url"])
+    url = f"{webui_url}/report_events"
+
+    resp = requests.post(url)
+    assert resp.status_code == 400
+    resp = requests.post(url, json={"Hello": "World"})
+    assert resp.status_code == 400
+
+    job_id = ray.JobID.from_int(100).hex()
+    sample_event = _get_event("Hello", job_id=job_id)
+    resp = requests.post(url, json=[json.dumps(sample_event)])
+    assert resp.status_code == 200
+
+    resp = requests.get(f"{webui_url}/events")
+    assert resp.status_code == 200
+    result = resp.json()
+    all_events = result["data"]["events"]
+    assert len(all_events) == 1
+    assert job_id in all_events
+    assert len(all_events[job_id]) == 1
+    assert all_events[job_id][0]["message"] == "Hello"
+
+
+@pytest.mark.asyncio
+async def test_monitor_events():
+    with tempfile.TemporaryDirectory() as temp_dir:
+        common = event_pb2.Event.SourceType.Name(event_pb2.Event.COMMON)
+        common_log = os.path.join(temp_dir, f"event_{common}.log")
+        test_logger = _test_logger(
+            __name__ + str(random.random()), common_log, max_bytes=10, backup_count=0
+        )
+        test_events1 = []
+        monitor_task = monitor_events(
+            temp_dir, lambda x: test_events1.extend(x), None, scan_interval_seconds=0.01
+        )
+        assert not monitor_task.done()
+        count = 10
+
+        async def _writer(*args, read_events, spin=True):
+            for x in range(*args):
+                test_logger.info("%s", x)
+                if spin:
+                    while str(x) not in read_events:
+                        await asyncio.sleep(0.01)
+
+        async def _check_events(expect_events, read_events, timeout=10):
+            start_time = time.time()
+            while True:
+                sorted_events = sorted(int(i) for i in read_events)
+                sorted_events = [str(i) for i in sorted_events]
+                if time.time() - start_time > timeout:
+                    raise TimeoutError(
+                        f"Timeout, read events: {sorted_events}, "
+                        f"expect events: {expect_events}"
+                    )
+                if len(sorted_events) == len(expect_events):
+                    if sorted_events == expect_events:
+                        break
+                await asyncio.sleep(1)
+
+        await asyncio.gather(
+            _writer(count, read_events=test_events1),
+            _check_events([str(i) for i in range(count)], read_events=test_events1),
+        )
+
+        monitor_task.cancel()
+        test_events2 = []
+        monitor_task = monitor_events(
+            temp_dir, lambda x: test_events2.extend(x), None, scan_interval_seconds=0.1
+        )
+
+        await _check_events([str(i) for i in range(count)], read_events=test_events2)
+
+        await _writer(count, count * 2, read_events=test_events2)
+        await _check_events(
+            [str(i) for i in range(count * 2)], read_events=test_events2
+        )
+
+        log_file_count = len(os.listdir(temp_dir))
+
+        test_logger = _test_logger(
+            __name__ + str(random.random()), common_log, max_bytes=1000, backup_count=0
+        )
+        assert len(os.listdir(temp_dir)) == log_file_count
+
+        await _writer(count * 2, count * 3, spin=False, read_events=test_events2)
+        await _check_events(
+            [str(i) for i in range(count * 3)], read_events=test_events2
+        )
+        await _writer(count * 3, count * 4, spin=False, read_events=test_events2)
+        await _check_events(
+            [str(i) for i in range(count * 4)], read_events=test_events2
+        )
+
+        # Test cancel monitor task.
+        monitor_task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await monitor_task
+        assert monitor_task.done()
+
+        assert len(os.listdir(temp_dir)) == 1, "There should just be 1 event log"
+
+
+@pytest.mark.parametrize("autoscaler_v2", [False, True], ids=["v1", "v2"])
+def test_autoscaler_cluster_events(autoscaler_v2, shutdown_only):
+    cluster = AutoscalingCluster(
+        head_resources={"CPU": 2},
+        worker_node_types={
+            "cpu_node": {
+                "resources": {
+                    "CPU": 4,
+                },
+                "node_config": {},
+                "min_workers": 0,
+                "max_workers": 1,
+            },
+            "gpu_node": {
+                "resources": {
+                    "CPU": 2,
+                    "GPU": 1,
+                },
+                "node_config": {},
+                "min_workers": 0,
+                "max_workers": 1,
+            },
+        },
+        autoscaler_v2=autoscaler_v2,
+        idle_timeout_minutes=1,
+    )
+
+    try:
+        cluster.start()
+        ray.init("auto")
+
+        # Triggers the addition of a GPU node.
+        @ray.remote(num_gpus=1)
+        def f():
+            print("gpu ok")
+
+        # Triggers the addition of a CPU node.
+        @ray.remote(num_cpus=3)
+        def g():
+            print("cpu ok")
+
+        wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 2)
+        ray.get(f.remote())
+        wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 4)
+        wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
+        ray.get(g.remote())
+        wait_for_condition(lambda: ray.cluster_resources()["CPU"] == 8)
+        wait_for_condition(lambda: ray.cluster_resources()["GPU"] == 1)
+
+        # Trigger an infeasible task
+        g.options(num_cpus=0, num_gpus=5).remote()
+
+        def verify():
+            cluster_events = list_cluster_events()
+            print(cluster_events)
+            messages = {(e["message"], e["source_type"]) for e in cluster_events}
+            if not autoscaler_v2:
+                # With head node resources, we don't actually resized. So this event is
+                # not really accurate.
+                assert ("Resized to 2 CPUs.", "AUTOSCALER") in messages, cluster_events
+            assert (
+                "Adding 1 node(s) of type gpu_node.",
+                "AUTOSCALER",
+            ) in messages, cluster_events
+            assert (
+                "Resized to 4 CPUs, 1 GPUs.",
+                "AUTOSCALER",
+            ) in messages, cluster_events
+            assert (
+                "Adding 1 node(s) of type cpu_node.",
+                "AUTOSCALER",
+            ) in messages, cluster_events
+            assert (
+                "Resized to 8 CPUs, 1 GPUs.",
+                "AUTOSCALER",
+            ) in messages, cluster_events
+            assert "No available node types can fulfill resource request" in "".join(
+                [t[0] for t in messages]
+            )
+
+            return True
+
+        wait_for_condition(verify, timeout=30)
+        pprint(list_cluster_events())
+    finally:
+        ray.shutdown()
+        cluster.shutdown()
+
+
+def test_filter_event_by_level(monkeypatch):
+    def gen_event(level: str):
+        return event_pb2.Event(
+            source_type=event_pb2.Event.AUTOSCALER,
+            severity=event_pb2.Event.Severity.Value(level),
+            message=level,
+        )
+
+    trace = gen_event("TRACE")
+    debug = gen_event("DEBUG")
+    info = gen_event("INFO")
+    warning = gen_event("WARNING")
+    error = gen_event("ERROR")
+    fatal = gen_event("FATAL")
+
+    def assert_events_filtered(events, expected, filter_level):
+        filtered = [e for e in events if filter_event_by_level(e, filter_level)]
+        print(filtered)
+        assert len(filtered) == len(expected)
+        assert {e.message for e in filtered} == {e.message for e in expected}
+
+    events = [trace, debug, info, warning, error, fatal]
+    assert_events_filtered(events, [], "TRACE")
+    assert_events_filtered(events, [trace], "DEBUG")
+    assert_events_filtered(events, [trace, debug], "INFO")
+    assert_events_filtered(events, [trace, debug, info], "WARNING")
+    assert_events_filtered(events, [trace, debug, info, warning], "ERROR")
+    assert_events_filtered(events, [trace, debug, info, warning, error], "FATAL")
+
+
+def test_jobs_cluster_events(shutdown_only):
+    ray.init()
+    address = ray._private.worker._global_node.webui_url
+    address = format_web_url(address)
+    client = JobSubmissionClient(address)
+    submission_id = client.submit_job(entrypoint="ls")
+
+    def verify():
+        events = list_cluster_events()
+        assert len(list_cluster_events()) == 2
+        start_event = events[0]
+        completed_event = events[1]
+
+        assert start_event["source_type"] == "JOBS"
+        assert f"Started a ray job {submission_id}" in start_event["message"]
+        assert start_event["severity"] == "INFO"
+        assert completed_event["source_type"] == "JOBS"
+        assert (
+            f"Completed a ray job {submission_id} with a status SUCCEEDED."
+            == completed_event["message"]
+        )
+        assert completed_event["severity"] == "INFO"
+        return True
+
+    print("Test successful job run.")
+    wait_for_condition(verify)
+    pprint(list_cluster_events())
+
+    # Test the failure case. In this part, job fails because the runtime env
+    # creation fails.
+    submission_id = client.submit_job(
+        entrypoint="ls",
+        runtime_env={"pip": ["nonexistent_dep"]},
+    )
+
+    def verify():
+        events = list_cluster_events(detail=True)
+        failed_events = []
+
+        for e in events:
+            if (
+                "submission_id" in e["custom_fields"]
+                and e["custom_fields"]["submission_id"] == submission_id
+            ):
+                failed_events.append(e)
+
+        assert len(failed_events) == 2
+        failed_start = failed_events[0]
+        failed_completed = failed_events[1]
+
+        assert failed_start["source_type"] == "JOBS"
+        assert f"Started a ray job {submission_id}" in failed_start["message"]
+        assert failed_completed["source_type"] == "JOBS"
+        assert failed_completed["severity"] == "ERROR"
+        assert (
+            f"Completed a ray job {submission_id} with a status FAILED."
+            in failed_completed["message"]
+        )
+
+        # Make sure the error message is included.
+        assert "ERROR: No matching distribution found" in failed_completed["message"]
+        return True
+
+    print("Test failed (runtime_env failure) job run.")
+    wait_for_condition(verify, timeout=30)
+    pprint(list_cluster_events())
+
+
+def test_core_events(shutdown_only):
+    # Test events recorded from core RAY_EVENT APIs.
+    ray.init()
+
+    @ray.remote
+    class Actor:
+        def getpid(self):
+            return os.getpid()
+
+    a = Actor.remote()
+    pid = ray.get(a.getpid.remote())
+    os.kill(pid, 9)
+    s = time.time()
+
+    def verify():
+        events = list_cluster_events(filters=[("source_type", "=", "RAYLET")])
+        print(events)
+        assert len(list_cluster_events()) == 1
+        event = events[0]
+        assert event["severity"] == "ERROR"
+        datetime_str = event["time"]
+        datetime_obj = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S")
+        timestamp = time.mktime(datetime_obj.timetuple())
+
+        # Make sure timestamp is not incorrect. Add sufficient buffer (60 seconds)
+        assert abs(timestamp - s) < 60
+        assert (
+            "A worker died or was killed while executing "
+            "a task by an unexpected system error" in event["message"]
+        )
+        return True
+
+    wait_for_condition(verify)
+    pprint(list_cluster_events())
+
+
+def test_cluster_events_retention(monkeypatch, shutdown_only):
+    with monkeypatch.context() as m:
+        # defer for 5s for the second node.
+        # This will help the API not return until the node is killed.
+        m.setenv("RAY_DASHBOARD_MAX_EVENTS_TO_CACHE", "10")
+        ray.init()
+        address = ray._private.worker._global_node.webui_url
+        address = format_web_url(address)
+        client = JobSubmissionClient(address)
+
+        submission_ids = []
+        for _ in range(12):
+            submission_ids.append(client.submit_job(entrypoint="ls"))
+        print(submission_ids)
+
+        def verify():
+            events = list_cluster_events()
+            assert len(list_cluster_events()) == 10
+
+            messages = [event["message"] for event in events]
+
+            # Make sure the first two has been GC'ed.
+            for m in messages:
+                assert submission_ids[0] not in m
+                assert submission_ids[1] not in m
+            return True
+
+        wait_for_condition(verify)
+        pprint(list_cluster_events())
+
+
+@pytest.mark.asyncio
+async def test_list_cluster_events_impl():
+    executor = ThreadPoolExecutor(
+        max_workers=1,
+        thread_name_prefix="event_head_executor",
+    )
+
+    event_id_1 = get_event_id()
+    event_id_2 = get_event_id()
+    events = {
+        "job_1": {
+            event_id_1: {
+                "timestamp": 10,
+                "severity": "DEBUG",
+                "message": "a",
+                "event_id": event_id_1,
+                "source_type": "GCS",
+            },
+            event_id_2: {
+                "timestamp": 10,
+                "severity": "INFO",
+                "message": "b",
+                "event_id": event_id_2,
+                "source_type": "GCS",
+            },
+        }
+    }
+    result = await _list_cluster_events_impl(
+        all_events=events, executor=executor, option=create_api_options()
+    )
+    data = result.result
+    data = data[0]
+    verify_schema(ClusterEventState, data)
+    assert result.total == 2
+
+    """
+    Test detail
+    """
+    # TODO(sang)
+
+    """
+    Test limit
+    """
+    assert len(result.result) == 2
+    result = await _list_cluster_events_impl(
+        all_events=events, executor=executor, option=create_api_options(limit=1)
+    )
+    data = result.result
+    assert len(data) == 1
+    assert result.total == 2
+
+    """
+    Test filters
+    """
+    # If the column is not supported for filtering, it should raise an exception.
+    with pytest.raises(ValueError):
+        result = await _list_cluster_events_impl(
+            all_events=events,
+            executor=executor,
+            option=create_api_options(filters=[("time", "=", "20")]),
+        )
+    result = await _list_cluster_events_impl(
+        all_events=events,
+        executor=executor,
+        option=create_api_options(filters=[("severity", "=", "INFO")]),
+    )
+    assert len(result.result) == 1
+
+
+def test_export_event_logger(tmp_path):
+    """
+    Unit test a mock export event of type ExportSubmissionJobEventData
+    is correctly written to file. This doesn't events are correctly generated.
+    """
+    logger = get_export_event_logger(EventLogType.SUBMISSION_JOB, str(tmp_path))
+    ExportSubmissionJobEventData = (
+        export_submission_job_event_pb2.ExportSubmissionJobEventData
+    )
+    event_data = ExportSubmissionJobEventData(
+        submission_job_id="submission_job_id0",
+        status=ExportSubmissionJobEventData.JobStatus.RUNNING,
+        entrypoint="ls",
+        metadata={},
+    )
+    logger.send_event(event_data)
+
+    event_dir = tmp_path / "export_events"
+    assert event_dir.exists()
+    event_file = event_dir / "event_EXPORT_SUBMISSION_JOB.log"
+    assert event_file.exists()
+
+    with event_file.open() as f:
+        lines = f.readlines()
+        assert len(lines) == 1
+
+        line = lines[0]
+        data = json.loads(line)
+        assert data["source_type"] == "EXPORT_SUBMISSION_JOB"
+        assert data["event_data"] == message_to_dict(
+            event_data,
+            always_print_fields_with_no_presence=True,
+            preserving_proto_field_name=True,
+            use_integers_for_enums=False,
+        )
+
+
+def test_event_logger_flushes_all_handlers():
+    mock_logger = MagicMock()
+    handlers = [MagicMock() for _ in range(3)]
+    mock_logger.handlers = handlers
+
+    adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
+    adapter.info("message")
+
+    for handler in handlers:
+        handler.flush.assert_called_once()
+
+
+def test_event_logger_allows_empty_handlers():
+    mock_logger = MagicMock()
+    mock_logger.handlers = []
+
+    adapter = EventLoggerAdapter(event_pb2.Event.GCS, mock_logger)
+    adapter.info("message")
+
+
+def test_export_event_logger_flushes_all_handlers():
+    mock_logger = MagicMock()
+    handlers = [MagicMock() for _ in range(3)]
+    mock_logger.handlers = handlers
+
+    adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
+    event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
+        submission_job_id="submission_job_id0",
+        status=(
+            export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
+        ),
+        entrypoint="ls",
+        metadata={},
+    )
+    adapter.send_event(event_data)
+
+    for handler in handlers:
+        handler.flush.assert_called_once()
+
+
+def test_export_event_logger_allows_empty_handlers():
+    mock_logger = MagicMock()
+    mock_logger.handlers = []
+
+    adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
+    event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
+        submission_job_id="submission_job_id0",
+        status=(
+            export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
+        ),
+        entrypoint="ls",
+        metadata={},
+    )
+    adapter.send_event(event_data)
+
+
+def test_export_event_logger_continues_flushing_after_handler_error():
+    mock_logger = MagicMock()
+    handler1 = MagicMock()
+    handler1.flush.side_effect = RuntimeError("flush failed")
+    handler2 = MagicMock()
+    mock_logger.handlers = [handler1, handler2]
+
+    adapter = ExportEventLoggerAdapter(EventLogType.SUBMISSION_JOB, mock_logger)
+    event_data = export_submission_job_event_pb2.ExportSubmissionJobEventData(
+        submission_job_id="submission_job_id0",
+        status=(
+            export_submission_job_event_pb2.ExportSubmissionJobEventData.JobStatus.RUNNING
+        ),
+        entrypoint="ls",
+        metadata={},
+    )
+    adapter.send_event(event_data)
+
+    handler2.flush.assert_called_once()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/event/tests/test_export_task.py

@@ -0,0 +1,66 @@
+import json
+import os
+import sys
+
+import pytest
+
+import ray
+from ray._common.test_utils import wait_for_condition
+from ray._private.test_utils import wait_until_server_available
+from ray.dashboard.tests.conftest import *  # noqa
+
+os.environ["RAY_enable_export_api_write"] = "1"
+os.environ["RAY_enable_core_worker_ray_event_to_aggregator"] = "0"
+
+
+@pytest.mark.asyncio
+async def test_task_labels(disable_aiohttp_cache, ray_start_with_dashboard):
+    """
+    Test task events are correctly generated and written to file
+    """
+    assert wait_until_server_available(ray_start_with_dashboard["webui_url"])
+    export_event_path = os.path.join(
+        ray_start_with_dashboard["session_dir"], "logs", "export_events"
+    )
+
+    # A simple task to trigger the export event
+    @ray.remote
+    def hi_w00t_task():
+        return 1
+
+    ray.get(hi_w00t_task.options(_labels={"hi": "w00t"}).remote())
+
+    def _verify():
+        # Verify export events are written
+        events = []
+        for filename in os.listdir(export_event_path):
+            if not filename.startswith("event_EXPORT_TASK"):
+                continue
+            with open(f"{export_event_path}/{filename}", "r") as f:
+                for line in f.readlines():
+                    events.append(json.loads(line))
+
+        hi_w00t_event = next(
+            (
+                event
+                for event in events
+                if event["source_type"] == "EXPORT_TASK"
+                and event["event_data"].get("task_info", {}).get("func_or_class_name")
+                == "hi_w00t_task"
+            ),
+            None,
+        )
+        return (
+            hi_w00t_event is not None
+            and hi_w00t_event["event_data"]
+            .get("task_info", {})
+            .get("labels", {})
+            .get("hi")
+            == "w00t"
+        )
+
+    wait_for_condition(_verify, timeout=30)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/event/tests/test_generate_export_events.py

@@ -0,0 +1,197 @@
+# isort: skip_file
+# ruff: noqa: E402
+import json
+import os
+import sys
+
+import pytest
+
+# RAY_enable_export_api_write_config env var must be set before importing
+# `ray` so the correct value is set for RAY_ENABLE_EXPORT_API_WRITE_CONFIG
+# even outside a Ray driver.
+os.environ["RAY_enable_export_api_write_config"] = "EXPORT_SUBMISSION_JOB"
+
+import ray
+from ray._common.test_utils import async_wait_for_condition
+from ray.dashboard.modules.job.job_manager import JobManager
+from ray.job_submission import JobStatus
+from ray.tests.conftest import call_ray_start  # noqa: F401
+
+
+async def check_job_succeeded(job_manager, job_id):
+    data = await job_manager.get_job_info(job_id)
+    status = data.status
+    if status == JobStatus.FAILED:
+        raise RuntimeError(f"Job failed! {data.message}")
+    assert status in {JobStatus.PENDING, JobStatus.RUNNING, JobStatus.SUCCEEDED}
+    if status == JobStatus.SUCCEEDED:
+        assert data.driver_exit_code == 0
+    else:
+        assert data.driver_exit_code is None
+    return status == JobStatus.SUCCEEDED
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "call_ray_start",
+    [
+        {
+            "env": {
+                "RAY_enable_export_api_write_config": "EXPORT_SUBMISSION_JOB,EXPORT_TASK",
+            },
+            "cmd": "ray start --head",
+        }
+    ],
+    indirect=True,
+)
+async def test_check_export_api_enabled(call_ray_start, tmp_path):  # noqa: F811
+    """
+    Test check_export_api_enabled is True for EXPORT_SUBMISSION_JOB and EXPORT_TASK but
+    not for EXPORT_ACTOR because of the value of RAY_enable_export_api_write_config.
+    """
+
+    @ray.remote
+    def test_check_export_api_enabled_remote():
+        from ray._private.event.export_event_logger import check_export_api_enabled
+        from ray.core.generated.export_event_pb2 import ExportEvent
+
+        success = True
+        success = success and check_export_api_enabled(
+            ExportEvent.SourceType.EXPORT_SUBMISSION_JOB
+        )
+        success = success and check_export_api_enabled(
+            ExportEvent.SourceType.EXPORT_TASK
+        )
+        success = success and (
+            not check_export_api_enabled(ExportEvent.SourceType.EXPORT_ACTOR)
+        )
+        return success
+
+    assert ray.get(test_check_export_api_enabled_remote.remote())
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "call_ray_start",
+    [
+        {
+            "env": {
+                "RAY_enable_export_api_write": "true",
+            },
+            "cmd": "ray start --head",
+        }
+    ],
+    indirect=True,
+)
+async def test_check_export_api_enabled_global(call_ray_start, tmp_path):  # noqa: F811
+    """
+    Test check_export_api_enabled always returns True because RAY_enable_export_api_write
+    is set to True.
+    """
+
+    @ray.remote
+    def test_check_export_api_enabled_remote():
+        from ray._private.event.export_event_logger import check_export_api_enabled
+        from ray.core.generated.export_event_pb2 import ExportEvent
+
+        success = True
+        success = success and check_export_api_enabled(
+            ExportEvent.SourceType.EXPORT_SUBMISSION_JOB
+        )
+        success = success and check_export_api_enabled(
+            ExportEvent.SourceType.EXPORT_ACTOR
+        )
+        return success
+
+    assert ray.get(test_check_export_api_enabled_remote.remote())
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "call_ray_start",
+    [
+        {
+            "env": {
+                "RAY_enable_export_api_write_config": "invalid source type",
+            },
+            "cmd": "ray start --head",
+        }
+    ],
+    indirect=True,
+)
+async def test_check_export_api_empty_config(call_ray_start, tmp_path):  # noqa: F811
+    """
+    Test check_export_api_enabled is False for all sources because
+    RAY_enable_export_api_write_config is not a vaild source type.
+    """
+
+    @ray.remote
+    def test_check_export_api_enabled_remote():
+        from ray._private.event.export_event_logger import check_export_api_enabled
+        from ray.core.generated.export_event_pb2 import ExportEvent
+
+        success = True
+        success = success and not (
+            check_export_api_enabled(ExportEvent.SourceType.EXPORT_SUBMISSION_JOB)
+        )
+        success = success and (
+            not check_export_api_enabled(ExportEvent.SourceType.EXPORT_ACTOR)
+        )
+        return success
+
+    assert ray.get(test_check_export_api_enabled_remote.remote())
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "call_ray_start",
+    [
+        {
+            "env": {
+                "RAY_enable_export_api_write_config": "EXPORT_SUBMISSION_JOB",
+            },
+            "cmd": "ray start --head",
+        }
+    ],
+    indirect=True,
+)
+async def test_submission_job_export_events(call_ray_start, tmp_path):  # noqa: F811
+    """
+    Test submission job events are correctly generated and written to file
+    as the job goes through various state changes in its lifecycle.
+    """
+
+    ray.init(address=call_ray_start)
+    gcs_client = ray._private.worker.global_worker.gcs_client
+    job_manager = JobManager(gcs_client, tmp_path)
+
+    # Submit a job.
+    submission_id = await job_manager.submit_job(
+        entrypoint="ls",
+    )
+
+    # Wait for the job to be finished.
+    await async_wait_for_condition(
+        check_job_succeeded, job_manager=job_manager, job_id=submission_id
+    )
+
+    # Verify export events are written
+    event_dir = f"{tmp_path}/export_events"
+    assert os.path.isdir(event_dir)
+    event_file = f"{event_dir}/event_EXPORT_SUBMISSION_JOB.log"
+    assert os.path.isfile(event_file)
+
+    with open(event_file, "r") as f:
+        lines = f.readlines()
+        assert len(lines) == 3
+        expected_status_values = ["PENDING", "RUNNING", "SUCCEEDED"]
+
+        for line, expected_status in zip(lines, expected_status_values):
+            data = json.loads(line)
+            assert data["source_type"] == "EXPORT_SUBMISSION_JOB"
+            assert data["event_data"]["submission_job_id"] == submission_id
+            assert data["event_data"]["status"] == expected_status
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/cli.py

@@ -0,0 +1,600 @@
+import json
+import os
+import pprint
+import shlex
+import sys
+import time
+from subprocess import list2cmdline
+from typing import Any, Dict, Optional, Tuple, Union
+
+import click
+
+import ray._private.ray_constants as ray_constants
+from ray._common.utils import (
+    get_or_create_event_loop,
+    load_class,
+)
+from ray._private.utils import (
+    parse_metadata_json,
+    parse_resources_json,
+)
+from ray.autoscaler._private.cli_logger import add_click_logging_options, cf, cli_logger
+from ray.dashboard.modules.dashboard_sdk import parse_runtime_env_args
+from ray.dashboard.modules.job.cli_utils import add_common_job_options
+from ray.dashboard.modules.job.utils import redact_url_password
+from ray.job_submission import JobStatus, JobSubmissionClient
+from ray.util.annotations import PublicAPI
+
+
+def _get_sdk_client(
+    address: Optional[str],
+    create_cluster_if_needed: bool = False,
+    headers: Optional[str] = None,
+    verify: Union[bool, str] = True,
+) -> JobSubmissionClient:
+    client = JobSubmissionClient(
+        address,
+        create_cluster_if_needed,
+        headers=_handle_headers(headers),
+        verify=verify,
+    )
+    client_address = client.get_address()
+    cli_logger.labeled_value(
+        "Job submission server address", redact_url_password(client_address)
+    )
+    return client
+
+
+def _handle_headers(headers: Optional[str]) -> Optional[Dict[str, Any]]:
+    if headers is None and "RAY_JOB_HEADERS" in os.environ:
+        headers = os.environ["RAY_JOB_HEADERS"]
+    if headers is not None:
+        try:
+            return json.loads(headers)
+        except Exception as exc:
+            raise ValueError(
+                """Failed to parse headers into JSON.
+                Expected format: {{"KEY": "VALUE"}}, got {}, {}""".format(
+                    headers, exc
+                )
+            )
+    return None
+
+
+def _log_big_success_msg(success_msg):
+    cli_logger.newline()
+    cli_logger.success("-" * len(success_msg))
+    cli_logger.success(success_msg)
+    cli_logger.success("-" * len(success_msg))
+    cli_logger.newline()
+
+
+def _log_big_error_msg(success_msg):
+    cli_logger.newline()
+    cli_logger.error("-" * len(success_msg))
+    cli_logger.error(success_msg)
+    cli_logger.error("-" * len(success_msg))
+    cli_logger.newline()
+
+
+def _log_job_status(client: JobSubmissionClient, job_id: str) -> JobStatus:
+    info = client.get_job_info(job_id)
+    if info.status == JobStatus.SUCCEEDED:
+        _log_big_success_msg(f"Job '{job_id}' succeeded")
+    elif info.status == JobStatus.STOPPED:
+        cli_logger.warning(f"Job '{job_id}' was stopped")
+    elif info.status == JobStatus.FAILED:
+        _log_big_error_msg(f"Job '{job_id}' failed")
+        if info.message is not None:
+            cli_logger.print(f"Status message: {info.message}", no_format=True)
+    else:
+        # Catch-all.
+        cli_logger.print(f"Status for job '{job_id}': {info.status}")
+        if info.message is not None:
+            cli_logger.print(f"Status message: {info.message}", no_format=True)
+    return info.status
+
+
+async def _tail_logs(client: JobSubmissionClient, job_id: str) -> JobStatus:
+    async for lines in client.tail_job_logs(job_id):
+        print(lines, end="")
+
+    return _log_job_status(client, job_id)
+
+
+@click.group("job")
+def job_cli_group():
+    """Submit, stop, delete, or list Ray jobs."""
+    pass
+
+
+@job_cli_group.command()
+@click.option(
+    "--address",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Address of the Ray cluster to connect to. Can also be specified "
+        "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)."
+    ),
+)
+@click.option(
+    "--job-id",
+    type=str,
+    default=None,
+    required=False,
+    help=("DEPRECATED: Use `--submission-id` instead."),
+)
+@click.option(
+    "--submission-id",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Submission ID to specify for the job. If not provided, one will be generated."
+    ),
+)
+@click.option(
+    "--runtime-env",
+    type=str,
+    default=None,
+    required=False,
+    help="Path to a local YAML file containing a runtime_env definition.",
+)
+@click.option(
+    "--runtime-env-json",
+    type=str,
+    default=None,
+    required=False,
+    help="JSON-serialized runtime_env dictionary.",
+)
+@click.option(
+    "--working-dir",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Directory containing files that your job will run in. Can be a "
+        "local directory or a remote URI to a .zip file (S3, GS, HTTP). "
+        "If specified, this overrides the option in `--runtime-env`."
+    ),
+)
+@click.option(
+    "--metadata-json",
+    type=str,
+    default=None,
+    required=False,
+    help="JSON-serialized dictionary of metadata to attach to the job.",
+)
+@click.option(
+    "--entrypoint-num-cpus",
+    required=False,
+    type=float,
+    help="the quantity of CPU cores to reserve for the entrypoint command, "
+    "separately from any tasks or actors that are launched by it",
+)
+@click.option(
+    "--entrypoint-num-gpus",
+    required=False,
+    type=float,
+    help="the quantity of GPUs to reserve for the entrypoint command, "
+    "separately from any tasks or actors that are launched by it",
+)
+@click.option(
+    "--entrypoint-memory",
+    required=False,
+    type=int,
+    help="the amount of memory to reserve "
+    "for the entrypoint command, separately from any tasks or actors that are "
+    "launched by it",
+)
+@click.option(
+    "--entrypoint-resources",
+    required=False,
+    type=str,
+    help="a JSON-serialized dictionary mapping resource name to resource quantity "
+    "describing resources to reserve for the entrypoint command, "
+    "separately from any tasks or actors that are launched by it",
+)
+@click.option(
+    "--entrypoint-label-selector",
+    required=False,
+    type=str,
+    help="a JSON-serialized dictionary mapping label keys to selector strings "
+    "describing placement constraints for the entrypoint command",
+)
+@click.option(
+    "--no-wait",
+    is_flag=True,
+    type=bool,
+    default=False,
+    help="If set, will not stream logs and wait for the job to exit.",
+)
+@add_common_job_options
+@add_click_logging_options
+@click.argument("entrypoint", nargs=-1, required=True, type=click.UNPROCESSED)
+@PublicAPI
+def submit(
+    address: Optional[str],
+    job_id: Optional[str],
+    submission_id: Optional[str],
+    runtime_env: Optional[str],
+    runtime_env_json: Optional[str],
+    metadata_json: Optional[str],
+    working_dir: Optional[str],
+    entrypoint: Tuple[str],
+    entrypoint_num_cpus: Optional[Union[int, float]],
+    entrypoint_num_gpus: Optional[Union[int, float]],
+    entrypoint_memory: Optional[int],
+    entrypoint_resources: Optional[str],
+    entrypoint_label_selector: Optional[str],
+    no_wait: bool,
+    verify: Union[bool, str],
+    headers: Optional[str],
+):
+    """Submits a job to be run on the cluster.
+
+    By default (if --no-wait is not set), streams logs to stdout until the job finishes.
+    If the job succeeded, exits with 0. If it failed, exits with 1.
+
+    Example:
+        `ray job submit -- python my_script.py --arg=val`
+
+    Args:
+        address: Job submission server address.
+        job_id: DEPRECATED. Use submission_id instead.
+        submission_id: Submission ID for the job.
+        runtime_env: Path to a runtime_env YAML file.
+        runtime_env_json: JSON-serialized runtime_env dictionary.
+        metadata_json: JSON-serialized metadata dictionary.
+        working_dir: Working directory for the job.
+        entrypoint: Entrypoint command.
+        entrypoint_num_cpus: CPU cores to reserve.
+        entrypoint_num_gpus: GPUs to reserve.
+        entrypoint_memory: Memory to reserve.
+        entrypoint_resources: JSON-serialized custom resources dict.
+        entrypoint_label_selector: JSON-serialized label selector dict.
+        no_wait: Do not wait for job completion.
+        verify: TLS verification flag or path.
+        headers: JSON-serialized headers.
+    """
+    if job_id:
+        cli_logger.warning(
+            "--job-id option is deprecated. Please use --submission-id instead."
+        )
+    if entrypoint_resources is not None:
+        entrypoint_resources = parse_resources_json(
+            entrypoint_resources, cli_logger, cf, command_arg="entrypoint-resources"
+        )
+    if entrypoint_label_selector is not None:
+        entrypoint_label_selector = parse_resources_json(
+            entrypoint_label_selector,
+            cli_logger,
+            cf,
+            command_arg="entrypoint-label-selector",
+        )
+    if metadata_json is not None:
+        metadata_json = parse_metadata_json(
+            metadata_json, cli_logger, cf, command_arg="metadata-json"
+        )
+
+    submission_id = submission_id or job_id
+
+    if ray_constants.RAY_JOB_SUBMIT_HOOK in os.environ:
+        # Submit all args as **kwargs per the JOB_SUBMIT_HOOK contract.
+        load_class(os.environ[ray_constants.RAY_JOB_SUBMIT_HOOK])(
+            address=address,
+            job_id=submission_id,
+            submission_id=submission_id,
+            runtime_env=runtime_env,
+            runtime_env_json=runtime_env_json,
+            metadata_json=metadata_json,
+            working_dir=working_dir,
+            entrypoint=entrypoint,
+            entrypoint_num_cpus=entrypoint_num_cpus,
+            entrypoint_num_gpus=entrypoint_num_gpus,
+            entrypoint_memory=entrypoint_memory,
+            entrypoint_resources=entrypoint_resources,
+            entrypoint_label_selector=entrypoint_label_selector,
+            no_wait=no_wait,
+        )
+
+    client = _get_sdk_client(
+        address, create_cluster_if_needed=True, headers=headers, verify=verify
+    )
+
+    final_runtime_env = parse_runtime_env_args(
+        runtime_env=runtime_env,
+        runtime_env_json=runtime_env_json,
+        working_dir=working_dir,
+    )
+    job_id = client.submit_job(
+        entrypoint=(
+            list2cmdline(entrypoint)
+            if sys.platform == "win32"
+            else shlex.join(entrypoint)
+        ),
+        submission_id=submission_id,
+        runtime_env=final_runtime_env,
+        metadata=metadata_json,
+        entrypoint_num_cpus=entrypoint_num_cpus,
+        entrypoint_num_gpus=entrypoint_num_gpus,
+        entrypoint_memory=entrypoint_memory,
+        entrypoint_resources=entrypoint_resources,
+        entrypoint_label_selector=entrypoint_label_selector,
+    )
+
+    _log_big_success_msg(f"Job '{job_id}' submitted successfully")
+
+    with cli_logger.group("Next steps"):
+        cli_logger.print("Query the logs of the job:")
+        with cli_logger.indented():
+            cli_logger.print(cf.bold(f"ray job logs {job_id}"))
+
+        cli_logger.print("Query the status of the job:")
+        with cli_logger.indented():
+            cli_logger.print(cf.bold(f"ray job status {job_id}"))
+
+        cli_logger.print("Request the job to be stopped:")
+        with cli_logger.indented():
+            cli_logger.print(cf.bold(f"ray job stop {job_id}"))
+
+    cli_logger.newline()
+    # Flush stdout to ensure the Ray job ID is output immediately
+    # for the kubectl plugin, ref PR #52780, Issue kuberay/#3508.
+    cli_logger.flush()
+    sdk_version = client.get_version()
+    # sdk version 0 does not have log streaming
+    if not no_wait:
+        if int(sdk_version) > 0:
+            cli_logger.print(
+                "Tailing logs until the job exits (disable with --no-wait):"
+            )
+            job_status = get_or_create_event_loop().run_until_complete(
+                _tail_logs(client, job_id)
+            )
+            if job_status == JobStatus.FAILED:
+                sys.exit(1)
+        else:
+            cli_logger.warning(
+                "Tailing logs is not enabled for job sdk client version "
+                f"{sdk_version}. Please upgrade Ray to the latest version "
+                "for this feature."
+            )
+
+
+@job_cli_group.command()
+@click.option(
+    "--address",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Address of the Ray cluster to connect to. Can also be specified "
+        "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)."
+    ),
+)
+@click.argument("job-id", type=str)
+@add_common_job_options
+@add_click_logging_options
+@PublicAPI(stability="stable")
+def status(
+    address: Optional[str],
+    job_id: str,
+    headers: Optional[str],
+    verify: Union[bool, str],
+):
+    """Queries for the current status of a job.
+
+    Example:
+        `ray job status <my_job_id>`
+
+    Args:
+        address: Address of the Ray cluster to connect to.
+        job_id: The submission ID of the job to query.
+        headers: JSON string of headers to attach to requests.
+        verify: Path to a CA bundle, or boolean toggling TLS verification.
+    """
+    client = _get_sdk_client(address, headers=headers, verify=verify)
+    _log_job_status(client, job_id)
+
+
+@job_cli_group.command()
+@click.option(
+    "--address",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Address of the Ray cluster to connect to. Can also be specified "
+        "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)."
+    ),
+)
+@click.option(
+    "--no-wait",
+    is_flag=True,
+    type=bool,
+    default=False,
+    help="If set, will not wait for the job to exit.",
+)
+@click.argument("job-id", type=str)
+@add_common_job_options
+@add_click_logging_options
+@PublicAPI(stability="stable")
+def stop(
+    address: Optional[str],
+    no_wait: bool,
+    job_id: str,
+    headers: Optional[str],
+    verify: Union[bool, str],
+):
+    """Attempts to stop a job.
+
+    Example:
+        `ray job stop <my_job_id>`
+
+    Args:
+        address: Address of the Ray cluster to connect to.
+        no_wait: If True, return immediately instead of waiting for the job to reach a terminal state.
+        job_id: The submission ID of the job to stop.
+        headers: JSON string of headers to attach to requests.
+        verify: Path to a CA bundle, or boolean toggling TLS verification.
+
+    Returns:
+        None. The function returns early when ``no_wait`` is True; otherwise it
+        polls until the job reaches a terminal state.
+    """
+    client = _get_sdk_client(address, headers=headers, verify=verify)
+    cli_logger.print(f"Attempting to stop job '{job_id}'")
+    client.stop_job(job_id)
+
+    if no_wait:
+        return
+    else:
+        cli_logger.print(
+            f"Waiting for job '{job_id}' to exit (disable with --no-wait):"
+        )
+
+    while True:
+        status = client.get_job_status(job_id)
+        if status in {JobStatus.STOPPED, JobStatus.SUCCEEDED, JobStatus.FAILED}:
+            _log_job_status(client, job_id)
+            break
+        else:
+            cli_logger.print(f"Job has not exited yet. Status: {status}")
+            time.sleep(1)
+
+
+@job_cli_group.command()
+@click.option(
+    "--address",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Address of the Ray cluster to connect to. Can also be specified "
+        "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)."
+    ),
+)
+@click.argument("job-id", type=str)
+@add_common_job_options
+@add_click_logging_options
+@PublicAPI(stability="stable")
+def delete(
+    address: Optional[str],
+    job_id: str,
+    headers: Optional[str],
+    verify: Union[bool, str],
+):
+    """Deletes a stopped job and its associated data from memory.
+
+    Only supported for jobs that are already in a terminal state.
+    Fails with exit code 1 if the job is not already stopped.
+    Does not delete job logs from disk.
+    Submitting a job with the same submission ID as a previously
+    deleted job is not supported and may lead to unexpected behavior.
+
+    Example:
+        ray job delete <my_job_id>
+
+    Args:
+        address: Address of the Ray cluster to connect to.
+        job_id: The submission ID of the job to delete.
+        headers: JSON string of headers to attach to requests.
+        verify: Path to a CA bundle, or boolean toggling TLS verification.
+    """
+    client = _get_sdk_client(address, headers=headers, verify=verify)
+    client.delete_job(job_id)
+    cli_logger.print(f"Job '{job_id}' deleted successfully")
+
+
+@job_cli_group.command()
+@click.option(
+    "--address",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Address of the Ray cluster to connect to. Can also be specified "
+        "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)."
+    ),
+)
+@click.argument("job-id", type=str)
+@click.option(
+    "-f",
+    "--follow",
+    is_flag=True,
+    type=bool,
+    default=False,
+    help="If set, follow the logs (like `tail -f`).",
+)
+@add_common_job_options
+@add_click_logging_options
+@PublicAPI(stability="stable")
+def logs(
+    address: Optional[str],
+    job_id: str,
+    follow: bool,
+    headers: Optional[str],
+    verify: Union[bool, str],
+):
+    """Gets the logs of a job.
+
+    Example:
+        `ray job logs <my_job_id>`
+
+    Args:
+        address: Address of the Ray cluster to connect to.
+        job_id: The submission ID of the job whose logs to fetch.
+        follow: If True, stream the logs (``tail -f`` style) instead of printing them once.
+        headers: JSON string of headers to attach to requests.
+        verify: Path to a CA bundle, or boolean toggling TLS verification.
+    """
+    client = _get_sdk_client(address, headers=headers, verify=verify)
+    sdk_version = client.get_version()
+    # sdk version 0 did not have log streaming
+    if follow:
+        if int(sdk_version) > 0:
+            get_or_create_event_loop().run_until_complete(_tail_logs(client, job_id))
+        else:
+            cli_logger.warning(
+                "Tailing logs is not enabled for the Jobs SDK client version "
+                f"{sdk_version}. Please upgrade Ray to latest version "
+                "for this feature."
+            )
+    else:
+        # Set no_format to True because the logs may have unescaped "{" and "}"
+        # and the CLILogger calls str.format().
+        cli_logger.print(client.get_job_logs(job_id), end="", no_format=True)
+
+
+@job_cli_group.command()
+@click.option(
+    "--address",
+    type=str,
+    default=None,
+    required=False,
+    help=(
+        "Address of the Ray cluster to connect to. Can also be specified "
+        "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)."
+    ),
+)
+@add_common_job_options
+@add_click_logging_options
+@PublicAPI(stability="stable")
+def list(address: Optional[str], headers: Optional[str], verify: Union[bool, str]):
+    """Lists all running jobs and their information.
+
+    Example:
+        `ray job list`
+
+    Args:
+        address: Address of the Ray cluster to connect to.
+        headers: JSON string of headers to attach to requests.
+        verify: Path to a CA bundle, or boolean toggling TLS verification.
+    """
+    client = _get_sdk_client(address, headers=headers, verify=verify)
+    # Set no_format to True because the logs may have unescaped "{" and "}"
+    # and the CLILogger calls str.format().
+    cli_logger.print(pprint.pformat(client.list_jobs()), no_format=True)

python/ray/dashboard/modules/job/cli_utils.py

@@ -0,0 +1,56 @@
+import functools
+from typing import Union
+
+import click
+
+
+def bool_cast(string: str) -> Union[bool, str]:
+    """Cast a string to a boolean if possible, otherwise return the string."""
+    if string.lower() == "true" or string == "1":
+        return True
+    elif string.lower() == "false" or string == "0":
+        return False
+    else:
+        return string
+
+
+class BoolOrStringParam(click.ParamType):
+    """A click parameter that can be either a boolean or a string."""
+
+    name = "BOOL | TEXT"
+
+    def convert(self, value, param, ctx):
+        if isinstance(value, bool):
+            return value
+        else:
+            return bool_cast(value)
+
+
+def add_common_job_options(func):
+    """Decorator for adding CLI flags shared by all `ray job` commands."""
+
+    @click.option(
+        "--verify",
+        default=True,
+        show_default=True,
+        type=BoolOrStringParam(),
+        help=(
+            "Boolean indication to verify the server's TLS certificate or a path to"
+            " a file or directory of trusted certificates."
+        ),
+    )
+    @click.option(
+        "--headers",
+        required=False,
+        type=str,
+        default=None,
+        help=(
+            "Used to pass headers through http/s to the Ray Cluster."
+            'please follow JSON formatting formatting {"key": "value"}'
+        ),
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper

python/ray/dashboard/modules/job/common.py

@@ -0,0 +1,599 @@
+import asyncio
+import json
+import logging
+import time
+from dataclasses import asdict, dataclass, replace
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple, Union
+
+from ray._private import ray_constants
+from ray._private.event.export_event_logger import (
+    EventLogType,
+    check_export_api_enabled,
+    get_export_event_logger,
+)
+from ray._private.runtime_env.packaging import parse_uri
+from ray._raylet import RAY_INTERNAL_NAMESPACE_PREFIX, GcsClient
+from ray.core.generated.export_event_pb2 import ExportEvent
+from ray.core.generated.export_submission_job_event_pb2 import (
+    ExportSubmissionJobEventData,
+)
+from ray.util.annotations import PublicAPI
+
+# NOTE(edoakes): these constants should be considered a public API because
+# they're exposed in the snapshot API.
+JOB_ID_METADATA_KEY = "job_submission_id"
+JOB_NAME_METADATA_KEY = "job_name"
+JOB_ACTOR_NAME_TEMPLATE = f"{RAY_INTERNAL_NAMESPACE_PREFIX}job_actor_" + "{job_id}"
+# In order to get information about SupervisorActors launched by different jobs,
+# they must be set to the same namespace.
+SUPERVISOR_ACTOR_RAY_NAMESPACE = "SUPERVISOR_ACTOR_RAY_NAMESPACE"
+JOB_LOGS_PATH_TEMPLATE = "job-driver-{submission_id}.log"
+
+logger = logging.getLogger(__name__)
+
+
+@PublicAPI(stability="stable")
+class JobStatus(str, Enum):
+    """An enumeration for describing the status of a job."""
+
+    #: The job has not started yet, likely waiting for the runtime_env to be set up.
+    PENDING = "PENDING"
+    #: The job is currently running.
+    RUNNING = "RUNNING"
+    #: The job was intentionally stopped by the user.
+    STOPPED = "STOPPED"
+    #: The job finished successfully.
+    SUCCEEDED = "SUCCEEDED"
+    #: The job failed.
+    FAILED = "FAILED"
+
+    def __str__(self) -> str:
+        return f"{self.value}"
+
+    def is_terminal(self) -> bool:
+        """Return whether or not this status is terminal.
+
+        A terminal status is one that cannot transition to any other status.
+        The terminal statuses are "STOPPED", "SUCCEEDED", and "FAILED".
+
+        Returns:
+            True if this status is terminal, otherwise False.
+        """
+        return self.value in {"STOPPED", "SUCCEEDED", "FAILED"}
+
+
+@PublicAPI(stability="stable")
+class JobErrorType(str, Enum):
+    """An enumeration for describing the error type of a job."""
+
+    # Runtime environment failed to be set up
+    RUNTIME_ENV_SETUP_FAILURE = "RUNTIME_ENV_SETUP_FAILURE"
+    # Job supervisor actor launched, but job failed to start within timeout
+    JOB_SUPERVISOR_ACTOR_START_TIMEOUT = "JOB_SUPERVISOR_ACTOR_START_TIMEOUT"
+    # Job supervisor actor failed to start
+    JOB_SUPERVISOR_ACTOR_START_FAILURE = "JOB_SUPERVISOR_ACTOR_START_FAILURE"
+    # Job supervisor actor failed to be scheduled
+    JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE = "JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE"
+    # Job supervisor actor failed for unknown exception
+    JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE = "JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE"
+    # Job supervisor actor died
+    JOB_SUPERVISOR_ACTOR_DIED = "JOB_SUPERVISOR_ACTOR_DIED"
+    # Job driver script failed to start due to exception
+    JOB_ENTRYPOINT_COMMAND_START_ERROR = "JOB_ENTRYPOINT_COMMAND_START_ERROR"
+    # Job driver script failed due to non-zero exit code
+    JOB_ENTRYPOINT_COMMAND_ERROR = "JOB_ENTRYPOINT_COMMAND_ERROR"
+
+
+# TODO(aguo): Convert to pydantic model
+@PublicAPI(stability="stable")
+@dataclass
+class JobInfo:
+    """A class for recording information associated with a job and its execution.
+
+    Please keep this in sync with the JobsAPIInfo proto in src/ray/protobuf/gcs.proto.
+    """
+
+    #: The status of the job.
+    status: JobStatus
+    #: The entrypoint command for this job.
+    entrypoint: str
+    #: A message describing the status in more detail.
+    message: Optional[str] = None
+    #: Internal error, user script error
+    error_type: Optional[JobErrorType] = None
+    #: The time when the job was started.  A Unix timestamp in ms.
+    start_time: Optional[int] = None
+    #: The time when the job moved into a terminal state.  A Unix timestamp in ms.
+    end_time: Optional[int] = None
+    #: Arbitrary user-provided metadata for the job.
+    metadata: Optional[Dict[str, str]] = None
+    #: The runtime environment for the job.
+    runtime_env: Optional[Dict[str, Any]] = None
+    #: The quantity of CPU cores to reserve for the entrypoint command.
+    entrypoint_num_cpus: Optional[Union[int, float]] = None
+    #: The number of GPUs to reserve for the entrypoint command.
+    entrypoint_num_gpus: Optional[Union[int, float]] = None
+    #: The amount of memory for workers requesting memory for the entrypoint command.
+    entrypoint_memory: Optional[int] = None
+    #: The quantity of various custom resources to reserve for the entrypoint command.
+    entrypoint_resources: Optional[Dict[str, float]] = None
+    #: Driver agent http address
+    driver_agent_http_address: Optional[str] = None
+    #: The node id that driver running on. It will be None only when the job status
+    # is PENDING, and this field will not be deleted or modified even if the driver dies
+    driver_node_id: Optional[str] = None
+    #: The driver process exit code after the driver executed. Return None if driver
+    #: doesn't finish executing
+    driver_exit_code: Optional[int] = None
+
+    def __post_init__(self):
+        if isinstance(self.status, str):
+            self.status = JobStatus(self.status)
+        if self.message is None:
+            if self.status == JobStatus.PENDING:
+                self.message = "Job has not started yet."
+                if any(
+                    [
+                        self.entrypoint_num_cpus is not None
+                        and self.entrypoint_num_cpus > 0,
+                        self.entrypoint_num_gpus is not None
+                        and self.entrypoint_num_gpus > 0,
+                        self.entrypoint_memory is not None
+                        and self.entrypoint_memory > 0,
+                        self.entrypoint_resources not in [None, {}],
+                    ]
+                ):
+                    self.message += (
+                        " It may be waiting for resources "
+                        "(CPUs, GPUs, memory, custom resources) to become available."
+                    )
+                if self.runtime_env not in [None, {}]:
+                    self.message += (
+                        " It may be waiting for the runtime environment to be set up."
+                    )
+            elif self.status == JobStatus.RUNNING:
+                self.message = "Job is currently running."
+            elif self.status == JobStatus.STOPPED:
+                self.message = "Job was intentionally stopped."
+            elif self.status == JobStatus.SUCCEEDED:
+                self.message = "Job finished successfully."
+            elif self.status == JobStatus.FAILED:
+                self.message = "Job failed."
+
+    def to_json(self) -> Dict[str, Any]:
+        """Convert this object to a JSON-serializable dictionary.
+
+        Note that the runtime_env field is converted to a JSON-serialized string
+        and the field is renamed to runtime_env_json.
+
+        Returns:
+            A JSON-serializable dictionary representing the JobInfo object.
+        """
+
+        json_dict = asdict(self)
+
+        # Convert enum values to strings.
+        json_dict["status"] = str(json_dict["status"])
+        json_dict["error_type"] = (
+            json_dict["error_type"].value if json_dict.get("error_type") else None
+        )
+
+        # Convert runtime_env to a JSON-serialized string.
+        if "runtime_env" in json_dict:
+            if json_dict["runtime_env"] is not None:
+                json_dict["runtime_env_json"] = json.dumps(json_dict["runtime_env"])
+            del json_dict["runtime_env"]
+
+        # Assert that the dictionary is JSON-serializable.
+        json.dumps(json_dict)
+
+        return json_dict
+
+    @classmethod
+    def from_json(cls, json_dict: Dict[str, Any]) -> None:
+        """Initialize this object from a JSON dictionary.
+
+        Note that the runtime_env_json field is converted to a dictionary and
+        the field is renamed to runtime_env.
+
+        Args:
+            json_dict: A JSON dictionary to use to initialize the JobInfo object.
+        """
+        # Convert enum values to enum objects.
+        json_dict["status"] = JobStatus(json_dict["status"])
+        json_dict["error_type"] = (
+            JobErrorType(json_dict["error_type"])
+            if json_dict.get("error_type")
+            else None
+        )
+
+        # Convert runtime_env from a JSON-serialized string to a dictionary.
+        if "runtime_env_json" in json_dict:
+            if json_dict["runtime_env_json"] is not None:
+                json_dict["runtime_env"] = json.loads(json_dict["runtime_env_json"])
+            del json_dict["runtime_env_json"]
+
+        return cls(**json_dict)
+
+
+class JobInfoStorageClient:
+    """
+    Interface to put and get job data from the Internal KV store.
+    """
+
+    # Please keep this format in sync with JobDataKey()
+    # in src/ray/gcs/gcs_server/gcs_job_manager.h.
+    JOB_DATA_KEY_PREFIX = f"{RAY_INTERNAL_NAMESPACE_PREFIX}job_info_"
+    JOB_DATA_KEY = f"{JOB_DATA_KEY_PREFIX}{{job_id}}"
+
+    def __init__(
+        self,
+        gcs_client: GcsClient,
+        export_event_log_dir_root: Optional[str] = None,
+    ):
+        """
+        Initialize the JobInfoStorageClient which manages data in the internal KV store.
+        Export Submission Job events are written when the KV store is updated if
+        the feature flag is on and a export_event_log_dir_root is passed.
+        export_event_log_dir_root doesn't need to be passed if the caller
+        is not modifying data in the KV store.
+        """
+        self._gcs_client = gcs_client
+        self._export_submission_job_event_logger: logging.Logger = None
+        try:
+            if (
+                check_export_api_enabled(ExportEvent.SourceType.EXPORT_SUBMISSION_JOB)
+                and export_event_log_dir_root is not None
+            ):
+                self._export_submission_job_event_logger = get_export_event_logger(
+                    EventLogType.SUBMISSION_JOB,
+                    export_event_log_dir_root,
+                )
+        except Exception:
+            logger.exception(
+                "Unable to initialize export event logger so no export "
+                "events will be written."
+            )
+
+    async def put_info(
+        self,
+        job_id: str,
+        job_info: JobInfo,
+        overwrite: bool = True,
+        timeout: Optional[int] = 30,
+    ) -> bool:
+        """Put job info to the internal kv store.
+
+        Args:
+            job_id: The job id.
+            job_info: The job info.
+            overwrite: Whether to overwrite the existing job info.
+            timeout: The timeout in seconds for the GCS operation.
+
+        Returns:
+            True if a new key is added.
+        """
+        added_num = await self._gcs_client.async_internal_kv_put(
+            self.JOB_DATA_KEY.format(job_id=job_id).encode(),
+            json.dumps(job_info.to_json()).encode(),
+            overwrite,
+            namespace=ray_constants.KV_NAMESPACE_JOB,
+            timeout=timeout,
+        )
+        if added_num == 1 or overwrite:
+            # Write export event if data was updated in the KV store
+            try:
+                self._write_submission_job_export_event(job_id, job_info)
+            except Exception:
+                logger.exception("Error while writing job submission export event.")
+        return added_num == 1
+
+    def _write_submission_job_export_event(
+        self, job_id: str, job_info: JobInfo
+    ) -> None:
+        """
+        Write Submission Job export event if _export_submission_job_event_logger
+        exists. The logger will exist if the export API feature flag is enabled
+        and a log directory was passed to JobInfoStorageClient.
+        """
+        if not self._export_submission_job_event_logger:
+            return
+
+        status_value_descriptor = (
+            ExportSubmissionJobEventData.JobStatus.DESCRIPTOR.values_by_name.get(
+                job_info.status.name
+            )
+        )
+        if status_value_descriptor is None:
+            logger.error(
+                f"{job_info.status.name} is not a valid "
+                "ExportSubmissionJobEventData.JobStatus enum value. This event "
+                "will not be written."
+            )
+            return
+        job_status = status_value_descriptor.number
+        submission_event_data = ExportSubmissionJobEventData(
+            submission_job_id=job_id,
+            status=job_status,
+            entrypoint=job_info.entrypoint,
+            message=job_info.message,
+            metadata=job_info.metadata,
+            error_type=job_info.error_type,
+            start_time=job_info.start_time,
+            end_time=job_info.end_time,
+            runtime_env_json=json.dumps(job_info.runtime_env),
+            driver_agent_http_address=job_info.driver_agent_http_address,
+            driver_node_id=job_info.driver_node_id,
+            driver_exit_code=job_info.driver_exit_code,
+        )
+        self._export_submission_job_event_logger.send_event(submission_event_data)
+
+    async def get_info(self, job_id: str, timeout: int = 30) -> Optional[JobInfo]:
+        serialized_info = await self._gcs_client.async_internal_kv_get(
+            self.JOB_DATA_KEY.format(job_id=job_id).encode(),
+            namespace=ray_constants.KV_NAMESPACE_JOB,
+            timeout=timeout,
+        )
+        if serialized_info is None:
+            return None
+        else:
+            return JobInfo.from_json(json.loads(serialized_info))
+
+    async def delete_info(self, job_id: str, timeout: int = 30):
+        await self._gcs_client.async_internal_kv_del(
+            self.JOB_DATA_KEY.format(job_id=job_id).encode(),
+            False,
+            namespace=ray_constants.KV_NAMESPACE_JOB,
+            timeout=timeout,
+        )
+
+    async def put_status(
+        self,
+        job_id: str,
+        status: JobStatus,
+        message: Optional[str] = None,
+        driver_exit_code: Optional[int] = None,
+        error_type: Optional[JobErrorType] = None,
+        jobinfo_replace_kwargs: Optional[Dict[str, Any]] = None,
+        timeout: Optional[int] = 30,
+    ):
+        """Puts or updates job status.  Sets end_time if status is terminal."""
+
+        old_info = await self.get_info(job_id, timeout=timeout)
+
+        if jobinfo_replace_kwargs is None:
+            jobinfo_replace_kwargs = dict()
+        jobinfo_replace_kwargs.update(
+            status=status,
+            message=message,
+            driver_exit_code=driver_exit_code,
+            error_type=error_type,
+        )
+        if old_info is not None:
+            if status != old_info.status and old_info.status.is_terminal():
+                raise RuntimeError(
+                    f"Attempted to change job status from a terminal state: "
+                    f"{old_info.status} -> {status}"
+                )
+            new_info = replace(old_info, **jobinfo_replace_kwargs)
+        else:
+            new_info = JobInfo(
+                entrypoint="Entrypoint not found.", **jobinfo_replace_kwargs
+            )
+
+        if status.is_terminal():
+            new_info.end_time = int(time.time() * 1000)
+
+        await self.put_info(job_id, new_info, timeout=timeout)
+
+    async def get_status(self, job_id: str, timeout: int = 30) -> Optional[JobStatus]:
+        job_info = await self.get_info(job_id, timeout)
+        if job_info is None:
+            return None
+        else:
+            return job_info.status
+
+    async def get_all_jobs(self, timeout: int = 30) -> Dict[str, JobInfo]:
+        raw_job_ids_with_prefixes = await self._gcs_client.async_internal_kv_keys(
+            self.JOB_DATA_KEY_PREFIX.encode(),
+            namespace=ray_constants.KV_NAMESPACE_JOB,
+            timeout=timeout,
+        )
+        job_ids_with_prefixes = [
+            job_id.decode() for job_id in raw_job_ids_with_prefixes
+        ]
+        job_ids = []
+        for job_id_with_prefix in job_ids_with_prefixes:
+            assert job_id_with_prefix.startswith(
+                self.JOB_DATA_KEY_PREFIX
+            ), "Unexpected format for internal_kv key for Job submission"
+            job_ids.append(job_id_with_prefix[len(self.JOB_DATA_KEY_PREFIX) :])
+
+        async def get_job_info(job_id: str):
+            job_info = await self.get_info(job_id, timeout)
+            return job_id, job_info
+
+        results = await asyncio.gather(*[get_job_info(job_id) for job_id in job_ids])
+        return {
+            job_id: job_info for job_id, job_info in results if job_info is not None
+        }
+
+
+def uri_to_http_components(package_uri: str) -> Tuple[str, str]:
+    suffix = Path(package_uri).suffix
+    if suffix not in {".zip", ".whl"}:
+        raise ValueError(f"package_uri ({package_uri}) does not end in .zip or .whl")
+    # We need to strip the <protocol>:// prefix to make it possible to pass
+    # the package_uri over HTTP.
+    protocol, package_name = parse_uri(package_uri)
+    return protocol.value, package_name
+
+
+def http_uri_components_to_uri(protocol: str, package_name: str) -> str:
+    return f"{protocol}://{package_name}"
+
+
+def validate_request_type(json_data: Dict[str, Any], request_type: dataclass) -> Any:
+    return request_type(**json_data)
+
+
+@dataclass
+class JobSubmitRequest:
+    # Command to start execution, ex: "python script.py"
+    entrypoint: str
+    # Optional submission_id to specify for the job. If the submission_id
+    # is not specified, one will be generated. If a job with the same
+    # submission_id already exists, it will be rejected.
+    submission_id: Optional[str] = None
+    # DEPRECATED. Use submission_id instead
+    job_id: Optional[str] = None
+    # Dict to setup execution environment.
+    runtime_env: Optional[Dict[str, Any]] = None
+    # Metadata to pass in to the JobConfig.
+    metadata: Optional[Dict[str, str]] = None
+    # The quantity of CPU cores to reserve for the execution
+    # of the entrypoint command, separately from any Ray tasks or actors
+    # that are created by it.
+    entrypoint_num_cpus: Optional[Union[int, float]] = None
+    # The quantity of GPUs to reserve for the execution
+    # of the entrypoint command, separately from any Ray tasks or actors
+    # that are created by it.
+    entrypoint_num_gpus: Optional[Union[int, float]] = None
+    # The amount of total available memory for workers requesting memory
+    # for the execution of the entrypoint command, separately from any Ray
+    # tasks or actors that are created by it.
+    entrypoint_memory: Optional[int] = None
+    # The quantity of various custom resources
+    # to reserve for the entrypoint command, separately from any Ray tasks
+    # or actors that are created by it.
+    entrypoint_resources: Optional[Dict[str, float]] = None
+    # Label selector for the entrypoint command.
+    entrypoint_label_selector: Optional[Dict[str, str]] = None
+
+    def __post_init__(self):
+        if not isinstance(self.entrypoint, str):
+            raise TypeError(f"entrypoint must be a string, got {type(self.entrypoint)}")
+
+        if self.submission_id is not None and not isinstance(self.submission_id, str):
+            raise TypeError(
+                "submission_id must be a string if provided, "
+                f"got {type(self.submission_id)}"
+            )
+
+        if self.job_id is not None and not isinstance(self.job_id, str):
+            raise TypeError(
+                "job_id must be a string if provided, " f"got {type(self.job_id)}"
+            )
+
+        if self.runtime_env is not None:
+            if not isinstance(self.runtime_env, dict):
+                raise TypeError(
+                    f"runtime_env must be a dict, got {type(self.runtime_env)}"
+                )
+            else:
+                for k in self.runtime_env.keys():
+                    if not isinstance(k, str):
+                        raise TypeError(
+                            f"runtime_env keys must be strings, got {type(k)}"
+                        )
+
+        if self.metadata is not None:
+            if not isinstance(self.metadata, dict):
+                raise TypeError(f"metadata must be a dict, got {type(self.metadata)}")
+            else:
+                for k in self.metadata.keys():
+                    if not isinstance(k, str):
+                        raise TypeError(f"metadata keys must be strings, got {type(k)}")
+                for v in self.metadata.values():
+                    if not isinstance(v, str):
+                        raise TypeError(
+                            f"metadata values must be strings, got {type(v)}"
+                        )
+
+        if self.entrypoint_num_cpus is not None and not isinstance(
+            self.entrypoint_num_cpus, (int, float)
+        ):
+            raise TypeError(
+                "entrypoint_num_cpus must be a number, "
+                f"got {type(self.entrypoint_num_cpus)}"
+            )
+
+        if self.entrypoint_num_gpus is not None and not isinstance(
+            self.entrypoint_num_gpus, (int, float)
+        ):
+            raise TypeError(
+                "entrypoint_num_gpus must be a number, "
+                f"got {type(self.entrypoint_num_gpus)}"
+            )
+
+        if self.entrypoint_memory is not None and not isinstance(
+            self.entrypoint_memory, int
+        ):
+            raise TypeError(
+                "entrypoint_memory must be an integer, "
+                f"got {type(self.entrypoint_memory)}"
+            )
+
+        if self.entrypoint_resources is not None:
+            if not isinstance(self.entrypoint_resources, dict):
+                raise TypeError(
+                    "entrypoint_resources must be a dict, "
+                    f"got {type(self.entrypoint_resources)}"
+                )
+            else:
+                for k in self.entrypoint_resources.keys():
+                    if not isinstance(k, str):
+                        raise TypeError(
+                            "entrypoint_resources keys must be strings, "
+                            f"got {type(k)}"
+                        )
+                for v in self.entrypoint_resources.values():
+                    if not isinstance(v, (int, float)):
+                        raise TypeError(
+                            "entrypoint_resources values must be numbers, "
+                            f"got {type(v)}"
+                        )
+
+        if self.entrypoint_label_selector is not None:
+            if not isinstance(self.entrypoint_label_selector, dict):
+                raise TypeError(
+                    "entrypoint_label_selector must be a dict, "
+                    f"got {type(self.entrypoint_label_selector)}"
+                )
+            else:
+                for k, v in self.entrypoint_label_selector.items():
+                    if not isinstance(k, str):
+                        raise TypeError(
+                            "entrypoint_label_selector keys must be strings, "
+                            f"got {type(k)}"
+                        )
+                    if not isinstance(v, str):
+                        raise TypeError(
+                            "entrypoint_label_selector values must be strings, "
+                            f"got {type(v)}"
+                        )
+
+
+@dataclass
+class JobSubmitResponse:
+    # DEPRECATED: Use submission_id instead.
+    job_id: str
+    submission_id: str
+
+
+@dataclass
+class JobStopResponse:
+    stopped: bool
+
+
+@dataclass
+class JobDeleteResponse:
+    deleted: bool
+
+
+# TODO(jiaodong): Support log streaming #19415
+@dataclass
+class JobLogsResponse:
+    logs: str

python/ray/dashboard/modules/job/component_activities_schema.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://github.com/ray-project/ray/dashboard/modules/job/component_activities_schema.json",
+  "type": "object",
+  "patternProperties": {
+    "[0-9a-f]*": {
+      "type": "object",
+      "properties": {
+        "is_active": {
+          "type": "string",
+          "enum": ["ACTIVE", "INACTIVE", "ERROR"]
+        },
+        "reason": {
+          "type": ["string", "null"]
+        },
+        "timestamp": {
+          "type": ["number"]
+        },
+        "last_activity_at": {
+          "type": ["number", "null"]
+        }
+      },
+      "required": ["is_active"]
+    }
+  }
+}

python/ray/dashboard/modules/job/job_agent.py

@@ -0,0 +1,210 @@
+import dataclasses
+import json
+import logging
+import traceback
+
+import aiohttp
+from aiohttp.web import Request, Response
+
+import ray
+import ray.dashboard.optional_utils as optional_utils
+import ray.dashboard.utils as dashboard_utils
+from ray.dashboard.modules.job.common import (
+    JobDeleteResponse,
+    JobLogsResponse,
+    JobStopResponse,
+    JobSubmitRequest,
+    JobSubmitResponse,
+)
+from ray.dashboard.modules.job.job_manager import JobManager
+from ray.dashboard.modules.job.pydantic_models import JobType
+from ray.dashboard.modules.job.utils import find_job_by_ids, parse_and_validate_request
+
+routes = optional_utils.DashboardAgentRouteTable
+logger = logging.getLogger(__name__)
+
+
+class JobAgent(dashboard_utils.DashboardAgentModule):
+    def __init__(self, dashboard_agent):
+        super().__init__(dashboard_agent)
+        self._job_manager = None
+
+    @routes.post("/api/job_agent/jobs/")
+    @optional_utils.init_ray_and_catch_exceptions()
+    async def submit_job(self, req: Request) -> Response:
+        result = await parse_and_validate_request(req, JobSubmitRequest)
+        # Request parsing failed, returned with Response object.
+        if isinstance(result, Response):
+            return result
+        else:
+            submit_request = result
+
+        request_submission_id = submit_request.submission_id or submit_request.job_id
+        try:
+            ray._common.usage.usage_lib.record_library_usage("job_submission")
+            submission_id = await self.get_job_manager().submit_job(
+                entrypoint=submit_request.entrypoint,
+                submission_id=request_submission_id,
+                runtime_env=submit_request.runtime_env,
+                metadata=submit_request.metadata,
+                entrypoint_num_cpus=submit_request.entrypoint_num_cpus,
+                entrypoint_num_gpus=submit_request.entrypoint_num_gpus,
+                entrypoint_memory=submit_request.entrypoint_memory,
+                entrypoint_resources=submit_request.entrypoint_resources,
+                entrypoint_label_selector=submit_request.entrypoint_label_selector,
+            )
+
+            resp = JobSubmitResponse(job_id=submission_id, submission_id=submission_id)
+        except (TypeError, ValueError):
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)),
+            content_type="application/json",
+            status=aiohttp.web.HTTPOk.status_code,
+        )
+
+    @routes.post("/api/job_agent/jobs/{job_or_submission_id}/stop")
+    @optional_utils.init_ray_and_catch_exceptions()
+    async def stop_job(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self._dashboard_agent.gcs_client,
+            self.get_job_manager().job_info_client(),
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only stop submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        try:
+            stopped = self.get_job_manager().stop_job(job.submission_id)
+            resp = JobStopResponse(stopped=stopped)
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)), content_type="application/json"
+        )
+
+    @routes.delete("/api/job_agent/jobs/{job_or_submission_id}")
+    @optional_utils.init_ray_and_catch_exceptions()
+    async def delete_job(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self._dashboard_agent.gcs_client,
+            self.get_job_manager().job_info_client(),
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only delete submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        try:
+            deleted = await self.get_job_manager().delete_job(job.submission_id)
+            resp = JobDeleteResponse(deleted=deleted)
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)), content_type="application/json"
+        )
+
+    @routes.get("/api/job_agent/jobs/{job_or_submission_id}/logs")
+    @optional_utils.init_ray_and_catch_exceptions()
+    async def get_job_logs(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self._dashboard_agent.gcs_client,
+            self.get_job_manager().job_info_client(),
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only get logs of submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        resp = JobLogsResponse(
+            logs=self.get_job_manager().get_job_logs(job.submission_id)
+        )
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)), content_type="application/json"
+        )
+
+    @routes.get("/api/job_agent/jobs/{job_or_submission_id}/logs/tail")
+    @optional_utils.init_ray_and_catch_exceptions()
+    async def tail_job_logs(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self._dashboard_agent.gcs_client,
+            self.get_job_manager().job_info_client(),
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only get logs of submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        ws = aiohttp.web.WebSocketResponse()
+        await ws.prepare(req)
+
+        async for lines in self._job_manager.tail_job_logs(job.submission_id):
+            await ws.send_str(lines)
+
+        return ws
+
+    def get_job_manager(self):
+        if not self._job_manager:
+            self._job_manager = JobManager(
+                self._dashboard_agent.gcs_client, self._dashboard_agent.log_dir
+            )
+        return self._job_manager
+
+    async def run(self, server):
+        pass
+
+    @staticmethod
+    def is_minimal_module():
+        return False

python/ray/dashboard/modules/job/job_head.py

@@ -0,0 +1,774 @@
+import asyncio
+import dataclasses
+import enum
+import json
+import logging
+import os
+import time
+import traceback
+from datetime import datetime
+from typing import AsyncIterator, Dict, Optional, Tuple
+
+import aiohttp.web
+from aiohttp.client import ClientResponse
+from aiohttp.web import Request, Response, StreamResponse
+
+import ray
+from ray import NodeID
+from ray._common.network_utils import build_address
+from ray._common.pydantic_compat import BaseModel, Extra, Field, validator
+from ray._common.utils import get_or_create_event_loop, load_class
+from ray._private.authentication.http_token_authentication import (
+    get_auth_headers_if_auth_enabled,
+)
+from ray._private.ray_constants import KV_NAMESPACE_DASHBOARD
+from ray._private.runtime_env.packaging import (
+    package_exists,
+    pin_runtime_env_uri,
+    upload_package_to_gcs,
+)
+from ray.dashboard.consts import (
+    DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX,
+    GCS_RPC_TIMEOUT_SECONDS,
+    RAY_CLUSTER_ACTIVITY_HOOK,
+    TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS,
+    WAIT_AVAILABLE_AGENT_TIMEOUT,
+)
+from ray.dashboard.modules.job.common import (
+    JobDeleteResponse,
+    JobInfoStorageClient,
+    JobLogsResponse,
+    JobStopResponse,
+    JobSubmitRequest,
+    JobSubmitResponse,
+    http_uri_components_to_uri,
+)
+from ray.dashboard.modules.job.pydantic_models import JobDetails, JobType
+from ray.dashboard.modules.job.utils import (
+    find_job_by_ids,
+    get_driver_jobs,
+    parse_and_validate_request,
+)
+from ray.dashboard.modules.version import CURRENT_VERSION, VersionResponse
+from ray.dashboard.subprocesses.module import SubprocessModule
+from ray.dashboard.subprocesses.routes import SubprocessRouteTable as routes
+from ray.dashboard.subprocesses.utils import ResponseType
+from ray.dashboard.utils import get_head_node_id
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+class RayActivityStatus(str, enum.Enum):
+    ACTIVE = "ACTIVE"
+    INACTIVE = "INACTIVE"
+    ERROR = "ERROR"
+
+
+class RayActivityResponse(BaseModel, extra=Extra.allow):
+    """
+    Pydantic model used to inform if a particular Ray component can be considered
+    active, and metadata about observation.
+    """
+
+    is_active: RayActivityStatus = Field(
+        ...,
+        description=(
+            "Whether the corresponding Ray component is considered active or inactive, "
+            "or if there was an error while collecting this observation."
+        ),
+    )
+    reason: Optional[str] = Field(
+        None, description="Reason if Ray component is considered active or errored."
+    )
+    timestamp: float = Field(
+        ...,
+        description=(
+            "Timestamp of when this observation about the Ray component was made. "
+            "This is in the format of seconds since unix epoch."
+        ),
+    )
+    last_activity_at: Optional[float] = Field(
+        None,
+        description=(
+            "Timestamp when last actvity of this Ray component finished in format of "
+            "seconds since unix epoch. This field does not need to be populated "
+            "for Ray components where it is not meaningful."
+        ),
+    )
+
+    @validator("reason", always=True)
+    def reason_required(cls, v, values, **kwargs):
+        if "is_active" in values and values["is_active"] != RayActivityStatus.INACTIVE:
+            if v is None:
+                raise ValueError(
+                    'Reason is required if is_active is "active" or "error"'
+                )
+        return v
+
+
+class JobAgentSubmissionClient:
+    """A local client for submitting and interacting with jobs on a specific node
+    in the remote cluster.
+    Submits requests over HTTP to the job agent on the specific node using the REST API.
+    """
+
+    def __init__(
+        self,
+        dashboard_agent_address: str,
+    ):
+        self._agent_address = dashboard_agent_address
+        self._session = aiohttp.ClientSession()
+
+    def _get_headers(self):
+        """Get auth headers if token authentication is enabled."""
+        return get_auth_headers_if_auth_enabled({})
+
+    async def _raise_error(self, resp: ClientResponse):
+        status = resp.status
+        error_text = await resp.text()
+        raise RuntimeError(f"Request failed with status code {status}: {error_text}.")
+
+    async def submit_job_internal(self, req: JobSubmitRequest) -> JobSubmitResponse:
+        logger.debug(f"Submitting job with submission_id={req.submission_id}.")
+
+        async with self._session.post(
+            f"{self._agent_address}/api/job_agent/jobs/",
+            json=dataclasses.asdict(req),
+            headers=self._get_headers(),
+        ) as resp:
+            if resp.status == 200:
+                result_json = await resp.json()
+                return JobSubmitResponse(**result_json)
+            else:
+                await self._raise_error(resp)
+
+    async def stop_job_internal(self, job_id: str) -> JobStopResponse:
+        logger.debug(f"Stopping job with job_id={job_id}.")
+
+        async with self._session.post(
+            f"{self._agent_address}/api/job_agent/jobs/{job_id}/stop",
+            headers=self._get_headers(),
+        ) as resp:
+            if resp.status == 200:
+                result_json = await resp.json()
+                return JobStopResponse(**result_json)
+            else:
+                await self._raise_error(resp)
+
+    async def delete_job_internal(self, job_id: str) -> JobDeleteResponse:
+        logger.debug(f"Deleting job with job_id={job_id}.")
+
+        async with self._session.delete(
+            f"{self._agent_address}/api/job_agent/jobs/{job_id}",
+            headers=self._get_headers(),
+        ) as resp:
+            if resp.status == 200:
+                result_json = await resp.json()
+                return JobDeleteResponse(**result_json)
+            else:
+                await self._raise_error(resp)
+
+    async def get_job_logs_internal(self, job_id: str) -> JobLogsResponse:
+        async with self._session.get(
+            f"{self._agent_address}/api/job_agent/jobs/{job_id}/logs",
+            headers=self._get_headers(),
+        ) as resp:
+            if resp.status == 200:
+                result_json = await resp.json()
+                return JobLogsResponse(**result_json)
+            else:
+                await self._raise_error(resp)
+
+    async def tail_job_logs(self, job_id: str) -> AsyncIterator[str]:
+        """Get an iterator that follows the logs of a job."""
+        ws = await self._session.ws_connect(
+            f"{self._agent_address}/api/job_agent/jobs/{job_id}/logs/tail",
+            headers=self._get_headers(),
+        )
+
+        while True:
+            msg = await ws.receive()
+
+            if msg.type == aiohttp.WSMsgType.TEXT:
+                yield msg.data
+            elif msg.type == aiohttp.WSMsgType.CLOSED:
+                logger.info(
+                    f"WebSocket to job agent closed for job {job_id} "
+                    f"with close code {ws.close_code}"
+                )
+                break
+            elif msg.type == aiohttp.WSMsgType.ERROR:
+                logger.warning(
+                    f"WebSocket to job agent received an error message "
+                    f"while tailing logs for job {job_id}: {ws.exception()!r}. "
+                )
+                pass
+
+    async def close(self, ignore_error=True):
+        try:
+            await self._session.close()
+        except Exception:
+            if not ignore_error:
+                raise
+
+
+class JobHead(SubprocessModule):
+    """Runs on the head node of a Ray cluster and handles Ray Jobs APIs.
+
+    NOTE(architkulkarni): Please keep this class in sync with the OpenAPI spec at
+    `doc/source/cluster/running-applications/job-submission/openapi.yml`.
+    We currently do not automatically check that the OpenAPI
+    spec is in sync with the implementation. If any changes are made to the
+    paths in the @route decorators or in the Responses returned by the
+    methods (or any nested fields in the Responses), you will need to find the
+    corresponding field of the OpenAPI yaml file and update it manually. Also,
+    bump the version number in the yaml file and in this class's `get_version`.
+    """
+
+    # Time that we sleep while tailing logs while waiting for
+    # the supervisor actor to start. We don't know which node
+    # to read the logs from until then.
+    WAIT_FOR_SUPERVISOR_ACTOR_INTERVAL_S = 1
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._job_info_client = None
+
+        # To make sure that the internal KV is initialized by getting the lazy property
+        assert self.gcs_client is not None
+        assert ray.experimental.internal_kv._internal_kv_initialized()
+
+        # It contains all `JobAgentSubmissionClient` that
+        # `JobHead` has ever used, and will not be deleted
+        # from it unless `JobAgentSubmissionClient` is no
+        # longer available (the corresponding agent process is dead)
+        # {node_id: JobAgentSubmissionClient}
+        self._agents: Dict[NodeID, JobAgentSubmissionClient] = dict()
+
+    async def get_target_agent(
+        self, timeout_s: float = WAIT_AVAILABLE_AGENT_TIMEOUT
+    ) -> JobAgentSubmissionClient:
+        """
+        Get a `JobAgentSubmissionClient`, which is a client for interacting with jobs
+        via an agent process.
+
+        Args:
+            timeout_s: The timeout for the operation.
+
+        Returns:
+            A `JobAgentSubmissionClient` for interacting with jobs via an agent process.
+
+        Raises:
+            TimeoutError: If the operation times out.
+        """
+        return await self._get_head_node_agent(timeout_s)
+
+    async def _get_head_node_agent_once(self) -> JobAgentSubmissionClient:
+        head_node_id_hex = await get_head_node_id(self.gcs_client)
+
+        if not head_node_id_hex:
+            raise Exception("Head node id has not yet been persisted in GCS")
+
+        head_node_id = NodeID.from_hex(head_node_id_hex)
+
+        if head_node_id not in self._agents:
+            ip, http_port, _ = await self._fetch_agent_info(head_node_id)
+            agent_http_address = f"http://{build_address(ip, http_port)}"
+            self._agents[head_node_id] = JobAgentSubmissionClient(agent_http_address)
+
+        return self._agents[head_node_id]
+
+    async def _get_head_node_agent(self, timeout_s: float) -> JobAgentSubmissionClient:
+        """Retrieves HTTP client for `JobAgent` running on the Head node. If the head
+        node does not have an agent, it will retry every
+        `TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS` seconds indefinitely.
+
+        Args:
+            timeout_s: The timeout for the operation.
+
+        Returns:
+            A `JobAgentSubmissionClient` for interacting with jobs via the head node's agent process.
+
+        Raises:
+            TimeoutError: If the operation times out.
+        """
+        timeout_point = time.time() + timeout_s
+        exception = None
+        while time.time() < timeout_point:
+            try:
+                return await self._get_head_node_agent_once()
+            except Exception as e:
+                exception = e
+                logger.exception(
+                    f"Failed to get head node agent, retrying in {TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS} seconds..."
+                )
+                await asyncio.sleep(TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS)
+        raise TimeoutError(
+            f"Failed to get head node agent within {timeout_s} seconds. The last exception is {exception}"
+        )
+
+    async def _fetch_agent_info(self, target_node_id: NodeID) -> Tuple[str, int, int]:
+        """
+        Fetches agent info by the Node ID. May raise exception if there's network error or the
+        agent info is not found.
+
+        Returns: (ip, http_port, grpc_port)
+        """
+        key = f"{DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}{target_node_id.hex()}"
+        value = await self.gcs_client.async_internal_kv_get(
+            key,
+            namespace=KV_NAMESPACE_DASHBOARD,
+            timeout=GCS_RPC_TIMEOUT_SECONDS,
+        )
+        if not value:
+            raise KeyError(
+                f"Agent info not found in internal KV for node {target_node_id}. "
+                "It's possible that the agent didn't launch successfully due to "
+                "port conflicts or other issues. Please check `dashboard_agent.log` "
+                "for more details."
+            )
+        return json.loads(value.decode())
+
+    @routes.get("/api/version")
+    async def get_version(self, req: Request) -> Response:
+        # NOTE(edoakes): CURRENT_VERSION should be bumped and checked on the
+        # client when we have backwards-incompatible changes.
+        resp = VersionResponse(
+            version=CURRENT_VERSION,
+            ray_version=ray.__version__,
+            ray_commit=ray.__commit__,
+            session_name=self.session_name,
+        )
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)),
+            content_type="application/json",
+            status=aiohttp.web.HTTPOk.status_code,
+        )
+
+    @routes.get("/api/packages/{protocol}/{package_name}")
+    async def get_package(self, req: Request) -> Response:
+        package_uri = http_uri_components_to_uri(
+            protocol=req.match_info["protocol"],
+            package_name=req.match_info["package_name"],
+        )
+
+        logger.debug(f"Adding temporary reference to package {package_uri}.")
+        try:
+            pin_runtime_env_uri(package_uri)
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        if not package_exists(package_uri):
+            return Response(
+                text=f"Package {package_uri} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+
+        return Response()
+
+    @routes.put("/api/packages/{protocol}/{package_name}")
+    async def upload_package(self, req: Request):
+        package_uri = http_uri_components_to_uri(
+            protocol=req.match_info["protocol"],
+            package_name=req.match_info["package_name"],
+        )
+        logger.info(f"Uploading package {package_uri} to the GCS.")
+        try:
+            data = await req.read()
+            await get_or_create_event_loop().run_in_executor(
+                None,
+                upload_package_to_gcs,
+                package_uri,
+                data,
+            )
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        return Response(status=aiohttp.web.HTTPOk.status_code)
+
+    @routes.post("/api/jobs/")
+    async def submit_job(self, req: Request) -> Response:
+        result = await parse_and_validate_request(req, JobSubmitRequest)
+        # Request parsing failed, returned with Response object.
+        if isinstance(result, Response):
+            return result
+        else:
+            submit_request: JobSubmitRequest = result
+
+        try:
+            job_agent_client = await self.get_target_agent()
+            resp = await job_agent_client.submit_job_internal(submit_request)
+        except asyncio.TimeoutError:
+            logger.warning(
+                "Timed out waiting for an available job agent to submit the job."
+            )
+            return Response(
+                text="No available agent to submit job, please try again later.",
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+        except (TypeError, ValueError):
+            logger.warning("Failed to submit job due to an invalid request.")
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+        except Exception:
+            logger.exception("Failed to submit job due to unexpected exception.")
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)),
+            content_type="application/json",
+            status=aiohttp.web.HTTPOk.status_code,
+        )
+
+    @routes.post("/api/jobs/{job_or_submission_id}/stop")
+    async def stop_job(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self.gcs_client,
+            self._job_info_client,
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only stop submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        try:
+            job_agent_client = await self.get_target_agent()
+            resp = await job_agent_client.stop_job_internal(job.submission_id)
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)), content_type="application/json"
+        )
+
+    @routes.delete("/api/jobs/{job_or_submission_id}")
+    async def delete_job(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self.gcs_client,
+            self._job_info_client,
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only delete submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        try:
+            job_agent_client = await self.get_target_agent()
+            resp = await job_agent_client.delete_job_internal(job.submission_id)
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+        return Response(
+            text=json.dumps(dataclasses.asdict(resp)), content_type="application/json"
+        )
+
+    @routes.get("/api/jobs/{job_or_submission_id}")
+    async def get_job_info(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self.gcs_client,
+            self._job_info_client,
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+
+        return Response(
+            text=json.dumps(job.dict()),
+            content_type="application/json",
+        )
+
+    # TODO(rickyx): This endpoint's logic is also mirrored in state API's endpoint.
+    # We should eventually unify the backend logic (and keep the logic in sync before
+    # that).
+    @routes.get("/api/jobs/")
+    async def list_jobs(self, req: Request) -> Response:
+        (driver_jobs, submission_job_drivers), submission_jobs = await asyncio.gather(
+            get_driver_jobs(self.gcs_client), self._job_info_client.get_all_jobs()
+        )
+
+        submission_jobs = [
+            JobDetails(
+                **dataclasses.asdict(job),
+                submission_id=submission_id,
+                job_id=submission_job_drivers.get(submission_id).id
+                if submission_id in submission_job_drivers
+                else None,
+                driver_info=submission_job_drivers.get(submission_id),
+                type=JobType.SUBMISSION,
+            )
+            for submission_id, job in submission_jobs.items()
+        ]
+        return Response(
+            text=json.dumps(
+                [
+                    *[submission_job.dict() for submission_job in submission_jobs],
+                    *[job_info.dict() for job_info in driver_jobs.values()],
+                ]
+            ),
+            content_type="application/json",
+        )
+
+    @routes.get("/api/jobs/{job_or_submission_id}/logs")
+    async def get_job_logs(self, req: Request) -> Response:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self.gcs_client,
+            self._job_info_client,
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only get logs of submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        try:
+            job_agent_client = self.get_job_driver_agent_client(job)
+            payload = (
+                await job_agent_client.get_job_logs_internal(job.submission_id)
+                if job_agent_client
+                else JobLogsResponse("")
+            )
+            return Response(
+                text=json.dumps(dataclasses.asdict(payload)),
+                content_type="application/json",
+            )
+        except Exception:
+            return Response(
+                text=traceback.format_exc(),
+                status=aiohttp.web.HTTPInternalServerError.status_code,
+            )
+
+    @routes.get(
+        "/api/jobs/{job_or_submission_id}/logs/tail", resp_type=ResponseType.WEBSOCKET
+    )
+    async def tail_job_logs(self, req: Request) -> StreamResponse:
+        job_or_submission_id = req.match_info["job_or_submission_id"]
+        job = await find_job_by_ids(
+            self.gcs_client,
+            self._job_info_client,
+            job_or_submission_id,
+        )
+        if not job:
+            return Response(
+                text=f"Job {job_or_submission_id} does not exist",
+                status=aiohttp.web.HTTPNotFound.status_code,
+            )
+
+        if job.type is not JobType.SUBMISSION:
+            return Response(
+                text="Can only get logs of submission type jobs",
+                status=aiohttp.web.HTTPBadRequest.status_code,
+            )
+
+        ws = aiohttp.web.WebSocketResponse()
+        await ws.prepare(req)
+
+        driver_agent_http_address = None
+        while driver_agent_http_address is None:
+            job = await find_job_by_ids(
+                self.gcs_client,
+                self._job_info_client,
+                job_or_submission_id,
+            )
+            driver_agent_http_address = job.driver_agent_http_address
+            status = job.status
+            if status.is_terminal() and driver_agent_http_address is None:
+                # Job exited before supervisor actor started.
+                return ws
+
+            await asyncio.sleep(self.WAIT_FOR_SUPERVISOR_ACTOR_INTERVAL_S)
+
+        job_agent_client = self.get_job_driver_agent_client(job)
+
+        async for lines in job_agent_client.tail_job_logs(job.submission_id):
+            await ws.send_str(lines)
+
+        return ws
+
+    def get_job_driver_agent_client(
+        self, job: JobDetails
+    ) -> Optional[JobAgentSubmissionClient]:
+        if job.driver_agent_http_address is None:
+            return None
+
+        driver_node_id = job.driver_node_id
+        if driver_node_id not in self._agents:
+            self._agents[driver_node_id] = JobAgentSubmissionClient(
+                job.driver_agent_http_address
+            )
+
+        return self._agents[driver_node_id]
+
+    @routes.get("/api/component_activities")
+    async def get_component_activities(
+        self, req: aiohttp.web.Request
+    ) -> aiohttp.web.Response:
+        timeout = req.query.get("timeout", None)
+        if timeout and timeout.isdigit():
+            timeout = int(timeout)
+        else:
+            timeout = 30
+
+        # Get activity information for driver
+        driver_activity_info = await self._get_job_activity_info(timeout=timeout)
+        resp = {"driver": dict(driver_activity_info)}
+
+        if RAY_CLUSTER_ACTIVITY_HOOK in os.environ:
+            try:
+                cluster_activity_callable = load_class(
+                    os.environ[RAY_CLUSTER_ACTIVITY_HOOK]
+                )
+                external_activity_output = cluster_activity_callable()
+                assert isinstance(external_activity_output, dict), (
+                    f"Output of hook {os.environ[RAY_CLUSTER_ACTIVITY_HOOK]} "
+                    "should be Dict[str, RayActivityResponse]. Got "
+                    f"output: {external_activity_output}"
+                )
+                for component_type in external_activity_output:
+                    try:
+                        component_activity_output = external_activity_output[
+                            component_type
+                        ]
+                        # Parse and validate output to type RayActivityResponse
+                        component_activity_output = RayActivityResponse(
+                            **dict(component_activity_output)
+                        )
+                        resp[component_type] = dict(component_activity_output)
+                    except Exception as e:
+                        logger.exception(
+                            f"Failed to get activity status of {component_type} "
+                            f"from user hook {os.environ[RAY_CLUSTER_ACTIVITY_HOOK]}."
+                        )
+                        resp[component_type] = {
+                            "is_active": RayActivityStatus.ERROR,
+                            "reason": repr(e),
+                            "timestamp": datetime.now().timestamp(),
+                        }
+            except Exception as e:
+                logger.exception(
+                    "Failed to get activity status from user "
+                    f"hook {os.environ[RAY_CLUSTER_ACTIVITY_HOOK]}."
+                )
+                resp["external_component"] = {
+                    "is_active": RayActivityStatus.ERROR,
+                    "reason": repr(e),
+                    "timestamp": datetime.now().timestamp(),
+                }
+
+        return aiohttp.web.Response(
+            text=json.dumps(resp),
+            content_type="application/json",
+            status=aiohttp.web.HTTPOk.status_code,
+        )
+
+    async def _get_job_activity_info(self, timeout: int) -> RayActivityResponse:
+        # Returns if there is Ray activity from drivers (job).
+        # Drivers in namespaces that start with _ray_internal_ are not
+        # considered activity.
+        # This includes the _ray_internal_dashboard job that gets automatically
+        # created with every cluster
+        try:
+            reply = await self.gcs_client.async_get_all_job_info(
+                skip_submission_job_info_field=True,
+                skip_is_running_tasks_field=True,
+                timeout=timeout,
+            )
+
+            num_active_drivers = 0
+            latest_job_end_time = 0
+            for job_table_entry in reply.values():
+                is_dead = bool(job_table_entry.is_dead)
+                in_internal_namespace = job_table_entry.config.ray_namespace.startswith(
+                    "_ray_internal_"
+                )
+                latest_job_end_time = (
+                    max(latest_job_end_time, job_table_entry.end_time)
+                    if job_table_entry.end_time
+                    else latest_job_end_time
+                )
+                if not is_dead and not in_internal_namespace:
+                    num_active_drivers += 1
+
+            current_timestamp = datetime.now().timestamp()
+            # Latest job end time must be before or equal to the current timestamp.
+            # Job end times may be provided in epoch milliseconds. Check if this
+            # is true, and convert to seconds
+            if latest_job_end_time > current_timestamp:
+                latest_job_end_time = latest_job_end_time / 1000
+                assert current_timestamp >= latest_job_end_time, (
+                    f"Most recent job end time {latest_job_end_time} must be "
+                    f"before or equal to the current timestamp {current_timestamp}"
+                )
+
+            is_active = (
+                RayActivityStatus.ACTIVE
+                if num_active_drivers > 0
+                else RayActivityStatus.INACTIVE
+            )
+            return RayActivityResponse(
+                is_active=is_active,
+                reason=f"Number of active drivers: {num_active_drivers}"
+                if num_active_drivers
+                else None,
+                timestamp=current_timestamp,
+                # If latest_job_end_time == 0, no jobs have finished yet so don't
+                # populate last_activity_at
+                last_activity_at=latest_job_end_time if latest_job_end_time else None,
+            )
+        except Exception as e:
+            logger.exception("Failed to get activity status of Ray drivers.")
+            return RayActivityResponse(
+                is_active=RayActivityStatus.ERROR,
+                reason=repr(e),
+                timestamp=datetime.now().timestamp(),
+            )
+
+    async def run(self):
+        await super().run()
+        if not self._job_info_client:
+            self._job_info_client = JobInfoStorageClient(self.gcs_client)

python/ray/dashboard/modules/job/job_log_storage_client.py

@@ -0,0 +1,57 @@
+import os
+from typing import AsyncIterator, List, Tuple
+
+import ray
+from ray.dashboard.modules.job.common import JOB_LOGS_PATH_TEMPLATE
+from ray.dashboard.modules.job.utils import fast_tail_last_n_lines, file_tail_iterator
+
+
+class JobLogStorageClient:
+    """
+    Disk storage for stdout / stderr of driver script logs.
+    """
+
+    # Number of last N lines to put in job message upon failure.
+    NUM_LOG_LINES_ON_ERROR = 10
+    # Maximum number of characters to print out of the logs to avoid
+    # HUGE log outputs that bring down the api server
+    MAX_LOG_SIZE = 20000
+
+    def get_logs(self, job_id: str) -> str:
+        try:
+            with open(self.get_log_file_path(job_id), "r") as f:
+                return f.read()
+        except FileNotFoundError:
+            return ""
+
+    def tail_logs(self, job_id: str) -> AsyncIterator[List[str]]:
+        return file_tail_iterator(self.get_log_file_path(job_id))
+
+    async def get_last_n_log_lines(
+        self, job_id: str, num_log_lines: int = NUM_LOG_LINES_ON_ERROR
+    ) -> str:
+        """Returns the last MAX_LOG_SIZE (20000) characters in the last ``num_log_lines`` lines.
+
+        Args:
+            job_id: The id of the job whose logs we want to return
+            num_log_lines: The number of lines to return.
+
+        Returns:
+            Up to ``MAX_LOG_SIZE`` characters drawn from the last
+            ``num_log_lines`` lines of the job's log file.
+        """
+        return fast_tail_last_n_lines(
+            path=self.get_log_file_path(job_id),
+            num_lines=num_log_lines,
+            max_chars=self.MAX_LOG_SIZE,
+        )
+
+    def get_log_file_path(self, job_id: str) -> Tuple[str, str]:
+        """
+        Get the file path to the logs of a given job. Example:
+            /tmp/ray/session_date/logs/job-driver-{job_id}.log
+        """
+        return os.path.join(
+            ray._private.worker._global_node.get_logs_dir_path(),
+            JOB_LOGS_PATH_TEMPLATE.format(submission_id=job_id),
+        )

python/ray/dashboard/modules/job/job_manager.py

@@ -0,0 +1,707 @@
+import asyncio
+import copy
+import logging
+import os
+import random
+import string
+import time
+import traceback
+from typing import Any, AsyncIterator, Dict, Optional, Union
+
+import ray
+import ray._private.ray_constants as ray_constants
+from ray._common.utils import Timer, run_background_task
+from ray._private.accelerators.npu import NOSET_ASCEND_RT_VISIBLE_DEVICES_ENV_VAR
+from ray._private.accelerators.nvidia_gpu import NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR
+from ray._private.event.event_logger import get_event_logger
+from ray._private.label_utils import validate_label_selector
+from ray._raylet import GcsClient
+from ray.actor import ActorHandle
+from ray.core.generated.event_pb2 import Event
+from ray.dashboard.consts import (
+    DEFAULT_JOB_START_TIMEOUT_SECONDS,
+    RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR,
+    RAY_JOB_START_TIMEOUT_SECONDS_ENV_VAR,
+    RAY_STREAM_RUNTIME_ENV_LOG_TO_JOB_DRIVER_LOG_ENV_VAR,
+)
+from ray.dashboard.modules.job.common import (
+    JOB_ACTOR_NAME_TEMPLATE,
+    SUPERVISOR_ACTOR_RAY_NAMESPACE,
+    JobInfo,
+    JobInfoStorageClient,
+)
+from ray.dashboard.modules.job.job_log_storage_client import JobLogStorageClient
+from ray.dashboard.modules.job.job_supervisor import JobSupervisor
+from ray.dashboard.utils import close_logger_file_descriptor, get_head_node_id
+from ray.exceptions import ActorDiedError, ActorUnschedulableError, RuntimeEnvSetupError
+from ray.job_submission import JobErrorType, JobStatus
+from ray.runtime_env import RuntimeEnvConfig
+
+logger = logging.getLogger(__name__)
+
+
+def generate_job_id() -> str:
+    """Returns a job_id of the form 'raysubmit_XYZ'.
+
+    Prefixed with 'raysubmit' to avoid confusion with Ray JobID (driver ID).
+    """
+    rand = random.SystemRandom()
+    possible_characters = list(
+        set(string.ascii_letters + string.digits)
+        - {"I", "l", "o", "O", "0"}  # No confusing characters
+    )
+    id_part = "".join(rand.choices(possible_characters, k=16))
+    return f"raysubmit_{id_part}"
+
+
+class JobManager:
+    """Provide python APIs for job submission and management.
+
+    It does not provide persistence, all info will be lost if the cluster
+    goes down.
+    """
+
+    # Time that we will sleep while tailing logs if no new log line is
+    # available.
+    LOG_TAIL_SLEEP_S = 1
+    JOB_MONITOR_LOOP_PERIOD_S = 1
+    WAIT_FOR_ACTOR_DEATH_TIMEOUT_S = 0.1
+
+    def __init__(
+        self, gcs_client: GcsClient, logs_dir: str, timeout_check_timer: Timer = None
+    ):
+        self._gcs_client = gcs_client
+        self._logs_dir = logs_dir
+        self._job_info_client = JobInfoStorageClient(gcs_client, logs_dir)
+        self._gcs_address = gcs_client.address
+        self._cluster_id_hex = gcs_client.cluster_id.hex()
+        self._log_client = JobLogStorageClient()
+        self._supervisor_actor_cls = ray.remote(JobSupervisor)
+        self._timeout_check_timer = timeout_check_timer or Timer()
+        self.monitored_jobs = set()
+        try:
+            self.event_logger = get_event_logger(Event.SourceType.JOBS, logs_dir)
+        except Exception:
+            self.event_logger = None
+
+        self._recover_running_jobs_event = asyncio.Event()
+        run_background_task(self._recover_running_jobs())
+
+    def _get_job_driver_logger(self, job_id: str) -> logging.Logger:
+        """Return job driver logger to log messages to the job driver log file.
+
+        If this function is called for the first time, configure the logger.
+        """
+        job_driver_logger = logging.getLogger(f"{__name__}.driver-{job_id}")
+
+        # Configure the logger if it's not already configured.
+        if not job_driver_logger.handlers:
+            job_driver_log_path = self._log_client.get_log_file_path(job_id)
+            job_driver_handler = logging.FileHandler(job_driver_log_path)
+            job_driver_formatter = logging.Formatter(ray_constants.LOGGER_FORMAT)
+            job_driver_handler.setFormatter(job_driver_formatter)
+            job_driver_logger.addHandler(job_driver_handler)
+
+        return job_driver_logger
+
+    async def _recover_running_jobs(self):
+        """Recovers all running jobs from the status client.
+
+        For each job, we will spawn a coroutine to monitor it.
+        Each will be added to self._running_jobs and reconciled.
+        """
+        try:
+            all_jobs = await self._job_info_client.get_all_jobs()
+            for job_id, job_info in all_jobs.items():
+                if not job_info.status.is_terminal():
+                    run_background_task(self._monitor_job(job_id))
+        finally:
+            # This event is awaited in `submit_job` to avoid race conditions between
+            # recovery and new job submission, so it must always get set even if there
+            # are exceptions.
+            self._recover_running_jobs_event.set()
+
+    def _get_actor_for_job(self, job_id: str) -> Optional[ActorHandle]:
+        try:
+            return ray.get_actor(
+                JOB_ACTOR_NAME_TEMPLATE.format(job_id=job_id),
+                namespace=SUPERVISOR_ACTOR_RAY_NAMESPACE,
+            )
+        except ValueError:  # Ray returns ValueError for nonexistent actor.
+            return None
+
+    async def _monitor_job(
+        self, job_id: str, job_supervisor: Optional[ActorHandle] = None
+    ):
+        """Monitors the specified job until it enters a terminal state.
+
+        This is necessary because we need to handle the case where the
+        JobSupervisor dies unexpectedly.
+        """
+        if job_id in self.monitored_jobs:
+            logger.debug(f"Job {job_id} is already being monitored.")
+            return
+
+        self.monitored_jobs.add(job_id)
+        try:
+            await self._monitor_job_internal(job_id, job_supervisor)
+        except Exception as e:
+            logger.error("Unhandled exception in job monitoring!", exc_info=e)
+            raise e
+        finally:
+            self.monitored_jobs.remove(job_id)
+
+    async def _monitor_job_internal(
+        self, job_id: str, job_supervisor: Optional[ActorHandle] = None
+    ):
+        timeout = float(
+            os.environ.get(
+                RAY_JOB_START_TIMEOUT_SECONDS_ENV_VAR,
+                DEFAULT_JOB_START_TIMEOUT_SECONDS,
+            )
+        )
+
+        job_status = None
+        job_info = None
+        ping_obj_ref = None
+
+        while True:
+            try:
+                # NOTE: Job monitoring loop sleeps before proceeding with monitoring
+                #       sequence to consolidate the control-flow of the pacing
+                #       in a single place, rather than having it spread across
+                #       many branches
+                await asyncio.sleep(self.JOB_MONITOR_LOOP_PERIOD_S)
+
+                job_status = await self._job_info_client.get_status(
+                    job_id, timeout=None
+                )
+                if job_status == JobStatus.PENDING:
+                    # Compare the current time with the job start time.
+                    # If the job is still pending, we will set the status
+                    # to FAILED.
+                    if job_info is None:
+                        job_info = await self._job_info_client.get_info(
+                            job_id, timeout=None
+                        )
+
+                    if (
+                        self._timeout_check_timer.time() - job_info.start_time / 1000
+                        > timeout
+                    ):
+                        err_msg = (
+                            "Job supervisor actor failed to start within "
+                            f"{timeout} seconds. This timeout can be "
+                            f"configured by setting the environment "
+                            f"variable {RAY_JOB_START_TIMEOUT_SECONDS_ENV_VAR}."
+                        )
+                        resources_specified = (
+                            (
+                                job_info.entrypoint_num_cpus is not None
+                                and job_info.entrypoint_num_cpus > 0
+                            )
+                            or (
+                                job_info.entrypoint_num_gpus is not None
+                                and job_info.entrypoint_num_gpus > 0
+                            )
+                            or (
+                                job_info.entrypoint_memory is not None
+                                and job_info.entrypoint_memory > 0
+                            )
+                            or (
+                                job_info.entrypoint_resources is not None
+                                and len(job_info.entrypoint_resources) > 0
+                            )
+                        )
+                        if resources_specified:
+                            err_msg += (
+                                " This may be because the job entrypoint's specified "
+                                "resources (entrypoint_num_cpus, entrypoint_num_gpus, "
+                                "entrypoint_resources, entrypoint_memory)"
+                                "aren't available on the cluster."
+                                " Try checking the cluster's available resources with "
+                                "`ray status` and specifying fewer resources for the "
+                                "job entrypoint."
+                            )
+                        await self._job_info_client.put_status(
+                            job_id,
+                            JobStatus.FAILED,
+                            message=err_msg,
+                            error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_TIMEOUT,
+                            timeout=None,
+                        )
+                        logger.error(err_msg)
+                        break
+
+                if job_supervisor is None:
+                    job_supervisor = self._get_actor_for_job(job_id)
+
+                if job_supervisor is None:
+                    if job_status == JobStatus.PENDING:
+                        # Maybe the job supervisor actor is not created yet.
+                        # We will wait for the next loop.
+                        continue
+                    else:
+                        # The job supervisor actor is not created, but the job
+                        # status is not PENDING. This means the job supervisor
+                        # actor is not created due to some unexpected errors.
+                        # We will set the job status to FAILED.
+                        logger.error(f"Failed to get job supervisor for job {job_id}.")
+                        await self._job_info_client.put_status(
+                            job_id,
+                            JobStatus.FAILED,
+                            message=(
+                                "Unexpected error occurred: "
+                                "failed to get job supervisor."
+                            ),
+                            error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_FAILURE,
+                            timeout=None,
+                        )
+                        break
+
+                # Check to see if `JobSupervisor` is alive and reachable
+                if ping_obj_ref is None:
+                    ping_obj_ref = job_supervisor.ping.options(
+                        max_task_retries=-1
+                    ).remote()
+                ready, _ = ray.wait([ping_obj_ref], timeout=0)
+                if ready:
+                    ray.get(ping_obj_ref)
+                    ping_obj_ref = None
+                else:
+                    continue
+
+            except Exception as e:
+                job_status = await self._job_info_client.get_status(
+                    job_id, timeout=None
+                )
+                target_job_error_message = ""
+                target_job_error_type: Optional[JobErrorType] = None
+                if job_status is not None and job_status.is_terminal():
+                    # If the job is already in a terminal state, then the actor
+                    # exiting is expected.
+                    pass
+                else:
+                    if isinstance(e, RuntimeEnvSetupError):
+                        logger.error(f"Failed to set up runtime_env for job {job_id}.")
+
+                        target_job_error_message = f"runtime_env setup failed: {e}"
+                        target_job_error_type = JobErrorType.RUNTIME_ENV_SETUP_FAILURE
+
+                    elif isinstance(e, ActorUnschedulableError):
+                        logger.error(
+                            f"Failed to schedule job {job_id} because the supervisor "
+                            f"actor could not be scheduled: {e}"
+                        )
+
+                        target_job_error_message = (
+                            f"Job supervisor actor could not be scheduled: {e}"
+                        )
+                        target_job_error_type = (
+                            JobErrorType.JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE
+                        )
+
+                    elif isinstance(e, ActorDiedError):
+                        logger.error(f"Job supervisor actor for {job_id} died: {e}")
+                        target_job_error_message = f"Job supervisor actor died: {e}"
+                        target_job_error_type = JobErrorType.JOB_SUPERVISOR_ACTOR_DIED
+
+                    else:
+                        logger.error(
+                            f"Job monitoring for job {job_id} failed "
+                            f"unexpectedly: {e}.",
+                            exc_info=e,
+                        )
+
+                        target_job_error_message = f"Unexpected error occurred: {e}"
+                        target_job_error_type = (
+                            JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE
+                        )
+
+                    job_status = JobStatus.FAILED
+                    await self._job_info_client.put_status(
+                        job_id,
+                        job_status,
+                        message=target_job_error_message,
+                        error_type=target_job_error_type
+                        or JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE,
+                        timeout=None,
+                    )
+
+                # Log error message to the job driver file for easy access.
+                if target_job_error_message:
+                    log_path = self._log_client.get_log_file_path(job_id)
+                    os.makedirs(os.path.dirname(log_path), exist_ok=True)
+                    with open(log_path, "a") as log_file:
+                        log_file.write(target_job_error_message)
+
+                # Log events
+                if self.event_logger:
+                    event_log = (
+                        f"Completed a ray job {job_id} with a status {job_status}."
+                    )
+                    if target_job_error_message:
+                        event_log += f" {target_job_error_message}"
+                        self.event_logger.error(event_log, submission_id=job_id)
+                    else:
+                        self.event_logger.info(event_log, submission_id=job_id)
+
+                break
+
+        # Kill the actor defensively to avoid leaking actors in unexpected error cases.
+        if job_supervisor is None:
+            job_supervisor = self._get_actor_for_job(job_id)
+        if job_supervisor is not None:
+            ray.kill(job_supervisor, no_restart=True)
+
+    def _handle_supervisor_startup(self, job_id: str, result: Optional[Exception]):
+        """Handle the result of starting a job supervisor actor.
+
+        If started successfully, result should be None. Otherwise it should be
+        an Exception.
+
+        On failure, the job will be marked failed with a relevant error
+        message.
+        """
+        if result is None:
+            return
+
+    def _get_supervisor_runtime_env(
+        self,
+        user_runtime_env: Dict[str, Any],
+        submission_id: str,
+        resources_specified: bool = False,
+    ) -> Dict[str, Any]:
+        """Configure and return the runtime_env for the supervisor actor.
+
+        Args:
+            user_runtime_env: The runtime_env specified by the user.
+            submission_id: The submission id of the job; used to derive the log
+                file path piped into the runtime env config.
+            resources_specified: Whether the user specified resources in the
+                submit_job() call. If so, we will skip the workaround introduced
+                in #24546 for GPU detection and just use the user's resource
+                requests, so that the behavior matches that of the user specifying
+                resources for any other actor.
+
+        Returns:
+            The runtime_env for the supervisor actor.
+        """
+        # Make a copy to avoid mutating passed runtime_env.
+        runtime_env = (
+            copy.deepcopy(user_runtime_env) if user_runtime_env is not None else {}
+        )
+
+        # NOTE(edoakes): Can't use .get(, {}) here because we need to handle the case
+        # where env_vars is explicitly set to `None`.
+        env_vars = runtime_env.get("env_vars")
+        if env_vars is None:
+            env_vars = {}
+
+        env_vars[ray_constants.RAY_WORKER_NICENESS] = "0"
+
+        if not resources_specified:
+            # Don't set CUDA_VISIBLE_DEVICES for the supervisor actor so the
+            # driver can use GPUs if it wants to. This will be removed from
+            # the driver's runtime_env so it isn't inherited by tasks & actors.
+            env_vars[NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR] = "1"
+            env_vars[NOSET_ASCEND_RT_VISIBLE_DEVICES_ENV_VAR] = "1"
+
+        runtime_env["env_vars"] = env_vars
+
+        if os.getenv(RAY_STREAM_RUNTIME_ENV_LOG_TO_JOB_DRIVER_LOG_ENV_VAR, "0") == "1":
+            config = runtime_env.get("config")
+            # Empty fields may be set to None, so we need to check for None explicitly.
+            if config is None:
+                config = RuntimeEnvConfig()
+            config["log_files"] = [self._log_client.get_log_file_path(submission_id)]
+            runtime_env["config"] = config
+        return runtime_env
+
+    async def _get_label_selector(self, resources_specified: bool) -> Dict:
+        """Determine the scheduling strategy for the job using a label selector.
+
+        If resources_specified is true, or if the environment variable is set to
+        allow the job to run on worker nodes, we will not use any label constraints.
+        Otherwise, we will force the job to use the head node via a label selector
+        specifying the head node id.
+
+        Args:
+            resources_specified: Whether the job specified any resources
+                (CPUs, GPUs, or custom resources).
+
+        Returns:
+            The label selector to use for the job.
+        """
+        if resources_specified:
+            return {}
+
+        if os.environ.get(RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR, "0") == "1":
+            logger.info(
+                f"{RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR} was set to 1. "
+                "Using Ray's default actor scheduling strategy for the job "
+                "driver instead of running it on the head node via a label selector."
+            )
+            return {}
+
+        # If the user did not specify any resources or set the driver on worker nodes
+        # env var, we will run the driver on the head node.
+
+        head_node_id = await get_head_node_id(self._gcs_client)
+        if head_node_id is None:
+            logger.info(
+                "Head node ID not found in GCS. Using Ray's default actor "
+                "scheduling strategy for the job driver instead of running "
+                "it on the head node via a label selector."
+            )
+            return {}
+
+        logger.info(
+            "Head node ID found in GCS; scheduling job driver on "
+            f"head node {head_node_id} using a label selector"
+        )
+        return {ray._raylet.RAY_NODE_ID_KEY: head_node_id}
+
+    async def submit_job(
+        self,
+        *,
+        entrypoint: str,
+        submission_id: Optional[str] = None,
+        runtime_env: Optional[Dict[str, Any]] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        entrypoint_num_cpus: Optional[Union[int, float]] = None,
+        entrypoint_num_gpus: Optional[Union[int, float]] = None,
+        entrypoint_memory: Optional[int] = None,
+        entrypoint_resources: Optional[Dict[str, float]] = None,
+        entrypoint_label_selector: Optional[Dict[str, str]] = None,
+        _start_signal_actor: Optional[ActorHandle] = None,
+    ) -> str:
+        """
+        Job execution happens asynchronously.
+
+        1) Generate a new unique id for this job submission, each call of this
+            method assumes they're independent submission with its own new
+            ID, job supervisor actor, and child process.
+        2) Create new detached actor with same runtime_env as job spec
+
+        Actual setting up runtime_env, subprocess group, driver command
+        execution, subprocess cleaning up and running status update to GCS
+        is all handled by job supervisor actor.
+
+        Args:
+            entrypoint: Driver command to execute in subprocess shell.
+                Represents the entrypoint to start user application.
+            submission_id: Optional caller-provided submission id. When None, a
+                new id is generated via ``generate_job_id()``.
+            runtime_env: Runtime environment used to execute driver command,
+                which could contain its own ray.init() to configure runtime
+                env at ray cluster, task and actor level.
+            metadata: Support passing arbitrary data to driver command in
+                case needed.
+            entrypoint_num_cpus: The quantity of CPU cores to reserve for the execution
+                of the entrypoint command, separately from any tasks or actors launched
+                by it. Defaults to 0.
+            entrypoint_num_gpus: The quantity of GPUs to reserve for
+                the entrypoint command, separately from any tasks or actors launched
+                by it. Defaults to 0.
+            entrypoint_memory: The amount of total available memory for workers
+                requesting memory the entrypoint command, separately from any tasks
+                or actors launched by it. Defaults to 0.
+            entrypoint_resources: The quantity of various custom resources
+                to reserve for the entrypoint command, separately from any tasks or
+                actors launched by it.
+            entrypoint_label_selector: Label selector for the entrypoint command.
+            _start_signal_actor: Used in testing only to capture state
+                transitions between PENDING -> RUNNING. Regular user shouldn't
+                need this.
+
+        Returns:
+            job_id: Generated uuid for further job management. Only valid
+                within the same ray cluster.
+        """
+        if entrypoint_num_cpus is None:
+            entrypoint_num_cpus = 0
+        if entrypoint_num_gpus is None:
+            entrypoint_num_gpus = 0
+        if entrypoint_memory is None:
+            entrypoint_memory = 0
+        if submission_id is None:
+            submission_id = generate_job_id()
+
+        # Wait for `_recover_running_jobs` to run before accepting submissions to
+        # avoid duplicate monitoring of the same job.
+        await self._recover_running_jobs_event.wait()
+
+        logger.info(f"Starting job with submission_id: {submission_id}")
+        if entrypoint_label_selector:
+            error_message = validate_label_selector(entrypoint_label_selector)
+            if error_message:
+                raise ValueError(error_message)
+        job_info = JobInfo(
+            entrypoint=entrypoint,
+            status=JobStatus.PENDING,
+            start_time=int(time.time() * 1000),
+            metadata=metadata,
+            runtime_env=runtime_env,
+            entrypoint_num_cpus=entrypoint_num_cpus,
+            entrypoint_num_gpus=entrypoint_num_gpus,
+            entrypoint_memory=entrypoint_memory,
+            entrypoint_resources=entrypoint_resources,
+        )
+        new_key_added = await self._job_info_client.put_info(
+            submission_id, job_info, overwrite=False
+        )
+        if not new_key_added:
+            raise ValueError(
+                f"Job with submission_id {submission_id} already exists. "
+                "Please use a different submission_id."
+            )
+
+        driver_logger = self._get_job_driver_logger(submission_id)
+        # Wait for the actor to start up asynchronously so this call always
+        # returns immediately and we can catch errors with the actor starting
+        # up.
+        try:
+            resources_specified = any(
+                [
+                    entrypoint_num_cpus is not None and entrypoint_num_cpus > 0,
+                    entrypoint_num_gpus is not None and entrypoint_num_gpus > 0,
+                    entrypoint_memory is not None and entrypoint_memory > 0,
+                    entrypoint_resources not in [None, {}],
+                    entrypoint_label_selector not in [None, {}],
+                ]
+            )
+            label_selector = await self._get_label_selector(resources_specified)
+            if entrypoint_label_selector:
+                label_selector = {**label_selector, **entrypoint_label_selector}
+
+            if self.event_logger:
+                self.event_logger.info(
+                    f"Started a ray job {submission_id}.", submission_id=submission_id
+                )
+
+            driver_logger.info("Runtime env is setting up.")
+            supervisor_options = dict(
+                lifetime="detached",
+                name=JOB_ACTOR_NAME_TEMPLATE.format(job_id=submission_id),
+                num_cpus=entrypoint_num_cpus,
+                num_gpus=entrypoint_num_gpus,
+                memory=entrypoint_memory,
+                resources=entrypoint_resources,
+                label_selector=label_selector,
+                runtime_env=self._get_supervisor_runtime_env(
+                    runtime_env, submission_id, resources_specified
+                ),
+                namespace=SUPERVISOR_ACTOR_RAY_NAMESPACE,
+                # Don't pollute task events with system actor tasks that users don't
+                # know about.
+                enable_task_events=False,
+            )
+            supervisor = self._supervisor_actor_cls.options(
+                **supervisor_options
+            ).remote(
+                submission_id,
+                entrypoint,
+                metadata or {},
+                self._gcs_address,
+                self._cluster_id_hex,
+                self._logs_dir,
+            )
+            supervisor.run.remote(
+                _start_signal_actor=_start_signal_actor,
+                resources_specified=resources_specified,
+            )
+
+            # Monitor the job in the background so we can detect errors without
+            # requiring a client to poll.
+            run_background_task(
+                self._monitor_job(submission_id, job_supervisor=supervisor)
+            )
+        except Exception as e:
+            tb_str = traceback.format_exc()
+            driver_logger.warning(
+                f"Failed to start supervisor actor for job {submission_id}: '{e}'"
+                f". Full traceback:\n{tb_str}"
+            )
+            await self._job_info_client.put_status(
+                submission_id,
+                JobStatus.FAILED,
+                message=(
+                    f"Failed to start supervisor actor {submission_id}: '{e}'"
+                    f". Full traceback:\n{tb_str}"
+                ),
+                error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_FAILURE,
+            )
+        finally:
+            close_logger_file_descriptor(driver_logger)
+
+        return submission_id
+
+    def stop_job(self, job_id) -> bool:
+        """Request a job to exit, fire and forget.
+
+        Returns whether or not the job was running.
+        """
+        job_supervisor_actor = self._get_actor_for_job(job_id)
+        if job_supervisor_actor is not None:
+            # Actor is still alive, signal it to stop the driver, fire and
+            # forget
+            job_supervisor_actor.stop.remote()
+            return True
+        else:
+            return False
+
+    async def delete_job(self, job_id):
+        """Delete a job's info and metadata from the cluster."""
+        job_status = await self._job_info_client.get_status(job_id)
+
+        if job_status is None or not job_status.is_terminal():
+            raise RuntimeError(
+                f"Attempted to delete job '{job_id}', "
+                f"but it is in a non-terminal state {job_status}."
+            )
+
+        await self._job_info_client.delete_info(job_id)
+        return True
+
+    def job_info_client(self) -> JobInfoStorageClient:
+        return self._job_info_client
+
+    async def get_job_status(self, job_id: str) -> Optional[JobStatus]:
+        """Get latest status of a job."""
+        return await self._job_info_client.get_status(job_id)
+
+    async def get_job_info(self, job_id: str) -> Optional[JobInfo]:
+        """Get latest info of a job."""
+        return await self._job_info_client.get_info(job_id)
+
+    async def list_jobs(self) -> Dict[str, JobInfo]:
+        """Get info for all jobs."""
+        return await self._job_info_client.get_all_jobs()
+
+    def get_job_logs(self, job_id: str) -> str:
+        """Get all logs produced by a job."""
+        return self._log_client.get_logs(job_id)
+
+    async def tail_job_logs(self, job_id: str) -> AsyncIterator[str]:
+        """Return an iterator following the logs of a job."""
+        if await self.get_job_status(job_id) is None:
+            raise RuntimeError(f"Job '{job_id}' does not exist.")
+
+        job_finished = False
+        async for lines in self._log_client.tail_logs(job_id):
+            if lines is None:
+                if job_finished:
+                    # Job has already finished and we have read EOF afterwards,
+                    # it's guaranteed that we won't get any more logs.
+                    return
+                else:
+                    status = await self.get_job_status(job_id)
+                    if status.is_terminal():
+                        job_finished = True
+                        # Continue tailing logs generated between the
+                        # last EOF read and the finish of the job.
+
+                await asyncio.sleep(self.LOG_TAIL_SLEEP_S)
+            else:
+                yield "".join(lines)

python/ray/dashboard/modules/job/job_supervisor.py

@@ -0,0 +1,484 @@
+import asyncio
+import json
+import logging
+import os
+import signal
+import subprocess
+import sys
+import traceback
+from asyncio.tasks import FIRST_COMPLETED
+from typing import Any, Dict, List, Optional
+
+import ray
+import ray._private.ray_constants as ray_constants
+from ray._common.filters import CoreContextFilter
+from ray._common.formatters import JSONFormatter, TextFormatter
+from ray._common.network_utils import build_address
+from ray._private.accelerators.npu import NOSET_ASCEND_RT_VISIBLE_DEVICES_ENV_VAR
+from ray._private.accelerators.nvidia_gpu import NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR
+from ray._private.runtime_env.constants import RAY_JOB_CONFIG_JSON_ENV_VAR
+from ray._private.utils import remove_ray_internal_flags_from_env
+from ray._raylet import GcsClient
+from ray.actor import ActorHandle
+from ray.dashboard.modules.job.common import (
+    JOB_ID_METADATA_KEY,
+    JOB_NAME_METADATA_KEY,
+    JobInfoStorageClient,
+)
+from ray.dashboard.modules.job.job_log_storage_client import JobLogStorageClient
+from ray.job_submission import JobErrorType, JobStatus
+
+import psutil
+
+# asyncio python version compatibility
+try:
+    create_task = asyncio.create_task
+except AttributeError:
+    create_task = asyncio.ensure_future
+
+# Windows requires additional packages for proper process control.
+if sys.platform == "win32":
+    try:
+        import win32api
+        import win32con
+        import win32job
+    except (ModuleNotFoundError, ImportError) as e:
+        win32api = None
+        win32con = None
+        win32job = None
+
+        logger = logging.getLogger(__name__)
+        logger.warning(
+            "Failed to Import win32api. For best usage experience run "
+            f"'conda install pywin32'. Import error: {e}"
+        )
+
+
+class JobSupervisor:
+    """
+    Ray actor created by JobManager for each submitted job, responsible to
+    setup runtime_env, execute given shell command in subprocess, update job
+    status, persist job logs and manage subprocess group cleaning.
+
+    One job supervisor actor maps to one subprocess, for one job_id.
+    Job supervisor actor should fate share with subprocess it created.
+    """
+
+    DEFAULT_RAY_JOB_STOP_WAIT_TIME_S = 3
+    SUBPROCESS_POLL_PERIOD_S = 0.1
+    VALID_STOP_SIGNALS = ["SIGINT", "SIGTERM"]
+
+    def __init__(
+        self,
+        job_id: str,
+        entrypoint: str,
+        user_metadata: Dict[str, str],
+        gcs_address: str,
+        cluster_id_hex: str,
+        logs_dir: Optional[str] = None,
+    ):
+        self._job_id = job_id
+        gcs_client = GcsClient(address=gcs_address, cluster_id=cluster_id_hex)
+        self._job_info_client = JobInfoStorageClient(gcs_client, logs_dir)
+        self._log_client = JobLogStorageClient()
+        self._entrypoint = entrypoint
+
+        # Default metadata if not passed by the user.
+        self._metadata = {JOB_ID_METADATA_KEY: job_id, JOB_NAME_METADATA_KEY: job_id}
+        self._metadata.update(user_metadata)
+
+        # Event used to signal that a job should be stopped.
+        # Set in the `stop_job` method.
+        self._stop_event = asyncio.Event()
+
+        # Windows Job Object used to handle stopping the child processes.
+        self._win32_job_object = None
+
+        # Logger object to persist JobSupervisor logs in separate file.
+        self._logger = logging.getLogger(f"{__name__}.supervisor-{job_id}")
+        self._configure_logger()
+
+    def _configure_logger(self) -> None:
+        """
+        Configure self._logger object to write logs to file based on job
+        submission ID and to console.
+        """
+        supervisor_log_file_name = os.path.join(
+            ray._private.worker._global_node.get_logs_dir_path(),
+            f"jobs/supervisor-{self._job_id}.log",
+        )
+        os.makedirs(os.path.dirname(supervisor_log_file_name), exist_ok=True)
+        self._logger.addFilter(CoreContextFilter())
+        stream_handler = logging.StreamHandler()
+        file_handler = logging.FileHandler(supervisor_log_file_name)
+        formatter = TextFormatter()
+        if ray_constants.env_bool(ray_constants.RAY_BACKEND_LOG_JSON_ENV_VAR, False):
+            formatter = JSONFormatter()
+        stream_handler.setFormatter(formatter)
+        file_handler.setFormatter(formatter)
+        self._logger.addHandler(stream_handler)
+        self._logger.addHandler(file_handler)
+        self._logger.propagate = False
+
+    def _get_driver_runtime_env(
+        self, resources_specified: bool = False
+    ) -> Dict[str, Any]:
+        """Get the runtime env that should be set in the job driver.
+
+        Args:
+            resources_specified: Whether the user specified resources (CPUs, GPUs,
+                custom resources) in the submit_job request. If so, we will skip
+                the workaround for GPU detection introduced in #24546, so that the
+                behavior matches that of the user specifying resources for any
+                other actor.
+
+        Returns:
+            The runtime env that should be set in the job driver.
+        """
+        # Get the runtime_env set for the supervisor actor.
+        curr_runtime_env = dict(ray.get_runtime_context().runtime_env)
+        if resources_specified:
+            return curr_runtime_env
+        # Allow CUDA_VISIBLE_DEVICES to be set normally for the driver's tasks
+        # & actors.
+        env_vars = curr_runtime_env.get("env_vars", {})
+        env_vars.pop(NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR)
+        env_vars.pop(NOSET_ASCEND_RT_VISIBLE_DEVICES_ENV_VAR)
+        env_vars.pop(ray_constants.RAY_WORKER_NICENESS)
+        curr_runtime_env["env_vars"] = env_vars
+        return curr_runtime_env
+
+    def ping(self):
+        """Used to check the health of the actor."""
+        pass
+
+    def _exec_entrypoint(self, env: dict, logs_path: str) -> subprocess.Popen:
+        """
+        Runs the entrypoint command as a child process, streaming stderr &
+        stdout to given log files.
+
+        Unix systems:
+        Meanwhile we start a demon process and group driver
+        subprocess in same pgid, such that if job actor dies, entire process
+        group also fate share with it.
+
+        Windows systems:
+        A jobObject is created to enable fate sharing for the entire process group.
+
+        Args:
+            env: Environment variables passed through to the driver subprocess.
+            logs_path: File path on head node's local disk to store driver
+                command's stdout & stderr.
+        Returns:
+            child_process: Child process that runs the driver command. Can be
+                terminated or killed upon user calling stop().
+        """
+        # Open in append mode to avoid overwriting runtime_env setup logs for the
+        # supervisor actor, which are also written to the same file.
+        with open(logs_path, "a") as logs_file:
+            logs_file.write(
+                f"Running entrypoint for job {self._job_id}: {self._entrypoint}\n"
+            )
+            child_process = subprocess.Popen(
+                self._entrypoint,
+                shell=True,
+                start_new_session=True,
+                stdout=logs_file,
+                stderr=subprocess.STDOUT,
+                env=env,
+                # Ray intentionally blocks SIGINT in all processes, so if the user wants
+                # to stop job through SIGINT, we need to unblock it in the child process
+                preexec_fn=(
+                    (
+                        lambda: signal.pthread_sigmask(
+                            signal.SIG_UNBLOCK, {signal.SIGINT}
+                        )
+                    )
+                    if sys.platform != "win32"
+                    and os.environ.get("RAY_JOB_STOP_SIGNAL") == "SIGINT"
+                    else None
+                ),
+            )
+            parent_pid = os.getpid()
+            child_pid = child_process.pid
+            # Create new pgid with new subprocess to execute driver command
+
+            if sys.platform != "win32":
+                try:
+                    child_pgid = os.getpgid(child_pid)
+                except ProcessLookupError:
+                    # Process died before we could get its pgid.
+                    return child_process
+
+                # Open a new subprocess to kill the child process when the parent
+                # process dies kill -s 0 parent_pid will succeed if the parent is
+                # alive. If it fails, SIGKILL the child process group and exit
+                subprocess.Popen(
+                    f"while kill -s 0 {parent_pid}; do sleep 1; done; kill -9 -{child_pgid}",  # noqa: E501
+                    shell=True,
+                    # Suppress output
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                )
+
+            elif sys.platform == "win32" and win32api:
+                # Create a JobObject to which the child process (and its children)
+                # will be connected. This job object can be used to kill the child
+                # processes explicitly or when the jobObject gets deleted during
+                # garbage collection.
+                self._win32_job_object = win32job.CreateJobObject(None, "")
+                win32_job_info = win32job.QueryInformationJobObject(
+                    self._win32_job_object, win32job.JobObjectExtendedLimitInformation
+                )
+                win32_job_info["BasicLimitInformation"][
+                    "LimitFlags"
+                ] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+                win32job.SetInformationJobObject(
+                    self._win32_job_object,
+                    win32job.JobObjectExtendedLimitInformation,
+                    win32_job_info,
+                )
+                child_handle = win32api.OpenProcess(
+                    win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA,
+                    False,
+                    child_pid,
+                )
+                win32job.AssignProcessToJobObject(self._win32_job_object, child_handle)
+
+            return child_process
+
+    def _get_driver_env_vars(self, resources_specified: bool) -> Dict[str, str]:
+        """Returns environment variables that should be set in the driver."""
+        # RAY_ADDRESS may be the dashboard URL but not the gcs address,
+        # so when the environment variable is not empty, we force set RAY_ADDRESS
+        # to "auto" to avoid function `canonicalize_bootstrap_address_or_die` returning
+        # the wrong GCS address.
+        # TODO(Jialing He, Archit Kulkarni): Definition of Specification RAY_ADDRESS
+        if ray_constants.RAY_ADDRESS_ENVIRONMENT_VARIABLE in os.environ:
+            os.environ[ray_constants.RAY_ADDRESS_ENVIRONMENT_VARIABLE] = "auto"
+        ray_addr = ray._private.services.canonicalize_bootstrap_address_or_die(
+            "auto", ray._private.worker._global_node._ray_params.temp_dir
+        )
+        assert ray_addr is not None
+        return {
+            # Set JobConfig for the child process (runtime_env, metadata).
+            RAY_JOB_CONFIG_JSON_ENV_VAR: json.dumps(
+                {
+                    "runtime_env": self._get_driver_runtime_env(resources_specified),
+                    "metadata": self._metadata,
+                }
+            ),
+            # Always set RAY_ADDRESS as find_bootstrap_address address for
+            # job submission. In case of local development, prevent user from
+            # re-using http://{address}:{dashboard_port} to interact with
+            # jobs SDK.
+            # TODO:(mwtian) Check why "auto" does not work in entrypoint script
+            ray_constants.RAY_ADDRESS_ENVIRONMENT_VARIABLE: ray_addr,
+            # Set PYTHONUNBUFFERED=1 to stream logs during the job instead of
+            # only streaming them upon completion of the job.
+            "PYTHONUNBUFFERED": "1",
+        }
+
+    async def _polling(self, child_process: subprocess.Popen) -> int:
+        while child_process is not None:
+            return_code = child_process.poll()
+            if return_code is not None:
+                # subprocess finished with return code
+                return return_code
+            else:
+                # still running, yield control, 0.1s by default
+                await asyncio.sleep(self.SUBPROCESS_POLL_PERIOD_S)
+
+    async def _poll_all(self, processes: List[psutil.Process]):
+        """Poll processes until all are completed."""
+        while True:
+            (_, alive) = psutil.wait_procs(processes, timeout=0)
+            if len(alive) == 0:
+                return
+            else:
+                await asyncio.sleep(self.SUBPROCESS_POLL_PERIOD_S)
+
+    def _kill_processes(self, processes: List[psutil.Process], sig: signal.Signals):
+        """Ensure each process is already finished or send a kill signal."""
+        for proc in processes:
+            try:
+                os.kill(proc.pid, sig)
+            except ProcessLookupError:
+                # Process is already dead
+                pass
+
+    async def run(
+        self,
+        # Signal actor used in testing to capture PENDING -> RUNNING cases
+        _start_signal_actor: Optional[ActorHandle] = None,
+        resources_specified: bool = False,
+    ):
+        """
+        Stop and start both happen asynchronously, coordinated by asyncio event
+        and coroutine, respectively.
+
+        1) Sets job status as running
+        2) Pass runtime env and metadata to subprocess as serialized env
+            variables.
+        3) Handle concurrent events of driver execution and
+        """
+        curr_info = await self._job_info_client.get_info(self._job_id)
+        if curr_info is None:
+            raise RuntimeError(f"Status could not be retrieved for job {self._job_id}.")
+        curr_status = curr_info.status
+        curr_message = curr_info.message
+        if curr_status == JobStatus.RUNNING:
+            raise RuntimeError(
+                f"Job {self._job_id} is already in RUNNING state. "
+                f"JobSupervisor.run() should only be called once. "
+            )
+        if curr_status != JobStatus.PENDING:
+            raise RuntimeError(
+                f"Job {self._job_id} is not in PENDING state. "
+                f"Current status is {curr_status} with message {curr_message}."
+            )
+
+        if _start_signal_actor:
+            # Block in PENDING state until start signal received.
+            await _start_signal_actor.wait.remote()
+
+        node = ray._private.worker.global_worker.node
+        driver_agent_http_address = f"http://{build_address(node.node_ip_address, node.dashboard_agent_listen_port)}"
+        driver_node_id = ray.get_runtime_context().get_node_id()
+
+        await self._job_info_client.put_status(
+            self._job_id,
+            JobStatus.RUNNING,
+            jobinfo_replace_kwargs={
+                "driver_agent_http_address": driver_agent_http_address,
+                "driver_node_id": driver_node_id,
+            },
+        )
+
+        try:
+            # Configure environment variables for the child process.
+            env = os.environ.copy()
+            # Remove internal Ray flags. They present because JobSuperVisor itself is
+            # a Ray worker process but we don't want to pass them to the driver.
+            remove_ray_internal_flags_from_env(env)
+            # These will *not* be set in the runtime_env, so they apply to the driver
+            # only, not its tasks & actors.
+            env.update(self._get_driver_env_vars(resources_specified))
+
+            self._logger.info(
+                "Submitting job with RAY_ADDRESS = "
+                f"{env[ray_constants.RAY_ADDRESS_ENVIRONMENT_VARIABLE]}"
+            )
+            log_path = self._log_client.get_log_file_path(self._job_id)
+            child_process = self._exec_entrypoint(env, log_path)
+            child_pid = child_process.pid
+
+            polling_task = create_task(self._polling(child_process))
+            finished, _ = await asyncio.wait(
+                [polling_task, create_task(self._stop_event.wait())],
+                return_when=FIRST_COMPLETED,
+            )
+
+            if self._stop_event.is_set():
+                polling_task.cancel()
+                if sys.platform == "win32" and self._win32_job_object:
+                    win32job.TerminateJobObject(self._win32_job_object, -1)
+                elif sys.platform != "win32":
+                    stop_signal = os.environ.get("RAY_JOB_STOP_SIGNAL", "SIGTERM")
+                    if stop_signal not in self.VALID_STOP_SIGNALS:
+                        self._logger.warning(
+                            f"{stop_signal} not a valid stop signal. Terminating "
+                            "job with SIGTERM."
+                        )
+                        stop_signal = "SIGTERM"
+
+                    job_process = psutil.Process(child_pid)
+                    proc_to_kill = [job_process] + job_process.children(recursive=True)
+
+                    # Send stop signal and wait for job to terminate gracefully,
+                    # otherwise SIGKILL job forcefully after timeout.
+                    self._kill_processes(proc_to_kill, getattr(signal, stop_signal))
+                    try:
+                        stop_job_wait_time = int(
+                            os.environ.get(
+                                "RAY_JOB_STOP_WAIT_TIME_S",
+                                self.DEFAULT_RAY_JOB_STOP_WAIT_TIME_S,
+                            )
+                        )
+                        poll_job_stop_task = create_task(self._poll_all(proc_to_kill))
+                        await asyncio.wait_for(poll_job_stop_task, stop_job_wait_time)
+                        self._logger.info(
+                            f"Job {self._job_id} has been terminated gracefully "
+                            f"with {stop_signal}."
+                        )
+                    except asyncio.TimeoutError:
+                        self._logger.warning(
+                            f"Attempt to gracefully terminate job {self._job_id} "
+                            f"through {stop_signal} has timed out after "
+                            f"{stop_job_wait_time} seconds. Job is now being "
+                            "force-killed with SIGKILL."
+                        )
+                        self._kill_processes(proc_to_kill, signal.SIGKILL)
+
+                await self._job_info_client.put_status(self._job_id, JobStatus.STOPPED)
+            else:
+                # Child process finished execution and no stop event is set
+                # at the same time
+                assert len(finished) == 1, "Should have only one coroutine done"
+                [child_process_task] = finished
+                return_code = child_process_task.result()
+                self._logger.info(
+                    f"Job {self._job_id} entrypoint command "
+                    f"exited with code {return_code}"
+                )
+                if return_code == 0:
+                    await self._job_info_client.put_status(
+                        self._job_id,
+                        JobStatus.SUCCEEDED,
+                        driver_exit_code=return_code,
+                    )
+                else:
+                    log_tail = await self._log_client.get_last_n_log_lines(self._job_id)
+                    if log_tail is not None and log_tail != "":
+                        message = (
+                            "Job entrypoint command "
+                            f"failed with exit code {return_code}, "
+                            "last available logs (truncated to 20,000 chars):\n"
+                            + log_tail
+                        )
+                    else:
+                        message = (
+                            "Job entrypoint command "
+                            f"failed with exit code {return_code}. No logs available."
+                        )
+                    await self._job_info_client.put_status(
+                        self._job_id,
+                        JobStatus.FAILED,
+                        message=message,
+                        driver_exit_code=return_code,
+                        error_type=JobErrorType.JOB_ENTRYPOINT_COMMAND_ERROR,
+                    )
+        except Exception:
+            self._logger.error(
+                "Got unexpected exception while trying to execute driver "
+                f"command. {traceback.format_exc()}"
+            )
+            try:
+                await self._job_info_client.put_status(
+                    self._job_id,
+                    JobStatus.FAILED,
+                    message=traceback.format_exc(),
+                    error_type=JobErrorType.JOB_ENTRYPOINT_COMMAND_START_ERROR,
+                )
+            except Exception:
+                self._logger.error(
+                    "Failed to update job status to FAILED. "
+                    f"Exception: {traceback.format_exc()}"
+                )
+        finally:
+            # clean up actor after tasks are finished
+            ray.actor.exit_actor()
+
+    def stop(self):
+        """Set step_event and let run() handle the rest in its asyncio.wait()."""
+        self._stop_event.set()

python/ray/dashboard/modules/job/pydantic_models.py

@@ -0,0 +1,110 @@
+from enum import Enum
+from typing import Any, Dict, Optional
+
+from ray._common.pydantic_compat import PYDANTIC_INSTALLED, BaseModel, Field
+from ray.dashboard.modules.job.common import JobStatus
+from ray.util.annotations import PublicAPI
+
+# Pydantic is not part of the minimal Ray installation.
+if PYDANTIC_INSTALLED:
+
+    @PublicAPI(stability="beta")
+    class DriverInfo(BaseModel):
+        """A class for recording information about the driver related to the job."""
+
+        id: str = Field(..., description="The id of the driver")
+        node_ip_address: str = Field(
+            ..., description="The IP address of the node the driver is running on."
+        )
+        pid: str = Field(
+            ..., description="The PID of the worker process the driver is using."
+        )
+        # TODO(aguo): Add node_id as a field.
+
+    @PublicAPI(stability="beta")
+    class JobType(str, Enum):
+        """An enumeration for describing the different job types.
+
+        NOTE:
+            This field is still experimental and may change in the future.
+        """
+
+        #: A job that was initiated by the Ray Jobs API.
+        SUBMISSION = "SUBMISSION"
+        #: A job that was initiated by a driver script.
+        DRIVER = "DRIVER"
+
+    @PublicAPI(stability="beta")
+    class JobDetails(BaseModel):
+        """
+        Job data with extra details about its driver and its submission.
+        """
+
+        type: JobType = Field(..., description="The type of job.")
+        job_id: Optional[str] = Field(
+            None,
+            description="The job ID. An ID that is created for every job that is "
+            "launched in Ray. This can be used to fetch data about jobs using Ray "
+            "Core APIs.",
+        )
+        submission_id: Optional[str] = Field(
+            None,
+            description="A submission ID is an ID created for every job submitted via"
+            "the Ray Jobs API. It can "
+            "be used to fetch data about jobs using the Ray Jobs API.",
+        )
+        driver_info: Optional[DriverInfo] = Field(
+            None,
+            description="The driver related to this job. For jobs submitted via "
+            "the Ray Jobs API, "
+            "it is the last driver launched by that job submission, "
+            "or None if there is no driver.",
+        )
+
+        # The following fields are copied from JobInfo.
+        # TODO(aguo): Inherit from JobInfo once it's migrated to pydantic.
+        status: JobStatus = Field(..., description="The status of the job.")
+        entrypoint: str = Field(..., description="The entrypoint command for this job.")
+        message: Optional[str] = Field(
+            None, description="A message describing the status in more detail."
+        )
+        error_type: Optional[str] = Field(
+            None, description="Internal error or user script error."
+        )
+        start_time: Optional[int] = Field(
+            None,
+            description="The time when the job was started. A Unix timestamp in ms.",
+        )
+        end_time: Optional[int] = Field(
+            None,
+            description="The time when the job moved into a terminal state. "
+            "A Unix timestamp in ms.",
+        )
+        metadata: Optional[Dict[str, str]] = Field(
+            None, description="Arbitrary user-provided metadata for the job."
+        )
+        runtime_env: Optional[Dict[str, Any]] = Field(
+            None, description="The runtime environment for the job."
+        )
+        # the node info where the driver running on.
+        #     - driver_agent_http_address: this node's agent http address
+        #     - driver_node_id: this node's id.
+        driver_agent_http_address: Optional[str] = Field(
+            None,
+            description="The HTTP address of the JobAgent on the node the job "
+            "entrypoint command is running on.",
+        )
+        driver_node_id: Optional[str] = Field(
+            None,
+            description="The ID of the node the job entrypoint command is running on.",
+        )
+        driver_exit_code: Optional[int] = Field(
+            None,
+            description="The driver process exit code after the driver executed. "
+            "Return None if driver doesn't finish executing.",
+        )
+
+else:
+    DriverInfo = None
+    JobType = None
+    JobDetails = None

python/ray/dashboard/modules/job/sdk.py

@@ -0,0 +1,542 @@
+import copy
+import dataclasses
+import logging
+from typing import Any, AsyncIterator, Dict, List, Optional, Union
+
+import packaging.version
+
+import ray
+from ray.dashboard.modules.dashboard_sdk import SubmissionClient
+from ray.dashboard.modules.job.common import (
+    JobDeleteResponse,
+    JobLogsResponse,
+    JobStatus,
+    JobStopResponse,
+    JobSubmitRequest,
+    JobSubmitResponse,
+)
+from ray.dashboard.modules.job.pydantic_models import JobDetails
+from ray.dashboard.modules.job.utils import strip_keys_with_value_none
+from ray.dashboard.utils import get_address_for_submission_client
+from ray.runtime_env import RuntimeEnv
+from ray.runtime_env.runtime_env import _validate_no_local_paths
+from ray.util.annotations import PublicAPI
+
+try:
+    import aiohttp
+    import requests
+except ImportError:
+    aiohttp = None
+    requests = None
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+class JobSubmissionClient(SubmissionClient):
+    """A local client for submitting and interacting with jobs on a remote cluster.
+
+    Submits requests over HTTP to the job server on the cluster using the REST API.
+
+
+    Args:
+        address: Either (1) the address of the Ray cluster, or (2) the HTTP address
+            of the dashboard server on the head node, e.g. "http://<head-node-ip>:8265".
+            In case (1) it must be specified as an address that can be passed to
+            ray.init(), e.g. a Ray Client address (ray://<head_node_host>:10001),
+            or "auto", or "localhost:<port>". If unspecified, will try to connect to
+            a running local Ray cluster. This argument is always overridden by the
+            RAY_API_SERVER_ADDRESS or RAY_ADDRESS environment variable.
+        create_cluster_if_needed: Indicates whether the cluster at the specified
+            address needs to already be running. Ray doesn't start a cluster
+            before interacting with jobs, but third-party job managers may do so.
+        cookies: Cookies to use when sending requests to the HTTP job server.
+        metadata: Arbitrary metadata to store along with all jobs.  New metadata
+            specified per job will be merged with the global metadata provided here
+            via a simple dict update.
+        headers: Headers to use when sending requests to the HTTP job server, used
+            for cases like authentication to a remote cluster.
+        verify: Boolean indication to verify the server's TLS certificate or a path to
+            a file or directory of trusted certificates. Default: True.
+        **kwargs: Additional keyword arguments forwarded to the cluster info
+            resolution function. For external module addresses (e.g.,
+            ``anyscale://``), these are passed through to the module's
+            ``get_job_submission_client_cluster_info()`` implementation.
+    """
+
+    def __init__(
+        self,
+        address: Optional[str] = None,
+        create_cluster_if_needed: bool = False,
+        cookies: Optional[Dict[str, Any]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, Any]] = None,
+        verify: Optional[Union[str, bool]] = True,
+        **kwargs,
+    ):
+        self._client_ray_version = ray.__version__
+        """Initialize a JobSubmissionClient and check the connection to the cluster."""
+        if requests is None:
+            raise RuntimeError(
+                "The Ray jobs CLI & SDK require the ray[default] "
+                "installation: `pip install 'ray[default]'`"
+            )
+        # Check types of arguments
+        if address is not None and not isinstance(address, str):
+            raise TypeError(f"address must be a string, got {type(address)}")
+        if not isinstance(create_cluster_if_needed, bool):
+            raise TypeError(
+                f"create_cluster_if_needed must be a bool, got"
+                f" {type(create_cluster_if_needed)}"
+            )
+        if cookies is not None and not isinstance(cookies, dict):
+            raise TypeError(f"cookies must be a dict, got {type(cookies)}")
+        if metadata is not None and not isinstance(metadata, dict):
+            raise TypeError(f"metadata must be a dict, got {type(metadata)}")
+        if headers is not None and not isinstance(headers, dict):
+            raise TypeError(f"headers must be a dict, got {type(headers)}")
+        if not (isinstance(verify, str) or isinstance(verify, bool)):
+            raise TypeError(f"verify must be a str or bool, got {type(verify)}")
+
+        api_server_url = get_address_for_submission_client(address)
+
+        super().__init__(
+            address=api_server_url,
+            create_cluster_if_needed=create_cluster_if_needed,
+            cookies=cookies,
+            metadata=metadata,
+            headers=headers,
+            verify=verify,
+            **kwargs,
+        )
+        self._check_connection_and_version(
+            min_version="1.9",
+            version_error_message="Jobs API is not supported on the Ray "
+            "cluster. Please ensure the cluster is "
+            "running Ray 1.9 or higher.",
+        )
+
+        # In ray>=2.0, the client sends the new kwarg `submission_id` to the server
+        # upon every job submission, which causes servers with ray<2.0 to error.
+        if packaging.version.parse(self._client_ray_version) > packaging.version.parse(
+            "2.0"
+        ):
+            self._check_connection_and_version(
+                min_version="2.0",
+                version_error_message=f"Client Ray version {self._client_ray_version} "
+                "is not compatible with the Ray cluster. Please ensure the cluster is "
+                "running Ray 2.0 or higher or downgrade the client Ray version.",
+            )
+
+    @PublicAPI(stability="stable")
+    def submit_job(
+        self,
+        *,
+        entrypoint: str,
+        job_id: Optional[str] = None,
+        runtime_env: Optional[Dict[str, Any]] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        submission_id: Optional[str] = None,
+        entrypoint_num_cpus: Optional[Union[int, float]] = None,
+        entrypoint_num_gpus: Optional[Union[int, float]] = None,
+        entrypoint_memory: Optional[int] = None,
+        entrypoint_resources: Optional[Dict[str, float]] = None,
+        entrypoint_label_selector: Optional[Dict[str, str]] = None,
+    ) -> str:
+        """Submit and execute a job asynchronously.
+
+        When a job is submitted, it runs once to completion or failure. Retries or
+        different runs with different parameters should be handled by the
+        submitter. Jobs are bound to the lifetime of a Ray cluster, so if the
+        cluster goes down, all running jobs on that cluster will be terminated.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP
+            >>> client.submit_job( # doctest: +SKIP
+            ...     entrypoint="python script.py",
+            ...     runtime_env={
+            ...         "working_dir": "./",
+            ...         "pip": ["requests==2.26.0"]
+            ...     }
+            ... )  # doctest: +SKIP
+            'raysubmit_4LamXRuQpYdSMg7J'
+
+        Args:
+            entrypoint: The shell command to run for this job.
+            job_id: DEPRECATED. This has been renamed to submission_id.
+            runtime_env: The runtime environment to install and run this job in.
+            metadata: Arbitrary data to store along with this job.
+            submission_id: A unique ID for this job.
+            entrypoint_num_cpus: The quantity of CPU cores to reserve for the execution
+                of the entrypoint command, separately from any tasks or actors launched
+                by it. Defaults to 0.
+            entrypoint_num_gpus: The quantity of GPUs to reserve for the execution
+                of the entrypoint command, separately from any tasks or actors launched
+                by it. Defaults to 0.
+            entrypoint_memory: The quantity of memory to reserve for the
+                execution of the entrypoint command, separately from any tasks or
+                actors launched by it. Defaults to 0.
+            entrypoint_resources: The quantity of custom resources to reserve for the
+                execution of the entrypoint command, separately from any tasks or
+                actors launched by it.
+            entrypoint_label_selector: Label selector for the entrypoint command.
+
+        Returns:
+            The submission ID of the submitted job.  If not specified,
+            this is a randomly generated unique ID.
+
+        Raises:
+            RuntimeError: If the request to the job server fails, or if the specified
+                submission_id has already been used by a job on this cluster.
+        """
+        if job_id:
+            logger.warning(
+                "job_id kwarg is deprecated. Please use submission_id instead."
+            )
+
+        if (
+            entrypoint_num_cpus
+            or entrypoint_num_gpus
+            or entrypoint_resources
+            or entrypoint_label_selector
+        ):
+            self._check_connection_and_version(
+                min_version="2.2",
+                version_error_message="`entrypoint_num_cpus`, `entrypoint_num_gpus`, "
+                "`entrypoint_resources`, and `entrypoint_label_selector` kwargs "
+                "are not supported on the Ray cluster. Please ensure the cluster is "
+                "running Ray 2.2 or higher.",
+            )
+
+        if entrypoint_memory:
+            self._check_connection_and_version(
+                min_version="2.8",
+                version_error_message="`entrypoint_memory` kwarg "
+                "is not supported on the Ray cluster. Please ensure the cluster is "
+                "running Ray 2.8 or higher.",
+            )
+
+        runtime_env = copy.deepcopy(runtime_env or {})
+        metadata = metadata or {}
+        metadata.update(self._default_metadata)
+
+        self._upload_working_dir_if_needed(runtime_env)
+        self._upload_py_modules_if_needed(runtime_env)
+
+        # Verify worker_process_setup_hook type.
+        setup_hook = runtime_env.get("worker_process_setup_hook")
+        if setup_hook and not isinstance(setup_hook, str):
+            raise ValueError(
+                f"Invalid type {type(setup_hook)} for `worker_process_setup_hook`. "
+                "When a job submission API is used, `worker_process_setup_hook` "
+                "only allows a string type (module name). "
+                "Specify `worker_process_setup_hook` via "
+                "ray.init within a driver to use a `Callable` type. "
+            )
+
+        # Run the RuntimeEnv constructor to parse local pip/conda requirements files.
+        runtime_env = RuntimeEnv(**runtime_env)
+        _validate_no_local_paths(runtime_env)
+        runtime_env = runtime_env.to_dict()
+
+        submission_id = submission_id or job_id
+        req = JobSubmitRequest(
+            entrypoint=entrypoint,
+            submission_id=submission_id,
+            runtime_env=runtime_env,
+            metadata=metadata,
+            entrypoint_num_cpus=entrypoint_num_cpus,
+            entrypoint_num_gpus=entrypoint_num_gpus,
+            entrypoint_memory=entrypoint_memory,
+            entrypoint_resources=entrypoint_resources,
+            entrypoint_label_selector=entrypoint_label_selector,
+        )
+
+        # Remove keys with value None so that new clients with new optional fields
+        # are still compatible with older servers.  This is also done on the server,
+        # but we do it here as well to be extra defensive.
+        json_data = strip_keys_with_value_none(dataclasses.asdict(req))
+
+        logger.debug(f"Submitting job with submission_id={submission_id}.")
+        r = self._do_request("POST", "/api/jobs/", json_data=json_data)
+
+        if r.status_code == 200:
+            return JobSubmitResponse(**r.json()).submission_id
+        else:
+            self._raise_error(r)
+
+    @PublicAPI(stability="stable")
+    def stop_job(
+        self,
+        job_id: str,
+    ) -> bool:
+        """Request a job to exit asynchronously.
+
+        Attempts to terminate process first, then kills process after timeout.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP
+            >>> sub_id = client.submit_job(entrypoint="sleep 10") # doctest: +SKIP
+            >>> client.stop_job(sub_id) # doctest: +SKIP
+            True
+
+        Args:
+            job_id: The job ID or submission ID for the job to be stopped.
+
+        Returns:
+            True if the job was running, otherwise False.
+
+        Raises:
+            RuntimeError: If the job does not exist or if the request to the
+                job server fails.
+        """
+        logger.debug(f"Stopping job with job_id={job_id}.")
+        r = self._do_request("POST", f"/api/jobs/{job_id}/stop")
+
+        if r.status_code == 200:
+            return JobStopResponse(**r.json()).stopped
+        else:
+            self._raise_error(r)
+
+    @PublicAPI(stability="stable")
+    def delete_job(
+        self,
+        job_id: str,
+    ) -> bool:
+        """Delete a job in a terminal state and all of its associated data.
+
+        If the job is not already in a terminal state, raises an error.
+        This does not delete the job logs from disk.
+        Submitting a job with the same submission ID as a previously
+        deleted job is not supported and may lead to unexpected behavior.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient() # doctest: +SKIP
+            >>> job_id = client.submit_job(entrypoint="echo hello") # doctest: +SKIP
+            >>> client.delete_job(job_id) # doctest: +SKIP
+            True
+
+        Args:
+            job_id: submission ID for the job to be deleted.
+
+        Returns:
+            True if the job was deleted, otherwise False.
+
+        Raises:
+            RuntimeError: If the job does not exist, if the request to the
+                job server fails, or if the job is not in a terminal state.
+        """
+        logger.debug(f"Deleting job with job_id={job_id}.")
+        r = self._do_request("DELETE", f"/api/jobs/{job_id}")
+
+        if r.status_code == 200:
+            return JobDeleteResponse(**r.json()).deleted
+        else:
+            self._raise_error(r)
+
+    @PublicAPI(stability="stable")
+    def get_job_info(
+        self,
+        job_id: str,
+    ) -> JobDetails:
+        """Get the latest status and other information associated with a job.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP
+            >>> submission_id = client.submit_job(entrypoint="sleep 1") # doctest: +SKIP
+            >>> client.get_job_info(submission_id) # doctest: +SKIP
+            JobDetails(status='SUCCEEDED',
+            job_id='03000000', type='submission',
+            submission_id='raysubmit_4LamXRuQpYdSMg7J',
+            message='Job finished successfully.', error_type=None,
+            start_time=1647388711, end_time=1647388712, metadata={}, runtime_env={})
+
+        Args:
+            job_id: The job ID or submission ID of the job whose information
+                is being requested.
+
+        Returns:
+            The JobDetails for the job.
+
+        Raises:
+            RuntimeError: If the job does not exist or if the request to the
+                job server fails.
+        """
+        r = self._do_request("GET", f"/api/jobs/{job_id}")
+
+        if r.status_code == 200:
+            if JobDetails is None:
+                raise RuntimeError(
+                    "The Ray jobs CLI & SDK require the ray[default] "
+                    "installation: `pip install 'ray[default]'`"
+                )
+            else:
+                return JobDetails(**r.json())
+        else:
+            self._raise_error(r)
+
+    @PublicAPI(stability="stable")
+    def list_jobs(self) -> List[JobDetails]:
+        """List all jobs along with their status and other information.
+
+        Lists all jobs that have ever run on the cluster, including jobs that are
+        currently running and jobs that are no longer running.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP
+            >>> client.submit_job(entrypoint="echo hello") # doctest: +SKIP
+            >>> client.submit_job(entrypoint="sleep 2") # doctest: +SKIP
+            >>> client.list_jobs() # doctest: +SKIP
+            [JobDetails(status='SUCCEEDED',
+            job_id='03000000', type='submission',
+            submission_id='raysubmit_4LamXRuQpYdSMg7J',
+            message='Job finished successfully.', error_type=None,
+            start_time=1647388711, end_time=1647388712, metadata={}, runtime_env={}),
+            JobDetails(status='RUNNING',
+            job_id='04000000', type='submission',
+            submission_id='raysubmit_1dxCeNvG1fCMVNHG',
+            message='Job is currently running.', error_type=None,
+            start_time=1647454832, end_time=None, metadata={}, runtime_env={})]
+
+        Returns:
+            A list of JobDetails containing the job status and other information.
+
+        Raises:
+            RuntimeError: If the request to the job server fails.
+        """
+        r = self._do_request("GET", "/api/jobs/")
+
+        if r.status_code == 200:
+            jobs_info_json = r.json()
+            jobs_info = [
+                JobDetails(**job_info_json) for job_info_json in jobs_info_json
+            ]
+            return jobs_info
+        else:
+            self._raise_error(r)
+
+    @PublicAPI(stability="stable")
+    def get_job_status(self, job_id: str) -> JobStatus:
+        """Get the most recent status of a job.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP
+            >>> client.submit_job(entrypoint="echo hello") # doctest: +SKIP
+            >>> client.get_job_status("raysubmit_4LamXRuQpYdSMg7J") # doctest: +SKIP
+            'SUCCEEDED'
+
+        Args:
+            job_id: The job ID or submission ID of the job whose status is being
+                requested.
+
+        Returns:
+            The JobStatus of the job.
+
+        Raises:
+            RuntimeError: If the job does not exist or if the request to the
+                job server fails.
+        """
+        return self.get_job_info(job_id).status
+
+    @PublicAPI(stability="stable")
+    def get_job_logs(self, job_id: str) -> str:
+        """Get all logs produced by a job.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP
+            >>> sub_id = client.submit_job(entrypoint="echo hello") # doctest: +SKIP
+            >>> client.get_job_logs(sub_id) # doctest: +SKIP
+            'hello\\n'
+
+        Args:
+            job_id: The job ID or submission ID of the job whose logs are being
+                requested.
+
+        Returns:
+            A string containing the full logs of the job.
+
+        Raises:
+            RuntimeError: If the job does not exist or if the request to the
+                job server fails.
+        """
+        r = self._do_request("GET", f"/api/jobs/{job_id}/logs")
+
+        if r.status_code == 200:
+            return JobLogsResponse(**r.json()).logs
+        else:
+            self._raise_error(r)
+
+    @PublicAPI(stability="stable")
+    async def tail_job_logs(self, job_id: str) -> AsyncIterator[str]:
+        """Get an iterator that follows the logs of a job.
+
+        Example:
+            >>> from ray.job_submission import JobSubmissionClient
+            >>> client = JobSubmissionClient("http://127.0.0.1:8265") # doctest: +SKIP
+            >>> submission_id = client.submit_job( # doctest: +SKIP
+            ...     entrypoint="echo hi && sleep 5 && echo hi2")
+            >>> async for lines in client.tail_job_logs( # doctest: +SKIP
+            ...           'raysubmit_Xe7cvjyGJCyuCvm2'):
+            ...     print(lines, end="") # doctest: +SKIP
+            hi
+            hi2
+
+        Args:
+            job_id: The job ID or submission ID of the job whose logs are being
+                requested.
+
+        Yields:
+            str: Successive chunks of the job's stdout/stderr as the driver
+            process produces them.
+
+        Raises:
+            RuntimeError: If the job does not exist, if the request to the
+                job server fails, or if the connection closes unexpectedly
+                before the job reaches a terminal state.
+        """
+        async with aiohttp.ClientSession(
+            cookies=self._cookies, headers=self._headers
+        ) as session:
+            ws = await session.ws_connect(
+                f"{self._address}/api/jobs/{job_id}/logs/tail",
+                headers=self._headers,
+                ssl=self._ssl_context,
+            )
+
+            while True:
+                msg = await ws.receive()
+
+                if msg.type == aiohttp.WSMsgType.TEXT:
+                    yield msg.data
+                elif msg.type == aiohttp.WSMsgType.CLOSED:
+                    logger.info(
+                        f"WebSocket closed for job {job_id} with close code "
+                        f"{ws.close_code}"
+                    )
+                    if ws.close_code == aiohttp.WSCloseCode.ABNORMAL_CLOSURE:
+                        raise RuntimeError(
+                            f"WebSocket connection closed unexpectedly with close code {ws.close_code}"
+                        )
+                    break
+                elif msg.type == aiohttp.WSMsgType.ERROR:
+                    # Old Ray versions (<=2.0.1) may send ERROR on connection close
+                    if self._server_ray_version is not None and packaging.version.parse(
+                        self._server_ray_version
+                    ) > packaging.version.parse("2.0.1"):
+                        raise RuntimeError(
+                            f"WebSocket error for job {job_id}: {ws.exception()}"
+                        )
+                    else:
+                        logger.warning(
+                            f"WebSocket error for job {job_id}, treating as "
+                            f"normal close. Err: {ws.exception()!r}"
+                        )
+                        break

python/ray/dashboard/modules/job/tests/backwards_compatibility_scripts/script.py

@@ -0,0 +1,26 @@
+import requests
+
+import ray
+
+ray.init()
+
+
+@ray.remote
+class Counter:
+    def __init__(self):
+        self.counter = 0
+
+    def inc(self):
+        self.counter += 1
+
+    def get_counter(self):
+        return self.counter
+
+
+counter = Counter.remote()
+
+for _ in range(5):
+    ray.get(counter.inc.remote())
+    print(ray.get(counter.get_counter.remote()))
+
+print(requests.__version__)

python/ray/dashboard/modules/job/tests/backwards_compatibility_scripts/test_backwards_compatibility.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -ex
+
+unset RAY_ADDRESS
+
+if ! [ -x "$(command -v conda)" ]; then
+    echo "conda doesn't exist. Please download conda for this machine"
+    exit 1
+else
+    echo "conda exists"
+fi
+
+# This is required to use conda activate
+source "$(conda info --base)/etc/profile.d/conda.sh"
+
+PYTHON_VERSION=$(python -c"from platform import python_version; print(python_version())")
+
+RAY_VERSIONS=("2.0.1")
+
+for RAY_VERSION in "${RAY_VERSIONS[@]}"
+do
+    env_name=${JOB_COMPATIBILITY_TEST_TEMP_ENV}
+
+    # Check if the conda env exists
+    if conda env list | grep -q "${env_name}"; then
+        # Clean up if env name is already taken from previous leaking runs
+        conda env remove --name="${env_name}"
+    fi
+
+    printf "\n\n\n"
+    echo "========================================================================================="
+    printf "Creating new conda environment with python %s for ray %s \n" "${PYTHON_VERSION}" "${RAY_VERSION}"
+    echo "========================================================================================="
+    printf "\n\n\n"
+
+    # Include `pip` explicitly: conda-forge's `python` package stopped
+    # bundling pip as a dep, and without it `conda activate` puts us in
+    # an env with python but no pip, so subsequent `pip install` falls
+    # back to the base miniforge env's pip. That clobbers the editable
+    # ray 3.0.0.dev0 in base with ray 2.0.1, and every subsequent
+    # dashboard test that imports `ray._common` fails because 2.0.1
+    # predates that module.
+    conda create -y -n "${env_name}" python="${PYTHON_VERSION}" pip=25.2
+    conda activate "${env_name}"
+    python -m pip install --upgrade pip
+
+    # Pin pydantic version due to: https://github.com/ray-project/ray/issues/36990.
+    # ray<2.9 is only compatible with pydantic<2 and setuptools < 70.
+    python -m pip install -U "pydantic<2" ray=="${RAY_VERSION}" ray[default]=="${RAY_VERSION}" setuptools==69.5.1
+
+    printf "\n\n\n"
+    echo "========================================================="
+    printf "Installed ray job server version: "
+    SERVER_RAY_VERSION=$(python -c "import ray; print(ray.__version__)")
+    printf "%s \n" "${SERVER_RAY_VERSION}"
+    echo "========================================================="
+    printf "\n\n\n"
+    ray stop --force
+    ray start --head
+
+    conda deactivate
+
+    CLIENT_RAY_VERSION=$(python -c "import ray; print(ray.__version__)")
+    CLIENT_RAY_COMMIT=$(python -c "import ray; print(ray.__commit__)")
+    printf "\n\n\n"
+    echo "========================================================================================="
+    printf "Using Ray %s on %s as job client \n" "${CLIENT_RAY_VERSION}" "${CLIENT_RAY_COMMIT}"
+    echo "========================================================================================="
+    printf "\n\n\n"
+
+    export RAY_ADDRESS="http://127.0.0.1:8265"
+
+    cleanup () {
+        unset RAY_ADDRESS
+        ray stop --force
+        conda remove -y --name "${env_name}" --all
+    }
+
+    JOB_ID=$(python -c "import uuid; print(uuid.uuid4().hex)")
+
+    # Get directory of current file. https://stackoverflow.com/questions/59895/
+    DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+    if ! ray job submit --job-id="${JOB_ID}" --working-dir="${DIR}" --runtime-env-json='{"pip": ["requests==2.26.0", "setuptools==69.5.1"]}' -- python script.py; then
+        cleanup
+        exit 1
+    fi
+
+    if ! ray job status "${JOB_ID}"; then
+        cleanup
+        exit 1
+    fi
+
+    if ! ray job logs "${JOB_ID}"; then
+        cleanup
+        exit 1
+    fi
+
+    if ! pytest -vs "${DIR}"/../test_backwards_compatibility.py::test_error_message; then
+        cleanup
+        exit 1
+    fi
+
+    cleanup
+done

python/ray/dashboard/modules/job/tests/conftest.py

@@ -0,0 +1,30 @@
+import os
+
+import ray
+from ray._raylet import GcsClient
+from ray.dashboard.modules.job.job_manager import JobManager
+
+TEST_NAMESPACE = "jobs_test_namespace"
+
+
+def create_ray_cluster(_tracing_startup_hook=None):
+    return ray.init(
+        num_cpus=16,
+        num_gpus=1,
+        resources={"Custom": 1},
+        namespace=TEST_NAMESPACE,
+        log_to_driver=True,
+        _tracing_startup_hook=_tracing_startup_hook,
+    )
+
+
+def create_job_manager(ray_cluster, tmp_path):
+    address_info = ray_cluster
+    gcs_client = GcsClient(address=address_info["gcs_address"])
+    return JobManager(gcs_client, tmp_path)
+
+
+def _driver_script_path(file_name: str) -> str:
+    return os.path.join(
+        os.path.dirname(__file__), "subprocess_driver_scripts", file_name
+    )

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_code_search_path_is_propagated.py

@@ -0,0 +1,31 @@
+"""
+A dummy ray driver script that executes in subprocess.
+Prints global worker's `load_code_from_local` property that ought to be set
+whenever `JobConfig.code_search_path` is specified
+"""
+
+
+def run():
+    import ray
+    from ray.job_config import JobConfig
+
+    ray.init(job_config=JobConfig(code_search_path=["/home/code/"]))
+
+    @ray.remote
+    def foo() -> bool:
+        return ray._private.worker.global_worker.load_code_from_local
+
+    load_code_from_local = ray.get(foo.remote())
+
+    statement = "propagated" if load_code_from_local else "NOT propagated"
+
+    # Step 1: Print the statement indicating that the code_search_path have been
+    #         properly respected
+    print(f"Code search path is {statement}")
+    # Step 2: Print the whole runtime_env to validate that it's been passed
+    #         appropriately from submit_job API
+    print(ray.get_runtime_context().runtime_env)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_cuda_devices.py

@@ -0,0 +1,22 @@
+import os
+
+import ray
+
+cuda_env = ray._private.accelerators.nvidia_gpu.NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR
+if os.environ.get("RAY_TEST_RESOURCES_SPECIFIED") == "1":
+    assert cuda_env not in os.environ
+    if os.environ.get("RAY_TEST_GPUS_SPECIFIED") == "1":
+        assert "CUDA_VISIBLE_DEVICES" in os.environ
+    else:
+        assert "CUDA_VISIBLE_DEVICES" not in os.environ
+else:
+    assert os.environ[cuda_env] == "1"
+
+
+@ray.remote
+def f():
+    assert cuda_env not in os.environ
+
+
+# Will raise if task fails.
+ray.get(f.remote())

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_niceness.py

@@ -0,0 +1,23 @@
+"""
+A dummy ray driver script that executes in subprocess.
+Checks that job manager's environment variable is different.
+"""
+
+import os
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        print("worker", os.nice(0))
+
+    ray.get(foo.remote())
+
+
+if __name__ == "__main__":
+    print("driver", os.nice(0))
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/consume_one_cpu.py

@@ -0,0 +1,13 @@
+import ray
+
+ray.init()
+
+
+@ray.remote(num_cpus=1)
+def f():
+    pass
+
+
+print("Hanging...")
+ray.get(f.remote())
+print("Success!")

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/driver_runtime_env_inheritance.py

@@ -0,0 +1,58 @@
+import argparse
+import sys
+import time
+
+import ray
+
+# This prefix is used to identify the output log line that contains the runtime env.
+RUNTIME_ENV_LOG_LINE_PREFIX = "ray_job_test_runtime_env_output:"
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Dashboard agent.")
+    parser.add_argument(
+        "--conflict",
+        type=str,
+    )
+    parser.add_argument(
+        "--worker-process-setup-hook",
+        type=str,
+    )
+
+    args = parser.parse_args()
+
+    if args.worker_process_setup_hook:
+        ray.init(
+            runtime_env={
+                "worker_process_setup_hook": lambda: print(
+                    args.worker_process_setup_hook
+                )
+            }
+        )
+
+        @ray.remote
+        def f():
+            pass
+
+        ray.get(f.remote())
+        time.sleep(5)
+        sys.exit(0)
+
+    if args.conflict == "pip":
+        ray.init(runtime_env={"pip": ["numpy"]})
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )
+    elif args.conflict == "env_vars":
+        ray.init(runtime_env={"env_vars": {"A": "1"}})
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )
+    else:
+        ray.init(
+            runtime_env={
+                "env_vars": {"C": "1"},
+            }
+        )
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/override_env_var.py

@@ -0,0 +1,27 @@
+"""
+Test script that attempts to set its own runtime_env, but we should ensure
+we ended up using job submission API call's runtime_env instead of scripts
+"""
+
+
+def run():
+    import os
+
+    import ray
+
+    ray.init(
+        runtime_env={
+            "env_vars": {"TEST_SUBPROCESS_JOB_CONFIG_ENV_VAR": "SHOULD_BE_OVERRIDEN"}
+        },
+    )
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+    print(os.environ.get("TEST_SUBPROCESS_JOB_CONFIG_ENV_VAR", None))
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/per_task_runtime_env.py

@@ -0,0 +1,26 @@
+import os
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote(runtime_env={"env_vars": {"FOO": "bar"}})
+    def get_task_working_dir():
+        # Check behavior of working_dir: The cwd should contain the
+        # current file, which is being used as a job entrypoint script.
+        assert os.path.exists("per_task_runtime_env.py")
+
+        return ray.get_runtime_context().runtime_env.working_dir()
+
+    driver_working_dir = ray.get_runtime_context().runtime_env.working_dir()
+    task_working_dir = ray.get(get_task_working_dir.remote())
+    assert driver_working_dir == task_working_dir, (
+        driver_working_dir,
+        task_working_dir,
+    )
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/print_namespace.py

@@ -0,0 +1,22 @@
+"""
+A dummy ray driver script that executes in subprocess. Prints namespace
+from ray's runtime context for job submission API testing.
+"""
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+
+    print(ray.get_runtime_context().namespace)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/print_runtime_env.py

@@ -0,0 +1,22 @@
+"""
+A dummy ray driver script that executes in subprocess. Prints runtime_env
+from ray's runtime context for job submission API testing.
+"""
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+
+    print(ray.get_runtime_context().runtime_env)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/ray_tune_basic.py

@@ -0,0 +1,15 @@
+"""Tests that Ray Tune works with the working_dir set in Jobs.
+
+Ray Tune internally sets environment variables using runtime_env.
+If the inherited internal runtime environment overwrites the working_dir
+from jobs with an empty working_dir, this test will fail.  See #25484"""
+from ray_tune_dependency import foo
+
+from ray import tune
+
+
+def objective(*args):
+    foo()
+
+
+tune.run(objective)

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/ray_tune_dependency.py

@@ -0,0 +1,5 @@
+"""A file dependency for testing working_dir behavior with Ray Tune."""
+
+
+def foo():
+    pass

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/script_with_exception.py

@@ -0,0 +1,6 @@
+def run():
+    raise Exception("Script failed with exception !")
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/test_backwards_compatibility.py

@@ -0,0 +1,124 @@
+import logging
+import os
+import subprocess
+import sys
+import uuid
+from contextlib import contextmanager
+
+import pytest
+
+from ray.job_submission import JobStatus, JobSubmissionClient
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def conda_env(env_name):
+    # Set env name for shell script
+    os.environ["JOB_COMPATIBILITY_TEST_TEMP_ENV"] = env_name
+    # Delete conda env if it already exists
+    try:
+        yield
+    finally:
+        # Clean up created conda env upon test exit to prevent leaking
+        del os.environ["JOB_COMPATIBILITY_TEST_TEMP_ENV"]
+        subprocess.run(
+            f"conda env remove -y --name {env_name}", shell=True, stdout=subprocess.PIPE
+        )
+
+
+def _compatibility_script_path(file_name: str) -> str:
+    return os.path.join(
+        os.path.dirname(__file__), "backwards_compatibility_scripts", file_name
+    )
+
+
+class TestBackwardsCompatibility:
+    @pytest.mark.skipif(
+        sys.platform == "darwin",
+        reason="ray 2.0.1 runs differently on apple silicon than today's.",
+    )
+    def test_cli(self):
+        """
+        Test that the current commit's CLI works with old server-side Ray versions.
+
+        1) Create a new conda environment with old ray version X installed;
+            inherits same env as current conda envionment except ray version
+        2) (Server) Start head node and dashboard with old ray version X
+        3) (Client) Use current commit's CLI code to do sample job submission flow
+        4) Deactivate the new conda environment and back to original place
+        """
+        # Shell script creates and cleans up tmp conda environment regardless
+        # of the outcome
+        env_name = f"jobs-backwards-compatibility-{uuid.uuid4().hex}"
+        with conda_env(env_name):
+            shell_cmd = f"{_compatibility_script_path('test_backwards_compatibility.sh')}"  # noqa: E501
+
+            try:
+                subprocess.check_output(shell_cmd, shell=True, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                logger.error(str(e))
+                logger.error(e.stdout.decode())
+                raise e
+
+
+@pytest.mark.skipif(
+    os.environ.get("JOB_COMPATIBILITY_TEST_TEMP_ENV") is None,
+    reason="This test is only meant to be run from the "
+    "test_backwards_compatibility.sh shell script.",
+)
+def test_error_message():
+    """
+    Check that we get a good error message when running against an old server version.
+    """
+    # Import lazily so the module still loads when the compatibility script
+    # installs an older Ray that does not expose `ray._common`.
+    from ray._common.test_utils import wait_for_condition
+
+    client = JobSubmissionClient("http://127.0.0.1:8265")
+
+    # Check that a basic job successfully runs.
+    job_id = client.submit_job(
+        entrypoint="echo 'hello world'",
+    )
+    wait_for_condition(lambda: client.get_job_status(job_id) == JobStatus.SUCCEEDED)
+
+    # `entrypoint_num_cpus`, `entrypoint_num_gpus`, `entrypoint_resources`, and
+    # `entrypoint_label_selector`
+    # are not supported in ray<2.2.0.
+    for unsupported_submit_kwargs in [
+        {"entrypoint_num_cpus": 1},
+        {"entrypoint_num_gpus": 1},
+        {"entrypoint_resources": {"custom": 1}},
+        {"entrypoint_label_selector": {"fragile_node": "!1"}},
+    ]:
+        with pytest.raises(
+            Exception,
+            match="Ray version 2.0.1 is running on the cluster. "
+            "`entrypoint_num_cpus`, `entrypoint_num_gpus`, "
+            "`entrypoint_resources`, and `entrypoint_label_selector` kwargs"
+            " are not supported on the Ray cluster. Please ensure the cluster is "
+            "running Ray 2.2 or higher.",
+        ):
+            client.submit_job(
+                entrypoint="echo hello",
+                **unsupported_submit_kwargs,
+            )
+
+    with pytest.raises(
+        Exception,
+        match="Ray version 2.0.1 is running on the cluster. "
+        "`entrypoint_memory` kwarg"
+        " is not supported on the Ray cluster. Please ensure the cluster is "
+        "running Ray 2.8 or higher.",
+    ):
+        client.submit_job(
+            entrypoint="echo hello",
+            entrypoint_memory=4,
+        )
+
+    assert True
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_cli.py

@@ -0,0 +1,695 @@
+import json
+import logging
+import os
+import shlex
+import sys
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from subprocess import list2cmdline
+from typing import Optional
+from unittest import mock
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from ray.dashboard.modules.job.cli import job_cli_group
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def mock_sdk_client():
+    class AsyncIterator:
+        def __init__(self, seq):
+            self._seq = seq
+            self.iter = iter(self._seq)
+
+        def __aiter__(self):
+            return self
+
+        async def __anext__(self):
+            try:
+                return next(self.iter)
+            except StopIteration:
+                self.iter = iter(self._seq)
+                raise StopAsyncIteration
+
+    if "RAY_ADDRESS" in os.environ:
+        del os.environ["RAY_ADDRESS"]
+    with mock.patch("ray.dashboard.modules.job.cli.JobSubmissionClient") as mock_client:
+        # In python 3.6 it will fail with error
+        # 'async for' requires an object with __aiter__ method, got MagicMock"
+        mock_client().tail_job_logs.return_value = AsyncIterator(range(10))
+
+        # We need to return a string for the address and not a MagicMock
+        mock_client().get_address.return_value = ""
+
+        yield mock_client
+
+
+@pytest.fixture
+def runtime_env_formats():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+
+        test_env = {
+            "working_dir": "s3://bogus.zip",
+            "conda": "conda_env",
+            "pip": ["pip-install-test"],
+            "env_vars": {"hi": "hi2"},
+        }
+
+        yaml_file = path / "env.yaml"
+        with yaml_file.open(mode="w") as f:
+            yaml.dump(test_env, f)
+
+        yield test_env, json.dumps(test_env), yaml_file
+
+
+@contextmanager
+def set_env_var(key: str, val: Optional[str] = None):
+    old_val = os.environ.get(key, None)
+    if val is not None:
+        os.environ[key] = val
+    elif key in os.environ:
+        del os.environ[key]
+
+    yield
+
+    if key in os.environ:
+        del os.environ[key]
+    if old_val is not None:
+        os.environ[key] = old_val
+
+
+def check_exit_code(result, exit_code):
+    assert result.exit_code == exit_code, result.output
+
+
+def _expected_entrypoint(*args):
+    """Return the expected entrypoint string for the current platform.
+
+    On Windows, the CLI uses subprocess.list2cmdline (double quotes).
+    On POSIX, it uses shlex.join (single quotes).
+    """
+    if sys.platform == "win32":
+        return list2cmdline(args)
+    return shlex.join(args)
+
+
+def _job_cli_group_test_address(mock_sdk_client, cmd, *args):
+    runner = CliRunner()
+
+    create_cluster_if_needed = True if cmd == "submit" else False
+    # Test passing address via command line.
+    result = runner.invoke(job_cli_group, [cmd, "--address=arg_addr", *args])
+    mock_sdk_client.assert_called_with(
+        "arg_addr", create_cluster_if_needed, headers=None, verify=True
+    )
+    with pytest.raises(AssertionError):
+        mock_sdk_client.assert_called_with(
+            "some_other_addr", True, headers=None, verify=True
+        )
+    check_exit_code(result, 0)
+    # Test passing address via env var.
+    with set_env_var("RAY_ADDRESS", "env_addr"):
+        result = runner.invoke(job_cli_group, [cmd, *args])
+        check_exit_code(result, 0)
+        # RAY_ADDRESS is read inside the SDK client.
+        mock_sdk_client.assert_called_with(
+            None, create_cluster_if_needed, headers=None, verify=True
+        )
+    # Test passing no address.
+    result = runner.invoke(job_cli_group, [cmd, *args])
+    check_exit_code(result, 0)
+    mock_sdk_client.assert_called_with(
+        None, create_cluster_if_needed, headers=None, verify=True
+    )
+
+
+class TestList:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "list")
+
+    def test_list(self, mock_sdk_client):
+        runner = CliRunner()
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["list"],
+            )
+            check_exit_code(result, 0)
+
+            result = runner.invoke(job_cli_group, ["submit", "--", "echo hello"])
+            check_exit_code(result, 0)
+
+            result = runner.invoke(
+                job_cli_group,
+                ["list"],
+            )
+            check_exit_code(result, 0)
+
+
+class TestSubmit:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "submit", "--", "echo", "hello")
+
+    def test_working_dir(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["submit", "--", "echo hello"])
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--working-dir", "blah", "--", "echo hello"],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={"working_dir": "blah"},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group, ["submit", "--working-dir='.'", "--", "echo hello"]
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={"working_dir": "'.'"},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_runtime_env(self, mock_sdk_client, runtime_env_formats):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+        env_dict, env_json, env_yaml = runtime_env_formats
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            # Test passing via file.
+            result = runner.invoke(
+                job_cli_group, ["submit", "--runtime-env", env_yaml, "--", "echo hello"]
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            # Test passing via json.
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--runtime-env-json", env_json, "--", "echo hello"],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            # Test passing both throws an error.
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--runtime-env",
+                    env_yaml,
+                    "--runtime-env-json",
+                    env_json,
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 1)
+            assert "Only one of" in str(result.exception)
+
+            # Test overriding working_dir.
+            env_dict.update(working_dir=".")
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--runtime-env",
+                    env_yaml,
+                    "--working-dir",
+                    ".",
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--runtime-env-json",
+                    env_json,
+                    "--working-dir",
+                    ".",
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_job_id(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["submit", "--", "echo hello"])
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--submission-id=my_job_id", "--", "echo hello"],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id="my_job_id",
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_entrypoint_num_cpus(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--entrypoint-num-cpus=2", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=2,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_entrypoint_num_gpus(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--entrypoint-num-gpus=2", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=2,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_entrypoint_memory(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--entrypoint-memory=4", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=4,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    @pytest.mark.parametrize(
+        "resources",
+        [
+            ("--entrypoint-num-cpus=2", {"entrypoint_num_cpus": 2}),
+            ("--entrypoint-num-gpus=2", {"entrypoint_num_gpus": 2}),
+            (
+                """--entrypoint-resources={"Custom":3}""",
+                {"entrypoint_resources": {"Custom": 3}},
+            ),
+        ],
+    )
+    def test_entrypoint_resources(self, mock_sdk_client, resources):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", resources[0], "--", "echo hello"],
+            )
+            print(result.output)
+            assert result.exit_code == 0
+            expected_kwargs = {
+                "entrypoint": _expected_entrypoint("echo hello"),
+                "submission_id": None,
+                "runtime_env": {},
+                "metadata": None,
+                "entrypoint_num_cpus": None,
+                "entrypoint_num_gpus": None,
+                "entrypoint_memory": None,
+                "entrypoint_resources": None,
+                "entrypoint_label_selector": None,
+            }
+            expected_kwargs.update(resources[1])
+            mock_client_instance.submit_job.assert_called_with(**expected_kwargs)
+
+    def test_entrypoint_resources_invalid_json(self, mock_sdk_client):
+        runner = CliRunner()
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    """--entrypoint-resources={"Custom":3""",
+                    "--",
+                    "echo hello world",
+                ],
+            )
+            print(result.output)
+            assert result.exit_code == 1
+            assert "not a valid JSON string" in result.output
+
+    def test_entrypoint_label_selector(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    """--entrypoint-label-selector={"fragile_node":"!1"}""",
+                    "--",
+                    "echo hello",
+                ],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector={"fragile_node": "!1"},
+            )
+
+    def test_metadata(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--metadata-json",
+                    '{"key": "value"}',
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+                metadata={"key": "value"},
+            )
+
+    def test_metadata_invalid_json(self, mock_sdk_client):
+        runner = CliRunner()
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--metadata-json",
+                    '{"key": "value"',
+                    "--",
+                    "echo hello",
+                ],
+            )
+            print(result.output)
+            check_exit_code(result, 1)
+            assert "not a valid JSON string" in result.output
+
+    @pytest.mark.parametrize(
+        "cli_val, verify_param",
+        [
+            ("True", True),
+            ("true", True),
+            ("1", True),
+            ("False", False),
+            ("false", False),
+            ("0", False),
+            ("a/rel/path", "a/rel/path"),
+            ("/an/abs/path", "/an/abs/path"),
+        ],
+    )
+    def test_entrypoint_verify(self, mock_sdk_client, cli_val, verify_param):
+        runner = CliRunner()
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", f"--verify={cli_val}", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_sdk_client.assert_called_with(
+                None, True, headers=None, verify=verify_param
+            )
+
+
+class TestDelete:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "delete", "fake_job_id")
+
+    def test_delete(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["delete", "job_id"])
+            check_exit_code(result, 0)
+            mock_client_instance.delete_job.assert_called_with("job_id")
+
+
+class TestStatus:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "status", "fake_job_id")
+
+    def test_status(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["status", "job_id"])
+            check_exit_code(result, 0)
+            mock_client_instance.get_job_info.assert_called_with("job_id")
+
+
+class TestEntrypointShellQuoting:
+    """Regression test for https://github.com/ray-project/ray/issues/56232.
+
+    `ray job submit` previously used `subprocess.list2cmdline` unconditionally
+    to join entrypoint arguments. That function wraps arguments in double
+    quotes, which causes POSIX shells on the server to expand $VAR references.
+
+    The fix uses `shlex.join` on POSIX platforms (which single-quotes
+    arguments to prevent expansion) and `list2cmdline` on Windows (which
+    double-quotes arguments as expected by cmd.exe).
+    """
+
+    def test_entrypoint_preserves_shell_variables(self, mock_sdk_client):
+        """Ensure $VAR in entrypoint is single-quoted on POSIX, not double-quoted."""
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            with mock.patch("ray.dashboard.modules.job.cli.sys") as mock_sys:
+                mock_sys.platform = "linux"
+                result = runner.invoke(
+                    job_cli_group,
+                    [
+                        "submit",
+                        "--",
+                        "python",
+                        "-m",
+                        "launcher",
+                        "--config",
+                        "$CONFIG_PATH",
+                    ],
+                )
+                check_exit_code(result, 0)
+                call_kwargs = mock_client_instance.submit_job.call_args
+                entrypoint = call_kwargs.kwargs["entrypoint"]
+
+                # shlex.join must single-quote the $VAR argument so that
+                # the server-side POSIX shell does NOT expand it.
+                assert "'$CONFIG_PATH'" in entrypoint, (
+                    f"Expected single-quoted $CONFIG_PATH in entrypoint, "
+                    f"got: {entrypoint!r}"
+                )
+                # Double quotes around $CONFIG_PATH would cause expansion.
+                assert '"$CONFIG_PATH"' not in entrypoint, (
+                    f"Double-quoted $CONFIG_PATH would be expanded by the "
+                    f"server shell, got: {entrypoint!r}"
+                )
+
+    def test_entrypoint_uses_list2cmdline_on_windows(self, mock_sdk_client):
+        """On Windows, entrypoint should use list2cmdline (double quotes)."""
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            with mock.patch("ray.dashboard.modules.job.cli.sys") as mock_sys:
+                mock_sys.platform = "win32"
+                result = runner.invoke(
+                    job_cli_group,
+                    [
+                        "submit",
+                        "--",
+                        "echo",
+                        "hello world",
+                    ],
+                )
+                check_exit_code(result, 0)
+                call_kwargs = mock_client_instance.submit_job.call_args
+                entrypoint = call_kwargs.kwargs["entrypoint"]
+
+                # list2cmdline wraps args with spaces in double quotes
+                assert (
+                    entrypoint == 'echo "hello world"'
+                ), f"Expected list2cmdline output on Windows, got: {entrypoint!r}"
+
+    def test_entrypoint_simple_args_not_over_quoted(self, mock_sdk_client):
+        """Simple arguments without special chars should not be quoted."""
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--", "echo", "hello"],
+            )
+            check_exit_code(result, 0)
+            call_kwargs = mock_client_instance.submit_job.call_args
+            entrypoint = call_kwargs.kwargs["entrypoint"]
+            assert entrypoint == "echo hello"
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_cli_integration.py

@@ -0,0 +1,356 @@
+import json
+import logging
+import os
+import subprocess
+import sys
+from contextlib import contextmanager
+from typing import Optional, Tuple
+
+import pytest
+
+import ray
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def shutdown_only():
+    yield None
+    # The code after the yield will run as teardown code.
+    ray.shutdown()
+    # Delete the cluster address just in case.
+    ray._common.utils.reset_ray_address()
+
+
+@contextmanager
+def set_env_var(key: str, val: Optional[str] = None):
+    old_val = os.environ.get(key, None)
+    if val is not None:
+        os.environ[key] = val
+    elif key in os.environ:
+        del os.environ[key]
+
+    try:
+        yield
+    finally:
+        if key in os.environ:
+            del os.environ[key]
+        if old_val is not None:
+            os.environ[key] = old_val
+
+
+@pytest.fixture
+def ray_start_stop():
+    subprocess.check_output(["ray", "start", "--head"])
+    try:
+        with set_env_var("RAY_ADDRESS", "http://127.0.0.1:8265"):
+            yield
+    finally:
+        subprocess.check_output(["ray", "stop", "--force"])
+
+
+@contextmanager
+def ray_cluster_manager():
+    """
+    Used not as fixture in case we want to set RAY_ADDRESS first.
+    """
+    subprocess.check_output(["ray", "start", "--head"])
+    try:
+        yield
+    finally:
+        subprocess.check_output(["ray", "stop", "--force"])
+
+
+def _run_cmd(cmd: str, should_fail=False) -> Tuple[str, str]:
+    """Convenience wrapper for subprocess.run.
+
+    We always run with shell=True to simulate the CLI.
+
+    Asserts that the process succeeds/fails depending on should_fail.
+
+    Returns (stdout, stderr).
+    """
+    print(f"Running command: '{cmd}'")
+    p: subprocess.CompletedProcess = subprocess.run(
+        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    if p.returncode == 0:
+        print("Command succeeded.")
+        if should_fail:
+            raise RuntimeError(
+                f"Expected command to fail, but got exit code: {p.returncode}."
+            )
+    else:
+        print(f"Command failed with exit code: {p.returncode}.")
+        if not should_fail:
+            raise RuntimeError(
+                f"Expected command to succeed, but got exit code: {p.returncode}."
+            )
+
+    return p.stdout.decode("utf-8"), p.stderr.decode("utf-8")
+
+
+class TestJobSubmitHook:
+    """Tests the RAY_JOB_SUBMIT_HOOK env var."""
+
+    def test_hook(self, ray_start_stop):
+        with set_env_var("RAY_JOB_SUBMIT_HOOK", "ray._private.test_utils.job_hook"):
+            stdout, _ = _run_cmd("ray job submit -- echo hello")
+            assert "hook intercepted: echo hello" in stdout
+
+
+class TestRayJobHeaders:
+    """
+    Integration version of job CLI test that ensures interaction with the
+    following components are working as expected:
+    1) Ray client: use of RAY_JOB_HEADERS and ray.init() in job_head.py
+    2) Ray dashboard: `ray start --head`
+    """
+
+    def test_empty_ray_job_headers(self, ray_start_stop):
+        with set_env_var("RAY_JOB_HEADERS", None):
+            stdout, _ = _run_cmd("ray job submit -- echo hello")
+            assert "hello" in stdout
+            assert "succeeded" in stdout
+
+    @pytest.mark.parametrize("ray_job_headers", ['{"key": "value"}'])
+    def test_ray_job_headers(self, ray_start_stop, ray_job_headers: str):
+        with set_env_var("RAY_JOB_HEADERS", ray_job_headers):
+            _run_cmd("ray job submit -- echo hello", should_fail=False)
+
+    @pytest.mark.parametrize("ray_job_headers", ["{key value}"])
+    def test_ray_incorrectly_formatted_job_headers(
+        self, ray_start_stop, ray_job_headers: str
+    ):
+        with set_env_var("RAY_JOB_HEADERS", ray_job_headers):
+            _run_cmd("ray job submit -- echo hello", should_fail=True)
+
+
+class TestRayAddress:
+    """
+    Integration version of job CLI test that ensures interaction with the
+    following components are working as expected:
+
+    1) Ray client: use of RAY_ADDRESS and ray.init() in job_head.py
+    2) Ray dashboard: `ray start --head`
+    """
+
+    def test_empty_ray_address(self, ray_start_stop):
+        with set_env_var("RAY_ADDRESS", None):
+            stdout, _ = _run_cmd("ray job submit -- echo hello")
+            assert "hello" in stdout
+            assert "succeeded" in stdout
+
+    @pytest.mark.parametrize(
+        "ray_api_server_address,should_fail",
+        [
+            ("http://127.0.0.1:8265", False),  # correct API server
+            ("127.0.0.1:8265", True),  # wrong format without http
+            ("http://127.0.0.1:9999", True),  # wrong port
+        ],
+    )
+    def test_ray_api_server_address(
+        self,
+        ray_start_stop,
+        ray_api_server_address: str,
+        should_fail: bool,
+    ):
+        # Set a `RAY_ADDRESS` that would not work with the `ray job submit` CLI because it uses the `ray://` prefix.
+        # This verifies that the `RAY_API_SERVER_ADDRESS` env var takes precedence.
+        with set_env_var("RAY_ADDRESS", "ray://127.0.0.1:8265"):
+            with set_env_var("RAY_API_SERVER_ADDRESS", ray_api_server_address):
+                _run_cmd("ray job submit -- echo hello", should_fail=should_fail)
+
+    @pytest.mark.parametrize(
+        "ray_client_address,should_fail",
+        [
+            ("127.0.0.1:8265", True),
+            ("ray://127.0.0.1:8265", True),
+            ("http://127.0.0.1:8265", False),
+        ],
+    )
+    def test_ray_client_address(
+        self, ray_start_stop, ray_client_address: str, should_fail: bool
+    ):
+        with set_env_var("RAY_ADDRESS", ray_client_address):
+            _run_cmd("ray job submit -- echo hello", should_fail=should_fail)
+
+    def test_valid_http_ray_address(self, ray_start_stop):
+        stdout, _ = _run_cmd("ray job submit -- echo hello")
+        assert "hello" in stdout
+        assert "succeeded" in stdout
+
+
+class TestJobSubmit:
+    def test_basic_submit(self, ray_start_stop):
+        """Should tail logs and wait for process to exit."""
+        cmd = "sleep 1 && echo hello && sleep 1 && echo hello"
+        stdout, _ = _run_cmd(f"ray job submit -- bash -c '{cmd}'")
+
+        # 'hello' should appear four times: twice when we print the entrypoint, then
+        # two more times in the logs from the `echo`.
+        assert stdout.count("hello") == 4
+        assert "succeeded" in stdout
+
+    def test_submit_no_wait(self, ray_start_stop):
+        """Should exit immediately w/o printing logs."""
+        cmd = "echo hello && sleep 1000"
+        stdout, _ = _run_cmd(f"ray job submit --no-wait -- bash -c '{cmd}'")
+        assert "hello" not in stdout
+        assert "Tailing logs until the job exits" not in stdout
+
+    def test_submit_with_logs_instant_job(self, ray_start_stop):
+        """Should exit immediately and print logs even if job returns instantly."""
+        cmd = "echo hello"
+        stdout, _ = _run_cmd(f"ray job submit -- bash -c '{cmd}'")
+
+        # 'hello' should appear twice: once when we print the entrypoint, then
+        # again from the `echo`.
+        assert stdout.count("hello") == 2
+
+    def test_multiple_ray_init(self, ray_start_stop):
+        cmd = (
+            "python -c 'import ray; ray.init(); ray.shutdown(); "
+            "ray.init(); ray.shutdown();'"
+        )
+        stdout, _ = _run_cmd(f"ray job submit -- {cmd}")
+        assert "succeeded" in stdout
+
+    def test_metadata(self, ray_start_stop):
+        cmd = "echo hello"
+        stdout, _ = _run_cmd(
+            f'ray job submit --metadata-json=\'{{"key": "value"}}\' -- {cmd}'
+        )
+        assert "hello" in stdout
+        assert "succeeded" in stdout
+
+    def test_job_failed(self, ray_start_stop):
+        cmd = "python -c 'import ray; ray.init(); assert 1 == 2;'"
+        _run_cmd(f"ray job submit -- {cmd}", should_fail=True)
+
+
+class TestRuntimeEnv:
+    def test_bad_runtime_env(self, ray_start_stop):
+        """Should fail with helpful error if runtime env setup fails."""
+        stdout, _ = _run_cmd(
+            'ray job submit --runtime-env-json=\'{"pip": '
+            '["does-not-exist"]}\' -- echo hi',
+            should_fail=True,
+        )
+        assert "Tailing logs until the job exits" in stdout
+        assert "runtime_env setup failed" in stdout
+        assert "No matching distribution found for does-not-exist" in stdout
+
+
+class TestJobStop:
+    def test_basic_stop(self, ray_start_stop):
+        """Should wait until the job is stopped."""
+        cmd = "sleep 1000"
+        job_id = "test_basic_stop"
+        _run_cmd(f"ray job submit --no-wait --job-id={job_id} -- {cmd}")
+
+        stdout, _ = _run_cmd(f"ray job stop {job_id}")
+        assert "Waiting for job" in stdout
+        assert f"Job '{job_id}' was stopped" in stdout
+
+    def test_stop_no_wait(self, ray_start_stop):
+        """Should not wait until the job is stopped."""
+        cmd = "echo hello && sleep 1000"
+        job_id = "test_stop_no_wait"
+        _run_cmd(f"ray job submit --no-wait --job-id={job_id} -- bash -c '{cmd}'")
+
+        stdout, _ = _run_cmd(f"ray job stop --no-wait {job_id}")
+        assert "Waiting for job" not in stdout
+        assert f"Job '{job_id}' was stopped" not in stdout
+
+
+class TestJobList:
+    def test_empty(self, ray_start_stop):
+        stdout, _ = _run_cmd("ray job list")
+        assert "[]" in stdout
+
+    def test_list(self, ray_start_stop):
+        _run_cmd("ray job submit --job-id='hello_id' -- echo hello")
+
+        runtime_env = {"env_vars": {"TEST": "123"}}
+        _run_cmd(
+            "ray job submit --job-id='hi_id' "
+            f"--runtime-env-json='{json.dumps(runtime_env)}' -- echo hi"
+        )
+        stdout, _ = _run_cmd("ray job list")
+        assert "123" in stdout
+        assert "hello_id" in stdout
+        assert "hi_id" in stdout
+
+
+class TestJobDelete:
+    def test_basic_delete(self, ray_start_stop):
+        cmd = "sleep 1000"
+        job_id = "test_basic_delete"
+        _run_cmd(f"ray job submit --no-wait --submission-id={job_id} -- {cmd}")
+
+        # Job shouldn't be able to be deleted because it is not in a terminal state.
+        stdout, stderr = _run_cmd(f"ray job delete {job_id}", should_fail=True)
+        assert "it is in a non-terminal state" in stderr
+
+        # Submit a job that finishes quickly.
+        cmd = "echo hello"
+        job_id = "test_basic_delete_quick"
+        _run_cmd(f"ray job submit --submission-id={job_id} -- bash -c '{cmd}'")
+
+        # Job should be able to be deleted because it is finished.
+        stdout, _ = _run_cmd(f"ray job delete {job_id}")
+        assert f"Job '{job_id}' deleted successfully" in stdout
+
+
+class TestJobStatus:
+    # `ray job status` should exit with 0 if the job exists and non-zero if it doesn't.
+    # This is the contract between Ray and KubRay v1.3.0.
+    def test_status_job_exists(self, ray_start_stop):
+        cmd = "echo hello"
+        job_id = "test_job_id"
+        _run_cmd(
+            f"ray job submit --submission-id={job_id} -- bash -c '{cmd}'",
+            should_fail=False,
+        )
+        _run_cmd(f"ray job status {job_id}", should_fail=False)
+
+    def test_status_job_does_not_exist(self, ray_start_stop):
+        job_id = "test_job_id"
+        _run_cmd(f"ray job status {job_id}", should_fail=True)
+
+
+def test_quote_escaping(ray_start_stop):
+    cmd = "echo \"hello 'world'\""
+    job_id = "test_quote_escaping"
+    stdout, _ = _run_cmd(
+        f"ray job submit --job-id={job_id} -- {cmd}",
+    )
+    assert "hello 'world'" in stdout
+
+
+def test_resources(shutdown_only):
+    ray.init(num_cpus=1, num_gpus=1, resources={"Custom": 1}, _memory=4)
+
+    # Check the case of too many resources.
+    for id, arg in [
+        ("entrypoint_num_cpus", "--entrypoint-num-cpus=2"),
+        ("entrypoint_num_gpus", "--entrypoint-num-gpus=2"),
+        ("entrypoint_memory", "--entrypoint-memory=5"),
+        ("entrypoint_resources", "--entrypoint-resources='{\"Custom\": 2}'"),
+    ]:
+        _run_cmd(f"ray job submit --submission-id={id} --no-wait {arg} -- echo hi")
+        stdout, _ = _run_cmd(f"ray job status {id}")
+        assert "waiting for resources" in stdout
+
+    # Check the case of sufficient resources.
+    stdout, _ = _run_cmd(
+        "ray job submit --entrypoint-num-cpus=1 "
+        "--entrypoint-num-gpus=1 --entrypoint-memory=4 --entrypoint-resources='{"
+        '"Custom": 1}\' -- echo hello',
+    )
+    assert "hello" in stdout
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_common.py

@@ -0,0 +1,302 @@
+import asyncio
+import json
+from dataclasses import asdict
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from google.protobuf.json_format import Parse
+
+from ray.core.generated.gcs_pb2 import JobsAPIInfo
+from ray.dashboard.modules.job.common import (
+    JobErrorType,
+    JobInfo,
+    JobInfoStorageClient,
+    JobStatus,
+    JobSubmitRequest,
+    http_uri_components_to_uri,
+    uri_to_http_components,
+    validate_request_type,
+)
+
+
+class TestJobSubmitRequestValidation:
+    def test_validate_entrypoint(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+
+        with pytest.raises(TypeError, match="required positional argument"):
+            validate_request_type({}, JobSubmitRequest)
+
+        with pytest.raises(TypeError, match="must be a string"):
+            validate_request_type({"entrypoint": 123}, JobSubmitRequest)
+
+    def test_validate_submission_id(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+        assert r.submission_id is None
+
+        r = validate_request_type(
+            {"entrypoint": "abc", "submission_id": "123"}, JobSubmitRequest
+        )
+        assert r.entrypoint == "abc"
+        assert r.submission_id == "123"
+
+        with pytest.raises(TypeError, match="must be a string"):
+            validate_request_type(
+                {"entrypoint": 123, "submission_id": 1}, JobSubmitRequest
+            )
+
+    def test_validate_runtime_env(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+        assert r.runtime_env is None
+
+        r = validate_request_type(
+            {"entrypoint": "abc", "runtime_env": {"hi": "hi2"}}, JobSubmitRequest
+        )
+        assert r.entrypoint == "abc"
+        assert r.runtime_env == {"hi": "hi2"}
+
+        with pytest.raises(TypeError, match="must be a dict"):
+            validate_request_type(
+                {"entrypoint": "abc", "runtime_env": 123}, JobSubmitRequest
+            )
+
+        with pytest.raises(TypeError, match="keys must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "runtime_env": {1: "hi"}}, JobSubmitRequest
+            )
+
+    def test_validate_metadata(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+        assert r.metadata is None
+
+        r = validate_request_type(
+            {"entrypoint": "abc", "metadata": {"hi": "hi2"}}, JobSubmitRequest
+        )
+        assert r.entrypoint == "abc"
+        assert r.metadata == {"hi": "hi2"}
+
+        with pytest.raises(TypeError, match="must be a dict"):
+            validate_request_type(
+                {"entrypoint": "abc", "metadata": 123}, JobSubmitRequest
+            )
+
+        with pytest.raises(TypeError, match="keys must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "metadata": {1: "hi"}}, JobSubmitRequest
+            )
+
+        with pytest.raises(TypeError, match="values must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "metadata": {"hi": 1}}, JobSubmitRequest
+            )
+
+    def test_validate_entrypoint_label_selector(self):
+        r = validate_request_type(
+            {
+                "entrypoint": "abc",
+                "entrypoint_label_selector": {"fragile_node": "!1"},
+            },
+            JobSubmitRequest,
+        )
+        assert r.entrypoint_label_selector == {"fragile_node": "!1"}
+
+        with pytest.raises(TypeError, match="must be a dict"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_label_selector": "bad"},
+                JobSubmitRequest,
+            )
+
+        with pytest.raises(TypeError, match="keys must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_label_selector": {1: "bad"}},
+                JobSubmitRequest,
+            )
+
+        with pytest.raises(TypeError, match="values must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_label_selector": {"k": 1}},
+                JobSubmitRequest,
+            )
+
+    def test_entrypoint_resources_disallow_strings(self):
+        with pytest.raises(TypeError, match="values must be numbers"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_resources": {"Custom": "1"}},
+                JobSubmitRequest,
+            )
+
+
+def test_uri_to_http_and_back():
+    assert uri_to_http_components("gcs://hello.zip") == ("gcs", "hello.zip")
+    assert uri_to_http_components("gcs://hello.whl") == ("gcs", "hello.whl")
+
+    with pytest.raises(ValueError, match="'blah' is not a valid Protocol"):
+        uri_to_http_components("blah://halb.zip")
+
+    with pytest.raises(ValueError, match="does not end in .zip or .whl"):
+        assert uri_to_http_components("gcs://hello.not_zip")
+
+    with pytest.raises(ValueError, match="does not end in .zip or .whl"):
+        assert uri_to_http_components("gcs://hello")
+
+    assert http_uri_components_to_uri("gcs", "hello.zip") == "gcs://hello.zip"
+    assert http_uri_components_to_uri("blah", "halb.zip") == "blah://halb.zip"
+    assert http_uri_components_to_uri("blah", "halb.whl") == "blah://halb.whl"
+
+    for original_uri in ["gcs://hello.zip", "gcs://fasdf.whl"]:
+        new_uri = http_uri_components_to_uri(*uri_to_http_components(original_uri))
+        assert new_uri == original_uri
+
+
+def test_dynamic_status_message():
+    info = JobInfo(
+        status=JobStatus.PENDING, entrypoint="echo hi", entrypoint_num_cpus=1
+    )
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(
+        status=JobStatus.PENDING, entrypoint="echo hi", entrypoint_num_gpus=1
+    )
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(status=JobStatus.PENDING, entrypoint="echo hi", entrypoint_memory=4)
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(
+        status=JobStatus.PENDING,
+        entrypoint="echo hi",
+        entrypoint_resources={"Custom": 1},
+    )
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(
+        status=JobStatus.PENDING, entrypoint="echo hi", runtime_env={"conda": "env"}
+    )
+    assert "may be waiting for the runtime environment" in info.message
+
+
+def test_job_info_to_json():
+    info = JobInfo(
+        status=JobStatus.PENDING,
+        entrypoint="echo hi",
+        entrypoint_num_cpus=1,
+        entrypoint_num_gpus=1,
+        entrypoint_memory=4,
+        entrypoint_resources={"Custom": 1},
+        runtime_env={"pip": ["pkg"]},
+    )
+    expected_items = {
+        "status": "PENDING",
+        "message": (
+            "Job has not started yet. It may be waiting for resources "
+            "(CPUs, GPUs, memory, custom resources) to become available. "
+            "It may be waiting for the runtime environment to be set up."
+        ),
+        "entrypoint": "echo hi",
+        "entrypoint_num_cpus": 1,
+        "entrypoint_num_gpus": 1,
+        "entrypoint_memory": 4,
+        "entrypoint_resources": {"Custom": 1},
+        "runtime_env_json": '{"pip": ["pkg"]}',
+    }
+
+    # Check that the expected items are in the JSON.
+    assert expected_items.items() <= info.to_json().items()
+
+    new_job_info = JobInfo.from_json(info.to_json())
+    assert new_job_info == info
+
+    # If `status` is just a string, then operations like status.is_terminal()
+    # would fail, so we should make sure that it's a JobStatus.
+    assert isinstance(new_job_info.status, JobStatus)
+
+
+def test_job_info_json_to_proto():
+    """Test that JobInfo JSON can be converted to JobsAPIInfo protobuf."""
+    info = JobInfo(
+        status=JobStatus.PENDING,
+        entrypoint="echo hi",
+        error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE,
+        start_time=123,
+        end_time=456,
+        metadata={"hi": "hi2"},
+        entrypoint_num_cpus=1,
+        entrypoint_num_gpus=1,
+        entrypoint_memory=4,
+        entrypoint_resources={"Custom": 1},
+        runtime_env={"pip": ["pkg"]},
+        driver_agent_http_address="http://localhost:1234",
+        driver_node_id="node_id",
+    )
+    info_json = json.dumps(info.to_json())
+    info_proto = Parse(info_json, JobsAPIInfo())
+    assert info_proto.status == "PENDING"
+    assert info_proto.entrypoint == "echo hi"
+    assert info_proto.start_time == 123
+    assert info_proto.end_time == 456
+    assert info_proto.metadata == {"hi": "hi2"}
+    assert info_proto.entrypoint_num_cpus == 1
+    assert info_proto.entrypoint_num_gpus == 1
+    assert info_proto.entrypoint_memory == 4
+    assert info_proto.entrypoint_resources == {"Custom": 1}
+    assert info_proto.runtime_env_json == '{"pip": ["pkg"]}'
+    assert info_proto.message == (
+        "Job has not started yet. It may be waiting for resources "
+        "(CPUs, GPUs, memory, custom resources) to become available. "
+        "It may be waiting for the runtime environment to be set up."
+    )
+    assert info_proto.error_type == "JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE"
+    assert info_proto.driver_agent_http_address == "http://localhost:1234"
+    assert info_proto.driver_node_id == "node_id"
+
+    minimal_info = JobInfo(status=JobStatus.PENDING, entrypoint="echo hi")
+    minimal_info_json = json.dumps(minimal_info.to_json())
+    minimal_info_proto = Parse(minimal_info_json, JobsAPIInfo())
+    assert minimal_info_proto.status == "PENDING"
+    assert minimal_info_proto.entrypoint == "echo hi"
+    for unset_optional_field in [
+        "entrypoint_num_cpus",
+        "entrypoint_num_gpus",
+        "entrypoint_memory",
+        "runtime_env_json",
+        "error_type",
+        "driver_agent_http_address",
+        "driver_node_id",
+    ]:
+        assert not minimal_info_proto.HasField(unset_optional_field)
+
+
+def test_get_all_jobs_filters_out_none_job_info():
+    prefix = JobInfoStorageClient.JOB_DATA_KEY_PREFIX
+    mock_gcs = MagicMock()
+    mock_gcs.async_internal_kv_keys = AsyncMock(
+        return_value=[
+            (prefix + "job1").encode(),
+            (prefix + "job2").encode(),
+        ]
+    )
+
+    storage = JobInfoStorageClient(mock_gcs)
+    job_info_1 = JobInfo(status=JobStatus.RUNNING, entrypoint="echo 1")
+
+    async def mock_get_info(job_id, timeout=30):
+        if job_id == "job1":
+            return job_info_1
+        return None
+
+    storage.get_info = mock_get_info
+
+    result = asyncio.run(storage.get_all_jobs())
+
+    assert result == {"job1": job_info_1}
+    for job_id, job_info in result.items():
+        asdict(job_info)  # This should not raise an exception
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_component_activities.py

@@ -0,0 +1,177 @@
+import json
+import os
+import pprint
+import sys
+
+import jsonschema
+import pytest
+import requests
+
+from ray._common.test_utils import (
+    run_string_as_driver,
+    wait_for_condition,
+)
+from ray._private.test_utils import (
+    format_web_url,
+    run_string_as_driver_nonblocking,
+)
+from ray.dashboard import dashboard
+from ray.dashboard.consts import RAY_CLUSTER_ACTIVITY_HOOK
+from ray.dashboard.modules.job.job_head import RayActivityResponse
+from ray.dashboard.tests.conftest import *  # noqa
+
+
+@pytest.fixture
+def set_ray_cluster_activity_hook(request):
+    """
+    Fixture that sets RAY_CLUSTER_ACTIVITY_HOOK environment variable
+    for test_e2e_component_activities_hook.
+    """
+    external_hook = request.param
+    assert (
+        external_hook
+    ), "Please pass value of RAY_CLUSTER_ACTIVITY_HOOK env var to this fixture"
+    old_hook = os.environ.get(RAY_CLUSTER_ACTIVITY_HOOK)
+    os.environ[RAY_CLUSTER_ACTIVITY_HOOK] = external_hook
+
+    yield external_hook
+
+    if old_hook is not None:
+        os.environ[RAY_CLUSTER_ACTIVITY_HOOK] = old_hook
+    else:
+        del os.environ[RAY_CLUSTER_ACTIVITY_HOOK]
+
+
+@pytest.mark.parametrize(
+    "set_ray_cluster_activity_hook",
+    [
+        "ray._private.test_utils.external_ray_cluster_activity_hook1",
+        "ray._private.test_utils.external_ray_cluster_activity_hook2",
+        "ray._private.test_utils.external_ray_cluster_activity_hook3",
+        "ray._private.test_utils.external_ray_cluster_activity_hook4",
+        "ray._private.test_utils.external_ray_cluster_activity_hook5",
+    ],
+    indirect=True,
+)
+def test_component_activities_hook(set_ray_cluster_activity_hook, call_ray_start):
+    """
+    Tests /api/component_activities returns correctly for various
+    responses of RAY_CLUSTER_ACTIVITY_HOOK.
+
+    Verify no active drivers are correctly reflected in response.
+    """
+    external_hook = set_ray_cluster_activity_hook
+
+    response = requests.get("http://127.0.0.1:8265/api/component_activities")
+    response.raise_for_status()
+
+    # Validate schema of response
+    data = response.json()
+    schema_path = os.path.join(
+        os.path.dirname(dashboard.__file__),
+        "modules/job/component_activities_schema.json",
+    )
+    pprint.pprint(data)
+    jsonschema.validate(instance=data, schema=json.load(open(schema_path)))
+
+    # Validate driver response can be cast to RayActivityResponse object
+    # and that there are no active drivers.
+    driver_ray_activity_response = RayActivityResponse(**data["driver"])
+    assert driver_ray_activity_response.is_active == "INACTIVE"
+    assert driver_ray_activity_response.reason is None
+
+    # Validate external component response can be cast to RayActivityResponse object
+    if external_hook[-1] == "5":
+        external_activity_response = RayActivityResponse(**data["test_component5"])
+        assert external_activity_response.is_active == "ACTIVE"
+        assert external_activity_response.reason == "Counter: 1"
+    elif external_hook[-1] == "4":
+        external_activity_response = RayActivityResponse(**data["external_component"])
+        assert external_activity_response.is_active == "ERROR"
+        assert (
+            "'Error in external cluster activity hook'"
+            in external_activity_response.reason
+        )
+    elif external_hook[-1] == "3":
+        external_activity_response = RayActivityResponse(**data["external_component"])
+        assert external_activity_response.is_active == "ERROR"
+    elif external_hook[-1] == "2":
+        external_activity_response = RayActivityResponse(**data["test_component2"])
+        assert external_activity_response.is_active == "ERROR"
+    elif external_hook[-1] == "1":
+        external_activity_response = RayActivityResponse(**data["test_component1"])
+        assert external_activity_response.is_active == "ACTIVE"
+        assert external_activity_response.reason == "Counter: 1"
+
+        # Call endpoint again to validate different response
+        response = requests.get("http://127.0.0.1:8265/api/component_activities")
+        response.raise_for_status()
+        data = response.json()
+        jsonschema.validate(instance=data, schema=json.load(open(schema_path)))
+
+        external_activity_response = RayActivityResponse(**data["test_component1"])
+        assert external_activity_response.is_active == "ACTIVE"
+        assert external_activity_response.reason == "Counter: 2"
+
+
+def test_active_component_activities(ray_start_with_dashboard):
+    # Verify drivers which don't have namespace starting with _ray_internal_
+    # are considered active.
+
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+
+    driver_template = """
+import ray
+
+ray.init(address="auto", namespace="{namespace}")
+import time
+time.sleep({sleep_time_s})
+    """
+    run_string_as_driver(
+        driver_template.format(namespace="my_namespace", sleep_time_s=0)
+    )
+
+    run_string_as_driver_nonblocking(
+        driver_template.format(namespace="my_namespace", sleep_time_s=5)
+    )
+    run_string_as_driver_nonblocking(
+        driver_template.format(namespace="_ray_internal_job_info_id1", sleep_time_s=5)
+    )
+    # Simulate the default driver that gets created by dashboard
+    run_string_as_driver_nonblocking(
+        driver_template.format(namespace="_ray_internal_dashboard", sleep_time_s=5)
+    )
+
+    def verify_driver_response():
+        # Verify drivers are considered active after running script
+        response = requests.get(f"{webui_url}/api/component_activities")
+        response.raise_for_status()
+
+        # Validate schema of response
+        data = response.json()
+        schema_path = os.path.join(
+            os.path.dirname(dashboard.__file__),
+            "modules/job/component_activities_schema.json",
+        )
+
+        jsonschema.validate(instance=data, schema=json.load(open(schema_path)))
+
+        # Validate ray_activity_response field can be cast to RayActivityResponse object
+        driver_ray_activity_response = RayActivityResponse(**data["driver"])
+        print(driver_ray_activity_response)
+
+        assert driver_ray_activity_response.is_active == "ACTIVE"
+        # Drivers with namespace starting with "_ray_internal" are not
+        # considered active drivers. Two active drivers are the second one
+        # run with namespace "my_namespace" and the one started
+        # from ray_start_with_dashboard
+        assert driver_ray_activity_response.reason == "Number of active drivers: 2"
+
+        return True
+
+    wait_for_condition(verify_driver_response)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_http_job_server.py

@@ -0,0 +1,777 @@
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Optional
+from unittest.mock import patch
+
+import pytest
+import requests
+import yaml
+
+import ray
+from ray._common.test_utils import wait_for_condition
+from ray._private.runtime_env.packaging import (
+    create_package,
+    download_and_unpack_package,
+    get_uri_for_file,
+)
+from ray._private.test_utils import (
+    chdir,
+    format_web_url,
+    wait_until_server_available,
+)
+from ray.dashboard.modules.dashboard_sdk import ClusterInfo, parse_cluster_info
+from ray.dashboard.modules.job.common import uri_to_http_components
+from ray.dashboard.modules.job.pydantic_models import JobDetails
+from ray.dashboard.modules.job.tests.test_cli_integration import set_env_var
+from ray.dashboard.modules.version import CURRENT_VERSION
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.job_submission import JobStatus, JobSubmissionClient
+from ray.runtime_env.runtime_env import RuntimeEnv, RuntimeEnvConfig
+from ray.tests.conftest import _ray_start
+
+# This test requires you have AWS credentials set up (any AWS credentials will
+# do, this test only accesses a public bucket).
+
+logger = logging.getLogger(__name__)
+
+DRIVER_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "subprocess_driver_scripts")
+
+
+@pytest.fixture(scope="module")
+def headers():
+    return {"Connection": "keep-alive", "Authorization": "TOK:<MY_TOKEN>"}
+
+
+@pytest.fixture(scope="module")
+def ray_start_context():
+    with _ray_start(include_dashboard=True, num_cpus=1) as ctx:
+        yield ctx
+
+
+@pytest.fixture(scope="module")
+def job_sdk_client(headers, ray_start_context) -> JobSubmissionClient:
+    address = ray_start_context.address_info["webui_url"]
+    assert wait_until_server_available(address)
+    yield JobSubmissionClient(format_web_url(address), headers=headers)
+
+
+@pytest.fixture
+def shutdown_only():
+    yield None
+    # The code after the yield will run as teardown code.
+    ray.shutdown()
+
+
+def test_submit_job_with_resources(shutdown_only):
+    ctx = ray.init(
+        include_dashboard=True,
+        num_cpus=1,
+        num_gpus=1,
+        resources={"Custom": 1},
+        dashboard_port=8269,
+        _memory=4,
+    )
+    address = ctx.address_info["webui_url"]
+    client = JobSubmissionClient(format_web_url(address))
+    # Check the case of too many resources.
+    for kwargs in [
+        {"entrypoint_num_cpus": 2},
+        {"entrypoint_num_gpus": 2},
+        {"entrypoint_memory": 4},
+        {"entrypoint_resources": {"Custom": 2}},
+    ]:
+        job_id = client.submit_job(entrypoint="echo hello", **kwargs)
+        data = client.get_job_info(job_id)
+        assert "waiting for resources" in data.message
+
+    # Check the case of sufficient resources.
+    job_id = client.submit_job(
+        entrypoint="echo hello",
+        entrypoint_num_cpus=1,
+        entrypoint_num_gpus=1,
+        entrypoint_memory=4,
+        entrypoint_resources={"Custom": 1},
+    )
+    wait_for_condition(_check_job_succeeded, client=client, job_id=job_id, timeout=10)
+
+
+@pytest.mark.parametrize("use_sdk", [True, False])
+def test_list_jobs_empty(headers, use_sdk: bool):
+    # Create a cluster using `ray start` instead of `ray.init` to avoid creating a job
+    subprocess.check_output(["ray", "start", "--head"])
+    address = "http://127.0.0.1:8265"
+    try:
+        with set_env_var("RAY_ADDRESS", address):
+            client = JobSubmissionClient(format_web_url(address), headers=headers)
+
+            if use_sdk:
+                assert client.list_jobs() == []
+            else:
+                r = client._do_request(
+                    "GET",
+                    "/api/jobs/",
+                )
+
+                assert r.status_code == 200
+                assert json.loads(r.text) == []
+
+    finally:
+        subprocess.check_output(["ray", "stop", "--force"])
+
+
+@pytest.mark.parametrize("use_sdk", [True, False])
+def test_list_jobs(job_sdk_client: JobSubmissionClient, use_sdk: bool):
+    client = job_sdk_client
+
+    runtime_env = {"env_vars": {"TEST": "123"}}
+    metadata = {"foo": "bar"}
+    entrypoint = "echo hello"
+    submission_id = client.submit_job(
+        entrypoint=entrypoint, runtime_env=runtime_env, metadata=metadata
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id=submission_id)
+    if use_sdk:
+        info: JobDetails = next(
+            job_info
+            for job_info in client.list_jobs()
+            if job_info.submission_id == submission_id
+        )
+    else:
+        r = client._do_request(
+            "GET",
+            "/api/jobs/",
+        )
+
+        assert r.status_code == 200
+        jobs_info_json = json.loads(r.text)
+        info_json = next(
+            job_info
+            for job_info in jobs_info_json
+            if job_info["submission_id"] == submission_id
+        )
+        info = JobDetails(**info_json)
+
+    assert info.entrypoint == entrypoint
+    assert info.status == JobStatus.SUCCEEDED
+    assert info.message is not None
+    assert info.end_time >= info.start_time
+    assert info.runtime_env == runtime_env
+    assert info.metadata == metadata
+
+    # Test get job status by job / driver id
+    status = client.get_job_status(info.submission_id)
+    assert status == JobStatus.SUCCEEDED
+
+
+def _check_job_succeeded(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    if status == JobStatus.FAILED:
+        logs = client.get_job_logs(job_id)
+        raise RuntimeError(
+            f"Job failed\nlogs:\n{logs}, info: {client.get_job_info(job_id)}"
+        )
+    assert status == JobStatus.SUCCEEDED
+    return True
+
+
+def _check_job_failed(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    return status == JobStatus.FAILED
+
+
+def _check_job_stopped(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    return status == JobStatus.STOPPED
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        "no_working_dir",
+        "local_working_dir",
+        "s3_working_dir",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+        "local_working_dir_zip",
+        "pip_txt",
+        "conda_yaml",
+        "local_py_modules",
+    ],
+)
+def runtime_env_option(request):
+    import_in_task_script = """
+import ray
+ray.init(address="auto")
+
+@ray.remote
+def f():
+    import pip_install_test
+
+ray.get(f.remote())
+"""
+    if request.param == "no_working_dir":
+        yield {
+            "runtime_env": {},
+            "entrypoint": "echo hello",
+            "expected_logs": "hello\n",
+        }
+    elif request.param in {
+        "local_working_dir",
+        "local_working_dir_zip",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+    }:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            path = Path(tmp_dir)
+
+            hello_file = path / "test.py"
+            with hello_file.open(mode="w") as f:
+                f.write("from test_module import run_test\n")
+                f.write("print(run_test())")
+
+            module_path = path / "test_module"
+            module_path.mkdir(parents=True)
+
+            test_file = module_path / "test.py"
+            with test_file.open(mode="w") as f:
+                f.write("def run_test():\n")
+                f.write("    return 'Hello from test_module!'\n")  # noqa: Q000
+
+            init_file = module_path / "__init__.py"
+            with init_file.open(mode="w") as f:
+                f.write("from test_module.test import run_test\n")
+
+            if request.param == "local_working_dir":
+                yield {
+                    "runtime_env": {"working_dir": tmp_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_working_dir_zip":
+                local_zipped_dir = shutil.make_archive(
+                    os.path.join(tmp_dir, "test"), "zip", tmp_dir
+                )
+                yield {
+                    "runtime_env": {"working_dir": local_zipped_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_py_modules":
+                yield {
+                    "runtime_env": {"py_modules": [str(Path(tmp_dir) / "test_module")]},
+                    "entrypoint": (
+                        "python -c 'import test_module;print(test_module.run_test())'"
+                    ),
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "working_dir_and_local_py_modules_whl":
+                yield {
+                    "runtime_env": {
+                        "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+                        "py_modules": [
+                            Path(os.path.dirname(__file__))
+                            / "pip_install_test-0.5-py3-none-any.whl"
+                        ],
+                    },
+                    "entrypoint": (
+                        "python script.py && python -c 'import pip_install_test'"
+                    ),
+                    "expected_logs": (
+                        "Executing main() from script.py !!\n"
+                        "Good job!  You installed a pip module."
+                    ),
+                }
+            else:
+                raise ValueError(f"Unexpected pytest fixture option {request.param}")
+    elif request.param == "s3_working_dir":
+        yield {
+            "runtime_env": {
+                "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+            },
+            "entrypoint": "python script.py",
+            "expected_logs": "Executing main() from script.py !!\n",
+        }
+    elif request.param == "pip_txt":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            pip_list = ["pip-install-test==0.5"]
+            relative_filepath = "requirements.txt"
+            pip_file = Path(relative_filepath)
+            pip_file.write_text("\n".join(pip_list))
+            runtime_env = {"pip": {"packages": relative_filepath, "pip_check": False}}
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": (
+                    f"python -c 'import pip_install_test' && "
+                    f"python -c '{import_in_task_script}'"
+                ),
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    elif request.param == "conda_yaml":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            conda_dict = {"dependencies": ["pip", {"pip": ["pip-install-test==0.5"]}]}
+            relative_filepath = "environment.yml"
+            conda_file = Path(relative_filepath)
+            conda_file.write_text(yaml.dump(conda_dict))
+            runtime_env = {"conda": relative_filepath}
+
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": f"python -c '{import_in_task_script}'",
+                # TODO(architkulkarni): Uncomment after #22968 is fixed.
+                # "entrypoint": "python -c 'import pip_install_test'",
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    else:
+        assert False, f"Unrecognized option: {request.param}."
+
+
+def test_submit_job(job_sdk_client, runtime_env_option, monkeypatch):
+    # This flag allows for local testing of runtime env conda functionality
+    # without needing a built Ray wheel.  Rather than insert the link to the
+    # wheel into the conda spec, it links to the current Python site.
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    client = job_sdk_client
+
+    job_id = client.submit_job(
+        entrypoint=runtime_env_option["entrypoint"],
+        runtime_env=runtime_env_option["runtime_env"],
+    )
+
+    try:
+        job_start_time = time.time()
+        wait_for_condition(
+            _check_job_succeeded, client=client, job_id=job_id, timeout=300
+        )
+        job_duration = time.time() - job_start_time
+        print(f"The job took {job_duration}s to succeed.")
+    except RuntimeError as e:
+        # If the job is still pending, include job logs and info in error.
+        if client.get_job_status(job_id) == JobStatus.PENDING:
+            logs = client.get_job_logs(job_id)
+            info = client.get_job_info(job_id)
+            raise RuntimeError(
+                f"Job was stuck in PENDING.\nLogs: {logs}\nInfo: {info}"
+            ) from e
+
+    logs = client.get_job_logs(job_id)
+    assert runtime_env_option["expected_logs"] in logs
+
+
+def test_timeout(job_sdk_client):
+    client = job_sdk_client
+
+    job_id = client.submit_job(
+        entrypoint="echo hello",
+        # Assume pip packages take > 1s to download, or this test will spuriously fail.
+        runtime_env=RuntimeEnv(
+            pip={
+                "packages": ["tensorflow", "requests", "botocore", "torch"],
+                "pip_check": False,
+                "pip_version": "==23.3.2;python_version=='3.9.16'",
+            },
+            config=RuntimeEnvConfig(setup_timeout_seconds=1),
+        ),
+    )
+
+    wait_for_condition(_check_job_failed, client=client, job_id=job_id, timeout=10)
+    data = client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+    assert "Timeout" in data.message
+    assert "setup_timeout_seconds" in data.message
+
+
+def test_per_task_runtime_env(job_sdk_client: JobSubmissionClient):
+    run_cmd = "python per_task_runtime_env.py"
+    job_id = job_sdk_client.submit_job(
+        entrypoint=run_cmd,
+        runtime_env={"working_dir": DRIVER_SCRIPT_DIR},
+    )
+
+    wait_for_condition(_check_job_succeeded, client=job_sdk_client, job_id=job_id)
+
+
+def test_ray_tune_basic(job_sdk_client: JobSubmissionClient):
+    run_cmd = "python ray_tune_basic.py"
+    job_id = job_sdk_client.submit_job(
+        entrypoint=run_cmd,
+        runtime_env={"working_dir": DRIVER_SCRIPT_DIR},
+    )
+    wait_for_condition(
+        _check_job_succeeded, timeout=30, client=job_sdk_client, job_id=job_id
+    )
+
+
+def test_http_bad_request(job_sdk_client):
+    """
+    Send bad requests to job http server and ensure right return code and
+    error message is returned via http.
+    """
+    client = job_sdk_client
+
+    # 400 - HTTPBadRequest
+    r = client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"key": "baaaad request"},
+    )
+
+    assert r.status_code == 400
+    assert "__init__() got an unexpected keyword argument" in r.text
+
+
+def test_invalid_runtime_env(job_sdk_client):
+    client = job_sdk_client
+    with pytest.raises(ValueError, match="Only .zip, .tar.gz, and .tgz files"):
+        client.submit_job(
+            entrypoint="echo hello", runtime_env={"working_dir": "s3://not_a_zip"}
+        )
+
+
+def test_runtime_env_setup_failure(job_sdk_client):
+    client = job_sdk_client
+    job_id = client.submit_job(
+        entrypoint="echo hello", runtime_env={"working_dir": "s3://does_not_exist.zip"}
+    )
+
+    wait_for_condition(_check_job_failed, client=client, job_id=job_id)
+    data = client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+
+
+def test_submit_job_with_exception_in_driver(job_sdk_client):
+    """
+    Submit a job that's expected to throw exception while executing.
+    """
+    client = job_sdk_client
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+print('Hello !')
+raise RuntimeError('Intentionally failed.')
+        """
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as file:
+            file.write(driver_script)
+
+        job_id = client.submit_job(
+            entrypoint="python test_script.py", runtime_env={"working_dir": tmp_dir}
+        )
+
+        wait_for_condition(_check_job_failed, client=client, job_id=job_id)
+        logs = client.get_job_logs(job_id)
+        assert "Hello !" in logs
+        assert "RuntimeError: Intentionally failed." in logs
+
+
+def test_stop_long_running_job(job_sdk_client):
+    """
+    Submit a job that runs for a while and stop it in the middle.
+    """
+    client = job_sdk_client
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+print('Hello !')
+import time
+time.sleep(300) # This should never finish
+raise RuntimeError('Intentionally failed.')
+        """
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as file:
+            file.write(driver_script)
+
+        job_id = client.submit_job(
+            entrypoint="python test_script.py", runtime_env={"working_dir": tmp_dir}
+        )
+        assert client.stop_job(job_id) is True
+        wait_for_condition(_check_job_stopped, client=client, job_id=job_id)
+
+
+def test_delete_job(job_sdk_client, capsys):
+    """
+    Submit a job and delete it.
+    """
+    client: JobSubmissionClient = job_sdk_client
+
+    job_id = client.submit_job(entrypoint="sleep 300 && echo hello")
+    with pytest.raises(Exception, match="but it is in a non-terminal state"):
+        # This should fail because the job is not in a terminal state.
+        client.delete_job(job_id)
+
+    # Check that the job appears in list_jobs
+    jobs = client.list_jobs()
+    assert job_id in [job.submission_id for job in jobs]
+
+    finished_job_id = client.submit_job(entrypoint="echo hello")
+    wait_for_condition(_check_job_succeeded, client=client, job_id=finished_job_id)
+    deleted = client.delete_job(finished_job_id)
+    assert deleted is True
+
+    # Check that the job no longer appears in list_jobs
+    jobs = client.list_jobs()
+    assert finished_job_id not in [job.submission_id for job in jobs]
+
+
+def test_job_metadata(job_sdk_client):
+    client = job_sdk_client
+
+    print_metadata_cmd = (
+        'python -c"'
+        "import ray;"
+        "ray.init();"
+        "job_config=ray._private.worker.global_worker.core_worker.get_job_config();"
+        "print(dict(sorted(job_config.metadata.items())))"
+        '"'
+    )
+
+    job_id = client.submit_job(
+        entrypoint=print_metadata_cmd, metadata={"key1": "val1", "key2": "val2"}
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id=job_id)
+
+    assert str(
+        {
+            "job_name": job_id,
+            "job_submission_id": job_id,
+            "key1": "val1",
+            "key2": "val2",
+        }
+    ) in client.get_job_logs(job_id)
+
+
+def test_pass_job_id(job_sdk_client):
+    client = job_sdk_client
+
+    job_id = "my_custom_id"
+    returned_id = client.submit_job(entrypoint="echo hello", job_id=job_id)
+
+    assert returned_id == job_id
+    wait_for_condition(_check_job_succeeded, client=client, job_id=returned_id)
+
+    # Test that a duplicate job_id is rejected.
+    with pytest.raises(Exception, match=f"{job_id} already exists"):
+        returned_id = client.submit_job(entrypoint="echo hello", job_id=job_id)
+
+
+def test_nonexistent_job(job_sdk_client):
+    client = job_sdk_client
+
+    with pytest.raises(RuntimeError, match="nonexistent_job does not exist"):
+        client.get_job_status("nonexistent_job")
+
+
+def test_submit_optional_args(job_sdk_client):
+    """Check that job_id, runtime_env, and metadata are optional."""
+    client = job_sdk_client
+
+    r = client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"entrypoint": "ls"},
+    )
+
+    wait_for_condition(
+        _check_job_succeeded, client=client, job_id=r.json()["submission_id"]
+    )
+
+
+def test_submit_still_accepts_job_id_or_submission_id(job_sdk_client):
+    """Check that job_id, runtime_env, and metadata are optional."""
+    client = job_sdk_client
+
+    client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"entrypoint": "ls", "job_id": "raysubmit_12345"},
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id="raysubmit_12345")
+
+    client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"entrypoint": "ls", "submission_id": "raysubmit_23456"},
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id="raysubmit_23456")
+
+
+def test_missing_resources(job_sdk_client):
+    """Check that 404s are raised for resources that don't exist."""
+    client = job_sdk_client
+
+    conditions = [
+        ("GET", "/api/jobs/fake_job_id"),
+        ("GET", "/api/jobs/fake_job_id/logs"),
+        ("POST", "/api/jobs/fake_job_id/stop"),
+        ("GET", "/api/packages/fake_package_uri"),
+    ]
+
+    for method, route in conditions:
+        assert client._do_request(method, route).status_code == 404
+
+
+def test_version_endpoint(job_sdk_client):
+    client = job_sdk_client
+
+    r = client._do_request("GET", "/api/version")
+    assert r.status_code == 200
+    body = r.json()
+    assert body == {
+        "version": CURRENT_VERSION,
+        "ray_version": ray.__version__,
+        "ray_commit": ray.__commit__,
+        "session_name": body["session_name"],
+    }
+
+
+def test_request_headers(job_sdk_client):
+    client = job_sdk_client
+    with patch("requests.request") as mock_request:
+        _ = client._do_request(
+            "POST",
+            "/api/jobs/",
+            json_data={"entrypoint": "ls"},
+        )
+        mock_request.assert_called_with(
+            "POST",
+            "http://127.0.0.1:8265/api/jobs/",
+            cookies=None,
+            data=None,
+            json={"entrypoint": "ls"},
+            headers={"Connection": "keep-alive", "Authorization": "TOK:<MY_TOKEN>"},
+            verify=True,
+        )
+
+
+@pytest.mark.parametrize("scheme", ["http", "https", "fake_module"])
+@pytest.mark.parametrize("host", ["127.0.0.1", "localhost", "fake.dns.name"])
+@pytest.mark.parametrize("port", [None, 8265, 10000])
+def test_parse_cluster_info(scheme: str, host: str, port: Optional[int]):
+    address = f"{scheme}://{host}"
+    if port is not None:
+        address += f":{port}"
+
+    if scheme in {"http", "https"}:
+        assert parse_cluster_info(address, False) == ClusterInfo(
+            address=address,
+            cookies=None,
+            metadata=None,
+            headers=None,
+        )
+    else:
+        with pytest.raises(RuntimeError):
+            parse_cluster_info(address, False)
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs(job_sdk_client):
+    client = job_sdk_client
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+import time
+for i in range(100):
+    print("Hello", i)
+    time.sleep(0.1)
+"""
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as f:
+            f.write(driver_script)
+
+        job_id = client.submit_job(
+            entrypoint="python test_script.py", runtime_env={"working_dir": tmp_dir}
+        )
+
+        st = time.time()
+        while time.time() - st <= 10:
+            try:
+                i = 0
+                async for lines in client.tail_job_logs(job_id):
+                    print(lines, end="")
+                    for line in lines.strip().split("\n"):
+                        assert line.split(" ") == ["Hello", str(i)]
+                        i += 1
+            except Exception as ex:
+                print("Exception:", ex)
+
+        wait_for_condition(_check_job_succeeded, client=client, job_id=job_id)
+
+
+def _hook(env):
+    with open(env["env_vars"]["TEMPPATH"], "w+") as f:
+        f.write(env["env_vars"]["TOKEN"])
+    return env
+
+
+def test_jobs_env_hook(job_sdk_client: JobSubmissionClient):
+    client = job_sdk_client
+
+    _, path = tempfile.mkstemp()
+    runtime_env = {"env_vars": {"TEMPPATH": path, "TOKEN": "Ray rocks!"}}
+    run_job_script = """
+import os
+import ray
+os.environ["RAY_RUNTIME_ENV_HOOK"] =\
+    "ray.dashboard.modules.job.tests.test_http_job_server._hook"
+ray.init(address="auto")
+"""
+    entrypoint = f"python -c '{run_job_script}'"
+    job_id = client.submit_job(entrypoint=entrypoint, runtime_env=runtime_env)
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id=job_id)
+
+    with open(path) as f:
+        assert f.read().strip() == "Ray rocks!"
+
+
+@pytest.mark.asyncio
+async def test_get_upload_package(ray_start_context, tmp_path):
+    assert wait_until_server_available(ray_start_context["webui_url"])
+    webui_url = format_web_url(ray_start_context["webui_url"])
+    gcs_client = ray._private.worker.global_worker.gcs_client
+    url = webui_url + "/api/packages/{protocol}/{package_name}"
+
+    pkg_dir = tmp_path / "pkg"
+    pkg_dir.mkdir()
+    filename = "task.py"
+
+    file_content = b"Hello world"
+    with (pkg_dir / filename).open("wb") as f:
+        f.write(file_content)
+
+    package_uri = get_uri_for_file(str(pkg_dir / filename))
+    protocol, package_name = uri_to_http_components(package_uri)
+    package_file = tmp_path / package_name
+    create_package(str(pkg_dir), package_file, include_gitignore=True)
+
+    resp = requests.get(url.format(protocol=protocol, package_name=package_name))
+    assert resp.status_code == 404
+
+    resp = requests.put(
+        url.format(protocol=protocol, package_name=package_name),
+        data=package_file.read_bytes(),
+    )
+    assert resp.status_code == 200
+
+    resp = requests.get(url.format(protocol=protocol, package_name=package_name))
+    assert resp.status_code == 200
+
+    await download_and_unpack_package(package_uri, str(tmp_path), gcs_client)
+    assert (package_file.with_suffix("") / filename).read_bytes() == file_content
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_https_connection.py

@@ -0,0 +1,47 @@
+import ssl
+import sys
+
+import pytest
+import trustme
+
+import ray
+from ray.job_submission import JobSubmissionClient
+
+
+@pytest.fixture(scope="session")
+def ca():
+    return trustme.CA()
+
+
+@pytest.fixture(scope="session")
+def httpserver_ssl_context(ca):
+    context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    localhost_cert = ca.issue_cert("localhost")
+    localhost_cert.configure_cert(context)
+    return context
+
+
+@pytest.fixture(scope="session")
+def httpclient_ssl_context(ca):
+    with ca.cert_pem.tempfile() as ca_temp_path:
+        return ssl.create_default_context(cafile=ca_temp_path)
+
+
+def test_mock_https_connection(httpserver, ca):
+    """Test connections to a mock HTTPS job submission server."""
+    httpserver.expect_request("/api/version").respond_with_json(
+        {"ray_version": ray.__version__}
+    )
+    mock_url = httpserver.url_for("/")
+    # Connection without SSL certificate should fail
+    with pytest.raises(ConnectionError):
+        JobSubmissionClient(mock_url)
+    # Connecton with SSL verification skipped should succeed
+    JobSubmissionClient(mock_url, verify=False)
+    # Connection with SSL verification should succeed
+    with ca.cert_pem.tempfile() as ca_temp_path:
+        JobSubmissionClient(mock_url, verify=ca_temp_path)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_job_agent.py

@@ -0,0 +1,692 @@
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import time
+from functools import partial
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+import requests
+import yaml
+
+import ray
+from ray._common.network_utils import build_address
+from ray._common.test_utils import async_wait_for_condition, wait_for_condition
+from ray._common.utils import get_or_create_event_loop
+from ray._private.ray_constants import DEFAULT_DASHBOARD_AGENT_LISTEN_PORT
+from ray._private.runtime_env.py_modules import upload_py_modules_if_needed
+from ray._private.runtime_env.working_dir import upload_working_dir_if_needed
+from ray._private.test_utils import (
+    chdir,
+    format_web_url,
+    get_current_unused_port,
+    run_string_as_driver_nonblocking,
+    wait_until_server_available,
+)
+from ray.dashboard.modules.job.common import (
+    JOB_ACTOR_NAME_TEMPLATE,
+    SUPERVISOR_ACTOR_RAY_NAMESPACE,
+    JobSubmitRequest,
+    validate_request_type,
+)
+from ray.dashboard.modules.job.job_head import JobAgentSubmissionClient
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.job_submission import JobStatus, JobSubmissionClient
+from ray.runtime_env.runtime_env import RuntimeEnv, RuntimeEnvConfig
+from ray.tests.conftest import _ray_start
+from ray.util.state import get_node, list_actors, list_nodes
+
+# This test requires you have AWS credentials set up (any AWS credentials will
+# do, this test only accesses a public bucket).
+
+logger = logging.getLogger(__name__)
+
+DRIVER_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "subprocess_driver_scripts")
+EVENT_LOOP = get_or_create_event_loop()
+
+
+def get_node_id_for_supervisor_actor_for_job(
+    address: str, job_submission_id: str
+) -> str:
+    actors = list_actors(
+        address=address,
+        filters=[("ray_namespace", "=", SUPERVISOR_ACTOR_RAY_NAMESPACE)],
+    )
+    for actor in actors:
+        if actor.name == JOB_ACTOR_NAME_TEMPLATE.format(job_id=job_submission_id):
+            return actor.node_id
+    raise ValueError(f"actor not found for job_submission_id {job_submission_id}")
+
+
+def get_node_ip_by_id(node_id: str) -> str:
+    node = get_node(id=node_id)
+    return node.node_ip
+
+
+class JobAgentSubmissionBrowserClient(JobAgentSubmissionClient):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._session.headers[
+            "User-Agent"
+        ] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"  # noqa: E501
+
+
+@pytest_asyncio.fixture
+async def job_sdk_client(make_sure_dashboard_http_port_unused):
+    with _ray_start(include_dashboard=True, num_cpus=1) as ctx:
+        node_ip = ctx.address_info["node_ip_address"]
+        agent_address = build_address(node_ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT)
+        assert wait_until_server_available(agent_address)
+        head_address = ctx.address_info["webui_url"]
+        assert wait_until_server_available(head_address)
+        yield (
+            JobAgentSubmissionClient(format_web_url(agent_address)),
+            JobSubmissionClient(format_web_url(head_address)),
+        )
+
+
+def _check_job(
+    client: JobSubmissionClient, job_id: str, status: JobStatus, timeout: int = 10
+) -> bool:
+    res_status = client.get_job_status(job_id)
+    assert res_status == status
+    return True
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        "no_working_dir",
+        "local_working_dir",
+        "s3_working_dir",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+        "local_working_dir_zip",
+        "pip_txt",
+        "conda_yaml",
+        "local_py_modules",
+    ],
+)
+def runtime_env_option(request):
+    import_in_task_script = """
+import ray
+ray.init(address="auto")
+
+@ray.remote
+def f():
+    import pip_install_test
+
+ray.get(f.remote())
+"""
+    if request.param == "no_working_dir":
+        yield {
+            "runtime_env": {},
+            "entrypoint": "echo hello",
+            "expected_logs": "hello\n",
+        }
+    elif request.param in {
+        "local_working_dir",
+        "local_working_dir_zip",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+    }:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            path = Path(tmp_dir)
+
+            hello_file = path / "test.py"
+            with hello_file.open(mode="w") as f:
+                f.write("from test_module import run_test\n")
+                f.write("print(run_test())")
+
+            module_path = path / "test_module"
+            module_path.mkdir(parents=True)
+
+            test_file = module_path / "test.py"
+            with test_file.open(mode="w") as f:
+                f.write("def run_test():\n")
+                f.write("    return 'Hello from test_module!'\n")  # noqa: Q000
+
+            init_file = module_path / "__init__.py"
+            with init_file.open(mode="w") as f:
+                f.write("from test_module.test import run_test\n")
+
+            if request.param == "local_working_dir":
+                yield {
+                    "runtime_env": {"working_dir": tmp_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_working_dir_zip":
+                local_zipped_dir = shutil.make_archive(
+                    os.path.join(tmp_dir, "test"), "zip", tmp_dir
+                )
+                yield {
+                    "runtime_env": {"working_dir": local_zipped_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_py_modules":
+                yield {
+                    "runtime_env": {"py_modules": [str(Path(tmp_dir) / "test_module")]},
+                    "entrypoint": (
+                        "python -c 'import test_module;print(test_module.run_test())'"
+                    ),
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "working_dir_and_local_py_modules_whl":
+                yield {
+                    "runtime_env": {
+                        "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+                        "py_modules": [
+                            Path(os.path.dirname(__file__))
+                            / "pip_install_test-0.5-py3-none-any.whl"
+                        ],
+                    },
+                    "entrypoint": (
+                        "python script.py && python -c 'import pip_install_test'"
+                    ),
+                    "expected_logs": (
+                        "Executing main() from script.py !!\n"
+                        "Good job!  You installed a pip module."
+                    ),
+                }
+            else:
+                raise ValueError(f"Unexpected pytest fixture option {request.param}")
+    elif request.param == "s3_working_dir":
+        yield {
+            "runtime_env": {
+                "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+            },
+            "entrypoint": "python script.py",
+            "expected_logs": "Executing main() from script.py !!\n",
+        }
+    elif request.param == "pip_txt":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            pip_list = ["pip-install-test==0.5"]
+            relative_filepath = "requirements.txt"
+            pip_file = Path(relative_filepath)
+            pip_file.write_text("\n".join(pip_list))
+            runtime_env = {"pip": {"packages": relative_filepath, "pip_check": False}}
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": (
+                    f"python -c 'import pip_install_test' && "
+                    f"python -c '{import_in_task_script}'"
+                ),
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    elif request.param == "conda_yaml":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            conda_dict = {"dependencies": ["pip", {"pip": ["pip-install-test==0.5"]}]}
+            relative_filepath = "environment.yml"
+            conda_file = Path(relative_filepath)
+            conda_file.write_text(yaml.dump(conda_dict))
+            runtime_env = {"conda": relative_filepath}
+
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": f"python -c '{import_in_task_script}'",
+                # TODO(architkulkarni): Uncomment after #22968 is fixed.
+                # "entrypoint": "python -c 'import pip_install_test'",
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    else:
+        assert False, f"Unrecognized option: {request.param}."
+
+
+@pytest.mark.asyncio
+async def test_submit_job(job_sdk_client, runtime_env_option, monkeypatch):
+    # This flag allows for local testing of runtime env conda functionality
+    # without needing a built Ray wheel.  Rather than insert the link to the
+    # wheel into the conda spec, it links to the current Python site.
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = runtime_env_option["runtime_env"]
+    runtime_env = upload_working_dir_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = upload_py_modules_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = RuntimeEnv(**runtime_env_option["runtime_env"]).to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": runtime_env_option["entrypoint"]},
+        JobSubmitRequest,
+    )
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    try:
+        job_start_time = time.time()
+        wait_for_condition(
+            partial(
+                _check_job,
+                client=head_client,
+                job_id=job_id,
+                status=JobStatus.SUCCEEDED,
+            ),
+            timeout=300,
+        )
+        job_duration = time.time() - job_start_time
+        print(f"The job took {job_duration}s to succeed.")
+    except RuntimeError as e:
+        # If the job is still pending, include job logs and info in error.
+        if head_client.get_job_status(job_id) == JobStatus.PENDING:
+            logs = head_client.get_job_logs(job_id)
+            info = head_client.get_job_info(job_id)
+            raise RuntimeError(
+                f"Job was stuck in PENDING.\nLogs: {logs}\nInfo: {info}"
+            ) from e
+
+    # There is only one node, so there is no need to replace the client of the JobAgent
+    resp = await agent_client.get_job_logs_internal(job_id)
+    assert runtime_env_option["expected_logs"] in resp.logs
+
+
+@pytest.mark.asyncio
+async def test_submit_job_rejects_browsers(
+    job_sdk_client, runtime_env_option, monkeypatch
+):
+    # This flag allows for local testing of runtime env conda functionality
+    # without needing a built Ray wheel.  Rather than insert the link to the
+    # wheel into the conda spec, it links to the current Python site.
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    agent_client, head_client = job_sdk_client
+    agent_address = agent_client._agent_address
+    agent_client = JobAgentSubmissionBrowserClient(agent_address)
+
+    runtime_env = runtime_env_option["runtime_env"]
+    runtime_env = upload_working_dir_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = upload_py_modules_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = RuntimeEnv(**runtime_env_option["runtime_env"]).to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": runtime_env_option["entrypoint"]},
+        JobSubmitRequest,
+    )
+
+    with pytest.raises(RuntimeError) as exc:
+        _ = await agent_client.submit_job_internal(request)
+
+    assert "status code 403" in str(exc.value)
+
+
+@pytest.mark.asyncio
+async def test_delete_job_rejects_browsers(job_sdk_client, monkeypatch):
+    """Test that DELETE job requests from browsers are rejected."""
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    agent_client, head_client = job_sdk_client
+
+    # First, submit a job using the normal client
+    runtime_env = RuntimeEnv().to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": "echo hello"},
+        JobSubmitRequest,
+    )
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    # Now try to delete the job using browser-like headers
+    agent_address = agent_client._agent_address
+    browser_client = JobAgentSubmissionBrowserClient(agent_address)
+
+    with pytest.raises(RuntimeError) as exc:
+        _ = await browser_client.delete_job_internal(job_id)
+
+    assert "status code 403" in str(exc.value)
+
+    await browser_client.close()
+
+
+@pytest.mark.asyncio
+async def test_timeout(job_sdk_client):
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = RuntimeEnv(
+        pip={
+            "packages": ["tensorflow", "requests", "botocore", "torch"],
+            "pip_check": False,
+            "pip_version": "==23.3.2;python_version=='3.9.16'",
+        },
+        config=RuntimeEnvConfig(setup_timeout_seconds=1),
+    ).to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": "echo hello"},
+        JobSubmitRequest,
+    )
+
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    wait_for_condition(
+        partial(_check_job, client=head_client, job_id=job_id, status=JobStatus.FAILED),
+        timeout=10,
+    )
+
+    data = head_client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+    assert "Timeout" in data.message
+    assert "setup_timeout_seconds" in data.message
+
+
+@pytest.mark.asyncio
+async def test_runtime_env_setup_failure(job_sdk_client):
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = RuntimeEnv(working_dir="s3://does_not_exist.zip").to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": "echo hello"},
+        JobSubmitRequest,
+    )
+
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    wait_for_condition(
+        partial(_check_job, client=head_client, job_id=job_id, status=JobStatus.FAILED),
+        timeout=10,
+    )
+
+    data = head_client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+
+
+@pytest.mark.asyncio
+async def test_stop_long_running_job(job_sdk_client):
+    """
+    Submit a job that runs for a while and stop it in the middle.
+    """
+    agent_client, head_client = job_sdk_client
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+print('Hello !')
+import time
+time.sleep(300) # This should never finish
+raise RuntimeError('Intentionally failed.')
+        """
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as file:
+            file.write(driver_script)
+
+        runtime_env = {"working_dir": tmp_dir}
+        runtime_env = upload_working_dir_if_needed(
+            runtime_env, include_gitignore=True, scratch_dir=tmp_dir, logger=logger
+        )
+        runtime_env = RuntimeEnv(**runtime_env).to_dict()
+
+        request = validate_request_type(
+            {"runtime_env": runtime_env, "entrypoint": "python test_script.py"},
+            JobSubmitRequest,
+        )
+        submit_result = await agent_client.submit_job_internal(request)
+        job_id = submit_result.submission_id
+
+        resp = await agent_client.stop_job_internal(job_id)
+        assert resp.stopped is True
+
+        wait_for_condition(
+            partial(
+                _check_job, client=head_client, job_id=job_id, status=JobStatus.STOPPED
+            ),
+            timeout=10,
+        )
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs_with_echo(job_sdk_client):
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = RuntimeEnv().to_dict()
+    entrypoint = "python -c \"import time; [(print('Hello', i), time.sleep(0.1)) for i in range(100)]\""  # noqa: E501
+    request = validate_request_type(
+        {
+            "runtime_env": runtime_env,
+            "entrypoint": entrypoint,
+        },
+        JobSubmitRequest,
+    )
+
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    i = 0
+    async for lines in agent_client.tail_job_logs(job_id):
+        print(lines, end="")
+        for line in lines.strip().split("\n"):
+            if (
+                "Runtime env is setting up." in line
+                or "Running entrypoint for job" in line
+            ):
+                continue
+            assert line.split(" ") == ["Hello", str(i)]
+            i += 1
+
+    wait_for_condition(
+        partial(
+            _check_job, client=head_client, job_id=job_id, status=JobStatus.SUCCEEDED
+        ),
+        timeout=10,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "ray_start_cluster_head",
+    [
+        {
+            "include_dashboard": True,
+            "dashboard_agent_listen_port": DEFAULT_DASHBOARD_AGENT_LISTEN_PORT,
+        }
+    ],
+    indirect=True,
+)
+async def test_job_log_in_multiple_node(
+    make_sure_dashboard_http_port_unused,
+    enable_test_module,
+    disable_aiohttp_cache,
+    ray_start_cluster_head,
+):
+    cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = cluster.webui_url
+    webui_url = format_web_url(webui_url)
+    cluster.add_node(
+        dashboard_agent_listen_port=DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 1
+    )
+    cluster.add_node(
+        dashboard_agent_listen_port=DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 2
+    )
+
+    node_ip = cluster.head_node.node_ip_address
+    agent_address = build_address(node_ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT)
+    assert wait_until_server_available(agent_address)
+    client = JobAgentSubmissionClient(format_web_url(agent_address))
+
+    def _check_nodes():
+        try:
+            assert len(list_nodes()) == 3
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_nodes, timeout=15)
+
+    job_ids = []
+    job_check_status = []
+    JOB_NUM = 10
+    job_agent_ports = [
+        DEFAULT_DASHBOARD_AGENT_LISTEN_PORT,
+        DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 1,
+        DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 2,
+    ]
+    for index in range(JOB_NUM):
+        runtime_env = RuntimeEnv().to_dict()
+        request = validate_request_type(
+            {
+                "runtime_env": runtime_env,
+                "entrypoint": f"while true; do echo hello index-{index}"
+                " && sleep 3600; done",
+            },
+            JobSubmitRequest,
+        )
+
+        submit_result = await client.submit_job_internal(request)
+        job_ids.append(submit_result.submission_id)
+        job_check_status.append(False)
+
+    async def _check_all_jobs_log():
+        response = requests.get(webui_url + "/nodes?view=summary")
+        response.raise_for_status()
+        summary = response.json()
+        assert summary["result"] is True, summary["msg"]
+        summary = summary["data"]["summary"]
+
+        for index, job_id in enumerate(job_ids):
+            if job_check_status[index]:
+                continue
+            result_log = f"hello index-{index}"
+            # Try to get the node id which supervisor actor running in.
+            node_id = get_node_id_for_supervisor_actor_for_job(cluster.address, job_id)
+            for node_info in summary:
+                if node_info["raylet"]["nodeId"] == node_id:
+                    break
+            assert node_info["raylet"]["nodeId"] == node_id, f"node id: {node_id}"
+
+            # Try to get the agent HTTP port by node id.
+            for agent_port in job_agent_ports:
+                if f"--listen-port={agent_port}" in " ".join(node_info["cmdline"]):
+                    break
+            assert f"--listen-port={agent_port}" in " ".join(
+                node_info["cmdline"]
+            ), f"port: {agent_port}"
+
+            # Finally, we got the whole agent address, and try to get the job log.
+            ip = get_node_ip_by_id(node_id)
+            agent_address = f"{ip}:{agent_port}"
+            assert wait_until_server_available(agent_address)
+            client = JobAgentSubmissionClient(format_web_url(agent_address))
+            resp = await client.get_job_logs_internal(job_id)
+            assert result_log in resp.logs, f"logs: {resp.logs}"
+
+            job_check_status[index] = True
+        return True
+
+    st = time.time()
+    while time.time() - st <= 30:
+        try:
+            await _check_all_jobs_log()
+            break
+        except Exception as ex:
+            print("error:", ex)
+            time.sleep(1)
+    assert all(job_check_status), job_check_status
+
+
+def test_agent_logs_not_streamed_to_drivers():
+    """Ensure when the job submission is used,
+    (ray.init is called from an agent), the agent logs are
+    not streamed to drivers.
+
+    Related: https://github.com/ray-project/ray/issues/29944
+    """
+    script = """
+import ray
+from ray.job_submission import JobSubmissionClient, JobStatus
+from ray._private.test_utils import format_web_url
+from ray._common.test_utils import wait_for_condition
+
+ray.init()
+address = ray._private.worker._global_node.webui_url
+address = format_web_url(address)
+client = JobSubmissionClient(address)
+submission_id = client.submit_job(entrypoint="ls")
+wait_for_condition(
+    lambda: client.get_job_status(submission_id) == JobStatus.SUCCEEDED
+)
+    """
+
+    proc = run_string_as_driver_nonblocking(script)
+    out_str = proc.stdout.read().decode("ascii")
+    err_str = proc.stderr.read().decode("ascii")
+
+    print(out_str, err_str)
+
+    assert "(raylet)" not in out_str
+    assert "(raylet)" not in err_str
+
+
+@pytest.mark.asyncio
+async def test_non_default_dashboard_agent_http_port(tmp_path):
+    """Test that we can connect to the dashboard agent with a non-default
+    http port.
+    """
+    import subprocess
+
+    dashboard_agent_port = get_current_unused_port()
+    cmd = f"ray start --head --dashboard-agent-listen-port {dashboard_agent_port}"
+    subprocess.check_output(cmd, shell=True)
+
+    try:
+        # We will need to wait for the ray to be started in the subprocess.
+        address_info = ray.init("auto", ignore_reinit_error=True).address_info
+
+        node_ip = address_info["node_ip_address"]
+
+        dashboard_agent_listen_port = address_info["dashboard_agent_listen_port"]
+        agent_address = build_address(node_ip, dashboard_agent_listen_port)
+        print("agent address = ", agent_address)
+
+        agent_client = JobAgentSubmissionClient(format_web_url(agent_address))
+        head_client = JobSubmissionClient(format_web_url(address_info["webui_url"]))
+
+        assert wait_until_server_available(agent_address)
+
+        # Submit a job through the agent.
+        runtime_env = RuntimeEnv().to_dict()
+        request = validate_request_type(
+            {
+                "runtime_env": runtime_env,
+                "entrypoint": "echo hello",
+            },
+            JobSubmitRequest,
+        )
+        submit_result = await agent_client.submit_job_internal(request)
+        job_id = submit_result.submission_id
+
+        async def verify():
+            # Wait for job finished.
+            wait_for_condition(
+                partial(
+                    _check_job,
+                    client=head_client,
+                    job_id=job_id,
+                    status=JobStatus.SUCCEEDED,
+                ),
+                timeout=10,
+            )
+
+            resp = await agent_client.get_job_logs_internal(job_id)
+            assert "hello" in resp.logs, resp.logs
+
+            return True
+
+        await async_wait_for_condition(verify, retry_interval_ms=2000)
+    finally:
+        subprocess.check_output("ray stop --force", shell=True)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_job_inheritance.py

@@ -0,0 +1,182 @@
+import json
+import sys
+import time
+
+import pytest
+
+from ray.cluster_utils import Cluster
+from ray.dashboard.consts import RAY_STREAM_RUNTIME_ENV_LOG_TO_JOB_DRIVER_LOG_ENV_VAR
+from ray.dashboard.modules.job.tests.conftest import _driver_script_path
+from ray.dashboard.modules.job.tests.subprocess_driver_scripts.driver_runtime_env_inheritance import (  # noqa: E501
+    RUNTIME_ENV_LOG_LINE_PREFIX,
+)
+from ray.job_submission import JobStatus, JobSubmissionClient
+
+
+def wait_until_status(client, job_id, status_to_wait_for, timeout_seconds=20):
+    start = time.time()
+    while time.time() - start <= timeout_seconds:
+        status = client.get_job_status(job_id)
+        print(f"status: {status}")
+        if status in status_to_wait_for:
+            return
+        time.sleep(1)
+    raise Exception
+
+
+def wait(client, job_id):
+    wait_until_status(
+        client,
+        job_id,
+        {JobStatus.SUCCEEDED, JobStatus.STOPPED, JobStatus.FAILED},
+        timeout_seconds=60,
+    )
+
+
+def get_runtime_env_from_logs(client, job_id):
+    wait(client, job_id)
+    logs = client.get_job_logs(job_id)
+    print(logs)
+    assert client.get_job_status(job_id) == JobStatus.SUCCEEDED
+    # Split logs by line, find the unique line that starts with
+    # RUNTIME_ENV_LOG_LINE_PREFIX, strip it and parse it as JSON.
+    lines = logs.strip().split("\n")
+    assert len(lines) > 0
+    for line in lines:
+        if line.startswith(RUNTIME_ENV_LOG_LINE_PREFIX):
+            return json.loads(line[len(RUNTIME_ENV_LOG_LINE_PREFIX) :])
+
+
+def test_job_driver_inheritance():
+    try:
+        c = Cluster()
+        c.add_node(num_cpus=1)
+        # If using a remote cluster, replace 127.0.0.1 with the head node's IP address.
+        client = JobSubmissionClient("http://127.0.0.1:8265")
+        driver_script_path = _driver_script_path("driver_runtime_env_inheritance.py")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path}",
+            runtime_env={
+                "env_vars": {"A": "1", "B": "2"},
+                "pip": ["requests"],
+            },
+        )
+
+        # Test key is merged
+        print("Test key merged")
+        runtime_env = get_runtime_env_from_logs(client, job_id)
+        assert runtime_env["env_vars"] == {"A": "1", "B": "2", "C": "1"}
+        assert runtime_env["pip"] == {"packages": ["requests"], "pip_check": False}
+
+        # Test worker process setuphook works.
+        print("Test key setup hook")
+        expected_str = "HELLOWORLD"
+        job_id = client.submit_job(
+            entrypoint=(
+                f"python {driver_script_path} "
+                f"--worker-process-setup-hook {expected_str}"
+            ),
+            runtime_env={
+                "env_vars": {"A": "1", "B": "2"},
+            },
+        )
+        wait(client, job_id)
+        logs = client.get_job_logs(job_id)
+        assert expected_str in logs
+
+        # Test raise an exception upon key conflict
+        print("Test conflicting pip")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=pip",
+            runtime_env={"pip": ["numpy"]},
+        )
+        wait(client, job_id)
+        status = client.get_job_status(job_id)
+        logs = client.get_job_logs(job_id)
+        assert status == JobStatus.FAILED
+        assert "Failed to merge the Job's runtime env" in logs
+
+        # Test raise an exception upon env var conflict
+        print("Test conflicting env vars")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=env_vars",
+            runtime_env={
+                "env_vars": {"A": "1"},
+            },
+        )
+        wait(client, job_id)
+        status = client.get_job_status(job_id)
+        logs = client.get_job_logs(job_id)
+        assert status == JobStatus.FAILED
+        assert "Failed to merge the Job's runtime env" in logs
+    finally:
+        c.shutdown()
+
+
+@pytest.mark.parametrize("stream_runtime_env_log", ["1", "0"])
+def test_runtime_env_logs_streamed_to_job_driver_log(
+    monkeypatch, stream_runtime_env_log
+):
+    monkeypatch.setenv(
+        RAY_STREAM_RUNTIME_ENV_LOG_TO_JOB_DRIVER_LOG_ENV_VAR, stream_runtime_env_log
+    )
+    try:
+        c = Cluster()
+        c.add_node(num_cpus=1)
+        client = JobSubmissionClient("http://127.0.0.1:8265")
+        job_id = client.submit_job(
+            entrypoint="echo hello world",
+            runtime_env={"pip": ["requests==2.25.1"]},
+        )
+        wait(client, job_id)
+        logs = client.get_job_logs(job_id)
+        if stream_runtime_env_log == "0":
+            assert "Creating virtualenv at" not in logs
+        else:
+            assert "Creating virtualenv at" in logs
+    finally:
+        c.shutdown()
+
+
+def test_job_driver_inheritance_override(monkeypatch):
+    monkeypatch.setenv("RAY_OVERRIDE_JOB_RUNTIME_ENV", "1")
+
+    try:
+        c = Cluster()
+        c.add_node(num_cpus=1)
+        # If using a remote cluster, replace 127.0.0.1 with the head node's IP address.
+        client = JobSubmissionClient("http://127.0.0.1:8265")
+        driver_script_path = _driver_script_path("driver_runtime_env_inheritance.py")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path}",
+            runtime_env={
+                "env_vars": {"A": "1", "B": "2"},
+                "pip": ["requests"],
+            },
+        )
+
+        # Test conflict resolution regular field
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=pip",
+            runtime_env={"pip": ["pip-install-test==0.5"]},
+        )
+        runtime_env = get_runtime_env_from_logs(client, job_id)
+        print(runtime_env)
+        assert runtime_env["pip"] == {"packages": ["numpy"], "pip_check": False}
+
+        # Test raise an exception upon env var conflict
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=env_vars",
+            runtime_env={
+                "env_vars": {"A": "2"},
+            },
+        )
+        runtime_env = get_runtime_env_from_logs(client, job_id)
+        print(runtime_env)
+        assert runtime_env["env_vars"]["A"] == "1"
+    finally:
+        c.shutdown()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_job_manager_standalone.py

@@ -0,0 +1,73 @@
+import sys
+
+import pytest
+
+from ray._common.test_utils import async_wait_for_condition
+from ray.dashboard.modules.job.tests.conftest import (
+    _driver_script_path,
+    create_job_manager,
+    create_ray_cluster,
+)
+from ray.dashboard.modules.job.tests.test_job_manager import check_job_succeeded
+
+
+@pytest.mark.asyncio
+class TestRuntimeEnvStandalone:
+    """NOTE: PLEASE READ CAREFULLY BEFORE MODIFYING
+    This test is extracted into a standalone module such that it can bootstrap its own
+    (standalone) Ray cluster while avoiding affecting the shared one used by other
+    JobManager tests
+    """
+
+    @pytest.mark.parametrize(
+        "tracing_enabled",
+        [
+            False,
+            # TODO(issues/38633): local code loading is broken when tracing is enabled
+            # True,
+        ],
+    )
+    async def test_user_provided_job_config_honored_by_worker(
+        self, tracing_enabled, tmp_path
+    ):
+        """Ensures that the JobConfig instance injected into ray.init in the driver
+        script is honored even in case when job is submitted via JobManager.submit_job
+        API (involving RAY_JOB_CONFIG_JSON_ENV_VAR being set in child process env)
+
+        """
+
+        if tracing_enabled:
+            tracing_startup_hook = (
+                "ray.util.tracing.setup_local_tmp_tracing:setup_tracing"
+            )
+        else:
+            tracing_startup_hook = None
+
+        with create_ray_cluster(_tracing_startup_hook=tracing_startup_hook) as cluster:
+            job_manager = create_job_manager(cluster, tmp_path)
+
+            driver_script_path = _driver_script_path(
+                "check_code_search_path_is_propagated.py"
+            )
+
+            job_id = await job_manager.submit_job(
+                entrypoint=f"python {driver_script_path}",
+                # NOTE: We inject runtime_env in here, but also specify the JobConfig in
+                #       the driver script: settings to JobConfig (other than the
+                #       runtime_env) passed in via ray.init(...) have to be respected
+                #       along with the runtime_env passed from submit_job API
+                runtime_env={"env_vars": {"TEST_SUBPROCESS_RANDOM_VAR": "0xDEEDDEED"}},
+            )
+
+            await async_wait_for_condition(
+                check_job_succeeded, job_manager=job_manager, job_id=job_id
+            )
+
+            logs = job_manager.get_job_logs(job_id)
+
+            assert "Code search path is propagated" in logs, logs
+            assert "0xDEEDDEED" in logs, logs
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_sdk.py

@@ -0,0 +1,435 @@
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import aiohttp
+import pytest
+
+import ray.experimental.internal_kv as kv
+from ray._common.test_utils import wait_for_condition
+from ray._private import worker
+from ray._private.ray_constants import (
+    KV_NAMESPACE_DASHBOARD,
+    PROCESS_TYPE_DASHBOARD,
+)
+from ray._private.test_utils import (
+    format_web_url,
+    wait_until_server_available,
+)
+from ray._raylet import GcsClient
+from ray.dashboard.consts import (
+    DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX,
+    GCS_RPC_TIMEOUT_SECONDS,
+    RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR,
+)
+from ray.dashboard.modules.dashboard_sdk import (
+    DEFAULT_DASHBOARD_ADDRESS,
+    ClusterInfo,
+    parse_cluster_info,
+)
+from ray.dashboard.modules.job.pydantic_models import JobType
+from ray.dashboard.modules.job.sdk import JobStatus, JobSubmissionClient
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.runtime_env.runtime_env import RuntimeEnv
+from ray.tests.conftest import _ray_start
+from ray.util.state import list_nodes
+
+import psutil
+
+
+def _check_job_succeeded(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    if status == JobStatus.FAILED:
+        logs = client.get_job_logs(job_id)
+        raise RuntimeError(f"Job failed\nlogs:\n{logs}")
+    return status == JobStatus.SUCCEEDED
+
+
+def check_internal_kv_gced():
+    return len(kv._internal_kv_list("gcs://")) == 0
+
+
+@pytest.mark.parametrize(
+    "address_param",
+    [
+        ("ray://1.2.3.4:10001", "ray", "1.2.3.4:10001"),
+        ("other_module://", "other_module", ""),
+        ("other_module://address", "other_module", "address"),
+    ],
+)
+@pytest.mark.parametrize("create_cluster_if_needed", [True, False])
+@pytest.mark.parametrize("cookies", [None, {"test_cookie_key": "test_cookie_val"}])
+@pytest.mark.parametrize("metadata", [None, {"test_metadata_key": "test_metadata_val"}])
+@pytest.mark.parametrize("headers", [None, {"test_headers_key": "test_headers_val"}])
+@pytest.mark.parametrize("extra_kwargs", [{}, {"cloud": "my-cloud"}])
+def test_parse_cluster_info(
+    address_param: Tuple[str, str, str],
+    create_cluster_if_needed: bool,
+    cookies: Optional[Dict[str, str]],
+    metadata: Optional[Dict[str, str]],
+    headers: Optional[Dict[str, str]],
+    extra_kwargs: Dict[str, str],
+):
+    """
+    Test ray.dashboard.modules.dashboard_sdk.parse_cluster_info for different
+    format of addresses.
+    """
+    mock_get_job_submission_client_cluster = Mock(return_value="Ray ClusterInfo")
+    mock_module = Mock()
+    mock_module.get_job_submission_client_cluster_info = Mock(
+        return_value="Other module ClusterInfo"
+    )
+    mock_import_module = Mock(return_value=mock_module)
+
+    address, module_string, inner_address = address_param
+
+    with (
+        patch.multiple(
+            "ray.dashboard.modules.dashboard_sdk",
+            get_job_submission_client_cluster_info=mock_get_job_submission_client_cluster,
+        ),
+        patch.multiple("importlib", import_module=mock_import_module),
+    ):
+        if module_string == "ray":
+            with pytest.raises(ValueError, match="ray://"):
+                parse_cluster_info(
+                    address,
+                    create_cluster_if_needed=create_cluster_if_needed,
+                    cookies=cookies,
+                    metadata=metadata,
+                    headers=headers,
+                    **extra_kwargs,
+                )
+        elif module_string == "other_module":
+            assert (
+                parse_cluster_info(
+                    address,
+                    create_cluster_if_needed=create_cluster_if_needed,
+                    cookies=cookies,
+                    metadata=metadata,
+                    headers=headers,
+                    **extra_kwargs,
+                )
+                == "Other module ClusterInfo"
+            )
+            mock_import_module.assert_called_once_with(module_string)
+            mock_module.get_job_submission_client_cluster_info.assert_called_once_with(
+                inner_address,
+                create_cluster_if_needed=create_cluster_if_needed,
+                cookies=cookies,
+                metadata=metadata,
+                headers=headers,
+                **extra_kwargs,
+            )
+
+
+def test_parse_cluster_info_default_address():
+    assert parse_cluster_info(
+        address=None,
+    ) == ClusterInfo(address=DEFAULT_DASHBOARD_ADDRESS)
+
+
+def test_submit_job_does_not_mutate_runtime_env():
+    class TestClient(JobSubmissionClient):
+        def __init__(self):
+            self._default_metadata = {}
+
+        def _upload_working_dir_if_needed(self, runtime_env):
+            runtime_env["working_dir"] = "gcs://test.zip"
+
+        def _upload_py_modules_if_needed(self, runtime_env):
+            runtime_env["py_modules"] = ["gcs://test_module.zip"]
+
+        def _do_request(self, method, endpoint, **kwargs):
+            return MagicMock(
+                status_code=200,
+                json=lambda: {"job_id": "test_job", "submission_id": "test_job"},
+            )
+
+    runtime_env = {"working_dir": "/tmp/test", "py_modules": ["/tmp/test_module"]}
+    original_runtime_env = {
+        "working_dir": runtime_env["working_dir"],
+        "py_modules": list(runtime_env["py_modules"]),
+    }
+
+    assert (
+        TestClient().submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+        == "test_job"
+    )
+    assert runtime_env == original_runtime_env
+
+
+@pytest.mark.parametrize("expiration_s", [0, 10])
+def test_temporary_uri_reference(monkeypatch, expiration_s):
+    """Test that temporary GCS URI references are deleted after expiration_s."""
+    monkeypatch.setenv(
+        "RAY_RUNTIME_ENV_TEMPORARY_REFERENCE_EXPIRATION_S", str(expiration_s)
+    )
+    # We can't use a fixture with a shared Ray runtime because we need to set the
+    # expiration_s env var before Ray starts.
+    with _ray_start(include_dashboard=True, num_cpus=1) as ctx:
+        headers = {"Connection": "keep-alive", "Authorization": "TOK:<MY_TOKEN>"}
+        address = ctx.address_info["webui_url"]
+        assert wait_until_server_available(address)
+        client = JobSubmissionClient(format_web_url(address), headers=headers)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            path = Path(tmp_dir)
+
+            hello_file = path / "hi.txt"
+            with hello_file.open(mode="w") as f:
+                f.write("hi\n")
+
+            start = time.time()
+
+            runtime_env = {"working_dir": tmp_dir}
+            job_id = client.submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+            assert runtime_env == {"working_dir": tmp_dir}
+            wait_for_condition(
+                _check_job_succeeded, client=client, job_id=job_id, timeout=30
+            )
+
+            # Give time for deletion to occur if expiration_s is 0.
+            time.sleep(2)
+            # Need to connect to Ray to check internal_kv.
+            # ray.init(address="auto")
+
+            print("Starting Internal KV checks at time ", time.time() - start)
+            if expiration_s > 0:
+                assert not check_internal_kv_gced()
+                wait_for_condition(check_internal_kv_gced, timeout=2 * expiration_s)
+                assert expiration_s < time.time() - start < 2 * expiration_s
+                print("Internal KV was GC'ed at time ", time.time() - start)
+            else:
+                wait_for_condition(check_internal_kv_gced)
+                print("Internal KV was GC'ed at time ", time.time() - start)
+
+                # Regression test for #46625: reusing the same runtime_env after
+                # the package has been GC'ed should re-upload the local working_dir.
+                job_id = client.submit_job(
+                    entrypoint="echo hi", runtime_env=runtime_env
+                )
+                wait_for_condition(
+                    _check_job_succeeded, client=client, job_id=job_id, timeout=30
+                )
+
+
+def get_register_agents_number(gcs_client):
+    keys = gcs_client.internal_kv_keys(
+        prefix=DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX,
+        namespace=KV_NAMESPACE_DASHBOARD,
+        timeout=GCS_RPC_TIMEOUT_SECONDS,
+    )
+    return len(keys)
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head_with_env_vars",
+    [
+        {
+            "include_dashboard": True,
+            "env_vars": {RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR: "1"},
+        },
+        {
+            "include_dashboard": True,
+            "env_vars": {RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR: "0"},
+        },
+    ],
+    indirect=True,
+)
+def test_jobs_run_on_head_by_default_E2E(ray_start_cluster_head_with_env_vars):
+    allow_driver_on_worker_nodes = (
+        os.environ.get(RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR) == "1"
+    )
+    # Cluster setup
+    cluster = ray_start_cluster_head_with_env_vars
+    cluster.add_node(dashboard_agent_listen_port=52366)
+    cluster.add_node(dashboard_agent_listen_port=52367)
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = cluster.webui_url
+    webui_url = format_web_url(webui_url)
+    client = JobSubmissionClient(webui_url)
+    gcs_client = GcsClient(address=cluster.gcs_address)
+
+    def _check_nodes(num_nodes):
+        try:
+            assert len(list_nodes()) == num_nodes
+            return True
+        except Exception as ex:
+            print(ex)
+            return False
+
+    wait_for_condition(lambda: _check_nodes(num_nodes=3), timeout=15)
+    wait_for_condition(lambda: get_register_agents_number(gcs_client) == 3, timeout=20)
+
+    # Submit 20 simple jobs.
+    for i in range(20):
+        client.submit_job(entrypoint="echo hi", submission_id=f"job_{i}")
+    import pprint
+
+    def check_all_jobs_succeeded():
+        submission_jobs = [
+            job for job in client.list_jobs() if job.type == JobType.SUBMISSION
+        ]
+        for job in submission_jobs:
+            pprint.pprint(job)
+            if job.status != JobStatus.SUCCEEDED:
+                return False
+        return True
+
+    # Wait until all jobs have finished.
+    wait_for_condition(check_all_jobs_succeeded, timeout=60, retry_interval_ms=1000)
+
+    # Check driver_node_id of all jobs.
+    submission_jobs = [
+        job for job in client.list_jobs() if job.type == JobType.SUBMISSION
+    ]
+    driver_node_ids = [job.driver_node_id for job in submission_jobs]
+
+    # Spuriously fails with probability (1/3)^20.
+    pprint.pprint(driver_node_ids)
+    num_ids = len(set(driver_node_ids))
+    assert (num_ids > 1) if allow_driver_on_worker_nodes else (num_ids == 1), [
+        id[:5] for id in driver_node_ids
+    ]
+
+
+@pytest.fixture
+def runtime_env_working_dir():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        working_dir = path / "working_dir"
+        working_dir.mkdir(parents=True)
+        yield working_dir
+
+
+@pytest.fixture
+def py_module_whl():
+    with tempfile.NamedTemporaryFile(suffix=".whl") as tmp_file:
+        yield tmp_file.name
+
+
+def test_job_submission_with_runtime_env_as_dict(
+    runtime_env_working_dir, py_module_whl
+):
+    working_dir_str = str(runtime_env_working_dir)
+    with _ray_start(num_cpus=1):
+        client = JobSubmissionClient()
+        runtime_env = {"working_dir": working_dir_str, "py_modules": [py_module_whl]}
+        job_id = client.submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+        job_details = client.get_job_info(job_id)
+        parsed_runtime_env = job_details.runtime_env
+        assert "gcs://" in parsed_runtime_env["working_dir"]
+        assert len(parsed_runtime_env["py_modules"]) == 1
+        assert "gcs://" in parsed_runtime_env["py_modules"][0]
+
+
+def test_job_submission_with_runtime_env_as_object(
+    runtime_env_working_dir, py_module_whl
+):
+    working_dir_str = str(runtime_env_working_dir)
+    with _ray_start(num_cpus=1):
+        client = JobSubmissionClient()
+        runtime_env = RuntimeEnv(
+            working_dir=working_dir_str, py_modules=[py_module_whl]
+        )
+        job_id = client.submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+        job_details = client.get_job_info(job_id)
+        parsed_runtime_env = job_details.runtime_env
+        assert "gcs://" in parsed_runtime_env["working_dir"]
+        assert len(parsed_runtime_env["py_modules"]) == 1
+        assert "gcs://" in parsed_runtime_env["py_modules"][0]
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs_passes_headers_to_websocket(ray_start_regular):
+    """
+    Test that authentication headers are passed to WebSocket connections.
+
+    This test verifies that headers provided to JobSubmissionClient are
+    explicitly passed to the ws_connect() method, not just to the ClientSession.
+    This is required because aiohttp's ClientSession does not automatically
+    include session headers in WebSocket upgrade requests.
+    """
+    dashboard_url = ray_start_regular.dashboard_url
+    test_headers = {"Authorization": "Bearer test-token"}
+    client = JobSubmissionClient(format_web_url(dashboard_url), headers=test_headers)
+
+    # Submit a simple job
+    job_id = client.submit_job(entrypoint="echo hello")
+
+    # Mock the aiohttp ClientSession and WebSocket
+    mock_ws = AsyncMock()
+    mock_ws.receive = AsyncMock()
+    mock_ws.receive.side_effect = [
+        # First call returns a text message
+        MagicMock(type=aiohttp.WSMsgType.TEXT, data="test log line\n"),
+        # Second call indicates WebSocket is closed
+        MagicMock(type=aiohttp.WSMsgType.CLOSED),
+    ]
+    mock_ws.close_code = 1000  # Normal closure
+
+    mock_session = AsyncMock()
+    mock_session.ws_connect = AsyncMock(return_value=mock_ws)
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+
+    # Patch ClientSession to use our mock
+    with patch("aiohttp.ClientSession", return_value=mock_session):
+        # Tail logs
+        log_lines = []
+        async for lines in client.tail_job_logs(job_id):
+            log_lines.append(lines)
+
+        # Verify ws_connect was called with headers
+        mock_session.ws_connect.assert_called_once()
+        call_args = mock_session.ws_connect.call_args
+
+        assert "headers" in call_args.kwargs
+        assert call_args.kwargs["headers"] == test_headers
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs_websocket_abnormal_closure(ray_start_regular):
+    """
+    Test that ABNORMAL_CLOSURE raises RuntimeError when tailing logs.
+
+    This test uses its own Ray cluster and kills the dashboard while tailing logs
+    to simulate an abnormal WebSocket closure.
+    """
+    dashboard_url = ray_start_regular.dashboard_url
+    client = JobSubmissionClient(format_web_url(dashboard_url))
+
+    # Submit a long-running job
+    driver_script = """
+import time
+for i in range(100):
+    print("Hello", i)
+    time.sleep(0.5)
+"""
+    entrypoint = f"python -c '{driver_script}'"
+    job_id = client.submit_job(entrypoint=entrypoint)
+
+    # Start tailing logs and stop Ray while tailing
+    # Expect RuntimeError when WebSocket closes abnormally
+    with pytest.raises(
+        RuntimeError,
+        match="WebSocket connection closed unexpectedly with close code",
+    ):
+        i = 0
+        async for lines in client.tail_job_logs(job_id):
+            print(lines, end="")
+            i += 1
+
+            # Kill the dashboard after receiving a few log lines
+            if i == 3:
+                print("\nKilling the dashboard to close websocket abnormally...")
+                dash_info = worker._global_node.all_processes[PROCESS_TYPE_DASHBOARD][0]
+                psutil.Process(dash_info.process.pid).kill()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_utils.py

@@ -0,0 +1,303 @@
+import os
+import sys
+from tempfile import NamedTemporaryFile
+
+import pytest
+
+from ray.dashboard.modules.job.common import JobSubmitRequest
+from ray.dashboard.modules.job.utils import (
+    fast_tail_last_n_lines,
+    file_tail_iterator,
+    parse_and_validate_request,
+    redact_url_password,
+    strip_keys_with_value_none,
+)
+
+
+# Polyfill anext() function for Python 3.9 compatibility
+# May raise StopAsyncIteration.
+async def anext_polyfill(iterator):
+    return await iterator.__anext__()
+
+
+# Use the built-in anext() for Python 3.10+, otherwise use our polyfilled function
+if sys.version_info < (3, 10):
+    anext = anext_polyfill
+
+
+@pytest.fixture
+def tmp():
+    with NamedTemporaryFile() as f:
+        yield f.name
+
+
+def test_strip_keys_with_value_none():
+    d = {"a": 1, "b": None, "c": 3}
+    assert strip_keys_with_value_none(d) == {"a": 1, "c": 3}
+    d = {"a": 1, "b": 2, "c": 3}
+    assert strip_keys_with_value_none(d) == d
+    d = {"a": 1, "b": None, "c": None}
+    assert strip_keys_with_value_none(d) == {"a": 1}
+
+
+def test_redact_url_password():
+    url = "http://user:password@host:port"
+    assert redact_url_password(url) == "http://user:<redacted>@host:port"
+    url = "http://user:password@host:port?query=1"
+    assert redact_url_password(url) == "http://user:<redacted>@host:port?query=1"
+    url = "http://user:password@host:port?query=1&password=2"
+    assert (
+        redact_url_password(url)
+        == "http://user:<redacted>@host:port?query=1&password=2"
+    )
+    url = "https://user:password@127.0.0.1:8080"
+    assert redact_url_password(url) == "https://user:<redacted>@127.0.0.1:8080"
+    url = "https://user:password@host:port?query=1"
+    assert redact_url_password(url) == "https://user:<redacted>@host:port?query=1"
+    url = "https://user:password@host:port?query=1&password=2"
+    assert (
+        redact_url_password(url)
+        == "https://user:<redacted>@host:port?query=1&password=2"
+    )
+
+
+# Mock for aiohttp.web.Request, which should not be constructed directly.
+class MockRequest:
+    def __init__(self, **kwargs):
+        self._json = kwargs
+
+    async def json(self):
+        return self._json
+
+
+@pytest.mark.asyncio
+async def test_mock_request():
+    request = MockRequest(a=1, b=2)
+    assert await request.json() == {"a": 1, "b": 2}
+    request = MockRequest(a=1, b=None)
+    assert await request.json() == {"a": 1, "b": None}
+
+
+# async test
+@pytest.mark.asyncio
+class TestParseAndValidateRequest:
+    async def test_basic(self):
+        request = MockRequest(entrypoint="echo hi")
+        expected = JobSubmitRequest(entrypoint="echo hi")
+        assert await parse_and_validate_request(request, JobSubmitRequest) == expected
+
+    async def test_forward_compatibility(self):
+        request = MockRequest(entrypoint="echo hi", new_client_field=None)
+        expected = JobSubmitRequest(entrypoint="echo hi")
+        assert await parse_and_validate_request(request, JobSubmitRequest) == expected
+
+
+class TestIterLine:
+    @pytest.mark.asyncio
+    async def test_invalid_type(self):
+        with pytest.raises(TypeError, match="path must be a string"):
+            await anext(file_tail_iterator(1))
+
+    @pytest.mark.asyncio
+    async def test_file_not_created(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+        f = open(tmp, "w")
+        f.write("hi\n")
+        f.flush()
+        assert await anext(it) is not None
+
+    @pytest.mark.asyncio
+    async def test_wait_for_newline(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+        f.write("no_newline_yet")
+        assert await anext(it) is None
+        f.write("\n")
+        f.flush()
+        assert await anext(it) == ["no_newline_yet\n"]
+
+    @pytest.mark.asyncio
+    async def test_multiple_lines(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        num_lines = 10
+        for i in range(num_lines):
+            s = f"{i}\n"
+            f.write(s)
+            f.flush()
+            assert await anext(it) == [s]
+
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_batching(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        # Write lines in batches of 10, check that we get them back in batches.
+        for _ in range(100):
+            num_lines = 10
+            for i in range(num_lines):
+                f.write(f"{i}\n")
+            f.flush()
+
+            assert await anext(it) == [f"{i}\n" for i in range(10)]
+
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_max_line_batching(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        # Write lines in batches of 50, check that we get them back in batches of 10.
+        for _ in range(100):
+            num_lines = 50
+            for i in range(num_lines):
+                f.write(f"{i}\n")
+            f.flush()
+
+            assert await anext(it) == [f"{i}\n" for i in range(10)]
+            assert await anext(it) == [f"{i}\n" for i in range(10, 20)]
+            assert await anext(it) == [f"{i}\n" for i in range(20, 30)]
+            assert await anext(it) == [f"{i}\n" for i in range(30, 40)]
+            assert await anext(it) == [f"{i}\n" for i in range(40, 50)]
+
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_max_char_batching(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        # Write a single line that is 60k characters
+        f.write(f"{'1234567890' * 6000}\n")
+        # Write a 4 lines that are 10k characters each
+        for _ in range(4):
+            f.write(f"{'1234567890' * 500}\n")
+        f.flush()
+
+        # First line will come in a batch of its own
+        assert await anext(it) == [f"{'1234567890' * 6000}\n"]
+        # Other 4 lines will be batched together
+        assert (
+            await anext(it)
+            == [
+                f"{'1234567890' * 500}\n",
+            ]
+            * 4
+        )
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_delete_file(self):
+        with NamedTemporaryFile() as tmp:
+            it = file_tail_iterator(tmp.name)
+            f = open(tmp.name, "w")
+
+            assert await anext(it) is None
+
+            f.write("hi\n")
+            f.flush()
+
+            assert await anext(it) == ["hi\n"]
+
+        # Calls should continue returning None after file deleted.
+        assert await anext(it) is None
+
+
+class TestFastTailLastNLines:
+    def test_nonexistent_path(self, tmp):
+        missing = tmp + ".missing"
+        assert not os.path.exists(missing)
+        with pytest.raises(FileNotFoundError):
+            fast_tail_last_n_lines(missing, num_lines=10, max_chars=1000)
+
+    def test_basic_last_n(self, tmp):
+        # Write 100 lines, check that we get the last 10 lines.
+        with open(tmp, "w") as f:
+            for i in range(100):
+                f.write(f"line-{i}\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=10, max_chars=1000)
+        expected = "".join([f"line-{i}\n" for i in range(90, 100)])
+        assert out == expected
+
+    def test_truncate_max_chars(self, tmp):
+        # Construct a log file with two lines, each over max_chars,
+        # check that we truncate to max_chars.
+        with open(tmp, "w") as f:
+            f.write("x" * 5000 + "\n")
+            f.write("y" * 5000 + "\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=2, max_chars=3000)
+        assert len(out) == 3000
+        # Check that we truncate to max_chars, and include the last line.
+        assert out.endswith("\n")
+
+    def test_partial_last_line(self, tmp):
+        # Write a log file with a partial last line, check that we include it.
+        with open(tmp, "w") as f:
+            f.write("a\n")
+            f.write("b\n")
+            f.write("partial_last_line")  # No newline at end
+        out = fast_tail_last_n_lines(tmp, num_lines=3, max_chars=1000)
+        assert out == "a\nb\npartial_last_line"
+
+    def test_small_block_size(self, tmp):
+        # Write 30 lines, check that we can read a small block size and get the last N lines.
+        with open(tmp, "w") as f:
+            for i in range(30):
+                f.write(f"{i}\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=5, max_chars=1000, block_size=16)
+        expected = "".join([f"{i}\n" for i in range(25, 30)])
+        assert out == expected
+
+    def test_mixed_long_lines(self, tmp):
+        # Write a log file with a mix of short and long lines, check that we get the last N lines.
+        with open(tmp, "w") as f:
+            f.write("short-1\n")
+            f.write("short-2\n")
+            f.write("long-" + ("Z" * 10000) + "\n")
+            f.write("short-3\n")
+            f.write("short-4\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=3, max_chars=20000)
+        # Check that we get the last 3 lines, including the long line.
+        assert out.splitlines()[-1] == "short-4"
+        assert out.splitlines()[-2] == "short-3"
+        assert out.splitlines()[-3].startswith("long-Z")
+
+    def test_sparse_large_file_tail_max_chars(self, tmp):
+        """Simulate ~8 GiB sparse file tail and verify max_chars=20000 truncation."""
+        size_8g = 8 * 1024 * 1024 * 1024
+        # Build tail of two extremely long lines
+        tail = "\n" + ("Q" * 25000 + "\n") + ("R" * 25000 + "\n")
+        tail_bytes = tail.encode("utf-8")
+
+        print("Start writing sparse file tail...")
+        # Create a sparse file: seek to near EOF then write only the tail.
+        with open(tmp, "wb") as f:
+            f.seek(size_8g - len(tail_bytes))
+            f.write(tail_bytes)
+            f.flush()
+
+        print("Finish writing sparse file tail.")
+        out = fast_tail_last_n_lines(tmp, num_lines=2, max_chars=20000)
+        print("Finish reading sparse file tail.")
+        assert len(out) == 20000
+        assert out.endswith("\n")
+        assert "R" * 100 in out  # sampling check for last line content
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/utils.py

@@ -0,0 +1,361 @@
+import asyncio
+import dataclasses
+import logging
+import os
+import re
+import traceback
+from dataclasses import dataclass
+from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union
+
+from ray._raylet import RAY_INTERNAL_NAMESPACE_PREFIX, GcsClient
+from ray.dashboard.modules.job.common import (
+    JOB_ID_METADATA_KEY,
+    JobInfoStorageClient,
+    JobStatus,
+    validate_request_type,
+)
+from ray.dashboard.modules.job.pydantic_models import DriverInfo, JobDetails, JobType
+from ray.runtime_env import RuntimeEnv
+
+try:
+    # package `aiohttp` is not in ray's minimal dependencies
+    import aiohttp
+    from aiohttp.web import Request, Response
+except Exception:
+    aiohttp = None
+    Request = None
+    Response = None
+
+
+logger = logging.getLogger(__name__)
+
+MAX_CHUNK_LINE_LENGTH = 10
+MAX_CHUNK_CHAR_LENGTH = 20000
+
+
+def strip_keys_with_value_none(d: Dict[str, Any]) -> Dict[str, Any]:
+    """Strip keys with value None from a dictionary."""
+    return {k: v for k, v in d.items() if v is not None}
+
+
+def redact_url_password(url: str) -> str:
+    """Redact any passwords in a URL."""
+    secret = re.findall(r"https?:\/\/.*:(.*)@.*", url)
+    if len(secret) > 0:
+        url = url.replace(f":{secret[0]}@", ":<redacted>@")
+
+    return url
+
+
+async def file_tail_iterator(path: str) -> AsyncIterator[Optional[List[str]]]:
+    """Yield lines from a file as it's written.
+
+    Returns lines in batches of up to 10 lines or 20000 characters,
+    whichever comes first. If it's a chunk of 20000 characters, then
+    the last line that is yielded could be an incomplete line.
+    New line characters are kept in the line string.
+
+    Returns None until the file exists or if no new line has been written.
+    """
+    if not isinstance(path, str):
+        raise TypeError(f"path must be a string, got {type(path)}.")
+
+    while not os.path.exists(path):
+        logger.debug(f"Path {path} doesn't exist yet.")
+        yield None
+
+    EOF = ""
+
+    with open(path, "r") as f:
+        lines = []
+
+        chunk_char_count = 0
+        curr_line = None
+
+        while True:
+            # We want to flush current chunk in following cases:
+            #   - We accumulated 10 lines
+            #   - We accumulated at least MAX_CHUNK_CHAR_LENGTH total chars
+            #   - We reached EOF
+            if (
+                len(lines) >= 10
+                or chunk_char_count > MAX_CHUNK_CHAR_LENGTH
+                or curr_line == EOF
+            ):
+                # Too many lines, return 10 lines in this chunk, and then
+                # continue reading the file.
+                yield lines or None
+
+                lines = []
+                chunk_char_count = 0
+
+            # Read next line
+            curr_line = f.readline()
+
+            # `readline` will return
+            #   - '' for EOF
+            #   - '\n' for an empty line in the file
+            if curr_line != EOF:
+                # Add line to current chunk
+                lines.append(curr_line)
+                chunk_char_count += len(curr_line)
+            else:
+                # If EOF is reached sleep for 1s before continuing
+                await asyncio.sleep(1)
+
+
+async def parse_and_validate_request(
+    req: Request, request_type: dataclass
+) -> Union[dataclass, Response]:
+    """Parse request and cast to request type.
+
+    Remove keys with value None to allow newer client versions with new optional fields
+    to work with older servers.
+
+    If parsing failed, return a Response object with status 400 and stacktrace instead.
+
+    Args:
+        req: aiohttp request object.
+        request_type: dataclass type to cast request to.
+
+    Returns:
+        Parsed request object or Response object with status 400 and stacktrace.
+    """
+    import aiohttp
+
+    json_data = strip_keys_with_value_none(await req.json())
+    try:
+        return validate_request_type(json_data, request_type)
+    except Exception as e:
+        logger.info(f"Got invalid request type: {e}")
+        return Response(
+            text=traceback.format_exc(),
+            status=aiohttp.web.HTTPBadRequest.status_code,
+        )
+
+
+async def get_driver_jobs(
+    gcs_client: GcsClient,
+    job_or_submission_id: Optional[str] = None,
+    timeout: Optional[int] = None,
+) -> Tuple[Dict[str, JobDetails], Dict[str, DriverInfo]]:
+    """Returns a tuple of dictionaries related to drivers.
+
+    The first dictionary contains all driver jobs and is keyed by the job's id.
+    The second dictionary contains drivers that belong to submission jobs.
+    It's keyed by the submission job's submission id.
+    Only the last driver of a submission job is returned.
+
+    An optional job_or_submission_id filter can be provided to only return
+    jobs with the job id or submission id.
+    """
+    job_infos = await gcs_client.async_get_all_job_info(
+        job_or_submission_id=job_or_submission_id,
+        skip_submission_job_info_field=True,
+        skip_is_running_tasks_field=True,
+        timeout=timeout,
+    )
+    # Sort jobs from GCS to follow convention of returning only last driver
+    # of submission job.
+    sorted_job_infos = sorted(
+        job_infos.values(), key=lambda job_table_entry: job_table_entry.job_id.hex()
+    )
+
+    jobs = {}
+    submission_job_drivers = {}
+    for job_table_entry in sorted_job_infos:
+        if job_table_entry.config.ray_namespace.startswith(
+            RAY_INTERNAL_NAMESPACE_PREFIX
+        ):
+            # Skip jobs in any _ray_internal_ namespace
+            continue
+        job_id = job_table_entry.job_id.hex()
+        metadata = dict(job_table_entry.config.metadata)
+        job_submission_id = metadata.get(JOB_ID_METADATA_KEY)
+        if not job_submission_id:
+            driver = DriverInfo(
+                id=job_id,
+                node_ip_address=job_table_entry.driver_address.ip_address,
+                pid=str(job_table_entry.driver_pid),
+            )
+            job = JobDetails(
+                job_id=job_id,
+                type=JobType.DRIVER,
+                status=JobStatus.SUCCEEDED
+                if job_table_entry.is_dead
+                else JobStatus.RUNNING,
+                entrypoint=job_table_entry.entrypoint,
+                start_time=job_table_entry.start_time,
+                end_time=job_table_entry.end_time,
+                metadata=metadata,
+                runtime_env=RuntimeEnv.deserialize(
+                    job_table_entry.config.runtime_env_info.serialized_runtime_env
+                ).to_dict(),
+                driver_info=driver,
+            )
+            jobs[job_id] = job
+        else:
+            driver = DriverInfo(
+                id=job_id,
+                node_ip_address=job_table_entry.driver_address.ip_address,
+                pid=str(job_table_entry.driver_pid),
+            )
+            submission_job_drivers[job_submission_id] = driver
+
+    return jobs, submission_job_drivers
+
+
+async def find_job_by_ids(
+    gcs_client: GcsClient,
+    job_info_client: JobInfoStorageClient,
+    job_or_submission_id: str,
+) -> Optional[JobDetails]:
+    """
+    Attempts to find the job with a given submission_id or job id.
+    """
+    # First try to find by job_id
+    driver_jobs, submission_job_drivers = await get_driver_jobs(
+        gcs_client, job_or_submission_id=job_or_submission_id
+    )
+    job = driver_jobs.get(job_or_submission_id)
+    if job:
+        return job
+    # Try to find a driver with the given id
+    submission_id = next(
+        (
+            id
+            for id, driver in submission_job_drivers.items()
+            if driver.id == job_or_submission_id
+        ),
+        None,
+    )
+
+    if not submission_id:
+        # If we didn't find a driver with the given id,
+        # then lets try to search for a submission with given id
+        submission_id = job_or_submission_id
+
+    job_info = await job_info_client.get_info(submission_id)
+    if job_info:
+        driver = submission_job_drivers.get(submission_id)
+        job = JobDetails(
+            **dataclasses.asdict(job_info),
+            submission_id=submission_id,
+            job_id=driver.id if driver else None,
+            driver_info=driver,
+            type=JobType.SUBMISSION,
+        )
+        return job
+
+    return None
+
+
+async def find_jobs_by_job_ids(
+    gcs_client: GcsClient,
+    job_info_client: JobInfoStorageClient,
+    job_ids: List[str],
+) -> Dict[str, JobDetails]:
+    """
+    Returns a dictionary of submission jobs with the given job ids, keyed by the job id.
+
+    This only accepts job ids and not submission ids.
+    """
+    driver_jobs, submission_job_drivers = await get_driver_jobs(gcs_client)
+
+    # Filter down to the request job_ids
+    driver_jobs = {key: job for key, job in driver_jobs.items() if key in job_ids}
+    submission_job_drivers = {
+        key: job for key, job in submission_job_drivers.items() if job.id in job_ids
+    }
+
+    # Fetch job details for each job
+    job_submission_ids = submission_job_drivers.keys()
+    job_infos = await asyncio.gather(
+        *[
+            job_info_client.get_info(submission_id)
+            for submission_id in job_submission_ids
+        ]
+    )
+
+    return {
+        **driver_jobs,
+        **{
+            submission_job_drivers.get(submission_id).id: JobDetails(
+                **dataclasses.asdict(job_info),
+                submission_id=submission_id,
+                job_id=submission_job_drivers.get(submission_id).id,
+                driver_info=submission_job_drivers.get(submission_id),
+                type=JobType.SUBMISSION,
+            )
+            for job_info, submission_id in zip(job_infos, job_submission_ids)
+        },
+    }
+
+
+def fast_tail_last_n_lines(
+    path: str,
+    num_lines: int,
+    max_chars: int,
+    block_size: int = 8192,
+) -> str:
+    """Return the last ``num_lines`` lines from a large log file efficiently.
+
+    This function avoids scanning the entire file. It seeks to the end of
+    the file and reads backwards in fixed-size blocks until enough lines are
+    collected. This is much faster for large files compared to using
+    ``file_tail_iterator()``, which performs a full sequential scan.
+
+    Args:
+        path: The file path to read.
+        num_lines: Number of lines to return.
+        max_chars: Maximum number of characters in the returned string.
+        block_size: Read size for each backward block.
+
+    Returns:
+        A string containing at most ``num_lines`` of the last lines in the file,
+        truncated to ``max_chars`` characters.
+    """
+    if num_lines < 0:
+        raise ValueError(f"num_lines must be non-negative, got {num_lines}")
+    if num_lines == 0:
+        return ""
+    if max_chars < 0:
+        raise ValueError(f"max_chars must be non-negative, got {max_chars}")
+    if max_chars == 0:
+        return ""
+    if block_size <= 0:
+        raise ValueError(f"block_size must be positive, got {block_size}")
+
+    logger.debug(
+        f"Start reading log file {path} with num_lines={num_lines} max_chars={max_chars} block_size={block_size}"
+    )
+    with open(path, "rb") as f:
+        f.seek(0, os.SEEK_END)
+        file_size = f.tell()
+        if file_size == 0:
+            return ""
+
+        chunks = []
+        position = file_size
+        newlines_found = 0
+
+        # We read backwards in chunks until we have enough newlines for num_lines.
+        # We may need one more newline to capture the content before the first newline.
+        while position > 0 and newlines_found < num_lines + 1:
+            read_size = min(block_size, position)
+            position -= read_size
+            f.seek(position)
+
+            chunk = f.read(read_size)
+            newlines_found += chunk.count(b"\n")
+            chunks.insert(0, chunk)
+
+    buffer = b"".join(chunks)
+    lines = buffer.decode("utf-8", errors="replace").splitlines(keepends=True)
+
+    if len(lines) <= num_lines:
+        result = "".join(lines)
+    else:
+        result = "".join(lines[-num_lines:])
+
+    return result[-max_chars:]

python/ray/dashboard/modules/log/log_agent.py

@@ -0,0 +1,405 @@
+import asyncio
+import concurrent.futures
+import io
+import logging
+import os
+from pathlib import Path
+from typing import AsyncIterator, Optional
+
+import grpc
+
+import ray.dashboard.modules.log.log_consts as log_consts
+import ray.dashboard.modules.log.log_utils as log_utils
+import ray.dashboard.optional_utils as dashboard_optional_utils
+import ray.dashboard.utils as dashboard_utils
+from ray._private.ray_constants import env_integer
+from ray.core.generated import reporter_pb2, reporter_pb2_grpc
+
+logger = logging.getLogger(__name__)
+routes = dashboard_optional_utils.DashboardAgentRouteTable
+
+# 64 KB
+BLOCK_SIZE = 1 << 16
+
+# Keep-alive interval for reading the file
+DEFAULT_KEEP_ALIVE_INTERVAL_SEC = 1
+
+RAY_DASHBOARD_LOG_TASK_LOG_SEARCH_MAX_WORKER_COUNT = env_integer(
+    "RAY_DASHBOARD_LOG_TASK_LOG_SEARCH_MAX_WORKER_COUNT", default=2
+)
+
+
+def find_offset_of_content_in_file(
+    file: io.BufferedIOBase, content: bytes, start_offset: int = 0
+) -> int:
+    """Find the offset of the first occurrence of content in a file.
+
+    Args:
+        file: File object
+        content: Content to find
+        start_offset: Start offset to read from, inclusive.
+
+    Returns:
+        Offset of the first occurrence of content in a file.
+    """
+    logger.debug(f"Finding offset of content {content} in file")
+    file.seek(start_offset, io.SEEK_SET)  # move file pointer to start of file
+    offset = start_offset
+    while True:
+        # Read in block
+        block_data = file.read(BLOCK_SIZE)
+        if block_data == b"":
+            # Stop reading
+            return -1
+        # Find the offset of the first occurrence of content in the block
+        block_offset = block_data.find(content)
+        if block_offset != -1:
+            # Found the offset in the block
+            return offset + block_offset
+        # Continue reading
+        offset += len(block_data)
+
+
+def find_end_offset_file(file: io.BufferedIOBase) -> int:
+    """
+    Find the offset of the end of a file without changing the file pointer.
+
+    Args:
+        file: File object
+
+    Returns:
+        Offset of the end of a file.
+    """
+    old_pos = file.tell()  # store old position
+    file.seek(0, io.SEEK_END)  # move file pointer to end of file
+    end = file.tell()  # return end of file offset
+    file.seek(old_pos, io.SEEK_SET)
+    return end
+
+
+def find_end_offset_next_n_lines_from_offset(
+    file: io.BufferedIOBase, start_offset: int, n: int
+) -> int:
+    """
+    Find the offsets of next n lines from a start offset.
+
+    Args:
+        file: File object
+        start_offset: Start offset to read from, inclusive.
+        n: Number of lines to find.
+
+    Returns:
+        Offset of the end of the next n line (exclusive)
+    """
+    file.seek(start_offset)  # move file pointer to start offset
+    end_offset = None
+    for _ in range(n):  # loop until we find n lines or reach end of file
+        line = file.readline()  # read a line and consume new line character
+        if not line:  # end of file
+            break
+        end_offset = file.tell()  # end offset.
+
+    logger.debug(f"Found next {n} lines from {start_offset} offset")
+    return (
+        end_offset if end_offset is not None else file.seek(0, io.SEEK_END)
+    )  # return last line offset or end of file offset if no lines found
+
+
+def find_start_offset_last_n_lines_from_offset(
+    file: io.BufferedIOBase, offset: int, n: int, block_size: int = BLOCK_SIZE
+) -> int:
+    """
+    Find the offset of the beginning of the line of the last X lines from an offset.
+
+    Args:
+        file: File object
+        offset: Start offset from which to find last X lines, -1 means end of file.
+            The offset is exclusive, i.e. data at the offset is not included
+            in the result.
+        n: Number of lines to find
+        block_size: Block size to read from file
+
+    Returns:
+        Offset of the beginning of the line of the last X lines from a start offset.
+    """
+    logger.debug(f"Finding last {n} lines from {offset} offset")
+    if offset == -1:
+        offset = file.seek(0, io.SEEK_END)  # move file pointer to end of file
+    else:
+        file.seek(offset, io.SEEK_SET)  # move file pointer to start offset
+
+    if n == 0:
+        return offset
+    nbytes_from_end = (
+        0  # Number of bytes that should be tailed from the end of the file
+    )
+    # Non new line terminating offset, adjust the line count and treat the non-newline
+    # terminated line as the last line. e.g. line 1\nline 2
+    file.seek(max(0, offset - 1), os.SEEK_SET)
+    if file.read(1) != b"\n":
+        n -= 1
+
+    # Remaining number of lines to tail
+    lines_more = n
+    read_offset = max(0, offset - block_size)
+    # So that we know how much to read on the last block (the block 0)
+    prev_offset = offset
+
+    while lines_more >= 0 and read_offset >= 0:
+        # Seek to the current block start
+        file.seek(read_offset, 0)
+        # Read the current block (or less than block) data
+        block_data = file.read(min(block_size, prev_offset - read_offset))
+        num_lines = block_data.count(b"\n")
+        if num_lines > lines_more:
+            # This is the last block to read.
+            # Need to find the offset of exact number of lines to tail
+            # in the block.
+            # Use `split` here to split away the extra lines, i.e.
+            # first `num_lines - lines_more` lines.
+            lines = block_data.split(b"\n", num_lines - lines_more)
+            # Added the len of those lines that at the end of the block.
+            nbytes_from_end += len(lines[-1])
+            break
+
+        # Need to read more blocks.
+        lines_more -= num_lines
+        nbytes_from_end += len(block_data)
+
+        if read_offset == 0:
+            # We have read all blocks (since the start)
+            break
+        # Continuing with the previous block
+        prev_offset = read_offset
+        read_offset = max(0, read_offset - block_size)
+
+    offset_read_start = offset - nbytes_from_end
+    assert (
+        offset_read_start >= 0
+    ), f"Read start offset({offset_read_start}) should be non-negative"
+    return offset_read_start
+
+
+async def _stream_log_in_chunk(
+    context: grpc.aio.ServicerContext,
+    file: io.BufferedIOBase,
+    start_offset: int,
+    end_offset: int = -1,
+    keep_alive_interval_sec: int = -1,
+    block_size: int = BLOCK_SIZE,
+) -> AsyncIterator[reporter_pb2.StreamLogReply]:
+    """Streaming log in chunk from start to end offset.
+
+    Stream binary file content in chunks from start offset to an end
+    offset if provided, else to the end of the file.
+
+    Args:
+        context: gRPC server side context
+        file: Binary file to stream
+        start_offset: File offset where streaming starts
+        end_offset: If -1, implying streaming til the EOF.
+        keep_alive_interval_sec: Duration for which streaming will be
+            retried when reaching the file end, -1 means no retry.
+        block_size: Number of bytes per chunk, exposed for testing
+
+    Yields:
+        reporter_pb2.StreamLogReply: Successive chunks of the file contents,
+        one per block.
+    """
+    assert "b" in file.mode, "Only binary file is supported."
+    assert not (
+        keep_alive_interval_sec >= 0 and end_offset != -1
+    ), "Keep-alive is not allowed when specifying an end offset"
+
+    file.seek(start_offset, 0)
+    cur_offset = start_offset
+
+    # Until gRPC is done
+    while not context.done():
+        # Read in block
+        if end_offset != -1:
+            to_read = min(end_offset - cur_offset, block_size)
+        else:
+            to_read = block_size
+
+        bytes = file.read(to_read)
+
+        if bytes == b"":
+            # Stop reading
+            if keep_alive_interval_sec >= 0:
+                await asyncio.sleep(keep_alive_interval_sec)
+                # Try reading again
+                continue
+
+            # Have read the entire file, done
+            break
+        logger.debug(f"Sending {len(bytes)} bytes at {cur_offset}")
+        yield reporter_pb2.StreamLogReply(data=bytes)
+
+        # Have read the requested section [start_offset, end_offset), done
+        cur_offset += len(bytes)
+        if end_offset != -1 and cur_offset >= end_offset:
+            break
+
+
+class LogAgent(dashboard_utils.DashboardAgentModule):
+    def __init__(self, dashboard_agent):
+        super().__init__(dashboard_agent)
+        log_utils.register_mimetypes()
+        routes.static("/logs", self._dashboard_agent.log_dir, show_index=True)
+
+    async def run(self, server):
+        pass
+
+    @staticmethod
+    def is_minimal_module():
+        return False
+
+
+_task_log_search_worker_pool = concurrent.futures.ThreadPoolExecutor(
+    max_workers=RAY_DASHBOARD_LOG_TASK_LOG_SEARCH_MAX_WORKER_COUNT
+)
+
+
+class LogAgentV1Grpc(dashboard_utils.DashboardAgentModule):
+    def __init__(self, dashboard_agent):
+        super().__init__(dashboard_agent)
+
+    async def run(self, server):
+        if server:
+            reporter_pb2_grpc.add_LogServiceServicer_to_server(self, server)
+
+    @property
+    def node_id(self) -> Optional[str]:
+        return self._dashboard_agent.get_node_id()
+
+    @staticmethod
+    def is_minimal_module():
+        # Dashboard is only available with non-minimal install now.
+        return False
+
+    async def ListLogs(self, request, context):
+        """
+        Lists all files in the active Ray logs directory.
+
+        Part of `LogService` gRPC.
+
+        NOTE: These RPCs are used by state_head.py, not log_head.py
+        """
+        path = Path(self._dashboard_agent.log_dir)
+        if not path.exists():
+            raise FileNotFoundError(
+                f"Could not find log dir at path: {self._dashboard_agent.log_dir}"
+                "It is unexpected. Please report an issue to Ray Github."
+            )
+        log_files = []
+        for p in path.glob(request.glob_filter):
+            log_files.append(str(p.relative_to(path)) + ("/" if p.is_dir() else ""))
+        return reporter_pb2.ListLogsReply(log_files=log_files)
+
+    @classmethod
+    def _resolve_filename(cls, root_log_dir: Path, filename: str) -> Path:
+        """
+        Resolves the file path relative to the root log directory.
+
+        Args:
+            root_log_dir: Root log directory.
+            filename: File path relative to the root log directory.
+
+        Raises:
+            FileNotFoundError: If the file path is invalid.
+
+        Returns:
+            The absolute file path resolved from the root log directory.
+        """
+        if not Path(filename).is_absolute():
+            filepath = root_log_dir / filename
+        else:
+            filepath = Path(filename)
+
+        # We want to allow relative paths that include symlinks pointing outside of the
+        # `root_log_dir`, so use `os.path.abspath` instead of `Path.resolve()` because
+        # `os.path.abspath` does not resolve symlinks.
+        filepath = Path(os.path.abspath(filepath))
+
+        if not filepath.is_file():
+            raise FileNotFoundError(f"A file is not found at: {filepath}")
+
+        try:
+            filepath.relative_to(root_log_dir)
+        except ValueError as e:
+            raise FileNotFoundError(f"{filepath} not in {root_log_dir}: {e}")
+
+        # Fully resolve the path before returning (including following symlinks).
+        return filepath.resolve()
+
+    async def StreamLog(self, request, context):
+        """
+        Streams the log in real time starting from `request.lines` number of lines from
+        the end of the file if `request.keep_alive == True`. Else, it terminates the
+        stream once there are no more bytes to read from the log file.
+
+        Part of `LogService` gRPC.
+
+        NOTE: These RPCs are used by state_head.py, not log_head.py
+        """
+        # NOTE: If the client side connection is closed, this handler will
+        # be automatically terminated.
+        lines = request.lines if request.lines else 1000
+
+        try:
+            filepath = self._resolve_filename(
+                Path(self._dashboard_agent.log_dir), request.log_file_name
+            )
+        except FileNotFoundError as e:
+            await context.send_initial_metadata([[log_consts.LOG_GRPC_ERROR, str(e)]])
+        else:
+            with open(filepath, "rb") as f:
+                await context.send_initial_metadata([])
+
+                # Default stream entire file
+                start_offset = (
+                    request.start_offset if request.HasField("start_offset") else 0
+                )
+                end_offset = (
+                    request.end_offset
+                    if request.HasField("end_offset")
+                    else find_end_offset_file(f)
+                )
+
+                if lines != -1:
+                    # If specified tail line number, cap the start offset
+                    # with lines from the current end offset
+                    start_offset = max(
+                        find_start_offset_last_n_lines_from_offset(
+                            f, offset=end_offset, n=lines
+                        ),
+                        start_offset,
+                    )
+
+                # If keep alive: following the log every 'interval'
+                keep_alive_interval_sec = -1
+                if request.keep_alive:
+                    keep_alive_interval_sec = (
+                        request.interval
+                        if request.interval
+                        else DEFAULT_KEEP_ALIVE_INTERVAL_SEC
+                    )
+
+                    # When following (keep_alive), it will read beyond the end
+                    end_offset = -1
+
+                logger.info(
+                    f"Tailing logs from {start_offset} to {end_offset} for "
+                    f"lines={lines}, with keep_alive={keep_alive_interval_sec}"
+                )
+
+                # Read and send the file data in chunk
+                async for chunk_res in _stream_log_in_chunk(
+                    context=context,
+                    file=f,
+                    start_offset=start_offset,
+                    end_offset=end_offset,
+                    keep_alive_interval_sec=keep_alive_interval_sec,
+                ):
+                    yield chunk_res

python/ray/dashboard/modules/log/log_consts.py

@@ -0,0 +1,8 @@
+MIME_TYPES = {
+    "text/plain": [".err", ".out", ".log"],
+}
+
+LOG_GRPC_ERROR = "log_grpc_status"
+
+# 10 seconds
+GRPC_TIMEOUT = 10

python/ray/dashboard/modules/log/log_manager.py

@@ -0,0 +1,478 @@
+import logging
+import re
+from collections import defaultdict
+from typing import AsyncIterable, Awaitable, Callable, Dict, List, Optional, Tuple
+
+from ray import ActorID, NodeID, WorkerID
+from ray._common.pydantic_compat import BaseModel
+from ray.core.generated.gcs_pb2 import ActorTableData
+from ray.dashboard.modules.job.common import JOB_LOGS_PATH_TEMPLATE
+from ray.util.state.common import (
+    DEFAULT_RPC_TIMEOUT,
+    GetLogOptions,
+    protobuf_to_task_state_dict,
+)
+from ray.util.state.state_manager import StateDataSourceClient
+
+if BaseModel is None:
+    raise ModuleNotFoundError("Please install pydantic via `pip install pydantic`.")
+
+
+logger = logging.getLogger(__name__)
+
+WORKER_LOG_PATTERN = re.compile(r".*worker-([0-9a-f]+)-([0-9a-f]+)-(\d+).(out|err)")
+
+
+class ResolvedStreamFileInfo(BaseModel):
+    # The node id where the log file is located.
+    node_id: str
+
+    # The log file path name. Could be a relative path relative to ray's logging folder,
+    # or an absolute path.
+    filename: str
+
+    # Start offset in the log file to stream from. None to indicate beginning of
+    # the file, or determined by last tail lines.
+    start_offset: Optional[int] = None
+
+    # End offset in the log file to stream from. None to indicate the end of the file.
+    end_offset: Optional[int] = None
+
+
+class LogsManager:
+    def __init__(self, data_source_client: StateDataSourceClient):
+        self.client = data_source_client
+
+    @property
+    def data_source_client(self) -> StateDataSourceClient:
+        return self.client
+
+    async def ip_to_node_id(self, node_ip: Optional[str]) -> Optional[str]:
+        """Resolve the node id in hex from a given node ip.
+
+        Args:
+            node_ip: The node ip.
+
+        Returns:
+            node_id if there's a node id that matches the given node ip and is alive.
+            None otherwise.
+        """
+        return await self.client.ip_to_node_id(node_ip)
+
+    async def list_logs(
+        self, node_id: str, timeout: int, glob_filter: str = "*"
+    ) -> Dict[str, List[str]]:
+        """Return a list of log files on a given node id filtered by the glob.
+
+        Args:
+            node_id: The node id where log files present.
+            timeout: The timeout of the API.
+            glob_filter: The glob filter to filter out log files.
+
+        Returns:
+            Dictionary of {component_name -> list of log files}
+
+        Raises:
+            ValueError: If a source is unresponsive.
+        """
+        reply = await self.client.list_logs(node_id, glob_filter, timeout=timeout)
+        return self._categorize_log_files(reply.log_files)
+
+    async def stream_logs(
+        self,
+        options: GetLogOptions,
+        get_actor_fn: Callable[[ActorID], Awaitable[Optional[ActorTableData]]],
+    ) -> AsyncIterable[bytes]:
+        """Generate a stream of logs in bytes.
+
+        Args:
+            options: The option for streaming logs.
+            get_actor_fn: Callable used to resolve actor metadata when the
+                request targets an actor's logs.
+
+        Yields:
+            bytes: Successive chunks of log content streamed from the agent.
+        """
+        node_id = options.node_id
+        if node_id is None:
+            node_id = await self.ip_to_node_id(options.node_ip)
+
+        res = await self.resolve_filename(
+            node_id=node_id,
+            log_filename=options.filename,
+            actor_id=options.actor_id,
+            task_id=options.task_id,
+            attempt_number=options.attempt_number,
+            pid=options.pid,
+            get_actor_fn=get_actor_fn,
+            timeout=options.timeout,
+            suffix=options.suffix,
+            submission_id=options.submission_id,
+        )
+
+        keep_alive = options.media_type == "stream"
+        stream = await self.client.stream_log(
+            node_id=res.node_id,
+            log_file_name=res.filename,
+            keep_alive=keep_alive,
+            lines=options.lines,
+            interval=options.interval,
+            # If we keepalive logs connection, we shouldn't have timeout
+            # otherwise the stream will be terminated forcefully
+            # after the deadline is expired.
+            timeout=options.timeout if not keep_alive else None,
+            start_offset=res.start_offset,
+            end_offset=res.end_offset,
+        )
+
+        async for streamed_log in stream:
+            yield streamed_log.data
+
+    async def _resolve_job_filename(self, sub_job_id: str) -> Tuple[str, str]:
+        """Return the log file name and node id for a given job submission id.
+
+        Args:
+            sub_job_id: The job submission id.
+
+        Returns:
+            The log file name and node id.
+        """
+        job_infos = await self.client.get_job_info(timeout=DEFAULT_RPC_TIMEOUT)
+        target_job = None
+        for job_info in job_infos:
+            if job_info.submission_id == sub_job_id:
+                target_job = job_info
+                break
+        if target_job is None:
+            logger.info(f"Submission job ID {sub_job_id} not found.")
+            return None, None
+
+        node_id = job_info.driver_node_id
+        if node_id is None:
+            raise ValueError(
+                f"Job {sub_job_id} has no driver node id info. "
+                "This is likely a bug. Please file an issue."
+            )
+
+        log_filename = JOB_LOGS_PATH_TEMPLATE.format(submission_id=sub_job_id)
+        return node_id, log_filename
+
+    async def _resolve_worker_file(
+        self,
+        node_id_hex: str,
+        worker_id_hex: Optional[str],
+        pid: Optional[int],
+        suffix: str,
+        timeout: int,
+    ) -> Optional[str]:
+        """Resolve worker log file."""
+        if worker_id_hex is not None and pid is not None:
+            raise ValueError(
+                f"Only one of worker id({worker_id_hex}) or pid({pid}) should be"
+                "provided."
+            )
+
+        if worker_id_hex is not None:
+            log_files = await self.list_logs(
+                node_id_hex, timeout, glob_filter=f"*{worker_id_hex}*{suffix}"
+            )
+        else:
+            log_files = await self.list_logs(
+                node_id_hex, timeout, glob_filter=f"*{pid}*{suffix}"
+            )
+
+        # Find matching worker logs.
+        for filename in [*log_files["worker_out"], *log_files["worker_err"]]:
+            # Worker logs look like worker-[worker_id]-[job_id]-[pid].out
+            if worker_id_hex is not None:
+                worker_id_from_filename = WORKER_LOG_PATTERN.match(filename).group(1)
+                if worker_id_from_filename == worker_id_hex:
+                    return filename
+            else:
+                worker_pid_from_filename = int(
+                    WORKER_LOG_PATTERN.match(filename).group(3)
+                )
+                if worker_pid_from_filename == pid:
+                    return filename
+        return None
+
+    async def _resolve_actor_filename(
+        self,
+        actor_id: ActorID,
+        get_actor_fn: Callable[[ActorID], Awaitable[Optional[ActorTableData]]],
+        suffix: str,
+        timeout: int,
+    ):
+        """Resolve actor log file.
+
+        Args:
+            actor_id: The actor id.
+            get_actor_fn: The function to get actor information.
+            suffix: The suffix of the log file.
+            timeout: Timeout in seconds.
+
+        Returns:
+            The log file name and node id.
+
+        Raises:
+            ValueError: If actor data is not found or get_actor_fn is not provided.
+        """
+        if get_actor_fn is None:
+            raise ValueError("get_actor_fn needs to be specified for actor_id")
+
+        actor_data = await get_actor_fn(actor_id)
+        if actor_data is None:
+            raise ValueError(f"Actor ID {actor_id} not found.")
+        # TODO(sang): Only the latest worker id can be obtained from
+        # actor information now. That means, if actors are restarted,
+        # there's no way for us to get the past worker ids.
+        worker_id_binary = actor_data.address.worker_id
+        if not worker_id_binary:
+            raise ValueError(
+                f"Worker ID for Actor ID {actor_id} not found. "
+                "Actor is not scheduled yet."
+            )
+        worker_id = WorkerID(worker_id_binary)
+        node_id_binary = actor_data.address.node_id
+        if not node_id_binary:
+            raise ValueError(
+                f"Node ID for Actor ID {actor_id} not found. "
+                "Actor is not scheduled yet."
+            )
+        node_id = NodeID(node_id_binary)
+        log_filename = await self._resolve_worker_file(
+            node_id_hex=node_id.hex(),
+            worker_id_hex=worker_id.hex(),
+            pid=None,
+            suffix=suffix,
+            timeout=timeout,
+        )
+        return node_id.hex(), log_filename
+
+    async def _resolve_task_filename(
+        self, task_id: str, attempt_number: int, suffix: str, timeout: int
+    ):
+        """Resolve log file for a task.
+
+        Args:
+            task_id: The task id.
+            attempt_number: The attempt number.
+            suffix: The suffix of the log file, e.g. out or err.
+            timeout: Timeout in seconds.
+
+        Returns:
+            The log file name, node id, the start and end offsets of the
+            corresponding task log in the file.
+
+        Raises:
+            FileNotFoundError: If the log file is not found.
+            ValueError: If the suffix is not out or err.
+        """
+        log_filename = None
+        node_id = None
+        start_offset = None
+        end_offset = None
+
+        if suffix not in ["out", "err"]:
+            raise ValueError(f"Suffix {suffix} is not supported.")
+
+        reply = await self.client.get_all_task_info(
+            filters=[("task_id", "=", task_id)], timeout=timeout
+        )
+        # Check if the task is found.
+        if len(reply.events_by_task) == 0:
+            raise FileNotFoundError(
+                f"Could not find log file for task: {task_id}"
+                f" (attempt {attempt_number}) with suffix: {suffix}"
+            )
+        task_event = None
+        for t in reply.events_by_task:
+            if t.attempt_number == attempt_number:
+                task_event = t
+                break
+
+        if task_event is None:
+            raise FileNotFoundError(
+                "Could not find log file for task attempt:"
+                f"{task_id}({attempt_number})"
+            )
+        # Get the worker id and node id.
+        task = protobuf_to_task_state_dict(task_event)
+
+        worker_id = task.get("worker_id", None)
+        node_id = task.get("node_id", None)
+        log_info = task.get("task_log_info", None)
+        actor_id = task.get("actor_id", None)
+
+        if node_id is None:
+            raise FileNotFoundError(
+                "Could not find log file for task attempt."
+                f"{task_id}({attempt_number}) due to missing node info."
+            )
+
+        if log_info is None and actor_id is not None:
+            # This is a concurrent actor task. The logs will be interleaved.
+            # So we return the log file of the actor instead.
+            raise FileNotFoundError(
+                f"For actor task, please query actor log for "
+                f"actor({actor_id}): e.g. ray logs actor --id {actor_id} . Or "
+                "set RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 in actor's runtime env "
+                "or when starting the cluster. Recording actor task's log could be "
+                "expensive, so Ray turns it off by default."
+            )
+        elif log_info is None:
+            raise FileNotFoundError(
+                "Could not find log file for task attempt:"
+                f"{task_id}({attempt_number})."
+                f"Worker id = {worker_id}, node id = {node_id},"
+                f"log_info = {log_info}"
+            )
+
+        filename_key = "stdout_file" if suffix == "out" else "stderr_file"
+        log_filename = log_info.get(filename_key, None)
+        if log_filename is None:
+            raise FileNotFoundError(
+                f"Missing log filename info in {log_info} for task {task_id},"
+                f"attempt {attempt_number}"
+            )
+
+        start_offset = log_info.get(f"std{suffix}_start", None)
+        end_offset = log_info.get(f"std{suffix}_end", None)
+
+        return node_id, log_filename, start_offset, end_offset
+
+    async def resolve_filename(
+        self,
+        *,
+        node_id: Optional[str] = None,
+        log_filename: Optional[str] = None,
+        actor_id: Optional[str] = None,
+        task_id: Optional[str] = None,
+        attempt_number: Optional[int] = None,
+        pid: Optional[str] = None,
+        get_actor_fn: Optional[
+            Callable[[ActorID], Awaitable[Optional[ActorTableData]]]
+        ] = None,
+        timeout: int = DEFAULT_RPC_TIMEOUT,
+        suffix: str = "out",
+        submission_id: Optional[str] = None,
+    ) -> ResolvedStreamFileInfo:
+        """Return the file name given all options.
+
+        Args:
+            node_id: The node's id from which logs are resolved.
+            log_filename: Filename of the log file.
+            actor_id: Id of the actor that generates the log file.
+            task_id: Id of the task that generates the log file.
+            attempt_number: The attempt number of the task. Used with
+                ``task_id`` to disambiguate retries.
+            pid: Id of the worker process that generates the log file.
+            get_actor_fn: Callback to get the actor's data by id.
+            timeout: Timeout for the gRPC to listing logs on the node
+                specified by `node_id`.
+            suffix: Log suffix if no `log_filename` is provided, when
+                resolving by other ids'. Default to "out".
+            submission_id: The submission id for a submission job.
+
+        Returns:
+            A ``ResolvedStreamFileInfo`` describing the resolved node id,
+            filename, and (optional) byte offsets to stream.
+        """
+        start_offset = None
+        end_offset = None
+        if suffix not in ["out", "err"]:
+            raise ValueError(f"Suffix {suffix} is not supported. ")
+
+        # TODO(rickyx): We should make sure we do some sort of checking on the log
+        # filename
+        if actor_id:
+            node_id, log_filename = await self._resolve_actor_filename(
+                ActorID.from_hex(actor_id), get_actor_fn, suffix, timeout
+            )
+
+        elif task_id:
+            (
+                node_id,
+                log_filename,
+                start_offset,
+                end_offset,
+            ) = await self._resolve_task_filename(
+                task_id, attempt_number, suffix, timeout
+            )
+
+        elif submission_id:
+            node_id, log_filename = await self._resolve_job_filename(submission_id)
+
+        elif pid:
+            if node_id is None:
+                raise ValueError(
+                    "Node id needs to be specified for resolving"
+                    f" filenames of pid {pid}"
+                )
+            log_filename = await self._resolve_worker_file(
+                node_id_hex=node_id,
+                worker_id_hex=None,
+                pid=pid,
+                suffix=suffix,
+                timeout=timeout,
+            )
+
+        if log_filename is None:
+            raise FileNotFoundError(
+                "Could not find a log file. Please make sure the given "
+                "option exists in the cluster.\n"
+                f"\tnode_id: {node_id}\n"
+                f"\tfilename: {log_filename}\n"
+                f"\tactor_id: {actor_id}\n"
+                f"\ttask_id: {task_id}\n"
+                f"\tpid: {pid}\n"
+                f"\tsuffix: {suffix}\n"
+                f"\tsubmission_id: {submission_id}\n"
+                f"\tattempt_number: {attempt_number}\n"
+            )
+
+        res = ResolvedStreamFileInfo(
+            node_id=node_id,
+            filename=log_filename,
+            start_offset=start_offset,
+            end_offset=end_offset,
+        )
+        logger.info(f"Resolved log file: {res}")
+        return res
+
+    def _categorize_log_files(self, log_files: List[str]) -> Dict[str, List[str]]:
+        """Categorize the given log files after filterieng them out using a given glob.
+
+        Args:
+            log_files: Filenames returned from a ``list_logs`` query, already
+                filtered by the caller's glob.
+
+        Returns:
+            Dictionary of {component_name -> list of log files}
+        """
+        result = defaultdict(list)
+        for log_file in log_files:
+            if "worker" in log_file and (log_file.endswith(".out")):
+                result["worker_out"].append(log_file)
+            elif "worker" in log_file and (log_file.endswith(".err")):
+                result["worker_err"].append(log_file)
+            elif "core-worker" in log_file and log_file.endswith(".log"):
+                result["core_worker"].append(log_file)
+            elif "core-driver" in log_file and log_file.endswith(".log"):
+                result["driver"].append(log_file)
+            elif "raylet." in log_file:
+                result["raylet"].append(log_file)
+            elif "gcs_server." in log_file:
+                result["gcs_server"].append(log_file)
+            elif "log_monitor" in log_file:
+                result["internal"].append(log_file)
+            elif "monitor" in log_file:
+                result["autoscaler"].append(log_file)
+            elif "agent." in log_file:
+                result["agent"].append(log_file)
+            elif "dashboard." in log_file:
+                result["dashboard"].append(log_file)
+            else:
+                result["internal"].append(log_file)
+
+        return result

python/ray/dashboard/modules/log/log_utils.py

@@ -0,0 +1,9 @@
+import mimetypes
+
+import ray.dashboard.modules.log.log_consts as log_consts
+
+
+def register_mimetypes():
+    for _type, extensions in log_consts.MIME_TYPES.items():
+        for ext in extensions:
+            mimetypes.add_type(_type, ext)

python/ray/dashboard/modules/metrics/dashboards/common.py

@@ -0,0 +1,615 @@
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from ray.util.annotations import DeveloperAPI
+
+
+@DeveloperAPI
+@dataclass
+class GridPos:
+    x: int
+    y: int
+    w: int
+    h: int
+
+
+GRAPH_TARGET_TEMPLATE = {
+    "exemplar": True,
+    "expr": "0",
+    "interval": "",
+    "legendFormat": "",
+    "queryType": "randomWalk",
+    "refId": "A",
+}
+
+HEATMAP_TARGET_TEMPLATE = {
+    "format": "heatmap",
+    "fullMetaSearch": False,
+    "includeNullMetadata": True,
+    "instant": False,
+    "range": True,
+    "useBackend": False,
+}
+
+HISTOGRAM_BAR_CHART_TARGET_TEMPLATE = {
+    "exemplar": True,
+    "format": "heatmap",
+    "fullMetaSearch": False,
+    "includeNullMetadata": True,
+    "instant": True,
+    "range": False,
+    "useBackend": False,
+}
+
+
+@DeveloperAPI
+class TargetTemplate(Enum):
+    GRAPH = GRAPH_TARGET_TEMPLATE
+    HEATMAP = HEATMAP_TARGET_TEMPLATE
+    HISTOGRAM_BAR_CHART = HISTOGRAM_BAR_CHART_TARGET_TEMPLATE
+
+
+@DeveloperAPI
+@dataclass
+class Target:
+    """Defines a Grafana target (time-series query) within a panel.
+
+    A panel will have one or more targets. By default, all targets are rendered as
+    stacked area charts, with the exception of legend="MAX", which is rendered as
+    a blue dotted line. Any legend="FINISHED|FAILED|DEAD|REMOVED" series will also be
+    rendered hidden by default.
+
+    Attributes:
+        expr: The prometheus query to evaluate.
+        legend: The legend string to format for each time-series.
+    """
+
+    expr: str
+    legend: str
+    template: Optional[TargetTemplate] = TargetTemplate.GRAPH
+
+
+HEATMAP_TEMPLATE = {
+    "datasource": r"${datasource}",
+    "description": "<Description>",
+    "fieldConfig": {"defaults": {}, "overrides": []},
+    "id": 12,
+    "options": {
+        "calculate": False,
+        "cellGap": 1,
+        "cellValues": {"unit": "none"},
+        "color": {
+            "exponent": 0.5,
+            "fill": "dark-orange",
+            "min": 0,
+            "mode": "scheme",
+            "reverse": False,
+            "scale": "exponential",
+            "scheme": "Spectral",
+            "steps": 64,
+        },
+        "exemplars": {"color": "rgba(255,0,255,0.7)"},
+        "filterValues": {"le": 1e-9},
+        "legend": {"show": True},
+        "rowsFrame": {"layout": "auto", "value": "Value"},
+        "tooltip": {"mode": "single", "showColorScale": False, "yHistogram": True},
+        "yAxis": {"axisPlacement": "left", "reverse": False, "unit": "none"},
+    },
+    "pluginVersion": "11.2.0",
+    "targets": [],
+    "title": "<Title>",
+    "type": "heatmap",
+    "yaxes": [
+        {
+            "$$hashKey": "object:628",
+            "format": "units",
+            "label": "",
+            "logBase": 1,
+            "max": None,
+            "min": "0",
+            "show": True,
+        },
+        {
+            "$$hashKey": "object:629",
+            "format": "short",
+            "label": None,
+            "logBase": 1,
+            "max": None,
+            "min": None,
+            "show": True,
+        },
+    ],
+}
+
+GRAPH_PANEL_TEMPLATE = {
+    "aliasColors": {},
+    "bars": False,
+    "dashLength": 10,
+    "dashes": False,
+    "datasource": r"${datasource}",
+    "description": "<Description>",
+    "fieldConfig": {"defaults": {}, "overrides": []},
+    # Setting height and width is important here to ensure the default panel has some size to it.
+    "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
+    "fill": 10,
+    "fillGradient": 0,
+    "hiddenSeries": False,
+    "id": 26,
+    "legend": {
+        "alignAsTable": True,
+        "avg": True,
+        "current": True,
+        "hideEmpty": False,
+        "hideZero": True,
+        "max": True,
+        "min": False,
+        "rightSide": False,
+        "show": True,
+        "sort": "current",
+        "sortDesc": True,
+        "total": False,
+        "values": True,
+    },
+    "lines": True,
+    "linewidth": 1,
+    "nullPointMode": None,
+    "options": {"alertThreshold": True},
+    "percentage": False,
+    "pluginVersion": "7.5.17",
+    "pointradius": 2,
+    "points": False,
+    "renderer": "flot",
+    # These series overrides are necessary to make the "MAX" and "MAX + PENDING" dotted lines
+    # instead of stacked filled areas.
+    "seriesOverrides": [
+        {
+            "$$hashKey": "object:2987",
+            "alias": "MAX",
+            "dashes": True,
+            "color": "#1F60C4",
+            "fill": 0,
+            "stack": False,
+        },
+        {
+            "$$hashKey": "object:78",
+            "alias": "/FINISHED|FAILED|DEAD|REMOVED|Failed Nodes:/",
+            "hiddenSeries": True,
+        },
+        {
+            "$$hashKey": "object:2987",
+            "alias": "MAX + PENDING",
+            "dashes": True,
+            "color": "#777777",
+            "fill": 0,
+            "stack": False,
+        },
+        {
+            "alias": "/Container/",
+            "hiddenSeries": True,
+        },
+        {
+            "alias": "Container MAX",
+            "dashes": True,
+            "color": "#73BF69",
+            "fill": 0,
+            "stack": False,
+            "hiddenSeries": True,
+        },
+    ],
+    "spaceLength": 10,
+    "stack": True,
+    "steppedLine": False,
+    "targets": [],
+    "thresholds": [],
+    "timeFrom": None,
+    "timeRegions": [],
+    "timeShift": None,
+    "title": "<Title>",
+    "tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
+    "type": "graph",
+    "xaxis": {
+        "buckets": None,
+        "mode": "time",
+        "name": None,
+        "show": True,
+        "values": [],
+    },
+    "yaxes": [
+        {
+            "$$hashKey": "object:628",
+            "format": "units",
+            "label": "",
+            "logBase": 1,
+            "max": None,
+            "min": "0",
+            "show": True,
+        },
+        {
+            "$$hashKey": "object:629",
+            "format": "short",
+            "label": None,
+            "logBase": 1,
+            "max": None,
+            "min": None,
+            "show": True,
+        },
+    ],
+    "yaxis": {"align": False, "alignLevel": None},
+}
+
+STAT_PANEL_TEMPLATE = {
+    "datasource": r"${datasource}",
+    "fieldConfig": {
+        "defaults": {
+            "color": {"mode": "thresholds"},
+            "mappings": [],
+            "min": 0,
+            "thresholds": {
+                "mode": "percentage",
+                "steps": [
+                    {"color": "super-light-yellow", "value": None},
+                    {"color": "super-light-green", "value": 50},
+                    {"color": "green", "value": 100},
+                ],
+            },
+            "unit": "short",
+        },
+        "overrides": [],
+    },
+    "id": 78,
+    "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
+        "text": {},
+        "textMode": "auto",
+    },
+    "pluginVersion": "7.5.17",
+    "targets": [],
+    "timeFrom": None,
+    "timeShift": None,
+    "title": "<Title>",
+    "type": "stat",
+    "yaxes": [
+        {
+            "$$hashKey": "object:628",
+            "format": "Tokens",
+            "label": "",
+            "logBase": 1,
+            "max": None,
+            "min": "0",
+            "show": True,
+        },
+        {
+            "$$hashKey": "object:629",
+            "format": "short",
+            "label": None,
+            "logBase": 1,
+            "max": None,
+            "min": None,
+            "show": True,
+        },
+    ],
+}
+
+
+GAUGE_PANEL_TEMPLATE = {
+    "datasource": r"${datasource}",
+    "fieldConfig": {
+        "defaults": {
+            "color": {"mode": "continuous-YlBl"},
+            "mappings": [],
+            "thresholds": {
+                "mode": "percentage",
+                "steps": [{"color": "rgb(230, 230, 230)", "value": None}],
+            },
+            "unit": "short",
+        },
+        "overrides": [],
+    },
+    "id": 10,
+    "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
+        "showThresholdLabels": False,
+        "showThresholdMarkers": False,
+        "text": {"titleSize": 12},
+    },
+    "pluginVersion": "7.5.17",
+    "targets": [],
+    "title": "<Title>",
+    "type": "gauge",
+    "yaxes": [
+        {
+            "$$hashKey": "object:628",
+            "format": "Tokens",
+            "label": "",
+            "logBase": 1,
+            "max": None,
+            "min": "0",
+            "show": True,
+        },
+        {
+            "$$hashKey": "object:629",
+            "format": "short",
+            "label": None,
+            "logBase": 1,
+            "max": None,
+            "min": None,
+            "show": True,
+        },
+    ],
+}
+
+PIE_CHART_TEMPLATE = {
+    "datasource": r"${datasource}",
+    "description": "<Description>",
+    "fieldConfig": {"defaults": {}, "overrides": []},
+    "id": 26,
+    "options": {
+        "displayLabels": [],
+        "legend": {
+            "displayMode": "table",
+            "placement": "right",
+            "values": ["percent", "value"],
+        },
+        "pieType": "pie",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
+        "text": {},
+    },
+    "pluginVersion": "7.5.17",
+    "targets": [],
+    "timeFrom": None,
+    "timeShift": None,
+    "title": "<Title>",
+    "type": "piechart",
+    "yaxes": [
+        {
+            "$$hashKey": "object:628",
+            "format": "units",
+            "label": "",
+            "logBase": 1,
+            "max": None,
+            "min": "0",
+            "show": True,
+        },
+        {
+            "$$hashKey": "object:629",
+            "format": "short",
+            "label": None,
+            "logBase": 1,
+            "max": None,
+            "min": None,
+            "show": True,
+        },
+    ],
+}
+
+BAR_CHART_PANEL_TEMPLATE = {
+    "aliasColors": {},
+    "dashLength": 10,
+    "dashes": False,
+    "datasource": r"${datasource}",
+    "description": "<Description>",
+    "fieldConfig": {"defaults": {}, "overrides": []},
+    # Setting height and width is important here to ensure the default panel has some size to it.
+    "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
+    "hiddenSeries": False,
+    "id": 26,
+    "legend": {
+        "alignAsTable": True,
+        "avg": False,
+        "current": True,
+        "hideEmpty": False,
+        "hideZero": True,
+        "max": False,
+        "min": False,
+        "rightSide": False,
+        "show": False,
+        "sort": "current",
+        "sortDesc": True,
+        "total": False,
+        "values": True,
+    },
+    "lines": False,
+    "linewidth": 1,
+    "bars": True,
+    "nullPointMode": None,
+    "options": {
+        "alertThreshold": True,
+        "legend": {
+            "showLegend": False,
+            "displayMode": "table",
+            "placement": "bottom",
+        },
+    },
+    "percentage": False,
+    "pluginVersion": "7.5.17",
+    "pointradius": 2,
+    "points": False,
+    "renderer": "flot",
+    "spaceLength": 10,
+    "stack": True,
+    "steppedLine": False,
+    "targets": [],
+    "thresholds": [],
+    "timeFrom": None,
+    "timeRegions": [],
+    "timeShift": None,
+    "title": "<Title>",
+    "tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
+    "type": "graph",
+    "xaxis": {
+        "buckets": None,
+        "mode": "series",
+        "name": None,
+        "show": True,
+        "values": [
+            "total",
+        ],
+    },
+    "yaxes": [
+        {
+            "$$hashKey": "object:628",
+            "format": "units",
+            "label": "",
+            "logBase": 1,
+            "max": None,
+            "min": "0",
+            "show": True,
+        },
+        {
+            "$$hashKey": "object:629",
+            "format": "short",
+            "label": None,
+            "logBase": 1,
+            "max": None,
+            "min": None,
+            "show": True,
+        },
+    ],
+    "yaxis": {"align": False, "alignLevel": None},
+}
+
+TABLE_PANEL_TEMPLATE = {
+    "datasource": r"${datasource}",
+    "description": "<Description>",
+    "fieldConfig": {
+        "defaults": {
+            "custom": {
+                "align": "auto",
+                "displayMode": "auto",
+            },
+            "mappings": [],
+        },
+        "overrides": [],
+    },
+    "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
+    "id": 26,
+    "options": {
+        "showHeader": True,
+        "footer": {
+            "show": False,
+            "reducer": ["sum"],
+            "fields": "",
+        },
+    },
+    "pluginVersion": "7.5.17",
+    "targets": [],
+    "title": "<Title>",
+    "type": "table",
+    "transformations": [{"id": "organize", "options": {}}],
+}
+
+
+@DeveloperAPI
+class PanelTemplate(Enum):
+    GRAPH = GRAPH_PANEL_TEMPLATE
+    HEATMAP = HEATMAP_TEMPLATE
+    PIE_CHART = PIE_CHART_TEMPLATE
+    STAT = STAT_PANEL_TEMPLATE
+    GAUGE = GAUGE_PANEL_TEMPLATE
+    BAR_CHART = BAR_CHART_PANEL_TEMPLATE
+    TABLE = TABLE_PANEL_TEMPLATE
+
+
+@DeveloperAPI
+@dataclass
+class Panel:
+    """Defines a Grafana panel (graph) for the Ray dashboard page.
+
+    A panel contains one or more targets (time-series queries).
+
+    Attributes:
+        title: Short name of the graph. Note: please keep this in sync with the title
+            definitions in Metrics.tsx.
+        description: Long form description of the graph.
+        id: Integer id used to reference the graph from Metrics.tsx.
+        unit: The unit to display on the y-axis of the graph.
+        targets: List of query targets.
+        fill: Whether or not the graph will be filled by a color.
+        stack: Whether or not the lines in the graph will be stacked.
+        linewidth: Width of the lines in the graph.
+        grid_pos: Grid position of the panel.
+        template: The panel template to use.
+        hideXAxis: Whether to hide the x-axis.
+        thresholds: Custom threshold configuration for stat/gauge panels.
+            Example: [
+                {"color": "green", "value": None},
+                {"color": "yellow", "value": 70},
+                {"color": "red", "value": 90}
+            ]
+        value_mappings: Value mappings for displaying text instead of numbers.
+            Used for status panels.
+        color_mode: Color mode for stat panels ("value", "background", "none").
+        legend_mode: Legend display mode ("list", "table", "hidden").
+        min_val: Minimum value for gauge/graph y-axis.
+        max_val: Maximum value for gauge/graph y-axis.
+        reduce_calc: Reduce calculation method for stat panels (default: "lastNotNull").
+        heatmap_color_scheme: Color scheme for heatmap panels (e.g., "Spectral", "RdYlGn").
+        heatmap_color_reverse: Whether to reverse the heatmap color scheme.
+        heatmap_yaxis_label: Y-axis label for heatmap panels.
+    """
+
+    title: str
+    description: str
+    id: int
+    unit: str
+    targets: List[Target]
+    fill: int = 10
+    stack: bool = True
+    linewidth: int = 1
+    grid_pos: Optional[GridPos] = None
+    template: Optional[PanelTemplate] = PanelTemplate.GRAPH
+    hideXAxis: bool = False
+    thresholds: Optional[List[Dict[str, Any]]] = None
+    value_mappings: Optional[List[Dict[str, Any]]] = None
+    color_mode: Optional[str] = None
+    legend_mode: Optional[str] = None
+    min_val: Optional[float] = None
+    max_val: Optional[float] = None
+    reduce_calc: Optional[str] = None
+    heatmap_color_scheme: Optional[str] = None
+    heatmap_color_reverse: Optional[bool] = None
+    heatmap_yaxis_label: Optional[str] = None
+
+
+@DeveloperAPI
+@dataclass
+class Row:
+    """Defines a Grafana row that can contain multiple panels.
+
+    Attributes:
+        title: The title of the row
+        panels: List of panels contained in this row
+        collapsed: Whether the row should be collapsed by default
+    """
+
+    title: str
+    id: int
+    panels: List[Panel]
+    collapsed: bool = False
+
+
+@DeveloperAPI
+@dataclass
+class DashboardConfig:
+    # This dashboard name is an internal key used to determine which env vars
+    # to check for customization
+    name: str
+    # The uid of the dashboard json if not overridden by a user
+    default_uid: str
+    # The global filters applied to all graphs in this dashboard. Users can
+    # add additional global_filters on top of this.
+    standard_global_filters: List[str]
+    base_json_file_name: str
+    # Panels can be specified in `panels`, or nested within `rows`.
+    # If both are specified, panels will be rendered before rows.
+    panels: List[Panel] = field(default_factory=list)
+    rows: List[Row] = field(default_factory=list)
+
+    def __post_init__(self):
+        if not self.panels and not self.rows:
+            raise ValueError("At least one of panels or rows must be specified")

python/ray/dashboard/modules/metrics/dashboards/data_grafana_dashboard_base.json

@@ -0,0 +1,183 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": "-- Grafana --",
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "gnetId": null,
+    "graphTooltip": 1,
+    "iteration": 1667344411089,
+    "links": [],
+    "panels": [],
+    "refresh": false,
+    "schemaVersion": 27,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "current": {
+            "selected": false
+          },
+          "description": "Filter queries of a specific Prometheus type.",
+          "hide": 2,
+          "includeAll": false,
+          "multi": false,
+          "name": "datasource",
+          "options": [],
+          "query": "prometheus",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "type": "datasource"
+        },
+        {
+          "allValue": ".+",
+          "current": {
+            "selected": false
+          },
+          "datasource": "${datasource}",
+          "definition": "query_result(count by (SessionName)(last_over_time(ray_data_output_bytes{{{global_filters}}}[$__range])))",
+          "description": "Filter queries to specific ray sessions.",
+          "error": null,
+          "hide": 0,
+          "includeAll": true,
+          "label": null,
+          "multi": false,
+          "name": "SessionName",
+          "options": [],
+          "query": {
+            "query": "query_result(count by (SessionName)(last_over_time(ray_data_output_bytes{{{global_filters}}}[$__range])))",
+            "refId": "StandardVariableQuery"
+          },
+          "refresh": 2,
+          "regex": "{SessionName=\"(?<value>.*)\".*",
+          "skipUrlSync": false,
+          "sort": 2,
+          "tagValuesQuery": "",
+          "tags": [],
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        },
+        {
+            "allValue": ".+",
+            "current": {
+              "selected": true,
+              "text": [
+                "All"
+              ],
+              "value": [
+                "$__all"
+              ]
+            },
+            "datasource": "${datasource}",
+            "definition": "query_result(count by (dataset)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
+            "description": null,
+            "error": null,
+            "hide": 0,
+            "includeAll": true,
+            "label": null,
+            "multi": true,
+            "name": "DatasetID",
+            "options": [],
+            "query": {
+              "query": "query_result(count by (dataset)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
+              "refId": "Prometheus-Dataset-Variable-Query"
+            },
+            "refresh": 2,
+            "regex": "{dataset=\"(?<value>.*)\".*",
+            "skipUrlSync": false,
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+          },
+          {
+            "allValue": ".+",
+            "current": {
+              "selected": true,
+              "text": [
+                "All"
+              ],
+              "value": [
+                "$__all"
+              ]
+            },
+            "datasource": "${datasource}",
+            "definition": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
+            "description": null,
+            "error": null,
+            "hide": 0,
+            "includeAll": true,
+            "label": null,
+            "multi": true,
+            "name": "Operator",
+            "options": [],
+            "query": {
+              "query": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))",
+              "refId": "Prometheus-Dataset-Variable-Query"
+            },
+            "refresh": 2,
+            "regex": "{operator=\"(?<value>.*)\".*",
+            "skipUrlSync": false,
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+          },
+          {
+            "allValue": ".*",
+            "current": {
+              "selected": false
+            },
+            "datasource": "${datasource}",
+            "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+            "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
+            "error": null,
+            "hide": 0,
+            "includeAll": true,
+            "label": null,
+            "multi": false,
+            "name": "Cluster",
+            "options": [],
+            "query": {
+              "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+              "refId": "StandardVariableQuery"
+            },
+            "refresh": 2,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 2,
+            "tagValuesQuery": "",
+            "tags": [],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+          }
+      ]
+    },
+    "rayMeta": ["excludesSystemRoutes"],
+    "time": {
+      "from": "now-30m",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Data Dashboard",
+    "uid": "rayDataDashboard",
+    "version": 1
+  }

python/ray/dashboard/modules/metrics/dashboards/data_llm_dashboard_panels.py

@@ -0,0 +1,325 @@
+# ruff: noqa: E501
+"""Ray Data LLM Dashboard panels for vLLM metrics visualization.
+
+This dashboard provides visibility into vLLM engine metrics when using Ray Data LLM,
+including latency metrics (TTFT, TPOT, E2E), throughput, cache utilization,
+prefix cache hit rate, and scheduler state.
+"""
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    GridPos,
+    Panel,
+    Target,
+)
+
+DATA_LLM_GRAFANA_PANELS = [
+    Panel(
+        id=1,
+        title="vLLM: Token Throughput",
+        description="Number of tokens processed per second",
+        unit="tokens/s",
+        targets=[
+            Target(
+                expr='sum by (model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
+                legend="Prompt Tokens/Sec - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='sum by (model_name, WorkerId) (rate(ray_vllm_generation_tokens_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
+                legend="Generation Tokens/Sec - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 0, 12, 8),
+    ),
+    Panel(
+        id=2,
+        title="vLLM: Time Per Output Token Latency",
+        description="P50, P90, P95, P99, and Mean TPOT latency",
+        unit="s",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P99 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P95 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P90 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P50 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_time_per_output_token_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="Mean - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 0, 12, 8),
+    ),
+    Panel(
+        id=3,
+        title="vLLM: Cache Utilization",
+        description="Percentage of used KV cache blocks by vLLM.",
+        unit="percentunit",
+        targets=[
+            Target(
+                expr='sum by (WorkerId) (ray_vllm_kv_cache_usage_perc{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}})',
+                legend="GPU Cache Usage - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 8, 12, 8),
+    ),
+    Panel(
+        id=4,
+        title="vLLM: Prefix Cache Hit Rate",
+        description="Percentage of prefix cache hits. Higher is better for repeated prefixes.",
+        unit="percent",
+        targets=[
+            Target(
+                expr='max(100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))))',
+                legend="Max Hit Rate",
+            ),
+            Target(
+                expr='min(100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))))',
+                legend="Min Hit Rate",
+            ),
+            Target(
+                expr='100 * (sum by (WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])) / sum by (WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="Hit Rate: worker {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(12, 8, 12, 8),
+    ),
+    Panel(
+        id=5,
+        title="vLLM: Time To First Token Latency",
+        description="P50, P90, P95, P99, and Mean TTFT latency",
+        unit="s",
+        targets=[
+            Target(
+                expr='(sum by(model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="Average - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P50 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P90 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P95 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId)(rate(ray_vllm_time_to_first_token_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P99 - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 16, 12, 8),
+    ),
+    Panel(
+        id=6,
+        title="vLLM: E2E Request Latency",
+        description="End-to-end request latency from arrival to completion.",
+        unit="s",
+        targets=[
+            Target(
+                expr='sum by(model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
+                legend="Average - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P50 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P90 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P95 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_e2e_request_latency_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P99 - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 16, 12, 8),
+    ),
+    Panel(
+        id=7,
+        title="vLLM: Scheduler State",
+        description="Number of requests in RUNNING, WAITING, and SWAPPED state",
+        unit="Requests",
+        targets=[
+            Target(
+                expr='ray_vllm_num_requests_running{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
+                legend="Num Running - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='ray_vllm_num_requests_swapped{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
+                legend="Num Swapped - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='ray_vllm_num_requests_waiting{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}',
+                legend="Num Waiting - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 24, 12, 8),
+    ),
+    Panel(
+        id=8,
+        title="vLLM: Queue Time",
+        description="P50, P90, P95, P99, and Mean time requests spend waiting in the queue.",
+        unit="s",
+        targets=[
+            Target(
+                expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="Mean - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P50 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P90 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.95, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P95 - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_queue_time_seconds_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P99 - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 24, 12, 8),
+    ),
+    Panel(
+        id=9,
+        title="vLLM: Prompt Length",
+        description="Distribution of prompt token lengths.",
+        unit="short",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.5, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P50-{{model_name}}-{{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.90, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P90-{{model_name}}-{{WorkerId}}",
+            ),
+            Target(
+                expr='(sum by(model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))\n/\nsum by(model_name, WorkerId) (rate(ray_vllm_request_prompt_tokens_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="Average-{{model_name}}-{{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(0, 32, 12, 8),
+    ),
+    Panel(
+        id=10,
+        title="vLLM: Generation Length",
+        description="Distribution of generated token lengths.",
+        unit="short",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.50, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P50-{{model_name}}-{{WorkerId}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.90, sum by(le, model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_bucket{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))',
+                legend="P90-{{model_name}}-{{WorkerId}}",
+            ),
+            Target(
+                expr=(
+                    '(sum by(model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))'
+                    "\n/\n"
+                    '(sum by(model_name, WorkerId) (rate(ray_vllm_request_generation_tokens_count{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval])))'
+                ),
+                legend="Average-{{model_name}}-{{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(12, 32, 12, 8),
+    ),
+    Panel(
+        id=11,
+        title="vLLM: Finish Reason",
+        description="Number of finished requests by their finish reason: EOS token or max length reached.",
+        unit="Requests",
+        targets=[
+            Target(
+                expr='sum by(finished_reason, model_name, WorkerId) (increase(ray_vllm_request_success_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
+                legend="{{finished_reason}} - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 40, 12, 8),
+    ),
+    Panel(
+        id=12,
+        title="vLLM: Prefill and Decode Time",
+        description="Time spent in prefill vs decode phases.",
+        unit="s",
+        targets=[
+            Target(
+                expr='sum by(model_name, WorkerId) (rate(ray_vllm_request_prefill_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
+                legend="Prefill - {{model_name}} - {{WorkerId}}",
+            ),
+            Target(
+                expr='sum by(model_name, WorkerId) (rate(ray_vllm_request_decode_time_seconds_sum{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", ReplicaId=""}}[$interval]))',
+                legend="Decode - {{model_name}} - {{WorkerId}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 40, 12, 8),
+    ),
+]
+
+data_llm_dashboard_config = DashboardConfig(
+    name="DATA_LLM",
+    default_uid="rayDataLlmDashboard",
+    panels=DATA_LLM_GRAFANA_PANELS,
+    standard_global_filters=[],
+    base_json_file_name="data_llm_grafana_dashboard_base.json",
+)

python/ray/dashboard/modules/metrics/dashboards/data_llm_grafana_dashboard_base.json

@@ -0,0 +1,144 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "iteration": 1667344411089,
+  "links": [],
+  "panels": [],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false
+        },
+        "description": "Filter queries of a specific Prometheus type.",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "name": "vllm_model_name",
+        "label": "vLLM Model Name",
+        "type": "query",
+        "hide": 0,
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
+        "query": {
+          "query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "name": "workerid",
+        "label": "Worker ID",
+        "type": "query",
+        "hide": 0,
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
+        "query": {
+          "query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "name": "interval",
+        "label": "Interval",
+        "type": "custom",
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "options": [
+          {
+            "selected": true,
+            "text": "30s",
+            "value": "30s"
+          },
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "current": {
+          "selected": true,
+          "text": "5m",
+          "value": "5m"
+        }
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Data LLM Dashboard",
+  "uid": "rayDataLlmDashboard",
+  "version": 1
+}

python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py

@@ -0,0 +1,797 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    Panel,
+    Row,
+    Target,
+)
+
+"""
+Queries for autoscaler resources.
+"""
+# Note: MAX & USED resources are reported from raylet to provide the most up to date information.
+# But MAX + PENDING data is coming from the autoscaler. That said, MAX + PENDING can be
+# more outdated. it is harmless because the actual MAX will catch up with MAX + PENDING
+# eventually.
+MAX_CPUS = 'sum(autoscaler_cluster_resources{{resource="CPU",{global_filters}}})'
+PENDING_CPUS = 'sum(autoscaler_pending_resources{{resource="CPU",{global_filters}}})'
+MAX_GPUS = 'sum(autoscaler_cluster_resources{{resource="GPU",{global_filters}}})'
+PENDING_GPUS = 'sum(autoscaler_pending_resources{{resource="GPU",{global_filters}}})'
+MAX_MEMORY = 'sum(autoscaler_cluster_resources{{resource="memory",{global_filters}}})'
+PENDING_MEMORY = (
+    'sum(autoscaler_pending_resources{{resource="memory",{global_filters}}})'
+)
+
+
+def max_plus_pending(max_resource, pending_resource):
+    return f"({max_resource} or vector(0)) + ({pending_resource} or vector(0))"
+
+
+MAX_PLUS_PENDING_CPUS = max_plus_pending(MAX_CPUS, PENDING_CPUS)
+MAX_PLUS_PENDING_GPUS = max_plus_pending(MAX_GPUS, PENDING_GPUS)
+MAX_PLUS_PENDING_MEMORY = max_plus_pending(MAX_MEMORY, PENDING_MEMORY)
+
+MAX_PERCENTAGE_EXPRESSION = (
+    "100"  # To help draw the max limit line on percentage panels
+)
+
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# IMPORTANT: Please keep this in sync with Metrics.tsx and ray-metrics.rst
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+OVERVIEW_AND_HEALTH_PANELS = [
+    Panel(
+        id=24,
+        title="Node Count",
+        description='Note: not impacted by "Instance" variable.\n\nA total number of active failed, and pending nodes from the cluster. \n\nACTIVE: A node is alive and available.\n\nFAILED: A node is dead and not available. The node is considered dead when the raylet process on the node is terminated. The node will get into the failed state if it cannot be provided (e.g., there\'s no available node from the cloud provider) or failed to setup (e.g., setup_commands have errors). \n\nPending: A node is being started by the Ray cluster launcher. The node is unavailable now because it is being provisioned and initialized.',
+        unit="nodes",
+        targets=[
+            Target(
+                expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Active Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Failed Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Pending Nodes: {{NodeType}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=41,
+        title="Cluster Utilization",
+        description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster.",
+        unit="%",
+        targets=[
+            # CPU
+            Target(
+                expr='avg(ray_node_cpu_utilization{{instance=~"$Instance",{global_filters}}})',
+                legend="CPU (physical)",
+            ),
+            # GPU
+            Target(
+                expr='sum(ray_node_gpus_utilization{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gpus_available{{instance=~"$Instance",{global_filters}}}) or vector(0))',
+                legend="GPU (physical)",
+            ),
+            # Memory
+            Target(
+                expr='sum(ray_node_mem_used_host{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_mem_total_host{{instance=~"$Instance",{global_filters}}})) * 100',
+                legend="Memory (RAM)",
+            ),
+            # GRAM
+            Target(
+                expr='sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 100',
+                legend="GRAM",
+            ),
+            # Object Store
+            Target(
+                expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) * 100',
+                legend="Object Store Memory",
+            ),
+            # Disk
+            Target(
+                expr='sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})) * 100',
+                legend="Disk",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=44,
+        title="Ray OOM Kills (Tasks and Actors)",
+        description="The number of tasks and actors killed by the Ray Out of Memory killer due to high memory pressure. Metrics are broken down by IP and the name. https://docs.ray.io/en/master/ray-core/scheduling/ray-oom-prevention.html. Note: The RayNodeType filter does not work on this graph.",
+        unit="failures",
+        targets=[
+            Target(
+                expr='sum(ray_memory_manager_worker_eviction_total{{instance=~"$Instance", {global_filters}}}) by (Name, instance)',
+                legend="OOM Killed: {{Name}}, {{instance}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=65,
+        title="Unexpected System Level Worker Failures",
+        description="The number of workers (potentially tasks or actors) that disconnected from the raylet unexpectedly. "
+        "This typically indicates the worker process unexpectedly failed due to "
+        "a Ray system error or a kernel kill (e.g. OOM, SIGKILL, Bad exit code). "
+        "Note that this metric only includes OOM kills from the kernel and does not "
+        "include OOM kills from Ray's memory monitor. "
+        "If errors of this type is encountered when the node is under memory pressure, "
+        "The failures are likely OOM kills.",
+        unit="failures",
+        targets=[
+            Target(
+                expr='sum(ray_node_manager_unexpected_worker_failure_total{{instance=~"$Instance", {global_filters}}}) by (Type, Name, instance)',
+                legend="Unexpected worker failure: {{Name}}, {{Type}}, {{instance}}",
+            ),
+        ],
+    ),
+]
+
+RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS = [
+    Panel(
+        id=26,
+        title="All Tasks by State",
+        description="Current count of tasks, grouped by scheduler state (e.g., pending, running, finished).\n\nState: the task state, as described by rpc::TaskStatus proto in common.proto. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr='sum(max_over_time(ray_tasks{{IsRetry="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
+                legend="{{State}}",
+            ),
+            Target(
+                expr='sum(max_over_time(ray_tasks{{IsRetry!="0",State=~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}[14d])) by (State) or clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (State), 0)',
+                legend="{{State}} (retry)",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=35,
+        title="Active Tasks by Name",
+        description="Current count of active tasks (i.e. pending or running; not finished), grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}}",
+            ),
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State!~"FINISHED|FAILED",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}} (retry)",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=38,
+        title="Running Tasks by Name",
+        description="Current count of tasks that are currently executing, grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}}",
+            ),
+            Target(
+                expr='clamp_min(sum(ray_tasks{{IsRetry!="0",State=~"RUNNING*",instance=~"$Instance",{global_filters}}}) by (Name), 0)',
+                legend="{{Name}} (retry)",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=64,
+        title="Running Tasks by Node",
+        description="Current count of tasks that are currently executing, grouped by node.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr='clamp_min(sum(ray_tasks{{State=~"RUNNING*",Source="executor",instance=~"$Instance",{global_filters}}}) by (instance), 0)',
+                legend="{{instance}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=33,
+        title="All Actors by State",
+        description='Note: not impacted by "Instance" variable.\n\nCurrent count of actors, grouped by lifecycle state (e.g., alive, restarting, dead/terminated).\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.',
+        unit="actors",
+        targets=[
+            Target(
+                expr='sum(ray_actors{{Source="gcs",{global_filters}}}) by (State)',
+                legend="{{State}}",
+            )
+        ],
+    ),
+    Panel(
+        id=42,
+        title="Alive Actors by State",
+        description="Current count of alive actors (i.e. not dead/terminated), grouped by state.\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.",
+        unit="actors",
+        targets=[
+            Target(
+                expr='sum(ray_actors{{Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (State)',
+                legend="{{State}}",
+            )
+        ],
+    ),
+    Panel(
+        id=36,
+        title="Alive Actors by Name",
+        description="Current count of alive actors, grouped by actor name.",
+        unit="actors",
+        targets=[
+            Target(
+                expr='sum(ray_actors{{State!="DEAD",Source="executor",NodeAddress=~"$Instance",{global_filters}}}) by (Name)',
+                legend="{{Name}}",
+            )
+        ],
+    ),
+    Panel(
+        id=40,
+        title="All Placement Groups by State",
+        description='Note: not impacted by "Instance" variable.\n\nCurrent count of placement groups, grouped by state.\n\nState: the placement group state, as described by the rpc::PlacementGroupTableData proto in gcs.proto.',
+        unit="placement groups",
+        targets=[
+            Target(
+                expr="sum(ray_placement_groups{{{global_filters}}}) by (State)",
+                legend="{{State}}",
+            )
+        ],
+    ),
+    Panel(
+        id=29,
+        title="Object Store Memory by Location",
+        description="Object store memory usage by location. The dotted line indicates the object store memory capacity. This metric can go over the max capacity in case of spillage to disk.\n\nLocation: where the memory was allocated, which is MMAP_SHM or MMAP_DISK to indicate memory-mapped page, SPILLED to indicate spillage to disk, and WORKER_HEAP for objects small enough to be inlined in worker memory.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (Location)',
+                legend="{{Location}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+]
+
+RAY_RESOURCES_PANELS = [
+    Panel(
+        id=27,
+        title="Logical CPUs Usage",
+        description="Logical CPU usage of Ray. The dotted line indicates the total number of CPUs. The logical CPU is allocated by `num_cpus` arguments from tasks and actors. PENDING means the number of CPUs that will be available when new nodes are up after the autoscaler scales up.\n\nNOTE: Ray's logical CPU is different from physical CPU usage. Ray's logical CPU is allocated by `num_cpus` arguments.",
+        unit="cores",
+        targets=[
+            Target(
+                expr='sum(ray_resources{{Name="CPU",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
+                legend="CPU Usage: {{instance}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="CPU",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+            # If max + pending > max, we display this value.
+            # (A and predicate) means to return A when the predicate satisfies in PromSql.
+            Target(
+                expr=f"({MAX_PLUS_PENDING_CPUS} and {MAX_PLUS_PENDING_CPUS} > ({MAX_CPUS} or vector(0)))",
+                legend="MAX + PENDING",
+            ),
+        ],
+    ),
+    Panel(
+        id=28,
+        title="Logical GPUs Usage",
+        description="Logical GPU usage of Ray. The dotted line indicates the total number of GPUs. The logical GPU is allocated by `num_gpus` arguments from tasks and actors. PENDING means the number of GPUs that will be available when new nodes are up after the autoscaler scales up.",
+        unit="GPUs",
+        targets=[
+            Target(
+                expr='sum(ray_resources{{Name="GPU",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
+                legend="GPU Usage: {{instance}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="GPU",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+            # If max + pending > max, we display this value.
+            # (A and predicate) means to return A when the predicate satisfies in PromSql.
+            Target(
+                expr=f"({MAX_PLUS_PENDING_GPUS} and {MAX_PLUS_PENDING_GPUS} > ({MAX_GPUS} or vector(0)))",
+                legend="MAX + PENDING",
+            ),
+        ],
+    ),
+    Panel(
+        id=61,
+        title="Logical Memory Usage",
+        description="Logical memory usage of Ray by node. The dotted line indicates the total amount of memory available. Logical memory refers to Ray's view of memory resources allocated to tasks and actors. PENDING means the amount of memory that will be available when new nodes are up after the autoscaler scales up.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_resources{{Name="memory",State="USED",instance=~"$Instance",{global_filters}}}) by (instance)',
+                legend="Memory Usage: {{instance}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="memory",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+            Target(
+                expr=f"({MAX_PLUS_PENDING_MEMORY} and {MAX_PLUS_PENDING_MEMORY} > ({MAX_MEMORY} or vector(0)))",
+                legend="MAX + PENDING",
+            ),
+        ],
+    ),
+    Panel(
+        id=58,
+        title="Object Store Memory Usage",
+        description="Object store memory usage by instance, including memory that has been spilled to disk. The dotted line indicates the object store memory capacity. This metric can go over the max capacity in case of spillage to disk.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (instance)',
+                legend="{{instance}}",
+            ),
+            Target(
+                expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=59,
+        title="Object Store Memory Usage %",
+        description="Object store memory usage % by instance, including memory that has been spilled to disk. This metric can go over 100% in case of spillage to disk.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (instance) * 100 / sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) by (instance)',
+                legend="{{instance}}",
+            ),
+            Target(
+                expr=MAX_PERCENTAGE_EXPRESSION,  # To show the memory limit visually
+                legend="MAX",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=60,
+        title="Object Store Memory Spilled to Disk",
+        description="Object store memory that has been spilled to disk, by instance.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_object_store_memory{{instance=~"$Instance",Location="SPILLED",{global_filters}}}) by (instance)',
+                legend="{{instance}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+]
+
+NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS = [
+    Panel(
+        id=37,
+        title="Node CPU Usage by Component",
+        description="The physical (hardware) CPU usage across the cluster, broken down by component. This reports the summed CPU usage per Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
+        unit="cores",
+        targets=[
+            Target(
+                # ray_component_cpu_percentage returns a percentage that can be > 100. It means that it uses more than 1 CPU.
+                expr='sum(ray_component_cpu_percentage{{instance=~"$Instance",{global_filters}}}) by (Component) / 100',
+                legend="{{Component}}",
+            ),
+            Target(
+                expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=34,
+        title="Node Memory Usage by Component",
+        description="The physical (hardware) memory usage across the cluster, broken down by component. This reports the summed RSS-SHM per Ray component, which corresponds to an approximate memory usage per proc. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='(sum(ray_component_rss_bytes{{instance=~"$Instance",{global_filters}}}) by (Component)) - (sum(ray_component_shared_bytes{{instance=~"$Instance",{global_filters}}}) by (Component))',
+                legend="{{Component}}",
+            ),
+            Target(
+                expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance",{global_filters}}})',
+                legend="shared_memory",
+            ),
+            Target(
+                expr='min(label_replace(sum(ray_node_mem_total_host{{instance=~"$Instance",{global_filters}}}), "mem_cap_source", "host", "", "") or label_replace(sum(ray_node_cgroup_mem_total{{instance=~"$Instance",{global_filters}}}), "mem_cap_source", "cgroup", "", ""))',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=45,
+        title="Node GPU Usage by Component",
+        description="The physical (hardware) GPU usage across the cluster, broken down by component. This reports the summed GPU usage per Ray component.",
+        unit="GPUs",
+        targets=[
+            Target(
+                expr='sum(ray_component_gpu_percentage{{instance=~"$Instance",{global_filters}}} / 100) by (Component)',
+                legend="{{Component}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=46,
+        title="Node GPU Memory Usage by Component",
+        description="The physical (hardware) GPU memory usage across the cluster, broken down by component. This reports the summed GPU memory usage per Ray component.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_component_gpu_memory_mb{{instance=~"$Instance",{global_filters}}} * 1024 * 1024) by (Component)',
+                legend="{{Component}}",
+            ),
+            Target(
+                expr='(sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 1024 * 1024',
+                legend="MAX",
+            ),
+        ],
+    ),
+]
+
+NODE_HARDWARE_UTILIZATION_PANELS = [
+    Panel(
+        id=2,
+        title="Node CPU Usage",
+        description="The physical (hardware) CPU usage for each node.",
+        unit="cores",
+        targets=[
+            Target(
+                expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType",{global_filters}}} / 100) by (instance, RayNodeType)',
+                legend="CPU Usage: {{instance}} ({{RayNodeType}})",
+            ),
+            Target(
+                expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=54,
+        title="Node CPU Usage %",
+        description="The percentage of physical (hardware) CPU usage for each node.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="CPU Usage: {{instance}} ({{RayNodeType}})",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=4,
+        title="Node Memory Usage (heap + object store)",
+        description="The physical (hardware) memory usage for each node. The dotted line means the total amount of memory from the cluster. "
+        "Node memory is a sum of object store memory (shared memory) and heap memory.\n\n"
+        "Host targets reflect memory as reported by the host machine. "
+        "Container targets reflect cgroup-limited memory and are only emitted when the ray node reside within a cgroup.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_node_mem_used_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Memory Used (Host): {{instance}} ({{RayNodeType}})",
+            ),
+            Target(
+                expr='sum(ray_node_mem_total_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+                legend="MAX",
+            ),
+            Target(
+                expr='sum(ray_node_cgroup_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Memory Used (Container): {{instance}} ({{RayNodeType}})",
+            ),
+            Target(
+                expr='sum(ray_node_cgroup_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+                legend="Container MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=48,
+        title="Node Memory Usage % (heap + object store)",
+        description="The percentage of physical (hardware) memory usage for each node.\n\n"
+        "Host targets reflect memory as reported by the host machine. "
+        "Container targets reflect cgroup-limited memory and are only emitted when the ray node reside within a cgroup.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_node_mem_used_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / sum(ray_node_mem_total_host{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Memory Used (Host): {{instance}} ({{RayNodeType}})",
+            ),
+            Target(
+                expr='sum(ray_node_cgroup_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / sum(ray_node_cgroup_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Memory Used (Container): {{instance}} ({{RayNodeType}})",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=6,
+        title="Node Disk Usage",
+        description="Node's physical (hardware) disk usage. The dotted line means the total amount of disk space from the cluster.\n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Disk Used: {{instance}} ({{RayNodeType}})",
+            ),
+            Target(
+                expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=57,
+        title="Node Disk Usage %",
+        description="Node's physical (hardware) disk usage. \n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) * 100 / (sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType))',
+                legend="Disk Used: {{instance}} ({{RayNodeType}})",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=8,
+        title="Node GPU Usage",
+        description="Node's physical (hardware) GPU usage. The dotted line means the total number of hardware GPUs from the cluster. ",
+        unit="GPUs",
+        targets=[
+            Target(
+                expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
+                legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+            Target(
+                expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=55,
+        title="Node GPU Usage %",
+        description="Node's physical (hardware) GPU usage.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
+                legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=18,
+        title="Node GPU Memory Usage (GRAM)",
+        description="The physical (hardware) GPU memory usage for each node. The dotted line means the total amount of GPU memory from the cluster.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
+                legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+            Target(
+                expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})) * 1024 * 1024',
+                legend="MAX",
+            ),
+        ],
+    ),
+    Panel(
+        id=56,
+        title="Node GPU Memory Usage (GRAM) %",
+        description="The percentage of physical (hardware) GPU memory usage for each node.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName) * 100 / (sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName))',
+                legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=62,
+        title="Node GPU Power",
+        description="Current GPU power draw per node. Reported in milliwatts; displayed in watts. Supported on NVIDIA and AMD GPUs.",
+        unit="mwatt",
+        targets=[
+            Target(
+                expr='sum(ray_node_gpu_power_milliwatts{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
+                legend="Power: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=63,
+        title="Node GPU Temperature",
+        description="Current GPU temperature per node in Celsius. Supported on NVIDIA GPUs.",
+        unit="celsius",
+        targets=[
+            Target(
+                expr='sum(ray_node_gpu_temperature_celsius{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
+                legend="Temperature: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+    ),
+    Panel(
+        id=32,
+        title="Node Disk IO Speed",
+        description="Disk IO per node.",
+        unit="Bps",
+        targets=[
+            Target(
+                expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Write: {{instance}} ({{RayNodeType}})",
+            ),
+            Target(
+                expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Read: {{instance}} ({{RayNodeType}})",
+            ),
+        ],
+    ),
+    Panel(
+        id=20,
+        title="Node Network",
+        description="Network speed per node",
+        unit="Bps",
+        targets=[
+            Target(
+                expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Recv: {{instance}} ({{RayNodeType}})",
+            ),
+            Target(
+                expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+                legend="Send: {{instance}} ({{RayNodeType}})",
+            ),
+        ],
+    ),
+]
+
+NODE_TPU_UTILIZATION_PANELS = [
+    Panel(
+        id=50,
+        title="Node TPU Tensorcore Utilization %",
+        description="Percentage of tensorcore utilization for the TPUs on this node. Computed by dividing the number of tensorcore operations by the maximum supported number of operations during the sample period.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_tpu_tensorcore_utilization{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology)',
+                legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=51,
+        title="Node TPU High Bandwidth Memory Utilization %",
+        description="Percentage of bandwidth memory utilization for the TPUs on this node. Computed by dividing the memory bandwidth used by the maximum supported memory bandwidth limit during the sample period.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_tpu_memory_bandwidth_utilization{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology)',
+                legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=52,
+        title="Node TPU Duty Cycle %",
+        description="Percentage of time over the sample period during which the TPU is actively processing.",
+        unit="%",
+        targets=[
+            Target(
+                expr='sum(ray_tpu_duty_cycle{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
+                legend="{{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
+            ),
+        ],
+    ),
+    Panel(
+        id=53,
+        title="Node TPU Memory Used",
+        description="Total memory used/allocated for the TPUs on this node.",
+        unit="bytes",
+        targets=[
+            Target(
+                expr='sum(ray_tpu_memory_used{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
+                legend="Memory Used: {{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
+            ),
+            Target(
+                expr='sum(ray_tpu_memory_total{{instance=~"$Instance",{global_filters}}}) by (instance, TpuIndex, TpuDeviceName, TpuType, TpuTopology) or vector(0)',
+                legend="Memory Total: {{instance}}, tpu.{{TpuIndex}}, {{TpuType}}, {{TpuTopology}}",
+            ),
+        ],
+    ),
+]
+
+DEFAULT_GRAFANA_ROWS = [
+    Row(
+        title="Overview and Health",
+        id=1001,
+        panels=OVERVIEW_AND_HEALTH_PANELS,
+        collapsed=False,
+    ),
+    Row(
+        title="Hardware Utilization by Node",
+        id=1005,
+        panels=NODE_HARDWARE_UTILIZATION_PANELS,
+        collapsed=False,
+    ),
+    Row(
+        title="Hardware Utilization by Ray Component",
+        id=1004,
+        panels=NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS,
+        collapsed=False,
+    ),
+    Row(
+        title="Ray Resources by Node",
+        id=1003,
+        panels=RAY_RESOURCES_PANELS,
+        collapsed=False,
+    ),
+    Row(
+        title="Ray Tasks, Actors and Placement Groups",
+        id=1002,
+        panels=RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS,
+        collapsed=False,
+    ),
+    Row(
+        title="TPU Utilization by Node",
+        id=1006,
+        panels=NODE_TPU_UTILIZATION_PANELS,
+        collapsed=True,
+    ),
+]
+
+ids = []
+for row in DEFAULT_GRAFANA_ROWS:
+    ids.append(row.id)
+    ids.extend(panel.id for panel in row.panels)
+
+ids.sort()
+
+assert len(ids) == len(
+    set(ids)
+), f"Duplicated id found. Use unique id for each panel. {ids}"
+
+default_dashboard_config = DashboardConfig(
+    name="DEFAULT",
+    default_uid="rayDefaultDashboard",
+    rows=DEFAULT_GRAFANA_ROWS,
+    standard_global_filters=[
+        'SessionName=~"$SessionName"',
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="default_grafana_dashboard_base.json",
+)

python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json

@@ -0,0 +1,177 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "iteration": 1667344411089,
+  "links": [],
+  "panels": [],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false
+        },
+        "description": "Filter queries of a specific Prometheus type.",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": false
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, SessionName)",
+        "description": "Filter queries to specific Ray sessions.",
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": false,
+        "name": "SessionName",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, SessionName)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{SessionName=~\"$SessionName\",{global_filters}}}, instance)",
+        "description": "Filter queries to specific Ray nodes by their IP address.",
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Instance",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{SessionName=~\"$SessionName\",{global_filters}}}, instance)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+        "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple Ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": false,
+        "name": "Cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "description": "Filter queries to specific Ray node types (head or worker).",
+        "includeAll": true,
+        "multi": true,
+        "name": "RayNodeType",
+        "options": [
+          {
+            "selected": false,
+            "text": "All",
+            "value": "$__all"
+          },
+          {
+            "selected": false,
+            "text": "Head Node",
+            "value": "head"
+          },
+          {
+            "selected": false,
+            "text": "Worker Node",
+            "value": "worker"
+          }
+        ],
+        "query": "head, worker",
+        "type": "custom"
+      }
+    ]
+  },
+  "rayMeta": ["supportsFullGrafanaView"],
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Default Dashboard",
+  "uid": "rayDefaultDashboard",
+  "version": 4
+}

python/ray/dashboard/modules/metrics/dashboards/serve_dashboard_panels.py

@@ -0,0 +1,423 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    GridPos,
+    Panel,
+    Target,
+)
+
+SERVE_GRAFANA_PANELS = [
+    Panel(
+        id=5,
+        title="Cluster Utilization",
+        description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster. Ignores application variable.",
+        unit="%",
+        targets=[
+            # CPU
+            Target(
+                expr="avg(ray_node_cpu_utilization{{{global_filters}}})",
+                legend="CPU (physical)",
+            ),
+            # GPU
+            Target(
+                expr="sum(ray_node_gpus_utilization{{{global_filters}}}) / on() (sum(autoscaler_cluster_resources{{resource='GPU',{global_filters}}}) or vector(0))",
+                legend="GPU (physical)",
+            ),
+            # Memory
+            Target(
+                expr="sum(ray_node_mem_used{{{global_filters}}}) / on() (sum(ray_node_mem_total{{{global_filters}}})) * 100",
+                legend="Memory (RAM)",
+            ),
+            # GRAM
+            Target(
+                expr="sum(ray_node_gram_used{{{global_filters}}}) / on() (sum(ray_node_gram_available{{{global_filters}}}) + sum(ray_node_gram_used{{{global_filters}}})) * 100",
+                legend="GRAM",
+            ),
+            # Object Store
+            Target(
+                expr='sum(ray_object_store_memory{{{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",{global_filters}}}) * 100',
+                legend="Object Store Memory",
+            ),
+            # Disk
+            Target(
+                expr="sum(ray_node_disk_usage{{{global_filters}}}) / on() (sum(ray_node_disk_free{{{global_filters}}}) + sum(ray_node_disk_usage{{{global_filters}}})) * 100",
+                legend="Disk",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 0, 12, 8),
+    ),
+    Panel(
+        id=7,
+        title="QPS per application",
+        description="QPS for each selected application.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_num_http_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='sum(rate(ray_serve_num_grpc_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
+                legend="{{application, method}}",
+            ),
+        ],
+        grid_pos=GridPos(12, 0, 12, 8),
+    ),
+    Panel(
+        id=8,
+        title="Error QPS per application",
+        description="Error QPS for each selected application.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route)',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method)',
+                legend="{{application, method}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 1, 12, 8),
+    ),
+    Panel(
+        id=17,
+        title="Error QPS per application per error code",
+        description="Error QPS for each selected application.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_num_http_error_requests_total{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, error_code)',
+                legend="{{application, route, error_code}}",
+            ),
+            Target(
+                expr='sum(rate(ray_serve_num_grpc_error_requests_total{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, error_code)',
+                legend="{{application, method, error_code}}",
+            ),
+        ],
+        grid_pos=GridPos(12, 1, 12, 8),
+    ),
+    Panel(
+        id=12,
+        title="P50 latency per application",
+        description="P50 latency for selected applications.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
+                legend="{{application, method}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 2, 8, 8),
+    ),
+    Panel(
+        id=15,
+        title="P90 latency per application",
+        description="P90 latency for selected applications.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
+                legend="{{application, method}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 2, 8, 8),
+    ),
+    Panel(
+        id=16,
+        title="P99 latency per application",
+        description="P99 latency for selected applications.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_http_request_latency_ms_bucket{{application=~"$Application",application!~"",route=~"$HTTP_Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, route, le))',
+                legend="{{application, route}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_grpc_request_latency_ms_bucket{{application=~"$Application",application!~"",method=~"$gRPC_Method",{global_filters}}}[5m])) by (application, method, le))',
+                legend="{{application, method}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate({{__name__=~ "ray_serve_(http|grpc)_request_latency_ms_bucket|ray_serve_grpc_request_latency_ms_bucket",application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 2, 8, 8),
+    ),
+    Panel(
+        id=2,
+        title="Replicas per deployment",
+        description='Number of replicas per deployment. Ignores "Application" variable.',
+        unit="replicas",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
+                legend="{{application, deployment}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 3, 8, 8),
+    ),
+    Panel(
+        id=13,
+        title="QPS per deployment",
+        description="QPS for each deployment.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_request_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
+                legend="{{application, deployment}}",
+            ),
+        ],
+        grid_pos=GridPos(8, 3, 8, 8),
+    ),
+    Panel(
+        id=14,
+        title="Error QPS per deployment",
+        description="Error QPS for each deplyoment.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_error_counter_total{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment)',
+                legend="{{application, deployment}}",
+            ),
+        ],
+        grid_pos=GridPos(16, 3, 8, 8),
+    ),
+    Panel(
+        id=9,
+        title="P50 latency per deployment",
+        description="P50 latency per deployment.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
+                legend="{{application, deployment}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 4, 8, 8),
+    ),
+    Panel(
+        id=10,
+        title="P90 latency per deployment",
+        description="P90 latency per deployment.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
+                legend="{{application, deployment}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 4, 8, 8),
+    ),
+    Panel(
+        id=11,
+        title="P99 latency per deployment",
+        description="P99 latency per deployment.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (application, deployment, le))',
+                legend="{{application, deployment}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{application=~"$Application",application!~"",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 4, 8, 8),
+    ),
+    Panel(
+        id=3,
+        title="Queue size per deployment",
+        description='Number of requests queued per deployment. Ignores "Application" variable.',
+        unit="requests",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
+                legend="{{application, deployment}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 5, 8, 8),
+    ),
+    Panel(
+        id=4,
+        title="Node count",
+        description='Number of nodes in this cluster. Ignores "Application" variable.',
+        unit="nodes",
+        targets=[
+            # TODO(aguo): Update this to use autoscaler metrics instead
+            Target(
+                expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Active Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Failed Nodes: {{NodeType}}",
+            ),
+            Target(
+                expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)",
+                legend="Pending Nodes: {{NodeType}}",
+            ),
+        ],
+        grid_pos=GridPos(8, 5, 8, 8),
+    ),
+    Panel(
+        id=6,
+        title="Node network",
+        description='Network speed per node. Ignores "Application" variable.',
+        unit="Bps",
+        targets=[
+            Target(
+                expr="sum(ray_node_network_receive_speed{{{global_filters}}}) by (instance)",
+                legend="Recv: {{instance}}",
+            ),
+            Target(
+                expr="sum(ray_node_network_send_speed{{{global_filters}}}) by (instance)",
+                legend="Send: {{instance}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(16, 5, 8, 8),
+    ),
+    Panel(
+        id=20,
+        title="Ongoing HTTP Requests",
+        description="The number of ongoing requests in the HTTP Proxy.",
+        unit="requests",
+        targets=[
+            Target(
+                expr="ray_serve_num_ongoing_http_requests{{{global_filters}}}",
+                legend="Ongoing HTTP Requests",
+            ),
+        ],
+        grid_pos=GridPos(0, 6, 8, 8),
+    ),
+    Panel(
+        id=21,
+        title="Ongoing gRPC Requests",
+        description="The number of ongoing requests in the gRPC Proxy.",
+        unit="requests",
+        targets=[
+            Target(
+                expr="ray_serve_num_ongoing_grpc_requests{{{global_filters}}}",
+                legend="Ongoing gRPC Requests",
+            ),
+        ],
+        grid_pos=GridPos(8, 6, 8, 8),
+    ),
+    Panel(
+        id=22,
+        title="Scheduling Tasks",
+        description="The number of request scheduling tasks in the router.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="ray_serve_num_scheduling_tasks{{{global_filters}}}",
+                legend="Scheduling Tasks",
+            ),
+        ],
+        grid_pos=GridPos(16, 6, 8, 8),
+    ),
+    Panel(
+        id=23,
+        title="Scheduling Tasks in Backoff",
+        description="The number of request scheduling tasks in the router that are undergoing backoff.",
+        unit="tasks",
+        targets=[
+            Target(
+                expr="ray_serve_num_scheduling_tasks_in_backoff{{{global_filters}}}",
+                legend="Scheduling Tasks in Backoff",
+            ),
+        ],
+        grid_pos=GridPos(0, 7, 8, 8),
+    ),
+    Panel(
+        id=24,
+        title="Controller Control Loop Duration",
+        description="The duration of the last control loop.",
+        unit="seconds",
+        targets=[
+            Target(
+                expr="ray_serve_controller_control_loop_duration_s{{{global_filters}}}",
+                legend="Control Loop Duration",
+            ),
+        ],
+        grid_pos=GridPos(8, 7, 8, 8),
+    ),
+    Panel(
+        id=25,
+        title="Number of Control Loops",
+        description="The number of control loops performed by the controller. Increases monotonically over the controller's lifetime.",
+        unit="loops",
+        targets=[
+            Target(
+                expr="ray_serve_controller_num_control_loops{{{global_filters}}}",
+                legend="Control Loops",
+            ),
+        ],
+        grid_pos=GridPos(16, 7, 8, 8),
+    ),
+]
+
+ids = []
+for panel in SERVE_GRAFANA_PANELS:
+    ids.append(panel.id)
+
+ids.sort()
+
+assert len(ids) == len(
+    set(ids)
+), f"Duplicated id found. Use unique id for each panel. {ids}"
+
+serve_dashboard_config = DashboardConfig(
+    name="SERVE",
+    default_uid="rayServeDashboard",
+    panels=SERVE_GRAFANA_PANELS,
+    standard_global_filters=[
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="serve_grafana_dashboard_base.json",
+)

python/ray/dashboard/modules/metrics/dashboards/serve_deployment_dashboard_panels.py

@@ -0,0 +1,262 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    GridPos,
+    Panel,
+    Target,
+)
+
+SERVE_DEPLOYMENT_GRAFANA_PANELS = [
+    Panel(
+        id=1,
+        title="Replicas per deployment",
+        description='Number of replicas per deployment. Ignores "Route" variable.',
+        unit="replicas",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_replica_healthy{{{global_filters}}}) by (application, deployment)",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 0, 8, 8),
+    ),
+    Panel(
+        id=2,
+        title="QPS per replica",
+        description="QPS for each replica.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_request_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        grid_pos=GridPos(8, 0, 8, 8),
+    ),
+    Panel(
+        id=3,
+        title="Error QPS per replica",
+        description="Error QPS for each replica.",
+        unit="qps",
+        targets=[
+            Target(
+                expr='sum(rate(ray_serve_deployment_error_counter_total{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica)',
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        grid_pos=GridPos(16, 0, 8, 8),
+    ),
+    Panel(
+        id=4,
+        title="P50 latency per replica",
+        description="P50 latency per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.5, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 1, 8, 8),
+    ),
+    Panel(
+        id=5,
+        title="P90 latency per replica",
+        description="P90 latency per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.9, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 1, 8, 8),
+    ),
+    Panel(
+        id=6,
+        title="P99 latency per replica",
+        description="P99 latency per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",route!~"/-/.*",{global_filters}}}[5m])) by (application, deployment, replica, le))',
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+            Target(
+                expr='histogram_quantile(0.99, sum(rate(ray_serve_deployment_processing_latency_ms_bucket{{route=~"$Route",{global_filters}}}[5m])) by (le))',
+                legend="Total",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 1, 8, 8),
+    ),
+    Panel(
+        id=7,
+        title="Queue size per deployment",
+        description='Number of requests queued per deployment. Ignores "Replica" and "Route" variable.',
+        unit="requests",
+        targets=[
+            Target(
+                expr="sum(ray_serve_deployment_queued_queries{{{global_filters}}}) by (application, deployment)",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 2, 12, 8),
+    ),
+    Panel(
+        id=8,
+        title="Running requests per replica",
+        description="Current running requests for each replica.",
+        unit="requests",
+        targets=[
+            Target(
+                expr="sum(ray_serve_replica_processing_queries{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(12, 2, 12, 8),
+    ),
+    Panel(
+        id=9,
+        title="Multiplexed models per replica",
+        description="The number of multiplexed models for each replica.",
+        unit="models",
+        targets=[
+            Target(
+                expr="sum(ray_serve_num_multiplexed_models{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 3, 8, 8),
+    ),
+    Panel(
+        id=10,
+        title="Multiplexed model loads per replica",
+        description="The number of times of multiplexed models loaded for each replica.",
+        unit="times",
+        targets=[
+            Target(
+                expr="sum(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 3, 8, 8),
+    ),
+    Panel(
+        id=11,
+        title="Multiplexed model unloads per replica",
+        description="The number of times of multiplexed models unloaded for each replica.",
+        unit="times",
+        targets=[
+            Target(
+                expr="sum(ray_serve_multiplexed_models_unload_counter_total{{{global_filters}}}) by (application, deployment, replica)",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(16, 3, 8, 8),
+    ),
+    Panel(
+        id=12,
+        title="P99 latency of multiplexed model loads per replica",
+        description="P99 latency of multiplexed model load per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_load_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(0, 4, 8, 8),
+    ),
+    Panel(
+        id=13,
+        title="P99 latency of multiplexed model unloads per replica",
+        description="P99 latency of multiplexed model unload per replica.",
+        unit="ms",
+        targets=[
+            Target(
+                expr="histogram_quantile(0.99, sum(rate(ray_serve_multiplexed_model_unload_latency_ms_bucket{{{global_filters}}}[5m])) by (application, deployment, replica, le))",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        fill=0,
+        stack=False,
+        grid_pos=GridPos(8, 4, 8, 8),
+    ),
+    Panel(
+        id=14,
+        title="Multiplexed model ids per replica",
+        description="The ids of multiplexed models for each replica.",
+        unit="model",
+        targets=[
+            Target(
+                expr="ray_serve_registered_multiplexed_model_id{{{global_filters}}}",
+                legend="{{replica}}:{{model_id}}",
+            ),
+        ],
+        grid_pos=GridPos(16, 4, 8, 8),
+        stack=False,
+    ),
+    Panel(
+        id=15,
+        title="Multiplexed model cache hit rate",
+        description="The cache hit rate of multiplexed models for the deployment.",
+        unit="%",
+        targets=[
+            Target(
+                expr="(1 - sum(rate(ray_serve_multiplexed_models_load_counter_total{{{global_filters}}}[5m]))/sum(rate(ray_serve_multiplexed_get_model_requests_counter_total{{{global_filters}}}[5m])))",
+                legend="{{application}}#{{deployment}}#{{replica}}",
+            ),
+        ],
+        grid_pos=GridPos(0, 5, 8, 8),
+    ),
+]
+
+ids = []
+for panel in SERVE_DEPLOYMENT_GRAFANA_PANELS:
+    ids.append(panel.id)
+
+ids.sort()
+
+assert len(ids) == len(
+    set(ids)
+), f"Duplicated id found. Use unique id for each panel. {ids}"
+
+serve_deployment_dashboard_config = DashboardConfig(
+    name="SERVE_DEPLOYMENT",
+    default_uid="rayServeDeploymentDashboard",
+    panels=SERVE_DEPLOYMENT_GRAFANA_PANELS,
+    standard_global_filters=[
+        'application=~"$Application"',
+        'deployment=~"$Deployment"',
+        'replica=~"$Replica"',
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="serve_deployment_grafana_dashboard_base.json",
+)

python/ray/dashboard/modules/metrics/dashboards/serve_deployment_grafana_dashboard_base.json

@@ -0,0 +1,208 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "iteration": 1667344411089,
+  "links": [],
+  "panels": [],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false
+        },
+        "description": "Filter queries to specific prometheus type.",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Application",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Deployment",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",{global_filters}}}, deployment)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Replica",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{application=~\"$Application\",deployment=~\"$Deployment\",{global_filters}}}, replica)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Route",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_request_counter{{deployment=~\"$Deployment\",{global_filters}}}, route)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+        "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": false,
+        "name": "Cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "rayMeta": ["excludesSystemRoutes"],
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Serve Deployment Dashboard",
+  "uid": "rayServeDeploymentDashboard",
+  "version": 1
+}

python/ray/dashboard/modules/metrics/dashboards/serve_grafana_dashboard_base.json

@@ -0,0 +1,177 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "iteration": 1667344411089,
+  "links": [],
+  "panels": [],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false
+        },
+        "description": "Filter queries of a specific Prometheus type.",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "Application",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_deployment_replica_healthy{{{global_filters}}}, application)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": "HTTP Route",
+        "multi": true,
+        "name": "HTTP_Route",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_num_http_requests_total{{{global_filters}}}, route)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_num_grpc_requests_total{{{global_filters}}}, method)",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": "gRPC Service Method",
+        "multi": true,
+        "name": "gRPC_Method",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_serve_num_grpc_requests_total{{{global_filters}}}, method)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+        "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": false,
+        "name": "Cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "rayMeta": ["excludesSystemRoutes"],
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Serve Dashboard",
+  "uid": "rayServeDashboard",
+  "version": 1
+}

python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py

@@ -0,0 +1,810 @@
+# ruff: noqa: E501
+
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    GridPos,
+    Panel,
+    PanelTemplate,
+    Row,
+    Target,
+)
+
+# ---------------------------------------------------------------------------
+# Reusable PromQL fragments
+# ---------------------------------------------------------------------------
+# WorkerId join: attaches deployment + replica labels to vLLM-only metrics.
+# The `* 0 + 1` trick turns the counter into a constant-1 lookup table keyed
+# by WorkerId so the join resolves deployment + replica labels without
+# affecting the numeric value of the left-hand side.
+# Keep `{global_filters}` trailing so an empty substitution leaves a tolerated
+# trailing comma instead of a leading/double comma that Prometheus rejects.
+_WORKER_JOIN = (
+    "\n* on(WorkerId) group_left(deployment, replica)"
+    "\nmax by(WorkerId, deployment, replica) ("
+    'ray_serve_deployment_request_counter_total{{deployment=~"$deployment", {global_filters}}} * 0 + 1)'
+)
+
+# Standard vLLM metric filter
+_VLLM_FILTER = 'model_name=~"$vllm_model_name", WorkerId=~"$workerid", {global_filters}'
+
+# vLLM filter scoped to a specific deployment (used for ray_serve_* metrics
+# that also carry model_name / WorkerId labels).
+_VLLM_DEPLOYMENT_FILTER = 'model_name=~"$vllm_model_name", WorkerId=~"$workerid", deployment=~"$deployment", {global_filters}'
+
+# Legend used by most per-worker panels
+_DEP_REPLICA = "{{deployment}}: {{replica}}"
+
+
+def _mean_with_join(metric_base: str) -> str:
+    """Mean = sum(_sum) / sum(_count) with NaN guard + WorkerId join."""
+    return (
+        "(\n"
+        "  (\n"
+        f"    sum by(WorkerId) (rate({metric_base}_sum{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
+        "    /\n"
+        f"    sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
+        "  )\n"
+        "  and on(WorkerId)\n"
+        "  (\n"
+        f"    sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
+        "  )\n"
+        ")" + _WORKER_JOIN
+    )
+
+
+def _percentile_with_join(metric_base: str, quantile: float) -> str:
+    """histogram_quantile with NaN guard + WorkerId join."""
+    return (
+        "(\n"
+        "  histogram_quantile(\n"
+        f"    {quantile},\n"
+        f"    sum by (le, WorkerId) (rate({metric_base}_bucket{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
+        "  )\n"
+        "  and on(WorkerId)\n"
+        "  (\n"
+        f"    sum by(WorkerId) (rate({metric_base}_count{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
+        "  )\n"
+        ")" + _WORKER_JOIN
+    )
+
+
+def _gauge_with_join(metric: str) -> str:
+    """Simple gauge metric with WorkerId join (no rate, no guard)."""
+    return f"sum by(WorkerId) ({metric}{{{{{_VLLM_FILTER}}}}})" + _WORKER_JOIN
+
+
+def _rate_with_join(metric: str, agg_fn: str = "rate") -> str:
+    """rate() or increase() of a metric summed by WorkerId, with join."""
+    return (
+        f"sum by(WorkerId) ({agg_fn}({metric}{{{{{_VLLM_FILTER}}}}}[$interval]))"
+        + _WORKER_JOIN
+    )
+
+
+def _ratio_with_join_and_guard(
+    numerator_metric: str,
+    denominator_metric: str,
+    *,
+    scale: str = "",
+    guard_metric: str | None = None,
+) -> str:
+    """Ratio of two rate metrics with NaN guard + WorkerId join.
+
+    Optionally applies a scale factor (e.g. '* 1000' or '/ 1024 / 1024 / 1024').
+    """
+    guard = guard_metric or denominator_metric
+    return (
+        "(\n"
+        "  (\n"
+        f"    sum by(WorkerId) (rate({numerator_metric}{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
+        "    /\n"
+        f"    sum by(WorkerId) (rate({denominator_metric}{{{{{_VLLM_FILTER}}}}}[$interval]))\n"
+        + (f"    {scale}\n" if scale else "")
+        + "  )\n"
+        "  and on(WorkerId)\n"
+        "  (\n"
+        f"    sum by(WorkerId) (rate({guard}{{{{{_VLLM_FILTER}}}}}[$interval])) > 0\n"
+        "  )\n"
+        ")" + _WORKER_JOIN
+    )
+
+
+# ---------------------------------------------------------------------------
+# Histogram helper: generates Mean / P50 / P90 panels for a given metric
+# ---------------------------------------------------------------------------
+def _histogram_panels(
+    metric_base: str,
+    label: str,
+    ids: tuple,
+    y: int,
+    unit: str = "s",
+    linewidth: int = 2,
+    description: str = "",
+) -> list:
+    """Return [Mean, P50, P90] panels for a histogram metric."""
+    return [
+        Panel(
+            id=ids[0],
+            title=f"{label} -- Mean",
+            description=description,
+            unit=unit,
+            targets=[Target(expr=_mean_with_join(metric_base), legend=_DEP_REPLICA)],
+            fill=1,
+            linewidth=linewidth,
+            stack=False,
+            grid_pos=GridPos(0, y, 8, 8),
+        ),
+        Panel(
+            id=ids[1],
+            title=f"{label} -- P50",
+            description=description,
+            unit=unit,
+            targets=[
+                Target(
+                    expr=_percentile_with_join(metric_base, 0.5), legend=_DEP_REPLICA
+                )
+            ],
+            fill=1,
+            linewidth=linewidth,
+            stack=False,
+            grid_pos=GridPos(8, y, 8, 8),
+        ),
+        Panel(
+            id=ids[2],
+            title=f"{label} -- P90",
+            description=description,
+            unit=unit,
+            targets=[
+                Target(
+                    expr=_percentile_with_join(metric_base, 0.9), legend=_DEP_REPLICA
+                )
+            ],
+            fill=1,
+            linewidth=linewidth,
+            stack=False,
+            grid_pos=GridPos(16, y, 8, 8),
+        ),
+    ]
+
+
+# ===================================================================
+# Row 1: Throughput
+# ===================================================================
+_throughput_panels = [
+    Panel(
+        id=2,
+        title="Requests / s",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"sum by (deployment, replica) (rate(ray_serve_deployment_request_counter_total{{{{{_VLLM_DEPLOYMENT_FILTER}}}}}[$interval]))",
+                legend=_DEP_REPLICA,
+            ),
+            Target(
+                expr=f"sum(rate(ray_serve_deployment_request_counter_total{{{{{_VLLM_DEPLOYMENT_FILTER}}}}}[$interval]))",
+                legend="Total QPS",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 1, 8, 8),
+    ),
+    Panel(
+        id=3,
+        title="Prompt Tokens/s",
+        description="Number of tokens processed per second",
+        unit="tokens/s",
+        targets=[
+            Target(
+                expr=_rate_with_join("ray_vllm_request_prompt_tokens_sum"),
+                legend=_DEP_REPLICA,
+            ),
+            Target(
+                expr=f"sum(rate(ray_vllm_request_prompt_tokens_sum{{{{{_VLLM_FILTER}}}}}[$interval]))",
+                legend="Total",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(8, 1, 8, 8),
+    ),
+    Panel(
+        id=4,
+        title="Generation Tokens/s",
+        description="Number of tokens processed per second",
+        unit="tokens/s",
+        targets=[
+            Target(
+                expr=_rate_with_join("ray_vllm_generation_tokens_total"),
+                legend=_DEP_REPLICA,
+            ),
+            Target(
+                expr=f"sum(rate(ray_vllm_generation_tokens_total{{{{{_VLLM_FILTER}}}}}[$interval]))",
+                legend="Total",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(16, 1, 8, 8),
+    ),
+]
+
+# ===================================================================
+# Row 2: Latency (3x3 grid)
+# ===================================================================
+_latency_panels_list = [
+    *_histogram_panels(
+        "ray_vllm_request_time_per_output_token_seconds", "TPOT", (6, 7, 8), 10
+    ),
+    *_histogram_panels("ray_vllm_time_to_first_token_seconds", "TTFT", (9, 10, 11), 18),
+    *_histogram_panels(
+        "ray_vllm_e2e_request_latency_seconds",
+        "Request Latency",
+        (12, 13, 14),
+        26,
+        description="Latency from request start to first token returned (in seconds).",
+    ),
+]
+
+# ===================================================================
+# Row 3: Cache
+# ===================================================================
+_cache_panels = [
+    Panel(
+        id=16,
+        title="Cache Utilization",
+        description="Percentage of used cache blocks by vLLM.",
+        unit="percentunit",
+        targets=[
+            Target(
+                expr=_gauge_with_join("ray_vllm_kv_cache_usage_perc"),
+                legend=_DEP_REPLICA,
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 35, 12, 8),
+    ),
+    Panel(
+        id=17,
+        title="GPU KV Cache Hit Rate",
+        description="",
+        unit="percent",
+        targets=[
+            Target(
+                expr=(
+                    f"100 * ("
+                    f"(sum by(WorkerId) (rate(ray_vllm_prefix_cache_hits_total{{{{{_VLLM_FILTER}}}}}[$interval])) "
+                    f"/ sum by(WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{{{{_VLLM_FILTER}}}}}[$interval])))"
+                    f" and on(WorkerId) "
+                    f"(sum by(WorkerId) (rate(ray_vllm_prefix_cache_queries_total{{{{{_VLLM_FILTER}}}}}[$interval])) > 0))"
+                    + _WORKER_JOIN
+                ),
+                legend=_DEP_REPLICA,
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(12, 35, 12, 8),
+    ),
+]
+
+# ===================================================================
+# Row 4: Request Length
+# ===================================================================
+_request_length_panels = [
+    *_histogram_panels(
+        "ray_vllm_request_prompt_tokens",
+        "Prompt Length",
+        (19, 20, 21),
+        44,
+        unit="short",
+        linewidth=1,
+    ),
+    *_histogram_panels(
+        "ray_vllm_request_generation_tokens",
+        "Generation Length",
+        (22, 23, 24),
+        52,
+        unit="short",
+        linewidth=1,
+    ),
+]
+
+# ===================================================================
+# Row 5: Scheduler
+# ===================================================================
+_scheduler_panels = [
+    Panel(
+        id=26,
+        title="Scheduler: Running",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=_gauge_with_join("ray_vllm_num_requests_running"),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(0, 61, 8, 8),
+    ),
+    Panel(
+        id=27,
+        title="Scheduler: Swapped",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=_gauge_with_join("ray_vllm_num_requests_swapped"),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(8, 61, 8, 8),
+    ),
+    Panel(
+        id=28,
+        title="Scheduler: Waiting",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=_gauge_with_join("ray_vllm_num_requests_waiting"),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(16, 61, 8, 8),
+    ),
+    Panel(
+        id=29,
+        title="Finish Reason",
+        description="Number of finished requests by their finish reason.",
+        unit="short",
+        targets=[
+            Target(
+                expr=(
+                    f"sum by(finished_reason, WorkerId) (increase(ray_vllm_request_success_total{{{{{_VLLM_FILTER}}}}}[$interval]))"
+                    + _WORKER_JOIN
+                ),
+                legend="{{finished_reason}} \u2014 {{deployment}}: {{replica}}",
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(0, 69, 12, 8),
+    ),
+    Panel(
+        id=30,
+        title="Queue Time",
+        description="",
+        unit="s",
+        targets=[
+            Target(
+                expr=_rate_with_join("ray_vllm_request_queue_time_seconds_sum"),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(12, 69, 12, 8),
+    ),
+    Panel(
+        id=31,
+        title="Prefill Time",
+        description="",
+        unit="s",
+        targets=[
+            Target(
+                expr=_rate_with_join("ray_vllm_request_prefill_time_seconds_sum"),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(0, 77, 12, 8),
+    ),
+    Panel(
+        id=32,
+        title="Decode Time",
+        description="",
+        unit="s",
+        targets=[
+            Target(
+                expr=_rate_with_join("ray_vllm_request_decode_time_seconds_sum"),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(12, 77, 12, 8),
+    ),
+]
+
+# ===================================================================
+# Row 6: NIXL
+# ===================================================================
+_nixl_panels = [
+    Panel(
+        id=34,
+        title="NIXL: Transfer Latency",
+        description="Average NIXL KV cache transfer latency in milliseconds.",
+        unit="ms",
+        targets=[
+            Target(
+                expr=_ratio_with_join_and_guard(
+                    "ray_vllm_nixl_xfer_time_seconds_sum",
+                    "ray_vllm_nixl_xfer_time_seconds_count",
+                    scale="* 1000",
+                ),
+                legend=_DEP_REPLICA,
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(0, 86, 8, 8),
+    ),
+    Panel(
+        id=35,
+        title="NIXL: Transfer Throughput",
+        description="NIXL KV cache transfer throughput in GB/s.",
+        unit="GBs",
+        targets=[
+            Target(
+                expr=_ratio_with_join_and_guard(
+                    "ray_vllm_nixl_bytes_transferred_sum",
+                    "ray_vllm_nixl_xfer_time_seconds_sum",
+                    scale="/ 1024 / 1024 / 1024",
+                ),
+                legend=_DEP_REPLICA,
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(8, 86, 8, 8),
+    ),
+    Panel(
+        id=36,
+        title="NIXL: Transfer Rate",
+        description="Number of NIXL KV cache transfers per second.",
+        unit="ops",
+        targets=[
+            Target(
+                expr=_rate_with_join("ray_vllm_nixl_xfer_time_seconds_count"),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(16, 86, 8, 8),
+    ),
+    Panel(
+        id=37,
+        title="NIXL: Avg Post Time",
+        description="Average time to post/initiate a NIXL transfer in milliseconds.",
+        unit="ms",
+        targets=[
+            Target(
+                expr=_ratio_with_join_and_guard(
+                    "ray_vllm_nixl_post_time_seconds_sum",
+                    "ray_vllm_nixl_post_time_seconds_count",
+                    scale="* 1000",
+                ),
+                legend=_DEP_REPLICA,
+            ),
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(0, 94, 8, 8),
+    ),
+    Panel(
+        id=38,
+        title="NIXL: KV Transfer Failures",
+        description="Number of failed NIXL KV cache transfers. Any non-zero value is concerning and indicates RDMA transfer errors.",
+        unit="short",
+        targets=[
+            Target(
+                expr=_rate_with_join(
+                    "ray_vllm_nixl_num_failed_transfers", agg_fn="increase"
+                ),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(8, 94, 8, 8),
+    ),
+    Panel(
+        id=39,
+        title="NIXL: KV Expired Requests",
+        description="Number of requests whose KV blocks expired before decode consumed them. Spikes indicate prefill is outrunning decode or the timeout is too short.",
+        unit="short",
+        targets=[
+            Target(
+                expr=_rate_with_join(
+                    "ray_vllm_nixl_num_kv_expired_reqs_total", agg_fn="increase"
+                ),
+                legend=_DEP_REPLICA,
+            )
+        ],
+        fill=1,
+        linewidth=1,
+        stack=False,
+        grid_pos=GridPos(16, 94, 8, 8),
+    ),
+]
+
+# ===================================================================
+# Row 7: Token Distribution (collapsed)
+# ===================================================================
+_WORKERID_FILTER = 'WorkerId=~"$workerid", {global_filters}'
+
+_token_distribution_panels = [
+    Panel(
+        id=41,
+        title="Tokens Last 24 Hours",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])))",
+                legend="Input: {{model_name}}",
+            ),
+            Target(
+                expr=f"(sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])))",
+                legend="Generated: {{model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 103, 12, 8),
+        template=PanelTemplate.STAT,
+    ),
+    Panel(
+        id=42,
+        title="Tokens Last Hour",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1h]))",
+                legend="Input: {{model_name}}",
+            ),
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1h]))",
+                legend="Generated: {{model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 103, 12, 8),
+        template=PanelTemplate.STAT,
+    ),
+    Panel(
+        id=43,
+        title="Ratio Input:Generated Tokens Last 24 Hours",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])) / sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
+                legend="{{model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 111, 12, 8),
+        template=PanelTemplate.STAT,
+    ),
+    Panel(
+        id=44,
+        title="Distribution of Requests Per Model Last 24 Hours",
+        description="",
+        unit="Requests",
+        targets=[
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
+                legend="{{model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 111, 12, 8),
+        template=PanelTemplate.PIE_CHART,
+    ),
+    Panel(
+        id=45,
+        title="Peak Tokens Per Second Per Model Last 24 Hours",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"max_over_time(sum by (model_name) (rate(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[2m]))[24h:1m])",
+                legend="{{model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 119, 12, 8),
+        template=PanelTemplate.STAT,
+    ),
+    Panel(
+        id=46,
+        title="Tokens Per Model Last 24 Hours",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d])) + sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1d]))",
+                legend="{{model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 119, 12, 8),
+        template=PanelTemplate.STAT,
+    ),
+    Panel(
+        id=47,
+        title="Avg Total Tokens Per Request Last 7 Days",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=(
+                    f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) +\n"
+                    f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])))"
+                    f" / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))"
+                ),
+                legend="{{ model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 127, 12, 8),
+        template=PanelTemplate.GAUGE,
+    ),
+    Panel(
+        id=48,
+        title="Requests Per Model Last Week",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
+                legend="{{ model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 127, 12, 8),
+        template=PanelTemplate.GAUGE,
+    ),
+    Panel(
+        id=49,
+        title="Tokens Per Model Last 7 Days",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
+                legend="In: {{ model_name}}",
+            ),
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
+                legend="Out: {{ model_name }}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 135, 12, 8),
+        template=PanelTemplate.GAUGE,
+    ),
+    Panel(
+        id=50,
+        title="Avg Total Tokens Per Request Per Model Last 7 Days",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=(
+                    f"(sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) "
+                    f"+ sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])))"
+                    f"/ sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))"
+                ),
+                legend="{{ model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(12, 135, 12, 8),
+        template=PanelTemplate.GAUGE,
+    ),
+    Panel(
+        id=51,
+        title="Tokens Per Request Per Model Last 7 Days",
+        description="",
+        unit="short",
+        targets=[
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_prompt_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
+                legend="In: {{ model_name}}",
+            ),
+            Target(
+                expr=f"sum by (model_name) (delta(ray_vllm_generation_tokens_total{{{{{_WORKERID_FILTER}}}}}[1w])) / sum by (model_name) (delta(ray_vllm_request_success_total{{{{{_WORKERID_FILTER}}}}}[1w]))",
+                legend="Out: {{ model_name}}",
+            ),
+        ],
+        fill=1,
+        linewidth=2,
+        stack=False,
+        grid_pos=GridPos(0, 143, 12, 8),
+        template=PanelTemplate.GAUGE,
+    ),
+]
+
+# ===================================================================
+# Assemble rows and config
+# ===================================================================
+_ALL_ROWS = [
+    Row(title="Throughput", id=501, panels=_throughput_panels),
+    Row(title="Latency", id=502, panels=_latency_panels_list),
+    Row(title="Cache", id=503, panels=_cache_panels),
+    Row(title="Request Length", id=504, panels=_request_length_panels),
+    Row(title="Scheduler", id=505, panels=_scheduler_panels),
+    Row(title="NIXL", id=506, panels=_nixl_panels),
+    Row(
+        title="Token Distribution",
+        id=507,
+        collapsed=True,
+        panels=_token_distribution_panels,
+    ),
+]
+
+# Validate uniqueness of panel IDs across all rows
+_all_ids = sorted(panel.id for row in _ALL_ROWS for panel in row.panels)
+assert len(_all_ids) == len(
+    set(_all_ids)
+), f"Duplicated id found. Use unique id for each panel. {_all_ids}"
+
+serve_llm_dashboard_config = DashboardConfig(
+    name="SERVE_LLM",
+    default_uid="rayServeLlmDashboard",
+    standard_global_filters=[
+        'ray_io_cluster=~"$Cluster"',
+    ],
+    base_json_file_name="serve_llm_grafana_dashboard_base.json",
+    rows=_ALL_ROWS,
+)

python/ray/dashboard/modules/metrics/dashboards/serve_llm_grafana_dashboard_base.json

@@ -0,0 +1,199 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 1,
+  "iteration": 1667344411089,
+  "links": [],
+  "panels": [],
+  "refresh": false,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false
+        },
+        "description": "Filter queries of a specific Prometheus type.",
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "name": "vllm_model_name",
+        "label": "vLLM Model Name",
+        "type": "query",
+        "hide": 0,
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
+        "query": {
+          "query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, model_name)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "name": "workerid",
+        "label": "Worker ID",
+        "type": "query",
+        "hide": 0,
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
+        "query": {
+          "query": "label_values(ray_vllm_request_prompt_tokens_sum{{{global_filters}}}, WorkerId)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "name": "deployment",
+        "label": "Deployment",
+        "type": "query",
+        "hide": 0,
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_serve_deployment_request_counter_total{{{global_filters}}}, deployment)",
+        "query": {
+          "query": "label_values(ray_serve_deployment_request_counter_total{{{global_filters}}}, deployment)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        }
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false
+        },
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+        "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.",
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": false,
+        "name": "Cluster",
+        "options": [],
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 2,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "name": "interval",
+        "label": "Interval",
+        "type": "custom",
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "options": [
+          {
+            "selected": true,
+            "text": "30s",
+            "value": "30s"
+          },
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "current": {
+          "selected": true,
+          "text": "5m",
+          "value": "5m"
+        }
+      }
+    ]
+  },
+  "rayMeta": ["excludesSystemRoutes"],
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Serve LLM Dashboard",
+  "uid": "rayServeLlmDashboard",
+  "version": 1
+}

python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py

@@ -0,0 +1,346 @@
+# flake8: noqa E501
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    Panel,
+    Row,
+    Target,
+)
+
+# Ray Train Metrics (Controller)
+CONTROLLER_STATE_PANEL = Panel(
+    id=1,
+    title="Controller State",
+    description="Current state of the Ray Train controller.",
+    unit="",
+    targets=[
+        Target(
+            expr='sum(ray_train_controller_state{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name, ray_train_controller_state)',
+            legend="Run Name: {{ray_train_run_name}}, Controller State: {{ray_train_controller_state}}",
+        ),
+    ],
+)
+
+CONTROLLER_OPERATION_TIME_PANEL = Panel(
+    id=2,
+    title="Cumulative Worker Group Start/Shutdown Time",
+    description="Cumulative time the controller spends starting and shutting down worker groups (re-created on worker failures and resizes).",
+    unit="seconds",
+    targets=[
+        Target(
+            expr='sum(ray_train_worker_group_start_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name)',
+            legend="Run Name: {{ray_train_run_name}}, Worker Group Start Time",
+        ),
+        Target(
+            expr='sum(ray_train_worker_group_shutdown_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", {global_filters}}}) by (ray_train_run_name)',
+            legend="Run Name: {{ray_train_run_name}}, Worker Group Shutdown Time",
+        ),
+    ],
+    fill=0,
+    stack=False,
+)
+
+# Ray Train Metrics (Worker)
+WORKER_TRAIN_REPORT_TIME_PANEL = Panel(
+    id=3,
+    title="Cumulative Time in ray.train.report",
+    description="Cumulative time workers spend blocked inside `ray.train.report()`. This includes the cross-rank checkpoint directory sync barrier, the checkpoint file transfer to storage, and the time waiting for the report queue ordering. See the Checkpoint Sync and Checkpoint Transfer panels for a breakdown.",
+    unit="seconds",
+    targets=[
+        Target(
+            expr='sum(ray_train_report_total_blocked_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
+            legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
+        )
+    ],
+    fill=0,
+    stack=False,
+)
+WORKER_CHECKPOINT_SYNC_TIME_PANEL = Panel(
+    id=16,
+    title="Cumulative Checkpoint Sync Time",
+    description="Cumulative time spent in the cross-rank barrier that synchronizes the checkpoint directory name across all workers. High values indicate workers are spending significant time waiting for each other to reach the synchronization point.",
+    unit="seconds",
+    targets=[
+        Target(
+            expr='sum(ray_train_checkpoint_sync_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
+            legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
+        )
+    ],
+    fill=0,
+    stack=False,
+)
+
+WORKER_CHECKPOINT_TRANSFER_TIME_PANEL = Panel(
+    id=17,
+    title="Cumulative Checkpoint Transfer Time",
+    description="Cumulative time spent transferring checkpoint files to storage. High values indicate slow storage throughput or large checkpoint sizes.",
+    unit="seconds",
+    targets=[
+        Target(
+            expr='sum(ray_train_checkpoint_transfer_total_time_s{{ray_train_run_name=~"$TrainRunName", ray_train_run_id=~"$TrainRunId", ray_train_worker_world_rank=~"$TrainWorkerWorldRank", ray_train_worker_actor_id=~"$TrainWorkerActorId", {global_filters}}}) by (ray_train_run_name, ray_train_worker_world_rank, ray_train_worker_actor_id)',
+            legend="Run Name: {{ray_train_run_name}}, World Rank: {{ray_train_worker_world_rank}}",
+        )
+    ],
+    fill=0,
+    stack=False,
+)
+
+# Core System Resources
+CPU_UTILIZATION_PANEL = Panel(
+    id=4,
+    title="CPU Usage",
+    description="CPU core utilization across all workers.",
+    unit="cores",
+    targets=[
+        Target(
+            expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType)',
+            legend="CPU Usage: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+            legend="MAX",
+        ),
+    ],
+)
+
+MEMORY_UTILIZATION_PANEL = Panel(
+    id=5,
+    title="Total Memory Usage",
+    description="Total physical memory used vs total available memory.",
+    unit="bytes",
+    targets=[
+        Target(
+            expr='sum(ray_node_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Memory Used: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+            legend="MAX",
+        ),
+    ],
+)
+
+MEMORY_DETAILED_PANEL = Panel(
+    id=6,
+    title="Memory Allocation Details",
+    description="Memory allocation details including available and shared memory.",
+    unit="bytes",
+    targets=[
+        Target(
+            expr='sum(ray_node_mem_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Available Memory: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Shared Memory: {{instance}} ({{RayNodeType}})",
+        ),
+    ],
+)
+
+# GPU Resources
+# TODO: Add GPU Device/Index as a filter.
+GPU_UTILIZATION_PANEL = Panel(
+    id=7,
+    title="GPU Usage",
+    description="GPU utilization across all workers.",
+    unit="GPUs",
+    targets=[
+        Target(
+            expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
+            legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+        ),
+        Target(
+            expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})',
+            legend="MAX",
+        ),
+    ],
+)
+
+GPU_MEMORY_UTILIZATION_PANEL = Panel(
+    id=8,
+    title="GPU Memory Usage",
+    description="GPU memory usage across all workers.",
+    unit="bytes",
+    targets=[
+        Target(
+            expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)',
+            legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}",
+        ),
+        Target(
+            expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})) * 1024 * 1024',
+            legend="MAX",
+        ),
+    ],
+)
+
+# Storage Resources
+DISK_UTILIZATION_PANEL = Panel(
+    id=9,
+    title="Disk Space Usage",
+    description="Disk space usage across all workers.",
+    unit="bytes",
+    targets=[
+        Target(
+            expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Disk Used: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})',
+            legend="MAX",
+        ),
+    ],
+)
+
+DISK_THROUGHPUT_PANEL = Panel(
+    id=10,
+    title="Disk Throughput",
+    description="Current disk read/write throughput.",
+    unit="Bps",
+    targets=[
+        Target(
+            expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Read Speed: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Write Speed: {{instance}} ({{RayNodeType}})",
+        ),
+    ],
+)
+
+DISK_OPERATIONS_PANEL = Panel(
+    id=11,
+    title="Disk Operations",
+    description="Current disk read/write operations per second.",
+    unit="ops/s",
+    targets=[
+        Target(
+            expr='sum(ray_node_disk_read_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Read IOPS: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_disk_write_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Write IOPS: {{instance}} ({{RayNodeType}})",
+        ),
+    ],
+)
+
+# Network Resources
+NETWORK_THROUGHPUT_PANEL = Panel(
+    id=12,
+    title="Network Throughput",
+    description="Current network send/receive throughput.",
+    unit="Bps",
+    targets=[
+        Target(
+            expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Receive Speed: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Send Speed: {{instance}} ({{RayNodeType}})",
+        ),
+    ],
+)
+
+NETWORK_TOTAL_PANEL = Panel(
+    id=13,
+    title="Network Total Traffic",
+    description="Total network traffic sent/received.",
+    unit="bytes",
+    targets=[
+        Target(
+            expr='sum(ray_node_network_sent{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Total Sent: {{instance}} ({{RayNodeType}})",
+        ),
+        Target(
+            expr='sum(ray_node_network_received{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)',
+            legend="Total Received: {{instance}} ({{RayNodeType}})",
+        ),
+    ],
+)
+
+TRAIN_GRAFANA_PANELS = []
+
+TRAIN_GRAFANA_ROWS = [
+    # Train Metrics Row
+    Row(
+        title="Train Metrics",
+        id=14,
+        panels=[
+            # Ray Train Metrics (Controller)
+            CONTROLLER_STATE_PANEL,
+            CONTROLLER_OPERATION_TIME_PANEL,
+            # Ray Train Metrics (Worker)
+            WORKER_TRAIN_REPORT_TIME_PANEL,
+            WORKER_CHECKPOINT_SYNC_TIME_PANEL,
+            WORKER_CHECKPOINT_TRANSFER_TIME_PANEL,
+        ],
+        collapsed=False,
+    ),
+    # System Resources Row
+    Row(
+        title="Resource Utilization",
+        id=15,
+        panels=[
+            CPU_UTILIZATION_PANEL,
+            MEMORY_UTILIZATION_PANEL,
+            MEMORY_DETAILED_PANEL,
+            # GPU Resources
+            GPU_UTILIZATION_PANEL,
+            GPU_MEMORY_UTILIZATION_PANEL,
+            # Storage Resources
+            DISK_UTILIZATION_PANEL,
+            DISK_THROUGHPUT_PANEL,
+            DISK_OPERATIONS_PANEL,
+            # Network Resources
+            NETWORK_THROUGHPUT_PANEL,
+            NETWORK_TOTAL_PANEL,
+        ],
+        collapsed=True,
+    ),
+]
+
+TRAIN_RUN_PANELS = [
+    # Ray Train Metrics (Controller)
+    CONTROLLER_STATE_PANEL,
+    CONTROLLER_OPERATION_TIME_PANEL,
+    # Ray Train Metrics (Worker)
+    WORKER_TRAIN_REPORT_TIME_PANEL,
+]
+
+TRAIN_WORKER_PANELS = [
+    # Ray Train Metrics (Worker)
+    WORKER_TRAIN_REPORT_TIME_PANEL,
+    WORKER_CHECKPOINT_SYNC_TIME_PANEL,
+    WORKER_CHECKPOINT_TRANSFER_TIME_PANEL,
+    # Core System Resources
+    CPU_UTILIZATION_PANEL,
+    MEMORY_UTILIZATION_PANEL,
+    # GPU Resources
+    GPU_UTILIZATION_PANEL,
+    GPU_MEMORY_UTILIZATION_PANEL,
+    # Storage Resources
+    DISK_UTILIZATION_PANEL,
+    # Network Resources
+    NETWORK_THROUGHPUT_PANEL,
+]
+
+# Get all panel IDs from both top-level panels and panels within rows
+all_panel_ids = [panel.id for panel in TRAIN_GRAFANA_PANELS]
+for row in TRAIN_GRAFANA_ROWS:
+    all_panel_ids.append(row.id)
+    all_panel_ids.extend(panel.id for panel in row.panels)
+
+all_panel_ids.sort()
+
+assert len(all_panel_ids) == len(
+    set(all_panel_ids)
+), f"Duplicated id found. Use unique id for each panel. {all_panel_ids}"
+
+train_dashboard_config = DashboardConfig(
+    name="TRAIN",
+    default_uid="rayTrainDashboard",
+    rows=TRAIN_GRAFANA_ROWS,
+    standard_global_filters=['SessionName=~"$SessionName"'],
+    base_json_file_name="train_grafana_dashboard_base.json",
+)

python/ray/dashboard/modules/metrics/dashboards/train_grafana_dashboard_base.json

@@ -0,0 +1,267 @@
+{
+  "title": "Train Dashboard",
+  "uid": "rayTrainDashboard",
+  "version": 1,
+  "schemaVersion": 27,
+  "style": "dark",
+  "editable": true,
+  "graphTooltip": 1,
+  "refresh": false,
+  "panels": [],
+
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+
+  "templating": {
+    "list": [
+      {
+        "name": "datasource",
+        "type": "datasource",
+        "description": "Filter queries of a specific Prometheus type.",
+        "datasource": null,
+        "query": "prometheus",
+        "refresh": 1,
+        "hide": 2,
+        "includeAll": false,
+        "multi": false,
+        "current": {
+          "selected": false
+        }
+      },
+
+      {
+        "name": "SessionName",
+        "type": "query",
+        "description": "Filter queries to specific ray sessions.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, SessionName)",
+        "query": {
+          "query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, SessionName)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "name": "TrainRunName",
+        "type": "query",
+        "description": "Filter queries to specific Ray Train run names.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_name)",
+        "query": {
+          "query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_name)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "name": "TrainRunId",
+        "type": "query",
+        "description": "Filter queries to specific Ray Train run ids.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_id)",
+        "query": {
+          "query": "label_values(ray_train_worker_group_start_total_time_s{{{global_filters}}}, ray_train_run_id)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 2,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "name": "TrainWorkerWorldRank",
+        "type": "query",
+        "description": "Filter queries to specific Ray Train worker world ranks.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_world_rank)",
+        "query": {
+          "query": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_world_rank)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "name": "TrainWorkerActorId",
+        "type": "query",
+        "description": "Filter queries to specific Ray Train worker actor ids.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_actor_id)",
+        "query": {
+          "query": "label_values(ray_train_report_total_blocked_time_s{{{global_filters}}}, ray_train_worker_actor_id)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 2,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "name": "Instance",
+        "type": "query",
+        "description": "Filter queries to specific node instances.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, instance)",
+        "query": {
+          "query": "label_values(ray_node_network_receive_speed{{{global_filters}}}, instance)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 2,
+        "includeAll": true,
+        "multi": false,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "name": "GpuIndex",
+        "type": "query",
+        "description": "Filter queries to specific GPU indices.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuIndex)",
+        "query": {
+          "query": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuIndex)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 2,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "name": "GpuDeviceName",
+        "type": "query",
+        "description": "Filter queries to specific GPU device names.",
+        "datasource": "${datasource}",
+        "definition": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuDeviceName)",
+        "query": {
+          "query": "label_values(ray_node_gpus_utilization{{{global_filters}}}, GpuDeviceName)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "hide": 2,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "sort": 2,
+        "current": {
+          "selected": true,
+          "text": ["All"],
+          "value": ["$__all"]
+        }
+      },
+
+      {
+        "current": {
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "description": "Filter queries to specific Ray node types (head or worker).",
+        "includeAll": true,
+        "multi": true,
+        "name": "RayNodeType",
+        "options": [
+          {
+            "selected": false,
+            "text": "All",
+            "value": "$__all"
+          },
+          {
+            "selected": false,
+            "text": "Head Node",
+            "value": "head"
+          },
+          {
+            "selected": false,
+            "text": "Worker Node",
+            "value": "worker"
+          }
+        ],
+        "query": "head, worker",
+        "type": "custom"
+      }
+    ]
+  }
+}

python/ray/dashboard/modules/metrics/default_impl.py

@@ -0,0 +1,12 @@
+from ray.dashboard.modules.metrics.dashboards.common import DashboardConfig
+
+
+def get_serve_dashboard_config() -> DashboardConfig:
+    from ray.dashboard.modules.metrics.dashboards.serve_dashboard_panels import (
+        serve_dashboard_config,
+    )
+
+    return serve_dashboard_config
+
+
+# Anyscale overrides

python/ray/dashboard/modules/metrics/export/prometheus/prometheus.yml

@@ -0,0 +1,12 @@
+# my global config
+global:
+  scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default is every 1 minute.
+  evaluation_interval: 10s # Evaluate rules every 10 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+scrape_configs:
+# Scrape from each Ray node as defined in the service_discovery.json provided by Ray.
+- job_name: 'ray'
+  file_sd_configs:
+  - files:
+    - '/tmp/ray/prom_metrics_service_discovery.json'

python/ray/dashboard/modules/metrics/grafana_dashboard_factory.py

@@ -0,0 +1,541 @@
+import copy
+import json
+import math
+import os
+import re
+from dataclasses import asdict
+from typing import List, Tuple
+
+import ray
+from ray.dashboard.modules.metrics.dashboards.common import (
+    DashboardConfig,
+    Panel,
+    PanelTemplate,
+)
+from ray.dashboard.modules.metrics.dashboards.data_dashboard_panels import (
+    data_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.data_llm_dashboard_panels import (
+    data_llm_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.default_dashboard_panels import (
+    default_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.serve_deployment_dashboard_panels import (
+    serve_deployment_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.serve_llm_dashboard_panels import (
+    serve_llm_dashboard_config,
+)
+from ray.dashboard.modules.metrics.dashboards.train_dashboard_panels import (
+    train_dashboard_config,
+)
+from ray.dashboard.modules.metrics.default_impl import get_serve_dashboard_config
+
+GRAFANA_DASHBOARD_UID_OVERRIDE_ENV_VAR_TEMPLATE = "RAY_GRAFANA_{name}_DASHBOARD_UID"
+GRAFANA_DASHBOARD_GLOBAL_FILTERS_OVERRIDE_ENV_VAR_TEMPLATE = (
+    "RAY_GRAFANA_{name}_DASHBOARD_GLOBAL_FILTERS"
+)
+GRAFANA_DASHBOARD_LOG_LINK_URL_ENV_VAR_TEMPLATE = "RAY_GRAFANA_{name}_LOG_LINK_URL"
+
+# Grafana dashboard layout constants
+# Dashboard uses a 24-column grid with 2-column panels
+ROW_WIDTH = 24  # Full dashboard width
+PANELS_PER_ROW = 2
+PANEL_WIDTH = ROW_WIDTH // PANELS_PER_ROW  # Width of each panel
+PANEL_HEIGHT = 8  # Height of each panel
+ROW_HEIGHT = 1  # Height of row container
+
+
+def _read_configs_for_dashboard(
+    dashboard_config: DashboardConfig,
+) -> Tuple[str, List[str], str]:
+    """Reads environment variable configs for overriding uid, global_filters, and the log link URL for a given dashboard.
+
+    Args:
+        dashboard_config: The dashboard whose env-var overrides are read.
+            ``dashboard_config.name`` selects the env-var suffix and
+            ``default_uid`` is used as a fallback.
+
+    Returns:
+      Tuple with format uid, global_filters, log_link_url
+    """
+    uid = (
+        os.environ.get(
+            GRAFANA_DASHBOARD_UID_OVERRIDE_ENV_VAR_TEMPLATE.format(
+                name=dashboard_config.name
+            )
+        )
+        or dashboard_config.default_uid
+    )
+    global_filters_str = (
+        os.environ.get(
+            GRAFANA_DASHBOARD_GLOBAL_FILTERS_OVERRIDE_ENV_VAR_TEMPLATE.format(
+                name=dashboard_config.name
+            )
+        )
+        or ""
+    )
+    if global_filters_str == "":
+        global_filters = []
+    else:
+        global_filters = global_filters_str.split(",")
+
+    log_link_url = (
+        os.environ.get(
+            GRAFANA_DASHBOARD_LOG_LINK_URL_ENV_VAR_TEMPLATE.format(
+                name=dashboard_config.name
+            )
+        )
+        or ""
+    )
+
+    return uid, global_filters, log_link_url
+
+
+def generate_default_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the default dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(default_dashboard_config)
+
+
+def generate_serve_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the serve dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(get_serve_dashboard_config())
+
+
+def generate_serve_deployment_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the serve dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(serve_deployment_dashboard_config)
+
+
+def generate_serve_llm_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the serve dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(serve_llm_dashboard_config)
+
+
+def generate_data_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the data dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(data_dashboard_config)
+
+
+def generate_data_llm_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the Data LLM dashboard and returns
+    both the content and the uid.
+
+    This dashboard provides vLLM metrics visibility for Ray Data LLM workloads,
+    including latency (TTFT, TPOT), throughput, cache utilization, and
+    prefix cache hit rate.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(data_llm_dashboard_config)
+
+
+def generate_train_grafana_dashboard() -> Tuple[str, str]:
+    """
+    Generates the dashboard output for the train dashboard and returns
+    both the content and the uid.
+
+    Returns:
+      Tuple with format content, uid
+    """
+    return _generate_grafana_dashboard(train_dashboard_config)
+
+
+def _generate_grafana_dashboard(dashboard_config: DashboardConfig) -> str:
+    """Render the Grafana dashboard JSON for the given config.
+
+    Args:
+        dashboard_config: Configuration describing the panels and base
+            template JSON file to use for rendering.
+
+    Returns:
+      Tuple with format dashboard_content, uid
+    """
+    uid, global_filters, log_link_url = _read_configs_for_dashboard(dashboard_config)
+    panels = _generate_grafana_panels(dashboard_config, global_filters, log_link_url)
+    base_file_name = dashboard_config.base_json_file_name
+
+    base_json = json.load(
+        open(os.path.join(os.path.dirname(__file__), "dashboards", base_file_name))
+    )
+    base_json["panels"] = panels
+    # Update variables to use global_filters
+    global_filters_str = ",".join(global_filters)
+    variables = base_json.get("templating", {}).get("list", [])
+    for variable in variables:
+        if "definition" not in variable:
+            continue
+        definition = variable["definition"].format(global_filters=global_filters_str)
+        query = variable["query"]["query"].format(global_filters=global_filters_str)
+        if not global_filters_str:
+            definition = _clean_empty_filters(definition)
+            query = _clean_empty_filters(query)
+        variable["definition"] = definition
+        variable["query"]["query"] = query
+
+    tags = base_json.get("tags", []) or []
+    tags.append(f"rayVersion:{ray.__version__}")
+    base_json["tags"] = tags
+    base_json["uid"] = uid
+    # Ray metadata can be used to put arbitrary metadata
+    ray_meta = base_json.get("rayMeta", []) or []
+    ray_meta.append("supportsGlobalFilterOverride")
+    base_json["rayMeta"] = ray_meta
+    return json.dumps(base_json, indent=4), uid
+
+
+def _generate_panel_template(
+    panel: Panel,
+    panel_global_filters: List[str],
+    panel_index: int,
+    base_y_position: int,
+    log_link_url: str,
+) -> dict:
+    """
+    Helper method to generate a panel template with common configuration.
+
+    Args:
+        panel: The panel configuration
+        panel_global_filters: List of global filters to apply
+        panel_index: The index of the panel within its row (0-based)
+        base_y_position: The base y-coordinate for the row in the dashboard grid
+        log_link_url: The URL to the log link for the panel
+
+    Returns:
+        dict: The configured panel template
+    """
+    # Create base template from panel configuration
+    template = copy.deepcopy(panel.template.value)
+    template.update(
+        {
+            "title": panel.title,
+            "description": panel.description,
+            "id": panel.id,
+            "targets": _generate_targets(panel, panel_global_filters),
+        }
+    )
+
+    # Set panel position and dimensions
+    if panel.grid_pos:
+        template["gridPos"] = asdict(panel.grid_pos)
+    else:
+        # Calculate panel position in 2-column grid layout
+        # x: 0 or 12 (left or right column)
+        # y: base position + (row number * panel height)
+        row_number = panel_index // PANELS_PER_ROW
+        template["gridPos"] = {
+            "h": PANEL_HEIGHT,
+            "w": PANEL_WIDTH,
+            "x": PANEL_WIDTH * (panel_index % PANELS_PER_ROW),
+            "y": base_y_position + (row_number * PANEL_HEIGHT),
+        }
+
+    # Set unit format for legacy graph-style panels (GRAPH, HEATMAP, STAT, GAUGE, PIE_CHART, BAR_CHART)
+    if panel.template in (
+        PanelTemplate.GRAPH,
+        PanelTemplate.HEATMAP,
+        PanelTemplate.STAT,
+        PanelTemplate.GAUGE,
+        PanelTemplate.PIE_CHART,
+        PanelTemplate.BAR_CHART,
+    ):
+        template["yaxes"][0]["format"] = panel.unit
+
+    # Set fieldConfig unit (for newer panel types with fieldConfig.defaults)
+    if panel.template in (
+        PanelTemplate.STAT,
+        PanelTemplate.GAUGE,
+        PanelTemplate.HEATMAP,
+        PanelTemplate.PIE_CHART,
+        PanelTemplate.BAR_CHART,
+        PanelTemplate.TABLE,
+        PanelTemplate.GRAPH,
+    ):
+        template["fieldConfig"]["defaults"]["unit"] = panel.unit
+
+    # Set fill, stack, linewidth, nullPointMode (only for GRAPH panels)
+    if panel.template == PanelTemplate.GRAPH:
+        template["fill"] = panel.fill
+        template["stack"] = panel.stack
+        template["linewidth"] = panel.linewidth
+        if panel.stack is True:
+            template["nullPointMode"] = "connected"
+
+    if panel.hideXAxis:
+        template.setdefault("xaxis", {})["show"] = False
+
+    # Handle optional panel customization fields
+
+    # Thresholds (for panels with fieldConfig.defaults.thresholds)
+    if panel.thresholds is not None:
+        if panel.template in (PanelTemplate.STAT, PanelTemplate.GAUGE):
+            template["fieldConfig"]["defaults"]["thresholds"][
+                "steps"
+            ] = panel.thresholds
+
+    # Value mappings (for panels with fieldConfig.defaults.mappings)
+    if panel.value_mappings is not None:
+        if panel.template in (
+            PanelTemplate.STAT,
+            PanelTemplate.GAUGE,
+            PanelTemplate.TABLE,
+        ):
+            template["fieldConfig"]["defaults"]["mappings"] = panel.value_mappings
+
+    # Color mode (for STAT panels with options.colorMode)
+    if panel.color_mode is not None:
+        if panel.template == PanelTemplate.STAT:
+            template["options"]["colorMode"] = panel.color_mode
+
+    # Legend mode
+    if panel.legend_mode is not None:
+        if panel.template in (PanelTemplate.GRAPH, PanelTemplate.BAR_CHART):
+            # For graph panels (legacy format with top-level legend object)
+            template["legend"]["show"] = panel.legend_mode != "hidden"
+            template["legend"]["alignAsTable"] = panel.legend_mode == "table"
+        elif panel.template == PanelTemplate.PIE_CHART:
+            # For PIE_CHART (options.legend.displayMode)
+            template["options"]["legend"]["displayMode"] = panel.legend_mode
+
+    # Min/max values (for panels with fieldConfig.defaults)
+    if panel.min_val is not None or panel.max_val is not None:
+        if panel.template in (
+            PanelTemplate.STAT,
+            PanelTemplate.GAUGE,
+            PanelTemplate.HEATMAP,
+            PanelTemplate.PIE_CHART,
+            PanelTemplate.BAR_CHART,
+            PanelTemplate.TABLE,
+            PanelTemplate.GRAPH,
+        ):
+            if panel.min_val is not None:
+                template["fieldConfig"]["defaults"]["min"] = panel.min_val
+            if panel.max_val is not None:
+                template["fieldConfig"]["defaults"]["max"] = panel.max_val
+
+    # Reduce calculation (for panels with options.reduceOptions)
+    if panel.reduce_calc is not None:
+        if panel.template in (
+            PanelTemplate.STAT,
+            PanelTemplate.GAUGE,
+            PanelTemplate.PIE_CHART,
+        ):
+            template["options"]["reduceOptions"]["calcs"] = [panel.reduce_calc]
+
+    # Handle heatmap-specific options
+    if panel.heatmap_color_scheme is not None:
+        if panel.template == PanelTemplate.HEATMAP:
+            template["options"]["color"]["scheme"] = panel.heatmap_color_scheme
+
+    if panel.heatmap_color_reverse is not None:
+        if panel.template == PanelTemplate.HEATMAP:
+            template["options"]["color"]["reverse"] = panel.heatmap_color_reverse
+
+    if panel.heatmap_yaxis_label is not None:
+        if panel.template in (
+            PanelTemplate.GRAPH,
+            PanelTemplate.HEATMAP,
+            PanelTemplate.STAT,
+            PanelTemplate.GAUGE,
+            PanelTemplate.PIE_CHART,
+            PanelTemplate.BAR_CHART,
+        ):
+            template["yaxes"][0]["label"] = panel.heatmap_yaxis_label
+
+    # Add log link if URL is provided via environment variable.
+    if log_link_url:
+        template["links"] = [
+            {
+                "targetBlank": True,
+                "title": "View Logs",
+                "url": log_link_url,
+            }
+        ]
+
+    return template
+
+
+def _create_row_panel(row: Panel, y_position: int) -> dict:
+    """
+    Creates a Grafana row panel that spans the full dashboard width.
+    Row panels can be collapsed to hide their contained panels.
+
+    Args:
+        row: Row config with title, id, and collapse state
+        y_position: Vertical position in dashboard grid
+
+    Returns:
+        Grafana row panel configuration
+    """
+    return {
+        "collapsed": row.collapsed,
+        "gridPos": {"h": ROW_HEIGHT, "w": ROW_WIDTH, "x": 0, "y": y_position},
+        "id": row.id,
+        "title": row.title,
+        "type": "row",
+        "panels": [],
+    }
+
+
+def _calculate_panel_heights(num_panels: int) -> int:
+    """
+    Calculate the total height needed for a set of panels.
+
+    Args:
+        num_panels: Number of panels to position
+
+    Returns:
+        Total height needed for the panels
+    """
+    rows_needed = math.ceil(num_panels / PANELS_PER_ROW)
+    return rows_needed * PANEL_HEIGHT
+
+
+def _generate_grafana_panels(
+    config: DashboardConfig, global_filters: List[str], log_link_url: str
+) -> List[dict]:
+    """
+    Generates Grafana panel configurations for a dashboard.
+
+    The dashboard layout follows these rules:
+    - Panels are arranged in 2 columns (12 units wide each)
+    - Each panel is 8 units high
+    - Rows are 1 unit high and can be collapsed
+    - Panels within rows follow the same 2-column layout
+    - Panel positions can be overridden via panel.grid_pos or auto-calculated
+
+    Args:
+        config: Dashboard configuration containing panels and rows
+        global_filters: List of filters to apply to all panels
+        log_link_url: Optional URL for panel log links. When set, each panel
+            gets a "View Logs" link pointing to this URL.
+
+    Returns:
+        List of Grafana panel configurations for the dashboard
+    """
+    panels = []
+    panel_global_filters = [*config.standard_global_filters, *global_filters]
+    current_y_position = 0
+
+    # Add top-level panels in 2-column grid
+    for panel_index, panel in enumerate(config.panels):
+        panel_template = _generate_panel_template(
+            panel, panel_global_filters, panel_index, current_y_position, log_link_url
+        )
+        panels.append(panel_template)
+
+    # Calculate space needed for top-level panels
+    current_y_position += _calculate_panel_heights(len(config.panels))
+
+    # Add rows and their panels
+    if not config.rows:
+        return panels
+
+    for row in config.rows:
+        # Create and add row panel
+        row_panel = _create_row_panel(row, current_y_position)
+        panels.append(row_panel)
+        current_y_position += ROW_HEIGHT
+
+        # Add panels within row using 2-column grid
+        for panel_index, panel in enumerate(row.panels):
+            panel_template = _generate_panel_template(
+                panel,
+                panel_global_filters,
+                panel_index,
+                current_y_position,
+                log_link_url,
+            )
+
+            # Add panel to row if collapsed, otherwise to main dashboard
+            if row.collapsed:
+                row_panel["panels"].append(panel_template)
+            else:
+                panels.append(panel_template)
+
+        # Update y position for next row based on actual panel positions
+        # when explicit grid_pos is used, or fallback to calculated height.
+        if any(p.grid_pos for p in row.panels):
+            max_y_bottom = max(
+                (p.grid_pos.y + p.grid_pos.h for p in row.panels if p.grid_pos),
+                default=current_y_position,
+            )
+            current_y_position = max_y_bottom
+        else:
+            current_y_position += _calculate_panel_heights(len(row.panels))
+
+    return panels
+
+
+def _clean_empty_filters(expr: str) -> str:
+    """Clean up malformed PromQL when global_filters is empty.
+
+    Removes artifacts like trailing/leading commas in label matchers:
+      ", ," → ","
+      ", }" → "}"
+      "{ ," → "{"
+    """
+    expr = re.sub(r",\s*,", ",", expr)
+    expr = re.sub(r",\s*}", "}", expr)
+    expr = re.sub(r"{\s*,", "{", expr)
+    return expr
+
+
+def gen_incrementing_alphabets(length):
+    assert 65 + length < 96, "we only support up to 26 targets at a time."
+    # 65: ascii code of 'A'.
+    return list(map(chr, range(65, 65 + length)))
+
+
+def _generate_targets(panel: Panel, panel_global_filters: List[str]) -> List[dict]:
+    targets = []
+    for target, ref_id in zip(
+        panel.targets, gen_incrementing_alphabets(len(panel.targets))
+    ):
+        template = copy.deepcopy(target.template.value)
+        global_filters_str = ",".join(panel_global_filters)
+        expr = target.expr.format(global_filters=global_filters_str)
+        if not global_filters_str:
+            expr = _clean_empty_filters(expr)
+        template.update(
+            {
+                "expr": expr,
+                "legendFormat": target.legend,
+                "refId": ref_id,
+            }
+        )
+        targets.append(template)
+    return targets

python/ray/dashboard/modules/metrics/install_and_start_prometheus.py

@@ -0,0 +1,203 @@
+import logging
+import os
+import platform
+import subprocess
+import sys
+import tarfile
+from pathlib import Path
+
+import requests
+
+from ray.dashboard.consts import PROMETHEUS_CONFIG_INPUT_PATH
+
+FALLBACK_PROMETHEUS_VERSION = "2.48.1"
+DOWNLOAD_BLOCK_SIZE = 8192  # 8 KB
+TEST_MODE_ENV_VAR = "RAY_PROMETHEUS_DOWNLOAD_TEST_MODE"
+
+
+def get_system_info():
+    os_type = platform.system().lower()
+    architecture = platform.machine()
+    if architecture == "x86_64":
+        # In the Prometheus filename, it's called amd64
+        architecture = "amd64"
+    elif architecture == "aarch64":
+        # In the Prometheus filename, it's called arm64
+        architecture = "arm64"
+    return os_type, architecture
+
+
+def download_file(url, filename):
+    logging.info(f"Downloading {url} to {Path(filename).absolute()}...")
+    try:
+        test_mode = os.environ.get(TEST_MODE_ENV_VAR, False)
+        request_method = requests.head if test_mode else requests.get
+        response = request_method(url, stream=True)
+        response.raise_for_status()
+
+        total_size_in_bytes = int(response.headers.get("content-length", 0))
+        total_size_in_mb = total_size_in_bytes / (1024 * 1024)
+
+        downloaded_size_in_mb = 0
+        block_size = DOWNLOAD_BLOCK_SIZE
+
+        with open(filename, "wb") as file:
+            for chunk in response.iter_content(chunk_size=block_size):
+                file.write(chunk)
+                downloaded_size_in_mb += len(chunk) / (1024 * 1024)
+                print(
+                    f"Downloaded: {downloaded_size_in_mb:.2f} MB / "
+                    f"{total_size_in_mb:.2f} MB",
+                    end="\r",
+                )
+
+        print("\nDownload completed.")
+        return True
+
+    except requests.RequestException as e:
+        logging.error(f"Error downloading file: {e}")
+        return False
+
+
+def install_prometheus(file_path):
+    try:
+        with tarfile.open(file_path) as tar:
+            tar.extractall()
+        logging.info("Prometheus installed successfully.")
+        return True
+    except Exception as e:
+        logging.error(f"Error installing Prometheus: {e}")
+        return False
+
+
+def start_prometheus(prometheus_dir):
+
+    # The function assumes the Ray cluster to be monitored by Prometheus uses the
+    # default configuration with "/tmp/ray" as the default root temporary directory.
+    #
+    # This is to support the `ray metrics launch-prometheus` command, when a Ray cluster
+    # hasn't started yet and the user doesn't have a way to get a `--temp-dir`
+    # anywhere. So we choose to use a hardcoded default value.
+
+    config_file = Path(PROMETHEUS_CONFIG_INPUT_PATH)
+
+    if not config_file.exists():
+        raise FileNotFoundError(f"Prometheus config file not found: {config_file}")
+
+    prometheus_cmd = [
+        f"{prometheus_dir}/prometheus",
+        "--config.file",
+        str(config_file),
+        "--web.enable-lifecycle",
+    ]
+    try:
+        process = subprocess.Popen(prometheus_cmd)
+        logging.info("Prometheus has started.")
+        return process
+    except Exception as e:
+        logging.error(f"Failed to start Prometheus: {e}")
+        return None
+
+
+def print_shutdown_message(process_id):
+    message = (
+        f"Prometheus is running with PID {process_id}.\n"
+        "To stop Prometheus, use the command: "
+        "`ray metrics shutdown-prometheus`, "
+        f"'kill {process_id}', or if you need to force stop, "
+        f"use 'kill -9 {process_id}'."
+    )
+    print(message)
+
+    debug_message = (
+        "To list all processes running Prometheus, use the command: "
+        "'ps aux | grep prometheus'."
+    )
+    print(debug_message)
+
+
+def get_latest_prometheus_version():
+    url = "https://api.github.com/repos/prometheus/prometheus/releases/latest"
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        data = response.json()
+        # Remove the leading 'v' from the version number
+        return data["tag_name"].lstrip("v")
+    except requests.RequestException as e:
+        logging.error(f"Error fetching latest Prometheus version: {e}")
+        return None
+
+
+def get_prometheus_filename(os_type=None, architecture=None, prometheus_version=None):
+    if os_type is None or architecture is None:
+        os_type, architecture = get_system_info()
+
+    if prometheus_version is None:
+        prometheus_version = get_latest_prometheus_version()
+        if prometheus_version is None:
+            logging.warning(
+                "Failed to retrieve the latest Prometheus version. Falling "
+                f"back to {FALLBACK_PROMETHEUS_VERSION}."
+            )
+            # Fall back to a hardcoded version
+            prometheus_version = FALLBACK_PROMETHEUS_VERSION
+
+    return (
+        f"prometheus-{prometheus_version}.{os_type}-{architecture}.tar.gz",
+        prometheus_version,
+    )
+
+
+def get_prometheus_download_url(
+    os_type=None, architecture=None, prometheus_version=None
+):
+    file_name, prometheus_version = get_prometheus_filename(
+        os_type, architecture, prometheus_version
+    )
+    return (
+        "https://github.com/prometheus/prometheus/releases/"
+        f"download/v{prometheus_version}/{file_name}"
+    )
+
+
+def download_prometheus(os_type=None, architecture=None, prometheus_version=None):
+    file_name, _ = get_prometheus_filename(os_type, architecture, prometheus_version)
+    download_url = get_prometheus_download_url(
+        os_type, architecture, prometheus_version
+    )
+
+    return download_file(download_url, file_name), file_name
+
+
+def main():
+    # Configure logging only when this script is run directly
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+    )
+    logging.warning("This script is not intended for production use.")
+
+    downloaded, file_name = download_prometheus()
+    if not downloaded:
+        logging.error("Failed to download Prometheus.")
+        sys.exit(1)
+
+    # TODO: Verify the checksum of the downloaded file
+
+    if not install_prometheus(file_name):
+        logging.error("Installation failed.")
+        sys.exit(1)
+
+    # TODO: Add a check to see if Prometheus is already running
+
+    assert file_name.endswith(".tar.gz")
+    process = start_prometheus(
+        # remove the .tar.gz extension
+        prometheus_dir=file_name.rstrip(".tar.gz")
+    )
+    if process:
+        print_shutdown_message(process.pid)
+
+
+if __name__ == "__main__":
+    main()