chore: import upstream snapshot with attribution

python/ray/dashboard/modules/aggregator/aggregator_agent.py

@@ -0,0 +1,230 @@
+import asyncio
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+import ray
+import ray.dashboard.utils as dashboard_utils
+from ray._private import ray_constants
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.core.generated import (
+    events_event_aggregator_service_pb2,
+    events_event_aggregator_service_pb2_grpc,
+)
+from ray.dashboard.modules.aggregator.constants import AGGREGATOR_AGENT_METRIC_PREFIX
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+from ray.dashboard.modules.aggregator.publisher.async_publisher_client import (
+    AsyncGCSTaskEventsPublisherClient,
+    AsyncHttpPublisherClient,
+)
+from ray.dashboard.modules.aggregator.publisher.ray_event_publisher import (
+    NoopPublisher,
+    RayEventPublisher,
+)
+from ray.dashboard.modules.aggregator.task_events_metadata_buffer import (
+    TaskEventsMetadataBuffer,
+)
+
+logger = logging.getLogger(__name__)
+
+# Max number of threads for the thread pool executor handling CPU intensive tasks
+THREAD_POOL_EXECUTOR_MAX_WORKERS = ray_constants.env_integer(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_THREAD_POOL_EXECUTOR_MAX_WORKERS", 1
+)
+# Interval to check the main thread liveness
+CHECK_MAIN_THREAD_LIVENESS_INTERVAL_SECONDS = ray_constants.env_float(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_CHECK_MAIN_THREAD_LIVENESS_INTERVAL_SECONDS", 0.1
+)
+# Maximum size of the event buffer in the aggregator agent
+# The default value was 1,000,000 but was reduced to 100,000 now to avoid being OOM Killed.
+# We observed that the previous 1,000,000 could take up to 20 GB of memory.
+# TODO (rueian): Find a better way for the event buffer to store events while avoiding being OOM Killed. For example:
+# 1. Store bytes instead of python objects and count the size in bytes.
+# 2. Compress the bytes before storing them in the buffer? (This will increase the CPU usage)
+# 3. Don't be fixed at 10,0000 but adjust the buffer size based on the available memory on startup.
+MAX_EVENT_BUFFER_SIZE = ray_constants.env_integer(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_BUFFER_SIZE", 100000
+)
+# Maximum number of events to send in a single batch to the destination
+MAX_EVENT_SEND_BATCH_SIZE = ray_constants.env_integer(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_SEND_BATCH_SIZE", 1000
+)
+# Address of the external service to send events with format of "http://<ip>:<port>"
+EVENTS_EXPORT_ADDR = os.environ.get(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR", ""
+)
+# flag to enable publishing events to the external HTTP service
+PUBLISH_EVENTS_TO_EXTERNAL_HTTP_SERVICE = ray_constants.env_bool(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISH_EVENTS_TO_EXTERNAL_HTTP_SERVICE", True
+)
+# flag to enable publishing events to GCS
+PUBLISH_EVENTS_TO_GCS = ray_constants.env_bool(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISH_EVENTS_TO_GCS", False
+)
+# flag to control whether preserve the proto field name when converting the events to
+# JSON. If True, the proto field name will be preserved. If False, the proto field name
+# will be converted to camel case.
+PRESERVE_PROTO_FIELD_NAME = ray_constants.env_bool(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_PRESERVE_PROTO_FIELD_NAME", False
+)
+
+
+class AggregatorAgent(
+    dashboard_utils.DashboardAgentModule,
+    events_event_aggregator_service_pb2_grpc.EventAggregatorServiceServicer,
+):
+    """
+    AggregatorAgent is a dashboard agent module that collects events sent with
+    gRPC from other components, buffers them, and periodically sends them to GCS and
+    an external service with HTTP POST requests for further processing or storage
+    """
+
+    def __init__(self, dashboard_agent) -> None:
+        super().__init__(dashboard_agent)
+        self._ip = dashboard_agent.ip
+        self._pid = os.getpid()
+
+        # common prometheus labels for aggregator-owned metrics
+        self._common_tags = {
+            "ip": self._ip,
+            "pid": str(self._pid),
+            "Version": ray.__version__,
+            "Component": "aggregator_agent",
+            "SessionName": self.session_name,
+        }
+
+        self._event_buffer = MultiConsumerEventBuffer(
+            max_size=MAX_EVENT_BUFFER_SIZE,
+            max_batch_size=MAX_EVENT_SEND_BATCH_SIZE,
+            common_metric_tags=self._common_tags,
+        )
+        self._executor = ThreadPoolExecutor(
+            max_workers=THREAD_POOL_EXECUTOR_MAX_WORKERS,
+            thread_name_prefix="aggregator_agent_executor",
+        )
+
+        # Task metadata buffer accumulates dropped task attempts for GCS publishing
+        self._task_metadata_buffer = TaskEventsMetadataBuffer(
+            common_metric_tags=self._common_tags
+        )
+
+        self._events_export_addr = (
+            dashboard_agent.events_export_addr or EVENTS_EXPORT_ADDR
+        )
+
+        self._event_processing_enabled = False
+        if PUBLISH_EVENTS_TO_EXTERNAL_HTTP_SERVICE and self._events_export_addr:
+            logger.info(
+                f"Publishing events to external HTTP service is enabled. events_export_addr: {self._events_export_addr}"
+            )
+            self._event_processing_enabled = True
+            self._http_endpoint_publisher = RayEventPublisher(
+                name="http_service",
+                publish_client=AsyncHttpPublisherClient(
+                    endpoint=self._events_export_addr,
+                    executor=self._executor,
+                    preserve_proto_field_name=PRESERVE_PROTO_FIELD_NAME,
+                ),
+                event_buffer=self._event_buffer,
+                common_metric_tags=self._common_tags,
+            )
+        else:
+            logger.info(
+                f"Event HTTP target is not enabled or publishing events to external HTTP service is disabled. Skipping sending events to external HTTP service. events_export_addr: {self._events_export_addr}"
+            )
+            self._http_endpoint_publisher = NoopPublisher()
+
+        if PUBLISH_EVENTS_TO_GCS:
+            logger.info("Publishing events to GCS is enabled")
+            self._event_processing_enabled = True
+            self._gcs_publisher = RayEventPublisher(
+                name="ray_gcs",
+                publish_client=AsyncGCSTaskEventsPublisherClient(
+                    gcs_client=self._dashboard_agent.gcs_client,
+                    executor=self._executor,
+                ),
+                event_buffer=self._event_buffer,
+                common_metric_tags=self._common_tags,
+                task_metadata_buffer=self._task_metadata_buffer,
+            )
+        else:
+            logger.info("Publishing events to GCS is disabled")
+            self._gcs_publisher = NoopPublisher()
+
+        # Metrics
+        self._open_telemetry_metric_recorder = OpenTelemetryMetricRecorder()
+
+        # Register counter metrics
+        self._events_received_metric_name = (
+            f"{AGGREGATOR_AGENT_METRIC_PREFIX}_events_received_total"
+        )
+        self._open_telemetry_metric_recorder.register_counter_metric(
+            self._events_received_metric_name,
+            "Total number of events received via AddEvents gRPC.",
+        )
+
+        self._events_failed_to_add_metric_name = (
+            f"{AGGREGATOR_AGENT_METRIC_PREFIX}_events_buffer_add_failures_total"
+        )
+        self._open_telemetry_metric_recorder.register_counter_metric(
+            self._events_failed_to_add_metric_name,
+            "Total number of events that failed to be added to the event buffer.",
+        )
+
+    async def AddEvents(self, request, context) -> None:
+        """
+        gRPC handler for adding events to the event aggregator. Receives events from the
+        request and adds them to the event buffer.
+        """
+        if not self._event_processing_enabled:
+            return events_event_aggregator_service_pb2.AddEventsReply()
+
+        received_count = len(request.events_data.events)
+        failed_count = 0
+        events_data = request.events_data
+
+        if PUBLISH_EVENTS_TO_GCS:
+            self._task_metadata_buffer.merge(events_data.task_events_metadata)
+
+        for event in events_data.events:
+            try:
+                await self._event_buffer.add_event(event)
+            except Exception as e:
+                failed_count += 1
+                logger.error(
+                    f"Failed to add event with id={event.event_id.decode()} to buffer. "
+                    "Error: %s",
+                    e,
+                )
+
+        if received_count > 0:
+            self._open_telemetry_metric_recorder.set_metric_value(
+                self._events_received_metric_name, self._common_tags, received_count
+            )
+        if failed_count > 0:
+            self._open_telemetry_metric_recorder.set_metric_value(
+                self._events_failed_to_add_metric_name, self._common_tags, failed_count
+            )
+
+        return events_event_aggregator_service_pb2.AddEventsReply()
+
+    async def run(self, server) -> None:
+        if server:
+            events_event_aggregator_service_pb2_grpc.add_EventAggregatorServiceServicer_to_server(
+                self, server
+            )
+        try:
+            await asyncio.gather(
+                self._http_endpoint_publisher.run_forever(),
+                self._gcs_publisher.run_forever(),
+            )
+        finally:
+            self._executor.shutdown()
+
+    @staticmethod
+    def is_minimal_module() -> bool:
+        return False

python/ray/dashboard/modules/aggregator/constants.py

@@ -0,0 +1,2 @@
+AGGREGATOR_AGENT_METRIC_PREFIX = "aggregator_agent"
+CONSUMER_TAG_KEY = "consumer"

python/ray/dashboard/modules/aggregator/multi_consumer_event_buffer.py

@@ -0,0 +1,194 @@
+import asyncio
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.core.generated import (
+    events_base_event_pb2,
+)
+from ray.core.generated.events_base_event_pb2 import RayEvent
+from ray.dashboard.modules.aggregator.constants import (
+    AGGREGATOR_AGENT_METRIC_PREFIX,
+    CONSUMER_TAG_KEY,
+)
+
+
+@dataclass
+class _ConsumerState:
+    # Index of the next event to be consumed by this consumer
+    cursor_index: int
+
+
+class MultiConsumerEventBuffer:
+    """A buffer which allows adding one event at a time and consuming events in batches.
+    Supports multiple consumers, each with their own cursor index. Tracks the number of events evicted for each consumer.
+
+    Buffer is not thread-safe but is asyncio-friendly. All operations must be called from within the same event loop.
+
+    Arguments:
+        max_size: Maximum number of events to store in the buffer.
+        max_batch_size: Maximum number of events to return in a batch when calling wait_for_batch.
+        common_metric_tags: Tags to add to all metrics.
+    """
+
+    def __init__(
+        self,
+        max_size: int,
+        max_batch_size: int,
+        common_metric_tags: Optional[Dict[str, str]] = None,
+    ):
+        self._buffer = deque(maxlen=max_size)
+        self._max_size = max_size
+        self._lock = asyncio.Lock()
+        self._has_new_events_to_consume = asyncio.Condition(self._lock)
+        self._consumers: Dict[str, _ConsumerState] = {}
+
+        self._max_batch_size = max_batch_size
+
+        self._common_metrics_tags = common_metric_tags or {}
+        self._metric_recorder = OpenTelemetryMetricRecorder()
+        self.evicted_events_metric_name = (
+            f"{AGGREGATOR_AGENT_METRIC_PREFIX}_queue_dropped_events"
+        )
+        self._metric_recorder.register_counter_metric(
+            self.evicted_events_metric_name,
+            "Total number of events dropped because the publish/buffer queue was full.",
+        )
+
+    async def add_event(self, event: events_base_event_pb2.RayEvent) -> None:
+        """Add an event to the buffer.
+
+        If the buffer is full, the oldest event is dropped.
+        """
+        async with self._lock:
+            dropped_event = None
+            if len(self._buffer) >= self._max_size:
+                dropped_event = self._buffer.popleft()
+            self._buffer.append(event)
+
+            if dropped_event is not None:
+                for consumer_name, consumer_state in self._consumers.items():
+                    # Update consumer cursor index and evicted events metric if an event was dropped
+                    if consumer_state.cursor_index == 0:
+                        # The dropped event was the next event this consumer would have consumed, publish eviction metric
+                        self._metric_recorder.set_metric_value(
+                            self.evicted_events_metric_name,
+                            {
+                                **self._common_metrics_tags,
+                                CONSUMER_TAG_KEY: consumer_name,
+                                "event_type": RayEvent.EventType.Name(
+                                    dropped_event.event_type
+                                ),
+                            },
+                            1,
+                        )
+                    else:
+                        # The dropped event was already consumed by the consumer, so we need to adjust the cursor
+                        consumer_state.cursor_index -= 1
+
+            # Signal the consumers that there are new events to consume
+            self._has_new_events_to_consume.notify_all()
+
+    def _evict_old_events(self) -> None:
+        """Clean the buffer by removing events from the buffer who have index lower than
+        all the cursor indexes of all consumers and updating the cursor index of all
+        consumers.
+        """
+        if not self._consumers:
+            return
+
+        min_cursor_index = min(
+            consumer_state.cursor_index for consumer_state in self._consumers.values()
+        )
+        for _ in range(min_cursor_index):
+            self._buffer.popleft()
+
+        # update the cursor index of all consumers
+        for consumer_state in self._consumers.values():
+            consumer_state.cursor_index -= min_cursor_index
+
+    async def wait_for_batch(
+        self, consumer_name: str, timeout_seconds: float = 1.0
+    ) -> List[events_base_event_pb2.RayEvent]:
+        """Wait for batch respecting self.max_batch_size and timeout_seconds.
+
+        Returns a batch of up to self.max_batch_size items. Waits for up to
+        timeout_seconds after receiving the first event that will be in
+        the next batch. After the timeout, returns as many items as are ready.
+
+        Always returns a batch with at least one item - will block
+        indefinitely until an item comes in.
+
+        Arguments:
+            consumer_name: name of the consumer consuming the batch
+            timeout_seconds: maximum time to wait for a batch
+
+        Returns:
+            A list of up to max_batch_size events ready for consumption.
+            The list always contains at least one event.
+        """
+        max_batch = self._max_batch_size
+        batch = []
+        async with self._has_new_events_to_consume:
+            consumer_state = self._consumers.get(consumer_name)
+            if consumer_state is None:
+                raise KeyError(f"unknown consumer '{consumer_name}'")
+
+            # Phase 1: read the first event, wait indefinitely until there is at least one event to consume
+            while consumer_state.cursor_index >= len(self._buffer):
+                await self._has_new_events_to_consume.wait()
+
+            # Add the first event to the batch
+            event = self._buffer[consumer_state.cursor_index]
+            consumer_state.cursor_index += 1
+            batch.append(event)
+
+            # Phase 2: add items to the batch up to timeout or until full
+            deadline = time.monotonic() + max(0.0, float(timeout_seconds))
+            while len(batch) < max_batch:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    break
+
+                # Drain whatever is available
+                while len(batch) < max_batch and consumer_state.cursor_index < len(
+                    self._buffer
+                ):
+                    batch.append(self._buffer[consumer_state.cursor_index])
+                    consumer_state.cursor_index += 1
+
+                if len(batch) >= max_batch:
+                    break
+
+                # There is still room in the batch, but no new events to consume; wait until notified or timeout
+                try:
+                    await asyncio.wait_for(
+                        self._has_new_events_to_consume.wait(), remaining
+                    )
+                except asyncio.TimeoutError:
+                    # Timeout, return the current batch
+                    break
+
+            self._evict_old_events()
+        return batch
+
+    async def register_consumer(self, consumer_name: str) -> None:
+        """Register a new consumer with a name.
+
+        Arguments:
+            consumer_name: A unique name for the consumer.
+
+        """
+        async with self._lock:
+            if self._consumers.get(consumer_name) is not None:
+                raise ValueError(f"consumer '{consumer_name}' already registered")
+
+            self._consumers[consumer_name] = _ConsumerState(cursor_index=0)
+
+    async def size(self) -> int:
+        """Get total number of events in the buffer. Does not take consumer cursors into account."""
+        return len(self._buffer)

python/ray/dashboard/modules/aggregator/publisher/async_publisher_client.py

@@ -0,0 +1,291 @@
+import json
+import logging
+from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import List, Optional
+
+import aiohttp
+
+import ray.dashboard.utils as dashboard_utils
+from ray._common.utils import get_or_create_event_loop
+from ray._private.protobuf_compat import message_to_json
+from ray._raylet import GcsClient
+from ray.core.generated import (
+    events_base_event_pb2,
+    events_event_aggregator_service_pb2,
+)
+from ray.dashboard.modules.aggregator.publisher.configs import (
+    GCS_EXPOSABLE_EVENT_TYPES,
+    HTTP_EXPOSABLE_EVENT_TYPES,
+    PUBLISHER_TIMEOUT_SECONDS,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PublishStats:
+    """Data class that represents stats of publishing a batch of events."""
+
+    # Whether the publish was successful
+    is_publish_successful: bool
+    # Number of events published
+    num_events_published: int
+    # Number of events filtered out
+    num_events_filtered_out: int
+
+
+@dataclass
+class PublishBatch:
+    """Data class that represents a batch of events to publish."""
+
+    # The list of events to publish
+    events: list[events_base_event_pb2.RayEvent]
+    # dropped task events metadata
+    task_events_metadata: Optional[
+        events_event_aggregator_service_pb2.TaskEventsMetadata
+    ] = None
+
+
+class PublisherClientInterface(ABC):
+    """Abstract interface for publishing Ray event batches to external destinations.
+
+    Implementations should handle the actual publishing logic, filtering,
+    and format conversion appropriate for their specific destination type.
+    """
+
+    def __init__(self):
+        self._exposable_event_types_list: List[str] = []
+        self._allow_all_event_types: bool = False
+
+    def count_num_events_in_batch(self, batch: PublishBatch) -> int:
+        """Count the number of events in a given PublishBatch."""
+        return len(batch.events)
+
+    def _can_expose_event(self, event) -> bool:
+        """
+        Check if an event should be allowed to be published.
+        """
+        if self._allow_all_event_types:
+            return True
+        if not self._exposable_event_types_list:
+            return False
+
+        event_type_name = events_base_event_pb2.RayEvent.EventType.Name(
+            event.event_type
+        )
+        return event_type_name in self._exposable_event_types_list
+
+    @abstractmethod
+    async def publish(self, batch: PublishBatch) -> PublishStats:
+        """Publish a batch of events to the destination."""
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        """Clean up any resources used by this client. Should be called when the publisherClient is no longer required"""
+        pass
+
+
+class AsyncHttpPublisherClient(PublisherClientInterface):
+    """Client for publishing ray event batches to an external HTTP service."""
+
+    def __init__(
+        self,
+        endpoint: str,
+        executor: ThreadPoolExecutor,
+        timeout: float = PUBLISHER_TIMEOUT_SECONDS,
+        preserve_proto_field_name: bool = False,
+    ) -> None:
+        super().__init__()
+        self._endpoint = endpoint
+        self._executor = executor
+        self._timeout = aiohttp.ClientTimeout(total=timeout)
+        self._session = None
+        self._preserve_proto_field_name = preserve_proto_field_name
+
+        if HTTP_EXPOSABLE_EVENT_TYPES.strip().upper() == "ALL":
+            self._allow_all_event_types = True
+            self._exposable_event_types_list = []
+        else:
+            self._exposable_event_types_list = [
+                event_type.strip()
+                for event_type in HTTP_EXPOSABLE_EVENT_TYPES.split(",")
+                if event_type.strip()
+            ]
+
+    async def publish(self, batch: PublishBatch) -> PublishStats:
+        events_batch: list[events_base_event_pb2.RayEvent] = batch.events
+        if not events_batch:
+            # Nothing to publish -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+        filtered = [e for e in events_batch if self._can_expose_event(e)]
+        num_filtered_out = len(events_batch) - len(filtered)
+        if not filtered:
+            # All filtered out -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=num_filtered_out,
+            )
+
+        # Convert protobuf objects to python dictionaries for HTTP POST. Run in executor to avoid blocking the event loop.
+        filtered_json = await get_or_create_event_loop().run_in_executor(
+            self._executor,
+            lambda: [
+                json.loads(
+                    message_to_json(
+                        e,
+                        always_print_fields_with_no_presence=True,
+                        preserving_proto_field_name=self._preserve_proto_field_name,
+                    )
+                )
+                for e in filtered
+            ],
+        )
+
+        try:
+            # Create session on first use (lazy initialization)
+            if not self._session:
+                self._session = aiohttp.ClientSession(timeout=self._timeout)
+
+            return await self._send_http_request(filtered_json, num_filtered_out)
+        except Exception as e:
+            logger.error("Failed to send events to external service. Error: %r", e)
+            return PublishStats(
+                is_publish_successful=False,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+
+    async def _send_http_request(self, json_data, num_filtered_out) -> PublishStats:
+        async with self._session.post(
+            self._endpoint,
+            json=json_data,
+        ) as resp:
+            resp.raise_for_status()
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=len(json_data),
+                num_events_filtered_out=num_filtered_out,
+            )
+
+    async def close(self) -> None:
+        """Closes the http session if one was created. Should be called when the publisherClient is no longer required"""
+        if self._session:
+            await self._session.close()
+            self._session = None
+
+    def set_session(self, session) -> None:
+        """Inject an HTTP client session.
+
+        If a session is set explicitly, it will be used and managed by close().
+        """
+        self._session = session
+
+
+class AsyncGCSTaskEventsPublisherClient(PublisherClientInterface):
+    """Client for publishing ray event batches to GCS."""
+
+    def __init__(
+        self,
+        gcs_client: GcsClient,
+        executor: ThreadPoolExecutor,
+        timeout_s: float = PUBLISHER_TIMEOUT_SECONDS,
+    ) -> None:
+        super().__init__()
+        self._gcs_client = gcs_client
+        self._executor = executor
+        self._timeout_s = timeout_s
+
+        self._exposable_event_types_list = GCS_EXPOSABLE_EVENT_TYPES
+
+    async def publish(
+        self,
+        batch: PublishBatch,
+    ) -> PublishStats:
+        events = batch.events
+        task_events_metadata = batch.task_events_metadata
+        has_dropped_task_attempts = (
+            task_events_metadata and task_events_metadata.dropped_task_attempts
+        )
+        if not events and not has_dropped_task_attempts:
+            # Nothing to publish -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+
+        # Filter events based on exposable event types
+        filtered_events = [e for e in events if self._can_expose_event(e)]
+        num_filtered_out = len(events) - len(filtered_events)
+
+        if not filtered_events and not has_dropped_task_attempts:
+            # all events filtered out and no task events metadata -> success but nothing published
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=0,
+                num_events_filtered_out=num_filtered_out,
+            )
+
+        try:
+            events_data = self._create_ray_events_data(
+                filtered_events, task_events_metadata
+            )
+            request = events_event_aggregator_service_pb2.AddEventsRequest(
+                events_data=events_data
+            )
+            serialized_request = await get_or_create_event_loop().run_in_executor(
+                self._executor,
+                lambda: request.SerializeToString(),
+            )
+            status_code = await self._gcs_client.async_add_events(
+                serialized_request, self._timeout_s, self._executor
+            )
+
+            if status_code != dashboard_utils.HTTPStatusCode.OK:
+                logger.error(f"GCS AddEvents failed: {status_code}")
+                return PublishStats(
+                    is_publish_successful=False,
+                    num_events_published=0,
+                    num_events_filtered_out=0,
+                )
+            return PublishStats(
+                is_publish_successful=True,
+                num_events_published=len(filtered_events),
+                num_events_filtered_out=num_filtered_out,
+            )
+        except Exception as e:
+            logger.error(f"Failed to send events to GCS: {e}")
+            return PublishStats(
+                is_publish_successful=False,
+                num_events_published=0,
+                num_events_filtered_out=0,
+            )
+
+    def _create_ray_events_data(
+        self,
+        event_batch: List[events_base_event_pb2.RayEvent],
+        task_events_metadata: Optional[
+            events_event_aggregator_service_pb2.TaskEventsMetadata
+        ] = None,
+    ) -> events_event_aggregator_service_pb2.RayEventsData:
+        """
+        Helper method to create RayEventsData from event batch and metadata.
+        """
+        events_data = events_event_aggregator_service_pb2.RayEventsData()
+        events_data.events.extend(event_batch)
+
+        if task_events_metadata:
+            events_data.task_events_metadata.CopyFrom(task_events_metadata)
+
+        return events_data
+
+    async def close(self) -> None:
+        pass

python/ray/dashboard/modules/aggregator/publisher/configs.py

@@ -0,0 +1,54 @@
+# Environment variables for the aggregator agent publisher component.
+import os
+
+from ray._private import ray_constants
+
+env_var_prefix = "RAY_DASHBOARD_AGGREGATOR_AGENT_PUBLISHER"
+# Timeout for the publisher to publish events to the destination
+PUBLISHER_TIMEOUT_SECONDS = ray_constants.env_integer(
+    f"{env_var_prefix}_TIMEOUT_SECONDS", 3
+)
+# Maximum number of retries for publishing events to the destination, if less than 0, will retry indefinitely
+PUBLISHER_MAX_RETRIES = ray_constants.env_integer(f"{env_var_prefix}_MAX_RETRIES", -1)
+# Initial backoff time for publishing events to the destination
+PUBLISHER_INITIAL_BACKOFF_SECONDS = ray_constants.env_float(
+    f"{env_var_prefix}_INITIAL_BACKOFF_SECONDS", 0.01
+)
+# Maximum backoff time for publishing events to the destination
+PUBLISHER_MAX_BACKOFF_SECONDS = ray_constants.env_float(
+    f"{env_var_prefix}_MAX_BACKOFF_SECONDS", 5.0
+)
+# Jitter ratio for publishing events to the destination
+PUBLISHER_JITTER_RATIO = ray_constants.env_float(f"{env_var_prefix}_JITTER_RATIO", 0.1)
+# Maximum sleep time between sending batches of events to the destination, should be greater than 0.0 to avoid busy looping
+PUBLISHER_MAX_BUFFER_SEND_INTERVAL_SECONDS = ray_constants.env_float(
+    f"{env_var_prefix}_MAX_BUFFER_SEND_INTERVAL_SECONDS", 0.1
+)
+
+# HTTP Publisher specific configurations
+# Comma-separated list of event types that are allowed to be exposed to external HTTP services
+# Valid values: TASK_DEFINITION_EVENT, TASK_LIFECYCLE_EVENT, ACTOR_TASK_DEFINITION_EVENT, etc.
+# Set to "ALL" to allow all event types.
+# The list of all supported event types can be found in src/ray/protobuf/public/events_base_event.proto (EventType enum)
+# By default TASK_PROFILE_EVENT is not exposed to external services
+DEFAULT_HTTP_EXPOSABLE_EVENT_TYPES = (
+    "TASK_DEFINITION_EVENT,TASK_LIFECYCLE_EVENT,ACTOR_TASK_DEFINITION_EVENT,"
+    "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT,"
+    "ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT,"
+    "NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT,"
+    "PLATFORM_EVENT,"
+)
+HTTP_EXPOSABLE_EVENT_TYPES = os.environ.get(
+    "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES",
+    DEFAULT_HTTP_EXPOSABLE_EVENT_TYPES,
+)
+
+# GCS Publisher specific configurations
+# List of event types that are allowed to be exposed to GCS, not overridden by environment variable
+# as GCS only supports Task event types
+GCS_EXPOSABLE_EVENT_TYPES = [
+    "TASK_DEFINITION_EVENT",
+    "TASK_LIFECYCLE_EVENT",
+    "TASK_PROFILE_EVENT",
+    "ACTOR_TASK_DEFINITION_EVENT",
+]

python/ray/dashboard/modules/aggregator/publisher/metrics.py

@@ -0,0 +1,53 @@
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.dashboard.modules.aggregator.constants import (
+    AGGREGATOR_AGENT_METRIC_PREFIX,
+)
+
+# OpenTelemetry metrics setup (registered once at import time)
+metric_recorder = OpenTelemetryMetricRecorder()
+
+# Counter metrics
+published_counter_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_published_events"
+metric_recorder.register_counter_metric(
+    published_counter_name,
+    "Total number of events successfully published to the destination.",
+)
+
+filtered_counter_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_filtered_events"
+metric_recorder.register_counter_metric(
+    filtered_counter_name,
+    "Total number of events filtered out before publishing to the destination.",
+)
+
+failed_counter_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_publish_failures"
+metric_recorder.register_counter_metric(
+    failed_counter_name,
+    "Total number of events that failed to publish after retries.",
+)
+
+# Histogram metric
+publish_latency_hist_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_publish_latency_seconds"
+metric_recorder.register_histogram_metric(
+    publish_latency_hist_name,
+    "Duration of publish calls in seconds.",
+    [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5],
+)
+
+# Gauge metrics
+consecutive_failures_gauge_name = (
+    f"{AGGREGATOR_AGENT_METRIC_PREFIX}_consecutive_failures_since_last_success"
+)
+metric_recorder.register_gauge_metric(
+    consecutive_failures_gauge_name,
+    "Number of consecutive failed publish attempts since the last success.",
+)
+
+time_since_last_success_gauge_name = (
+    f"{AGGREGATOR_AGENT_METRIC_PREFIX}_time_since_last_success_seconds"
+)
+metric_recorder.register_gauge_metric(
+    time_since_last_success_gauge_name,
+    "Seconds since the last successful publish to the destination.",
+)

python/ray/dashboard/modules/aggregator/publisher/ray_event_publisher.py

@@ -0,0 +1,289 @@
+import asyncio
+import logging
+import random
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+from ray.dashboard.modules.aggregator.constants import (
+    CONSUMER_TAG_KEY,
+)
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+from ray.dashboard.modules.aggregator.publisher.async_publisher_client import (
+    PublishBatch,
+    PublisherClientInterface,
+)
+from ray.dashboard.modules.aggregator.publisher.configs import (
+    PUBLISHER_INITIAL_BACKOFF_SECONDS,
+    PUBLISHER_JITTER_RATIO,
+    PUBLISHER_MAX_BACKOFF_SECONDS,
+    PUBLISHER_MAX_BUFFER_SEND_INTERVAL_SECONDS,
+    PUBLISHER_MAX_RETRIES,
+)
+from ray.dashboard.modules.aggregator.publisher.metrics import (
+    consecutive_failures_gauge_name,
+    failed_counter_name,
+    filtered_counter_name,
+    metric_recorder,
+    publish_latency_hist_name,
+    published_counter_name,
+    time_since_last_success_gauge_name,
+)
+from ray.dashboard.modules.aggregator.task_events_metadata_buffer import (
+    TaskEventsMetadataBuffer,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class RayEventPublisherInterface(ABC):
+    """Abstract interface for publishing Ray event batches to external destinations."""
+
+    @abstractmethod
+    async def run_forever(self) -> None:
+        """Run the publisher forever until cancellation or process death."""
+        pass
+
+    @abstractmethod
+    async def wait_until_running(self, timeout: Optional[float] = None) -> bool:
+        """Wait until the publisher has started."""
+        pass
+
+
+class RayEventPublisher(RayEventPublisherInterface):
+    """RayEvents publisher that publishes batches of events to a destination by running a worker loop.
+
+    The worker loop continuously pulls batches from the event buffer and publishes them to the destination.
+    """
+
+    # Cap the exponent to avoid computing unnecessarily large intermediate
+    _MAX_BACKOFF_EXPONENT = 30
+
+    def __init__(
+        self,
+        name: str,
+        publish_client: PublisherClientInterface,
+        event_buffer: MultiConsumerEventBuffer,
+        common_metric_tags: Optional[Dict[str, str]] = None,
+        task_metadata_buffer: Optional[TaskEventsMetadataBuffer] = None,
+        max_retries: int = PUBLISHER_MAX_RETRIES,
+        initial_backoff: float = PUBLISHER_INITIAL_BACKOFF_SECONDS,
+        max_backoff: float = PUBLISHER_MAX_BACKOFF_SECONDS,
+        jitter_ratio: float = PUBLISHER_JITTER_RATIO,
+    ) -> None:
+        """Initialize a RayEventsPublisher.
+
+        Args:
+            name: Name identifier for this publisher instance
+            publish_client: Client for publishing events to the destination
+            event_buffer: Buffer for reading batches of events
+            common_metric_tags: Common labels for all prometheus metrics
+            task_metadata_buffer: Buffer for reading a batch of droppedtask metadata
+            max_retries: Maximum number of retries for failed publishes
+            initial_backoff: Initial backoff time between retries in seconds
+            max_backoff: Maximum backoff time between retries in seconds
+            jitter_ratio: Random jitter ratio to add to backoff times
+        """
+        self._name = name
+        self._common_metric_tags = dict(common_metric_tags or {})
+        self._common_metric_tags[CONSUMER_TAG_KEY] = name
+        self._max_retries = int(max_retries)
+        self._initial_backoff = float(initial_backoff)
+        self._max_backoff = float(max_backoff)
+        self._jitter_ratio = float(jitter_ratio)
+        self._publish_client = publish_client
+        self._event_buffer = event_buffer
+        self._task_metadata_buffer = task_metadata_buffer
+
+        # Event set once the publisher has registered as a consumer and is ready to publish events
+        self._started_event: asyncio.Event = asyncio.Event()
+
+    async def run_forever(self) -> None:
+        """Run the publisher forever until cancellation or process death.
+
+        Registers as a consumer, starts the worker loop, and handles cleanup on cancellation.
+        """
+        await self._event_buffer.register_consumer(self._name)
+
+        # Signal that the publisher is ready to publish events
+        self._started_event.set()
+
+        try:
+            logger.info(f"Starting publisher {self._name}")
+            while True:
+                events_batch = await self._event_buffer.wait_for_batch(
+                    self._name,
+                    PUBLISHER_MAX_BUFFER_SEND_INTERVAL_SECONDS,
+                )
+                publish_batch = PublishBatch(events=events_batch)
+
+                if self._task_metadata_buffer is not None:
+                    task_metadata_batch = self._task_metadata_buffer.get()
+                    publish_batch.task_events_metadata = task_metadata_batch
+
+                await self._async_publish_with_retries(publish_batch)
+        except asyncio.CancelledError:
+            logger.info(f"Publisher {self._name} cancelled, shutting down gracefully")
+            raise
+        except Exception as e:
+            logger.error(f"Publisher {self._name} encountered error: {e}")
+            raise
+        finally:
+            self._started_event.clear()
+            await self._publish_client.close()
+
+    async def wait_until_running(self, timeout: Optional[float] = None) -> bool:
+        """Wait until the publisher has started.
+
+        Args:
+            timeout: Maximum time to wait in seconds. If None, waits indefinitely.
+
+        Returns:
+            True if the publisher started before the timeout, False otherwise.
+            If timeout is None, waits indefinitely.
+        """
+        if timeout is None:
+            await self._started_event.wait()
+            return True
+        try:
+            await asyncio.wait_for(self._started_event.wait(), timeout)
+            return True
+        except asyncio.TimeoutError:
+            return False
+
+    async def _async_publish_with_retries(self, batch) -> None:
+        """Attempts to publish a batch with retries.
+
+        Will retry failed publishes up to max_retries times with increasing delays.
+        """
+        num_events_in_batch = self._publish_client.count_num_events_in_batch(batch)
+        failed_attempts_since_last_success = 0
+        while True:
+            start = asyncio.get_running_loop().time()
+            result = await self._publish_client.publish(batch)
+            duration = asyncio.get_running_loop().time() - start
+
+            if result.is_publish_successful:
+                await self._record_success(
+                    num_published=int(result.num_events_published),
+                    num_filtered=int(result.num_events_filtered_out),
+                    duration=float(duration),
+                )
+                failed_attempts_since_last_success = 0
+                return
+
+            # Failed attempt
+            # case 1: if max retries are exhausted mark as failed and break out, retry indefinitely if max_retries is less than 0
+            if (
+                self._max_retries >= 0
+                and failed_attempts_since_last_success >= self._max_retries
+            ):
+                await self._record_final_failure(
+                    num_failed_events=int(num_events_in_batch),
+                    duration=float(duration),
+                )
+                return
+
+            # case 2: max retries not exhausted, increment failed attempts counter and add latency to failure list, retry publishing batch with backoff
+            failed_attempts_since_last_success += 1
+            await self._record_retry_failure(
+                duration=float(duration),
+                failed_attempts=int(failed_attempts_since_last_success),
+            )
+
+            await self._async_sleep_with_backoff(failed_attempts_since_last_success)
+
+    async def _async_sleep_with_backoff(self, attempt: int) -> None:
+        """Sleep with exponential backoff and optional jitter.
+
+        Args:
+            attempt: The current attempt number (0-based)
+        """
+        capped_attempt = min(attempt, self._MAX_BACKOFF_EXPONENT)
+        delay = min(
+            self._max_backoff,
+            self._initial_backoff * (2**capped_attempt),
+        )
+        if self._jitter_ratio > 0:
+            jitter = delay * self._jitter_ratio
+            delay = max(0.0, random.uniform(delay - jitter, delay + jitter))
+        await asyncio.sleep(delay)
+
+    async def _record_success(
+        self, num_published: int, num_filtered: int, duration: float
+    ) -> None:
+        """Update in-memory stats and Prometheus metrics for a successful publish."""
+        if num_published > 0:
+            metric_recorder.set_metric_value(
+                published_counter_name,
+                self._common_metric_tags,
+                int(num_published),
+            )
+        if num_filtered > 0:
+            metric_recorder.set_metric_value(
+                filtered_counter_name, self._common_metric_tags, int(num_filtered)
+            )
+        metric_recorder.set_metric_value(
+            consecutive_failures_gauge_name, self._common_metric_tags, 0
+        )
+        metric_recorder.set_metric_value(
+            time_since_last_success_gauge_name, self._common_metric_tags, 0
+        )
+        metric_recorder.set_metric_value(
+            publish_latency_hist_name,
+            {**self._common_metric_tags, "Outcome": "success"},
+            float(duration),
+        )
+
+    async def _record_retry_failure(
+        self, duration: float, failed_attempts: int
+    ) -> None:
+        """Update Prometheus metrics for a retryable failure attempt."""
+        metric_recorder.set_metric_value(
+            consecutive_failures_gauge_name,
+            self._common_metric_tags,
+            int(failed_attempts),
+        )
+        metric_recorder.set_metric_value(
+            publish_latency_hist_name,
+            {**self._common_metric_tags, "Outcome": "failure"},
+            float(duration),
+        )
+
+    async def _record_final_failure(
+        self, num_failed_events: int, duration: float
+    ) -> None:
+        """Update in-memory stats and Prometheus metrics for a final (non-retryable) failure."""
+        if num_failed_events > 0:
+            metric_recorder.set_metric_value(
+                failed_counter_name,
+                self._common_metric_tags,
+                int(num_failed_events),
+            )
+        metric_recorder.set_metric_value(
+            consecutive_failures_gauge_name, self._common_metric_tags, 0
+        )
+        metric_recorder.set_metric_value(
+            publish_latency_hist_name,
+            {**self._common_metric_tags, "Outcome": "failure"},
+            float(duration),
+        )
+
+
+class NoopPublisher(RayEventPublisherInterface):
+    """A no-op publisher that adheres to the minimal interface used by AggregatorAgent.
+
+    Used when a destination is disabled. It runs forever but does nothing.
+    """
+
+    async def run_forever(self) -> None:
+        """Run forever doing nothing until cancellation."""
+        try:
+            await asyncio.Event().wait()
+        except asyncio.CancelledError:
+            logger.info("NoopPublisher cancelled")
+            raise
+
+    async def wait_until_running(self, timeout: Optional[float] = None) -> bool:
+        return True

python/ray/dashboard/modules/aggregator/task_events_metadata_buffer.py

@@ -0,0 +1,85 @@
+from collections import deque
+from typing import Dict, Optional
+
+from ray._private.telemetry.open_telemetry_metric_recorder import (
+    OpenTelemetryMetricRecorder,
+)
+from ray.core.generated import events_event_aggregator_service_pb2
+from ray.dashboard.modules.aggregator.constants import AGGREGATOR_AGENT_METRIC_PREFIX
+
+
+class TaskEventsMetadataBuffer:
+    """Buffer for accumulating task event metadata and batching it into a bounded queue.
+
+    This buffer is used to construct TaskEventsMetadata protobuf messages (defined in events_event_aggregator_service.proto).
+    """
+
+    def __init__(
+        self,
+        max_buffer_size: int = 1000,
+        max_dropped_attempts_per_metadata_entry: int = 100,
+        common_metric_tags: Optional[Dict[str, str]] = None,
+    ):
+        self._buffer_maxlen = max(
+            max_buffer_size - 1, 1
+        )  # -1 to account for the current batch
+        self._buffer = deque(maxlen=self._buffer_maxlen)
+        self._current_metadata_batch = (
+            events_event_aggregator_service_pb2.TaskEventsMetadata()
+        )
+        self._max_dropped_attempts = max_dropped_attempts_per_metadata_entry
+
+        self._common_metric_tags = common_metric_tags or {}
+        self._metric_recorder = OpenTelemetryMetricRecorder()
+        self._dropped_metadata_count_metric_name = f"{AGGREGATOR_AGENT_METRIC_PREFIX}_task_metadata_buffer_dropped_attempts_total"
+        self._metric_recorder.register_counter_metric(
+            self._dropped_metadata_count_metric_name,
+            "Total number of dropped task attempt metadata entries which were dropped due to buffer being full",
+        )
+
+    def merge(
+        self,
+        new_metadata: Optional[events_event_aggregator_service_pb2.TaskEventsMetadata],
+    ) -> None:
+        """Merge new task event metadata into the current entry, enqueuing when limits are reached."""
+        if new_metadata is None:
+            return
+
+        for new_attempt in new_metadata.dropped_task_attempts:
+            if (
+                len(self._current_metadata_batch.dropped_task_attempts)
+                >= self._max_dropped_attempts
+            ):
+                # Add current metadata to buffer, if buffer is full, drop the oldest entry
+                if len(self._buffer) >= self._buffer_maxlen:
+                    # Record the number of dropped attempts
+                    oldest_entry = self._buffer.popleft()
+                    self._metric_recorder.set_metric_value(
+                        self._dropped_metadata_count_metric_name,
+                        self._common_metric_tags,
+                        len(oldest_entry.dropped_task_attempts),
+                    )
+
+                # Enqueue current metadata batch and start a new batch
+                metadata_copy = events_event_aggregator_service_pb2.TaskEventsMetadata()
+                metadata_copy.CopyFrom(self._current_metadata_batch)
+                self._buffer.append(metadata_copy)
+                self._current_metadata_batch.Clear()
+
+            # Now add the new attempt
+            new_entry = self._current_metadata_batch.dropped_task_attempts.add()
+            new_entry.CopyFrom(new_attempt)
+
+    def get(self) -> events_event_aggregator_service_pb2.TaskEventsMetadata:
+        """Return the next buffered metadata entry or a snapshot of the current one and reset state."""
+        if len(self._buffer) == 0:
+            # create a copy of the current metadata and return it
+            current_metadata = events_event_aggregator_service_pb2.TaskEventsMetadata()
+            current_metadata.CopyFrom(self._current_metadata_batch)
+
+            # Reset the current metadata and start merging afresh
+            self._current_metadata_batch.Clear()
+
+            return current_metadata
+
+        return self._buffer.popleft()

python/ray/dashboard/modules/aggregator/tests/test_multi_consumer_event_buffer.py

@@ -0,0 +1,270 @@
+import asyncio
+import random
+import sys
+
+import pytest
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from ray.core.generated.events_base_event_pb2 import RayEvent
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+
+
+def _create_test_event(
+    event_id: bytes = b"test",
+    event_type_enum=RayEvent.EventType.TASK_DEFINITION_EVENT,
+    message: str = "test message",
+):
+    """Helper function to create a test RayEvent."""
+    event = RayEvent()
+    event.event_id = event_id
+    event.source_type = RayEvent.SourceType.CORE_WORKER
+    event.event_type = event_type_enum
+    event.severity = RayEvent.Severity.INFO
+    event.message = message
+    event.session_name = "test_session"
+
+    # Set timestamp
+    timestamp = Timestamp()
+    timestamp.GetCurrentTime()
+    event.timestamp.CopyFrom(timestamp)
+
+    return event
+
+
+class TestMultiConsumerEventBuffer:
+    @pytest.mark.asyncio
+    async def test_add_and_consume_event_basic(self):
+        """Test basic event addition."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+        assert await buffer.size() == 0
+
+        event = _create_test_event(b"event1")
+        await buffer.add_event(event)
+
+        assert await buffer.size() == 1
+
+        batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0)
+        assert len(batch) == 1
+        assert batch[0] == event
+
+    @pytest.mark.asyncio
+    async def test_add_event_buffer_overflow(self):
+        """Test buffer overflow behavior and eviction logic."""
+        buffer = MultiConsumerEventBuffer(max_size=3, max_batch_size=2)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        # Add events to fill buffer
+        events = []
+        event_types = [
+            RayEvent.EventType.TASK_DEFINITION_EVENT,
+            RayEvent.EventType.TASK_LIFECYCLE_EVENT,
+            RayEvent.EventType.ACTOR_TASK_DEFINITION_EVENT,
+        ]
+        for i in range(3):
+            event = _create_test_event(f"event{i}".encode(), event_types[i])
+            events.append(event)
+            await buffer.add_event(event)
+
+        assert await buffer.size() == 3
+
+        # Add one more event to trigger eviction
+        overflow_event = _create_test_event(
+            b"overflow", RayEvent.EventType.TASK_PROFILE_EVENT
+        )
+        await buffer.add_event(overflow_event)
+
+        assert await buffer.size() == 3  # Still max size
+
+    @pytest.mark.asyncio
+    async def test_wait_for_batch_multiple_events(self):
+        """Test waiting for batch when multiple events are immediately available and when when not all events are available."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=3)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        # Add multiple events
+        events = []
+        for i in range(5):
+            event = _create_test_event(f"event{i}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        # Should get max_batch_size events immediately
+        batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0.1)
+        assert len(batch) == 3  # max_batch_size
+        assert batch == events[:3]
+        # should now get the leftover events (< max_batch_size)
+        batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0.1)
+        assert len(batch) == 2
+        assert batch == events[3:]
+
+    @pytest.mark.asyncio
+    async def test_wait_for_batch_unknown_consumer(self):
+        """Test error handling for unknown consumer."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+
+        with pytest.raises(KeyError, match="unknown consumer"):
+            await buffer.wait_for_batch("nonexistent_consumer", timeout_seconds=0)
+
+    @pytest.mark.asyncio
+    async def test_register_consumer_duplicate(self):
+        """Test error handling for duplicate consumer registration."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+        with pytest.raises(
+            ValueError, match="consumer 'test_consumer' already registered"
+        ):
+            await buffer.register_consumer(consumer_name)
+
+    @pytest.mark.asyncio
+    async def test_multiple_consumers_independent_cursors(self):
+        """Test that multiple consumers have independent cursors."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=2)
+        consumer_name_1 = "test_consumer_1"
+        consumer_name_2 = "test_consumer_2"
+        await buffer.register_consumer(consumer_name_1)
+        await buffer.register_consumer(consumer_name_2)
+
+        # Add events
+        events = []
+        for i in range(10):
+            event = _create_test_event(f"event{i}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        # Consumer 1 reads first batch
+        batch1 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch1 == events[:2]
+
+        # Consumer 2 reads from beginning
+        batch2 = await buffer.wait_for_batch(consumer_name_2, timeout_seconds=0.1)
+        assert batch2 == events[:2]
+
+        # consumer 1 reads another batch
+        batch3 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch3 == events[2:4]
+
+        # more events are added leading to events not consumed by consumer 2 getting evicted
+        # 4 events get evicted, consumer 1 has processed all 4 evicted events previously
+        # but consumer 2 has only processed 2 out of the 4 evicted events
+        for i in range(4):
+            event = _create_test_event(f"event{i + 10}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        # Just ensure buffer remains at max size
+        assert await buffer.size() == 10
+
+        # consumer 1 will read the next 2 events, not affected by the evictions
+        # consumer 1's cursor is adjusted internally to account for the evicted events
+        batch4 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch4 == events[4:6]
+
+        # consumer 2 will read 2 events, skipping the evicted events
+        batch5 = await buffer.wait_for_batch(consumer_name_2, timeout_seconds=0.1)
+        assert batch5 == events[4:6]  # events[2:4] are lost
+
+    @pytest.mark.asyncio
+    async def test_wait_for_batch_blocks_until_event_available(self):
+        """Test that wait_for_batch blocks until at least one event is available."""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=5)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        # Start waiting for batch (should block)
+        async def wait_for_batch():
+            return await buffer.wait_for_batch(consumer_name, timeout_seconds=2.0)
+
+        wait_task = asyncio.create_task(wait_for_batch())
+
+        # Wait a bit to ensure the task is waiting
+        await asyncio.sleep(4.0)
+        assert not wait_task.done()
+
+        # Add an event
+        event = _create_test_event(b"event1")
+        await buffer.add_event(event)
+
+        # Now the task should complete
+        batch = await wait_task
+        assert len(batch) == 1
+        assert batch[0] == event
+
+    @pytest.mark.asyncio
+    async def test_concurrent_producer_consumer_random_sleeps_with_overall_timeout(
+        self,
+    ):
+        """Producer with random sleeps and consumer reading until all events are received.
+
+        Uses an overall asyncio timeout to ensure the test fails if it hangs
+        before consuming all events.
+        """
+        total_events = 40
+        max_batch_size = 2
+        buffer = MultiConsumerEventBuffer(max_size=100, max_batch_size=max_batch_size)
+        consumer_name = "test_consumer"
+        await buffer.register_consumer(consumer_name)
+
+        produced_events = []
+        consumed_events = []
+
+        random.seed(0)
+
+        async def producer():
+            for i in range(total_events):
+                event = _create_test_event(f"e{i}".encode())
+                produced_events.append(event)
+                await buffer.add_event(event)
+                await asyncio.sleep(random.uniform(0.0, 0.02))
+
+        async def consumer():
+            while len(consumed_events) < total_events:
+                batch = await buffer.wait_for_batch(consumer_name, timeout_seconds=0.1)
+                consumed_events.extend(batch)
+
+        # The test should fail if this times out before all events are consumed
+        await asyncio.wait_for(asyncio.gather(producer(), consumer()), timeout=5.0)
+
+        assert len(consumed_events) == total_events
+        assert consumed_events == produced_events
+
+    @pytest.mark.asyncio
+    async def test_events_are_evicted_once_consumed_by_all_consumers(self):
+        """Test events are evicted from the buffer once they are consumed by all consumers"""
+        buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=2)
+        consumer_name_1 = "test_consumer_1"
+        consumer_name_2 = "test_consumer_2"
+        await buffer.register_consumer(consumer_name_1)
+        await buffer.register_consumer(consumer_name_2)
+
+        # Add events
+        events = []
+        for i in range(10):
+            event = _create_test_event(f"event{i}".encode())
+            events.append(event)
+            await buffer.add_event(event)
+
+        assert await buffer.size() == 10
+        # Consumer 1 reads first batch
+        batch1 = await buffer.wait_for_batch(consumer_name_1, timeout_seconds=0.1)
+        assert batch1 == events[:2]
+
+        # buffer size does not change as consumer 2 is yet to consume these events
+        assert await buffer.size() == 10
+
+        # Consumer 2 reads from beginning
+        batch2 = await buffer.wait_for_batch(consumer_name_2, timeout_seconds=0.1)
+        assert batch2 == events[:2]
+
+        # size reduces by 2 as both consumers have consumed 2 events
+        assert await buffer.size() == 8
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_actor_events.py

@@ -0,0 +1,125 @@
+import base64
+import json
+import sys
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_ACTOR_EVENT_PORT = 12346
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _ACTOR_EVENT_PORT)
+
+
+def test_ray_actor_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_ACTOR_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    head_node_id = cluster.head_node.node_id
+    all_nodes_ids = [node.node_id for node in cluster.list_all_nodes()]
+
+    class A:
+        def ping(self):
+            return "pong"
+
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Create an actor to trigger definition + lifecycle events
+    a = ray.remote(A).options(name="actor-test").remote()
+    ray.get(a.ping.remote())
+
+    # Check that an actor definition and a lifecycle event are published.
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 1)
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+    # We expect batched events containing definition then lifecycle
+    assert len(req_json) >= 2
+    # Verify event types and IDs exist
+    assert (
+        base64.b64decode(req_json[0]["actorDefinitionEvent"]["actorId"]).hex()
+        == a._actor_id.hex()
+    )
+    assert base64.b64decode(req_json[0]["nodeId"]).hex() == head_node_id
+    # Verify ActorId and state for ActorLifecycleEvents
+    has_alive_state = False
+    for actorLifeCycleEvent in req_json[1:]:
+        assert base64.b64decode(actorLifeCycleEvent["nodeId"]).hex() == head_node_id
+        assert (
+            base64.b64decode(
+                actorLifeCycleEvent["actorLifecycleEvent"]["actorId"]
+            ).hex()
+            == a._actor_id.hex()
+        )
+        for stateTransition in actorLifeCycleEvent["actorLifecycleEvent"][
+            "stateTransitions"
+        ]:
+            assert stateTransition["state"] in [
+                "DEPENDENCIES_UNREADY",
+                "PENDING_CREATION",
+                "ALIVE",
+                "RESTARTING",
+                "DEAD",
+            ]
+            if stateTransition["state"] == "ALIVE":
+                has_alive_state = True
+                assert (
+                    base64.b64decode(stateTransition["nodeId"]).hex() in all_nodes_ids
+                )
+                assert base64.b64decode(stateTransition["workerId"]).hex() != ""
+    assert has_alive_state
+
+    # Kill the actor and verify we get a DEAD state with death cause
+    ray.kill(a)
+
+    # Wait for the death event to be published
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 2)
+
+    has_dead_state = False
+    for death_req, _ in httpserver.log:
+        death_req_json = json.loads(death_req.data)
+
+        for actorLifeCycleEvent in death_req_json:
+            if "actorLifecycleEvent" in actorLifeCycleEvent:
+                assert (
+                    base64.b64decode(
+                        actorLifeCycleEvent["actorLifecycleEvent"]["actorId"]
+                    ).hex()
+                    == a._actor_id.hex()
+                )
+
+                for stateTransition in actorLifeCycleEvent["actorLifecycleEvent"][
+                    "stateTransitions"
+                ]:
+                    if stateTransition["state"] == "DEAD":
+                        has_dead_state = True
+                        assert (
+                            stateTransition["deathCause"]["actorDiedErrorContext"][
+                                "reason"
+                            ]
+                            == "RAY_KILL"
+                        )
+
+    assert has_dead_state
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_event_publisher.py

@@ -0,0 +1,164 @@
+import asyncio
+import sys
+import uuid
+
+import pytest
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from ray._common.test_utils import async_wait_for_condition
+from ray.core.generated import events_base_event_pb2
+from ray.dashboard.modules.aggregator.multi_consumer_event_buffer import (
+    MultiConsumerEventBuffer,
+)
+from ray.dashboard.modules.aggregator.publisher.async_publisher_client import (
+    PublisherClientInterface,
+    PublishStats,
+)
+from ray.dashboard.modules.aggregator.publisher.ray_event_publisher import (
+    NoopPublisher,
+    RayEventPublisher,
+)
+
+
+class MockPublisherClient(PublisherClientInterface):
+    """Test implementation of PublisherClientInterface."""
+
+    def __init__(
+        self,
+        batch_size: int = 1,
+        side_effect=lambda batch: PublishStats(True, 1, 0),
+    ):
+        self.batch_size = batch_size
+        self.publish_calls = []
+        self._side_effect = side_effect
+
+    async def publish(self, batch) -> PublishStats:
+        self.publish_calls.append(batch)
+        return self._side_effect(batch)
+
+    def count_num_events_in_batch(self, batch) -> int:
+        return self.batch_size
+
+    async def close(self) -> None:
+        pass
+
+
+@pytest.fixture
+def base_kwargs():
+    """Common kwargs for publisher initialization."""
+    return {
+        "name": "test",
+        "max_retries": 2,
+        "initial_backoff": 0,
+        "max_backoff": 0,
+        "jitter_ratio": 0,
+        "enable_publisher_stats": True,
+    }
+
+
+class TestRayEventPublisher:
+    """Test the main RayEventsPublisher functionality."""
+
+    @pytest.mark.asyncio
+    async def test_publish_with_retries_failure_then_success(self, base_kwargs):
+        """Test publish that fails then succeeds."""
+        call_count = {"count": 0}
+
+        # fail the first publish call but succeed on retry
+        def side_effect(batch):
+            call_count["count"] += 1
+            if call_count["count"] == 1:
+                return PublishStats(False, 0, 0)
+            return PublishStats(True, 1, 0)
+
+        client = MockPublisherClient(side_effect=side_effect)
+        event_buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=10)
+        publisher = RayEventPublisher(
+            name=base_kwargs["name"] + str(uuid.uuid4()),
+            publish_client=client,
+            event_buffer=event_buffer,
+            max_retries=base_kwargs["max_retries"],
+            initial_backoff=base_kwargs["initial_backoff"],
+            max_backoff=base_kwargs["max_backoff"],
+            jitter_ratio=base_kwargs["jitter_ratio"],
+        )
+
+        task = asyncio.create_task(publisher.run_forever())
+        try:
+            # ensure consumer is registered
+            assert await publisher.wait_until_running(2.0)
+            # Enqueue one event into buffer
+            e = events_base_event_pb2.RayEvent(
+                event_id=b"1",
+                source_type=events_base_event_pb2.RayEvent.SourceType.CORE_WORKER,
+                event_type=events_base_event_pb2.RayEvent.EventType.TASK_DEFINITION_EVENT,
+                timestamp=Timestamp(seconds=123, nanos=0),
+                severity=events_base_event_pb2.RayEvent.Severity.INFO,
+                message="hello",
+            )
+            await event_buffer.add_event(e)
+
+            # wait for two publish attempts (failure then success)
+            await async_wait_for_condition(lambda: len(client.publish_calls) == 2)
+        finally:
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+
+    @pytest.mark.asyncio
+    async def test_publish_with_retries_max_retries_exceeded(self, base_kwargs):
+        """Test publish that fails all retries and records failed events."""
+        client = MockPublisherClient(
+            side_effect=lambda batch: PublishStats(False, 0, 0)
+        )
+        event_buffer = MultiConsumerEventBuffer(max_size=10, max_batch_size=10)
+        publisher = RayEventPublisher(
+            name=base_kwargs["name"] + str(uuid.uuid4()),
+            publish_client=client,
+            event_buffer=event_buffer,
+            max_retries=2,  # override to finite retries
+            initial_backoff=0,
+            max_backoff=0,
+            jitter_ratio=0,
+        )
+
+        task = asyncio.create_task(publisher.run_forever())
+        try:
+            # ensure consumer is registered
+            assert await publisher.wait_until_running(2.0)
+            e = events_base_event_pb2.RayEvent(
+                event_id=b"1",
+                source_type=events_base_event_pb2.RayEvent.SourceType.CORE_WORKER,
+                event_type=events_base_event_pb2.RayEvent.EventType.TASK_DEFINITION_EVENT,
+                timestamp=Timestamp(seconds=123, nanos=0),
+                severity=events_base_event_pb2.RayEvent.Severity.INFO,
+                message="hello",
+            )
+            await event_buffer.add_event(e)
+
+            # wait for publish attempts (initial + 2 retries)
+            await async_wait_for_condition(lambda: len(client.publish_calls) == 3)
+            assert len(client.publish_calls) == 3
+        finally:
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+
+
+class TestNoopPublisher:
+    """Test no-op publisher implementation."""
+
+    @pytest.mark.asyncio
+    async def test_all_methods_noop(self):
+        """Test that run_forever can be cancelled and metrics return expected values."""
+        publisher = NoopPublisher()
+
+        # Start and cancel run_forever
+        task = asyncio.create_task(publisher.run_forever())
+        task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_job_events.py

@@ -0,0 +1,58 @@
+import base64
+import json
+import sys
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_RAY_EVENT_PORT = 12345
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _RAY_EVENT_PORT)
+
+
+def test_ray_job_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_RAY_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_LIFECYCLE_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Submit a ray job
+    @ray.remote
+    def f():
+        return 1
+
+    ray.get(f.remote())
+
+    # Check that a driver job event with the correct job id is published.
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 1)
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+    head_node_id = cluster.head_node.node_id
+    assert base64.b64decode(req_json[0]["nodeId"]).hex() == head_node_id
+    assert (
+        base64.b64decode(req_json[0]["driverJobDefinitionEvent"]["jobId"]).hex()
+        == ray.get_runtime_context().get_job_id()
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_node_events.py

@@ -0,0 +1,75 @@
+import base64
+import json
+import os
+import socket
+import sys
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_RAY_EVENT_PORT = 12345
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _RAY_EVENT_PORT)
+
+
+def test_ray_node_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        node_name="test-head-node",
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_RAY_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "NODE_DEFINITION_EVENT,NODE_LIFECYCLE_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    head_node_id = cluster.head_node.node_id
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Check that a node definition and a node lifecycle event are published.
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+    wait_for_condition(lambda: len(httpserver.log) >= 1)
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+    assert len(req_json) == 2
+    assert base64.b64decode(req_json[0]["nodeId"]).hex() == head_node_id
+    assert (
+        base64.b64decode(req_json[0]["nodeDefinitionEvent"]["nodeId"]).hex()
+        == cluster.head_node.node_id
+    )
+
+    node_def_event = req_json[0]["nodeDefinitionEvent"]
+    assert node_def_event["hostname"] == socket.gethostname()
+    assert node_def_event["nodeName"] == "test-head-node"
+    # instanceId and instanceTypeName are set via env vars by cloud providers.
+    # In local/CI environments these are typically empty.
+    assert node_def_event["instanceId"] == os.environ.get("RAY_CLOUD_INSTANCE_ID", "")
+    assert node_def_event["instanceTypeName"] == os.environ.get(
+        "RAY_CLOUD_INSTANCE_TYPE_NAME", ""
+    )
+    assert base64.b64decode(req_json[1]["nodeId"]).hex() == head_node_id
+    assert (
+        base64.b64decode(req_json[1]["nodeLifecycleEvent"]["nodeId"]).hex()
+        == cluster.head_node.node_id
+    )
+    assert req_json[1]["nodeLifecycleEvent"]["stateTransitions"][0]["state"] == "ALIVE"
+    assert (
+        req_json[1]["nodeLifecycleEvent"]["stateTransitions"][0]["aliveSubState"]
+        == "UNSPECIFIED"
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_ray_platform_events.py

@@ -0,0 +1,117 @@
+import base64
+import json
+import sys
+import time
+
+import pytest
+
+import ray
+from ray._private.test_utils import (
+    wait_for_condition,
+    wait_for_dashboard_agent_available,
+)
+from ray.dashboard.tests.conftest import *  # noqa
+
+_PLATFORM_EVENT_PORT = 12348
+
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    return ("127.0.0.1", _PLATFORM_EVENT_PORT)
+
+
+def test_ray_platform_events(ray_start_cluster, httpserver):
+    cluster = ray_start_cluster
+    cluster.add_node(
+        env_vars={
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_PLATFORM_EVENT_PORT}",
+            "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "PLATFORM_EVENT",
+            "RAY_ENABLE_PYTHON_RAY_EVENT_TYPES": "PLATFORM_EVENT",
+        },
+        _system_config={
+            "enable_ray_event": True,
+        },
+    )
+    cluster.wait_for_nodes()
+    head_node_id = cluster.head_node.node_id
+
+    ray.init(address=cluster.address)
+    wait_for_dashboard_agent_available(cluster)
+
+    # Define a task that explicitly initializes and emits a platform event via EventRecorder
+    @ray.remote
+    def emit_test_platform_event(aggregator_port, node_ip, node_id):
+        from ray._common.observability.platform_events import PlatformEventBuilder
+        from ray._raylet import EventRecorder
+        from ray.core.generated.events_base_event_pb2 import RayEvent
+        from ray.core.generated.platform_event_pb2 import Source
+
+        EventRecorder.initialize(
+            aggregator_port=aggregator_port,
+            node_ip=node_ip,
+            node_id_hex=node_id,
+            max_buffer_size=1000,
+            metric_source="platform_events",
+        )
+
+        builder = PlatformEventBuilder(
+            event_uid="uid-test-platform-e2e",
+            platform=Source.Platform.KUBERNETES,
+            object_kind="Pod",
+            object_name="test-pod-name",
+            reason="OOMKilled",
+            message="Container exited with code 137",
+            severity=RayEvent.Severity.WARNING,
+            component="kubelet",
+        )
+        cython_event = builder.build(
+            event_id=b"uid-test-platform-e2e",
+            timestamp_ns=int(time.time() * 1e9),
+        )
+        EventRecorder.emit(cython_event)
+        EventRecorder.shutdown()
+        return True
+
+    # Expect the POST request on the HTTP server
+    httpserver.expect_request("/", method="POST").respond_with_data("", status=200)
+
+    # Fetch the aggregator agent's address from GCS
+    from ray._private.test_utils import GcsClient, get_dashboard_agent_address
+
+    gcs_client = GcsClient(address=cluster.address)
+    agent_address = get_dashboard_agent_address(gcs_client, head_node_id)
+    ip, port_str = agent_address.split(":")
+    aggregator_port = int(port_str)
+
+    # Execute the remote task to emit the event on the node
+    ray.get(emit_test_platform_event.remote(aggregator_port, ip, head_node_id))
+
+    # Wait for the HTTP log collector to receive the batched payload
+    wait_for_condition(lambda: len(httpserver.log) >= 1, timeout=20)
+
+    # Validate the captured POST payload
+    req, _ = httpserver.log[0]
+    req_json = json.loads(req.data)
+
+    assert len(req_json) >= 1
+    platform_event_entry = None
+    for entry in req_json:
+        if "platformEvent" in entry:
+            platform_event_entry = entry
+            break
+
+    assert platform_event_entry is not None
+    assert platform_event_entry["eventType"] == "PLATFORM_EVENT"
+    assert base64.b64decode(platform_event_entry["nodeId"]).hex() == head_node_id
+
+    pe_data = platform_event_entry["platformEvent"]
+    assert pe_data["objectKind"] == "Pod"
+    assert pe_data["objectName"] == "test-pod-name"
+    assert pe_data["reason"] == "OOMKilled"
+    assert pe_data["message"] == "Container exited with code 137"
+    assert pe_data["source"]["platform"] == "KUBERNETES"
+    assert pe_data["source"]["component"] == "kubelet"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/aggregator/tests/test_task_events_metadata_buffer.py

@@ -0,0 +1,115 @@
+import sys
+
+import pytest
+
+from ray.core.generated.events_event_aggregator_service_pb2 import TaskEventsMetadata
+from ray.dashboard.modules.aggregator.task_events_metadata_buffer import (
+    TaskEventsMetadataBuffer,
+)
+
+
+def _create_test_metadata(dropped_task_ids: list = None, attempt_number=1):
+    """Helper function to create test metadata"""
+    metadata = TaskEventsMetadata()
+    if dropped_task_ids:
+        for task_id in dropped_task_ids:
+            attempt = metadata.dropped_task_attempts.add()
+            attempt.task_id = task_id.encode()
+            attempt.attempt_number = attempt_number
+    return metadata
+
+
+def _result_to_attempts_list(result):
+    """Normalize return value from buffer.get() to a python list of attempts."""
+    if hasattr(result, "dropped_task_attempts"):
+        attempts = result.dropped_task_attempts
+    else:
+        attempts = result
+    return list(attempts)
+
+
+def _drain_all_attempts(buffer: TaskEventsMetadataBuffer):
+    """Drain the buffer completely via public API and return list of bytes task_ids.
+
+    Continues calling get() until it returns an empty set of attempts.
+    """
+    collected_ids = []
+    num_metadata_entries = 0
+    while True:
+        result = buffer.get()
+        attempts = _result_to_attempts_list(result)
+        if len(attempts) == 0:
+            break
+
+        num_metadata_entries += 1
+        collected_ids.extend([a.task_id for a in attempts])
+    return collected_ids, num_metadata_entries
+
+
+class TestTaskMetadataBuffer:
+    """tests for TaskMetadataBuffer class"""
+
+    def test_merge_and_get(self):
+        """Test merging multiple metadata objects and verify task attempts are combined."""
+        buffer = TaskEventsMetadataBuffer(
+            max_buffer_size=100, max_dropped_attempts_per_metadata_entry=10
+        )
+
+        # Create two separate metadata objects with different task IDs
+        metadata1 = _create_test_metadata(["task_1", "task_2"])
+        metadata2 = _create_test_metadata(["task_3", "task_4"])
+
+        # Merge both metadata objects
+        buffer.merge(metadata1)
+        buffer.merge(metadata2)
+
+        # Get the merged results
+        result = buffer.get()
+        attempts = _result_to_attempts_list(result)
+
+        # Verify we have all 4 task attempts
+        assert len(attempts) == 4
+
+        # Verify all expected task IDs are present
+        task_ids = [attempt.task_id for attempt in attempts]
+        assert sorted(task_ids) == [b"task_1", b"task_2", b"task_3", b"task_4"]
+
+    @pytest.mark.parametrize(
+        "max_attempts_per_metadata_entry,num_tasks,max_buffer_size,expected_drop_attempts,expected_num_metadata_entries",
+        [
+            # No overflow, two metadata entries should be created
+            (2, 3, 100, 0, 2),
+            # No overflow, three metadata entries should be created
+            (5, 15, 100, 0, 3),
+            # Overflow scenario: buffer too small, ensure drop count is tracked.
+            (1, 4, 2, 2, 2),
+        ],
+    )
+    def test_buffer_merge_and_overflow(
+        self,
+        max_attempts_per_metadata_entry,
+        num_tasks,
+        max_buffer_size,
+        expected_drop_attempts,
+        expected_num_metadata_entries,
+    ):
+        buffer = TaskEventsMetadataBuffer(
+            max_buffer_size=max_buffer_size,
+            max_dropped_attempts_per_metadata_entry=max_attempts_per_metadata_entry,
+        )
+
+        for i in range(num_tasks):
+            test_metadata = _create_test_metadata([f"task_{i}"])
+            buffer.merge(test_metadata)
+
+        # Drain everything and verify number of attempts in buffer is as expected
+        drained_ids, num_metadata_entries = _drain_all_attempts(buffer)
+        assert len(drained_ids) == num_tasks - expected_drop_attempts
+        assert num_metadata_entries == expected_num_metadata_entries
+
+        # Buffer should now be empty
+        assert len(_result_to_attempts_list(buffer.get())) == 0
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))