chore: import upstream snapshot with attribution

python/ray/llm/_internal/serve/observability/__init__.py

@@ -0,0 +1,23 @@
+from ray.llm._internal.common.observability.logging_utils import (
+    disable_datasets_logger,
+    disable_vllm_custom_ops_logger_on_cpu_nodes,
+)
+from ray.llm._internal.common.observability.telemetry_utils import Once
+from ray.llm._internal.serve.observability.logging.setup import (
+    setup_logging,
+)
+
+_setup_observability_once = Once()
+
+
+def _setup_observability():
+    setup_logging()
+    disable_datasets_logger()
+    disable_vllm_custom_ops_logger_on_cpu_nodes()
+
+
+def setup_observability():
+    _setup_observability_once.do_once(_setup_observability)
+
+
+__all__ = ["setup_observability"]

python/ray/llm/_internal/serve/observability/logging/__init__.py

@@ -0,0 +1,39 @@
+import logging
+from typing import Optional
+
+from ray._common.filters import CoreContextFilter
+from ray.serve._private.logging_utils import ServeContextFilter
+
+
+def _setup_logger(logger_name: str):
+    """Setup logger given the logger name.
+
+    This function is idempotent and won't set up the same logger multiple times. It will
+    Also skip the setup if Serve logger is already setup and has handlers.
+    """
+    logger = logging.getLogger(logger_name)
+    serve_logger = logging.getLogger("ray.serve")
+
+    # Skip setup if the logger already has handlers setup or if the parent (Serve
+    # logger) has handlers.
+    if logger.handlers or serve_logger.handlers:
+        return
+
+    # Set up stream handler, which logs to console as plaintext.
+    stream_handler = logging.StreamHandler()
+    stream_handler.addFilter(CoreContextFilter())
+    stream_handler.addFilter(ServeContextFilter())
+    logger.addHandler(stream_handler)
+    logger.setLevel(logging.INFO)
+    logger.propagate = False
+
+
+def get_logger(name: Optional[str] = None):
+    """Get a structured logger inherited from the Ray Serve logger.
+
+    Loggers by default are logging to stdout, and are expected to be scraped by an
+    external process.
+    """
+    logger_name = f"ray.serve.{name}"
+    _setup_logger(logger_name)
+    return logging.getLogger(logger_name)

python/ray/llm/_internal/serve/observability/logging/setup.py

@@ -0,0 +1,28 @@
+import logging
+
+from ray._common.filters import CoreContextFilter
+from ray._common.formatters import JSONFormatter
+from ray.serve._private.logging_utils import ServeContextFilter
+
+
+def _configure_stdlib_logging():
+    """Configures stdlib root logger to make sure stdlib loggers (created as
+    `logging.getLogger(...)`) are using Ray's `JSONFormatter` with Core and Serve
+     context filters.
+    """
+
+    handler = logging.StreamHandler()
+    handler.addFilter(CoreContextFilter())
+    handler.addFilter(ServeContextFilter())
+    handler.setFormatter(JSONFormatter())
+
+    root_logger = logging.getLogger()
+    # NOTE: It's crucial we reset all the handlers of the root logger,
+    #       to make sure that logs aren't emitted twice
+    root_logger.handlers = []
+    root_logger.addHandler(handler)
+    root_logger.setLevel(logging.INFO)
+
+
+def setup_logging():
+    _configure_stdlib_logging()

python/ray/llm/_internal/serve/observability/metrics/event_loop_monitoring.py

@@ -0,0 +1,77 @@
+import asyncio
+import time
+from typing import Dict
+
+from ray.llm._internal.serve.observability.logging import get_logger
+from ray.util import metrics
+
+logger = get_logger(__name__)
+
+_METRICS_LOOP_INTERVAL = 5  # 5 seconds
+EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES = [
+    0.05,
+    0.1,
+    0.15,
+    0.20,
+    0.25,
+    0.5,
+    0.75,
+    1.0,
+    1.5,
+    2.0,
+    3.0,
+    5.0,
+    10.0,
+    15.0,
+    20.0,
+    30.0,
+    45.0,
+    60.0,
+    90.0,
+    120.0,
+    150.0,
+    180.0,
+    300.0,
+    600.0,
+]
+
+
+def setup_event_loop_monitoring(
+    loop: asyncio.AbstractEventLoop,
+    scheduling_latency: metrics.Histogram,
+    iterations: metrics.Counter,
+    tasks: metrics.Gauge,
+    tags: Dict[str, str],
+) -> asyncio.Task:
+    return asyncio.create_task(
+        _run_monitoring_loop(
+            loop,
+            schedule_latency=scheduling_latency,
+            iterations=iterations,
+            task_gauge=tasks,
+            tags=tags,
+        )
+    )
+
+
+async def _run_monitoring_loop(
+    loop: asyncio.AbstractEventLoop,
+    schedule_latency: metrics.Histogram,
+    iterations: metrics.Counter,
+    task_gauge: metrics.Gauge,
+    tags: Dict[str, str],
+) -> None:
+    while loop.is_running():
+        iterations.inc(1, tags)
+        num_tasks = len(asyncio.all_tasks())
+        task_gauge.set(num_tasks, tags)
+        yield_time = time.monotonic()
+        await asyncio.sleep(_METRICS_LOOP_INTERVAL)
+        elapsed_time = time.monotonic() - yield_time
+
+        # Historically, Ray's implementation of histograms are extremely finicky
+        # with non-positive values (https://github.com/ray-project/ray/issues/26698).
+        # Technically it shouldn't be possible for this to be negative, add the
+        # max just to be safe.
+        latency = max(0.0, elapsed_time - _METRICS_LOOP_INTERVAL)
+        schedule_latency.observe(latency, tags)

python/ray/llm/_internal/serve/observability/metrics/fast_api_metrics.py

@@ -0,0 +1,133 @@
+import asyncio
+import os
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from ray.llm._internal.serve.constants import ENABLE_VERBOSE_TELEMETRY
+from ray.llm._internal.serve.observability.logging import get_logger
+from ray.llm._internal.serve.observability.metrics.event_loop_monitoring import (
+    EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES,
+    setup_event_loop_monitoring,
+)
+from ray.llm._internal.serve.observability.metrics.fastapi_utils import (
+    FASTAPI_API_SERVER_TAG_KEY,
+    FASTAPI_BASE_HTTP_METRIC_TAG_KEYS,
+    get_app_name,
+)
+from ray.llm._internal.serve.observability.metrics.middleware import (
+    MeasureHTTPRequestMetricsMiddleware,
+)
+from ray.util import metrics
+
+logger = get_logger(__name__)
+
+ray_llm_build_info_gauge = metrics.Gauge(
+    "ray_llm_build_info",
+    description="Metadata about the ray-llm build.",
+    tag_keys=("git_commit",),
+)
+
+
+_HTTP_HANDLER_LATENCY_S_HISTOGRAM_BUCKETS = [
+    0.01,
+    0.05,
+    0.1,
+    0.25,
+    0.5,
+    0.75,
+    1,
+    1.5,
+    2,
+    5,
+    10,
+    30,
+    60,
+    120,
+    300,
+]
+
+
+async def add_fastapi_event_loop_monitoring(app: FastAPI):
+    tags = {FASTAPI_API_SERVER_TAG_KEY: get_app_name(app)}
+    tag_keys = tuple(tags.keys())
+
+    # Store the task handle to prevent it from being garbage collected
+    app.state.fastapi_event_loop_schedule_latency = metrics.Histogram(
+        "fastapi_event_loop_schedule_latency",
+        description="Latency of getting yielded control on the FastAPI event loop in seconds",
+        boundaries=EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES,
+        tag_keys=tag_keys,
+    )
+    app.state.fastapi_event_loop_monitoring_iterations = metrics.Counter(
+        "fastapi_event_loop_monitoring_iterations",
+        description="Number of times the FastAPI event loop has iterated to get anyscale_fastapi_event_loop_schedule_latency.",
+        tag_keys=tag_keys,
+    )
+    app.state.fastapi_event_loop_monitoring_tasks = metrics.Gauge(
+        "fastapi_event_loop_monitoring_tasks",
+        description="Number of outstanding tasks on the FastAPI event loop.",
+        tag_keys=tag_keys,
+    )
+
+    app.state.fastapi_event_loop_schedule_latency_metrics_task = (
+        setup_event_loop_monitoring(
+            asyncio.get_running_loop(),
+            app.state.fastapi_event_loop_schedule_latency,
+            app.state.fastapi_event_loop_monitoring_iterations,
+            app.state.fastapi_event_loop_monitoring_tasks,
+            tags,
+        )
+    )
+
+
+def add_http_metrics_middleware(app: FastAPI):
+    if not ENABLE_VERBOSE_TELEMETRY:
+        logger.debug(
+            "ENABLE_VERBOSE_TELEMETRY is false, not setting up FastAPI telemetry"
+        )
+        return
+
+    logger.info("ENABLE_VERBOSE_TELEMETRY is true, setting up FastAPI telemetry")
+    base_tag_keys = FASTAPI_BASE_HTTP_METRIC_TAG_KEYS
+
+    logger.debug("Setting up FastAPI telemetry")
+
+    app.state.http_requests_metrics = metrics.Counter(
+        "http_requests",
+        description=(
+            "Total number of HTTP requests by status code, handler and method."
+        ),
+        tag_keys=base_tag_keys,
+    )
+    # NOTE: Custom decorators are not applied to histogram-based metrics
+    #       to make sure we can keep cardinality of those in check
+    app.state.http_requests_latency_metrics = metrics.Histogram(
+        "http_request_duration_seconds",
+        description="Duration in seconds of HTTP requests.",
+        boundaries=_HTTP_HANDLER_LATENCY_S_HISTOGRAM_BUCKETS,
+        tag_keys=base_tag_keys,
+    )
+
+    app.add_middleware(MeasureHTTPRequestMetricsMiddleware)
+
+    logger.debug("Setting up FastAPI telemetry completed")
+
+
+async def set_ray_llm_build_info():
+    git_commit = os.environ.get("GIT_COMMIT")
+    if git_commit:
+        tags = {"git_commit": git_commit}
+        ray_llm_build_info_gauge.set(1, tags)
+
+
+@asynccontextmanager
+async def metrics_lifespan(app: FastAPI):
+    """Lifespan for a FastAPI app that sets up metrics observability."""
+
+    if ENABLE_VERBOSE_TELEMETRY:
+        await add_fastapi_event_loop_monitoring(app)
+
+    await set_ray_llm_build_info()
+
+    yield

python/ray/llm/_internal/serve/observability/metrics/fastapi_utils.py

@@ -0,0 +1,26 @@
+"""This file contains constants and utility functions for FastAPI."""
+
+from fastapi import FastAPI
+
+# These tag keys are used in metrics for the FastAPI app.
+FASTAPI_HTTP_RESPONSE_CODE_TAG_KEY = "code"
+FASTAPI_HTTP_HANDLER_TAG_KEY = "handler"
+FASTAPI_HTTP_METHOD_TAG_KEY = "method"
+FASTAPI_HTTP_PATH_TAG_KEY = "path"
+FASTAPI_HTTP_USER_ID_TAG_KEY = "user_id"
+FASTAPI_API_SERVER_TAG_KEY = "api_server"
+
+FASTAPI_BASE_HTTP_METRIC_TAG_KEYS = (
+    FASTAPI_HTTP_RESPONSE_CODE_TAG_KEY,
+    FASTAPI_HTTP_HANDLER_TAG_KEY,
+    FASTAPI_HTTP_METHOD_TAG_KEY,
+    FASTAPI_HTTP_PATH_TAG_KEY,
+    FASTAPI_HTTP_USER_ID_TAG_KEY,
+    FASTAPI_API_SERVER_TAG_KEY,
+)
+
+
+def get_app_name(app: FastAPI) -> str:
+    """Gets the FastAPI app name."""
+
+    return getattr(app.state, "name", "unknown")

python/ray/llm/_internal/serve/observability/metrics/middleware.py

@@ -0,0 +1,150 @@
+import time
+from asyncio import CancelledError
+from typing import Dict, Optional
+
+from fastapi import FastAPI
+from starlette.requests import Request
+from starlette.types import Message
+
+from ray.llm._internal.serve.core.ingress.middleware import (
+    get_request_id,
+    get_user_id,
+)
+from ray.llm._internal.serve.observability.logging import get_logger
+from ray.llm._internal.serve.observability.metrics.fastapi_utils import (
+    FASTAPI_API_SERVER_TAG_KEY,
+    FASTAPI_HTTP_HANDLER_TAG_KEY,
+    FASTAPI_HTTP_METHOD_TAG_KEY,
+    FASTAPI_HTTP_PATH_TAG_KEY,
+    FASTAPI_HTTP_RESPONSE_CODE_TAG_KEY,
+    FASTAPI_HTTP_USER_ID_TAG_KEY,
+    get_app_name,
+)
+from ray.serve._private.thirdparty.get_asgi_route_name import _get_route_name
+
+logger = get_logger("ray.serve")
+
+
+class MeasureHTTPRequestMetricsMiddleware:
+    """Measures and stores HTTP request metrics."""
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] not in ("http", "websocket"):
+            await self.app(scope, receive, send)
+
+        # If the status_code isn't set by send_wrapper,
+        # we should consider that an error.
+        status_code = 500
+        send_wrapper_failed_exc_info = "Status code was never set by send_wrapper."
+        exception_info = send_wrapper_failed_exc_info
+
+        async def send_wrapper(message: Message) -> None:
+            """Wraps the send message.
+
+            Enables this middleware to access the response headers.
+            """
+
+            nonlocal status_code, exception_info
+            if message["type"] == "http.response.start":
+                status_code = message.get("status", 500)
+
+                # Clear the send_wrapper_failed_exc_info.
+                if exception_info == send_wrapper_failed_exc_info:
+                    exception_info = None
+
+            await send(message)
+
+        request = Request(scope)
+        req_id = get_request_id(request)
+        now = time.monotonic()
+        try:
+            logger.info(f"Starting handling of the request {req_id}")
+            await self.app(scope, receive, send_wrapper)
+
+        except CancelledError as ce:
+            status_code = -1
+            exception_info = ce
+            raise
+
+        except BaseException as e:
+            status_code = 500
+            exception_info = e
+            raise
+
+        finally:
+            duration_s = time.monotonic() - now
+
+            tags = _get_tags(request, status_code, request.app)
+            # NOTE: Custom decorators are not applied to histogram-based metrics
+            #       to make sure we can keep cardinality of those in check
+            truncated_tags = {
+                **tags,
+                FASTAPI_HTTP_USER_ID_TAG_KEY: "truncated",
+            }
+
+            request.app.state.http_requests_metrics.inc(1, tags)
+            request.app.state.http_requests_latency_metrics.observe(
+                duration_s, truncated_tags
+            )
+
+            extra_context = {
+                "status_code": status_code,
+                "duration_ms": duration_s * 1000,
+            }
+
+            if status_code >= 400:
+                log = logger.error if status_code >= 500 else logger.warning
+                log(
+                    f"Handling of the request {req_id} failed",
+                    exc_info=exception_info,
+                    extra={"ray_serve_extra_fields": extra_context},
+                )
+            elif status_code == -1:
+                logger.info(
+                    f"Handling of the request {req_id} have been cancelled",
+                    extra={"ray_serve_extra_fields": extra_context},
+                )
+            else:
+                logger.info(
+                    f"Handling of the request {req_id} successfully completed",
+                    extra={"ray_serve_extra_fields": extra_context},
+                )
+
+
+def _get_route_details(scope: dict) -> Optional[str]:
+    """
+    Function to retrieve Starlette route from scope.
+    TODO: there is currently no way to retrieve http.route from
+    a starlette application from scope.
+    See: https://github.com/encode/starlette/pull/804
+    Args:
+        scope: A Starlette scope
+    Returns:
+        A string containing the route or None
+    """
+    # Delegate to Serve's shared route-name resolver, which walks the route tree
+    # and handles FastAPI >= 0.137 `_IncludedRouter` nodes (added by
+    # `include_router`) that have no `.path` attribute of their own. Accessing
+    # `.path` on such a node previously raised AttributeError here (#64245).
+    return _get_route_name(scope, scope["app"].routes)
+
+
+def _get_tags(request: Request, status_code: int, app: FastAPI) -> Dict[str, str]:
+    """Generates tags for the request's metrics."""
+
+    route = str(_get_route_details(request.scope)) or "unknown"
+    path = str(request.url.path) or "unknown"
+    method = str(request.method) or "unknown"
+    user_id = str(get_user_id(request) or "unknown")
+
+    return {
+        FASTAPI_API_SERVER_TAG_KEY: get_app_name(app),
+        FASTAPI_HTTP_RESPONSE_CODE_TAG_KEY: str(status_code),
+        FASTAPI_HTTP_PATH_TAG_KEY: path,
+        FASTAPI_HTTP_HANDLER_TAG_KEY: route,
+        FASTAPI_HTTP_METHOD_TAG_KEY: method,
+        FASTAPI_HTTP_USER_ID_TAG_KEY: user_id,
+    }

python/ray/llm/_internal/serve/observability/metrics/utils.py

@@ -0,0 +1,144 @@
+import time
+from enum import Enum
+from typing import (
+    AsyncGenerator,
+    Callable,
+    List,
+    Set,
+    TypeVar,
+)
+
+from ray.util import metrics
+
+# Histogram buckets for short-range latencies measurements:
+# Min=1ms, Max=30s
+#
+# NOTE: Number of buckets have to be bounded (and not exceed 30)
+#       to avoid overloading metrics sub-system
+SHORT_RANGE_LATENCY_HISTOGRAM_BUCKETS_MS: List[float] = [
+    1,
+    5,
+    10,
+    25,
+    50,
+    100,
+    150,
+    250,
+    500,
+    1000,
+    1500,
+    2500,
+    5000,
+    7500,
+    10000,
+    20000,
+    30000,
+]
+
+# Histogram buckets for long-range latencies measurements:
+# Min=10ms, Max=300s
+LONG_RANGE_LATENCY_HISTOGRAM_BUCKETS_MS = [
+    x * 10 for x in SHORT_RANGE_LATENCY_HISTOGRAM_BUCKETS_MS
+]
+
+
+class ClockUnit(int, Enum):
+    ms = 1000
+    s = 1
+
+
+class MsClock:
+    """A clock that tracks intervals in milliseconds"""
+
+    def __init__(self, unit: ClockUnit = ClockUnit.ms):
+        self.reset()
+        self.unit = unit.value
+        self.start_time = time.perf_counter()
+
+    def reset(self):
+        self.start_time = time.perf_counter()
+
+    def interval(self):
+        return (time.perf_counter() - self.start_time) * self.unit
+
+    def reset_interval(self):
+        interval = self.interval()
+        self.reset()
+        return interval
+
+
+T = TypeVar("T")
+
+
+class InstrumentTokenAsyncGenerator:
+    """This class instruments an asynchronous generator.
+
+    It gathers 3 metrics:
+    1. Time to first time
+    2. Time between tokens
+    3. Total completion time
+
+    Usage:
+
+    @InstrumentTokenAsyncGenerator("my_special_fn")
+    async def to_instrument():
+        yield ...
+    """
+
+    all_instrument_names: Set[str] = set()
+
+    def __init__(
+        self, generator_name: str, latency_histogram_buckets: List[float] = None
+    ):
+        self.generator_name = f"rayllm_{generator_name}"
+
+        target_latency_histogram_buckets = (
+            latency_histogram_buckets or SHORT_RANGE_LATENCY_HISTOGRAM_BUCKETS_MS
+        )
+
+        assert (
+            self.generator_name not in self.all_instrument_names
+        ), "This generator name was already used elsewhere. Please specify another one."
+        self.all_instrument_names.add(self.generator_name)
+
+        self.token_latency_histogram = metrics.Histogram(
+            f"{self.generator_name}_per_token_latency_ms",
+            f"Generator metrics for {self.generator_name}",
+            boundaries=target_latency_histogram_buckets,
+        )
+
+        self.first_token_latency_histogram = metrics.Histogram(
+            f"{self.generator_name}_first_token_latency_ms",
+            f"Generator metrics for {self.generator_name}",
+            boundaries=target_latency_histogram_buckets,
+        )
+        self.total_latency_histogram = metrics.Histogram(
+            f"{self.generator_name}_total_latency_ms",
+            f"Generator metrics for {self.generator_name}",
+            boundaries=target_latency_histogram_buckets,
+        )
+
+    def __call__(
+        self, async_generator_fn: Callable[..., AsyncGenerator[T, None]]
+    ) -> Callable[..., AsyncGenerator[T, None]]:
+        async def new_gen(*args, **kwargs):
+            interval_clock = MsClock()
+            total_clock = MsClock()
+            is_first_token = True
+            try:
+                async for x in async_generator_fn(*args, **kwargs):
+                    if is_first_token:
+                        self.first_token_latency_histogram.observe(
+                            total_clock.interval()
+                        )
+                        interval_clock.reset()
+                        is_first_token = False
+                    else:
+                        self.token_latency_histogram.observe(
+                            interval_clock.reset_interval()
+                        )
+                    yield x
+            finally:
+                self.total_latency_histogram.observe(total_clock.interval())
+
+        return new_gen

python/ray/llm/_internal/serve/observability/usage_telemetry/usage.py

@@ -0,0 +1,353 @@
+import hashlib
+import random
+import time
+from enum import Enum
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence
+
+import ray
+from ray import serve
+from ray._common.constants import HEAD_NODE_RESOURCE_NAME
+from ray._common.usage.usage_lib import (
+    get_hardware_usages_to_report,
+    record_extra_usage_tag,
+)
+from ray.llm._internal.common.base_pydantic import BaseModelExtended
+from ray.llm._internal.common.observability.telemetry_utils import DEFAULT_GPU_TYPE
+from ray.llm._internal.common.utils.lora_utils import get_lora_model_ids
+from ray.llm._internal.serve.observability.logging import get_logger
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+if TYPE_CHECKING:
+    from ray.llm._internal.serve.core.configs.llm_config import LLMConfig
+
+LLM_SERVE_TELEMETRY_NAMESPACE = "llm_serve_telemetry"
+LLM_SERVE_TELEMETRY_ACTOR_NAME = "llm_serve_telemetry"
+
+logger = get_logger(__name__)
+
+
+class TelemetryTags(str, Enum):
+    """Telemetry tags for LLM SERVE."""
+
+    LLM_SERVE_SERVE_MULTIPLE_MODELS = "LLM_SERVE_SERVE_MULTIPLE_MODELS"
+    LLM_SERVE_SERVE_MULTIPLE_APPS = "LLM_SERVE_SERVE_MULTIPLE_APPS"
+    LLM_SERVE_LORA_BASE_MODELS = "LLM_SERVE_LORA_BASE_MODELS"
+    LLM_SERVE_INITIAL_NUM_LORA_ADAPTERS = "LLM_SERVE_INITIAL_NUM_LORA_ADAPTERS"
+    LLM_SERVE_AUTOSCALING_ENABLED_MODELS = "LLM_SERVE_AUTOSCALING_ENABLED_MODELS"
+    LLM_SERVE_AUTOSCALING_MIN_REPLICAS = "LLM_SERVE_AUTOSCALING_MIN_REPLICAS"
+    LLM_SERVE_AUTOSCALING_MAX_REPLICAS = "LLM_SERVE_AUTOSCALING_MAX_REPLICAS"
+    LLM_SERVE_TENSOR_PARALLEL_DEGREE = "LLM_SERVE_TENSOR_PARALLEL_DEGREE"
+    LLM_SERVE_NUM_REPLICAS = "LLM_SERVE_NUM_REPLICAS"
+    LLM_SERVE_MODELS = "LLM_SERVE_MODELS"
+    LLM_SERVE_GPU_TYPE = "LLM_SERVE_GPU_TYPE"
+    LLM_SERVE_NUM_GPUS = "LLM_SERVE_NUM_GPUS"
+
+
+class TelemetryModel(BaseModelExtended):
+    """Telemetry model for LLM Serve.
+
+    ``model_id_hash`` is the dedup identity used by the telemetry agent and is
+    never recorded as a tag value. It is a hash of the model id so the raw model
+    name never reaches the head-node actor.
+    """
+
+    model_id_hash: str
+    model_architecture: str
+    num_replicas: int
+    use_lora: bool
+    initial_num_lora_adapters: int
+    use_autoscaling: bool
+    min_replicas: int
+    max_replicas: int
+    tensor_parallel_degree: int
+    gpu_type: str
+    num_gpus: int
+
+
+@ray.remote(
+    name=LLM_SERVE_TELEMETRY_ACTOR_NAME,
+    namespace=LLM_SERVE_TELEMETRY_NAMESPACE,
+    num_cpus=0,
+    lifetime="detached",
+)
+class TelemetryAgent:
+    """Named Actor to keep the state of all deployed models and record telemetry."""
+
+    def __init__(self):
+        # Keyed by model_id_hash so repeated reports from replicas/restarts of
+        # the same model overwrite rather than accumulate.
+        self.models: Dict[str, TelemetryModel] = {}
+        self.record_tag_func = record_extra_usage_tag
+
+    def _update_record_tag_func(self, record_tag_func: Callable) -> None:
+        """This method is only used in tests to record the telemetry tags to a different
+        object than Ray's default `record_extra_usage_tag` function."""
+        self.record_tag_func = record_tag_func
+
+    def _reset_models(self):
+        """This method is only used in tests to clean up the models list."""
+        self.models = {}
+
+    def _multiple_models(self) -> str:
+        unique_models = {model.model_architecture for model in self.models.values()}
+        return "1" if len(unique_models) > 1 else "0"
+
+    @staticmethod
+    def _multiple_apps() -> str:
+        try:
+            try:
+                serve_status = serve.status()
+            except ray.exceptions.ActorDiedError:
+                # In a workspace with multiple Serve sessions, the long-running
+                # telemetry agent may still be connected to a previous, now-dead
+                # session. Shut down so we can reconnect to the live one.
+                serve.shutdown()
+                serve_status = serve.status()
+            return "1" if len(serve_status.applications) > 1 else "0"
+        except Exception:
+            # Telemetry must never fail; fall back to "not multiple".
+            logger.debug("Failed to query serve.status() for telemetry", exc_info=True)
+            return "0"
+
+    def _lora_base_nodes(self) -> str:
+        return ",".join(
+            [
+                model.model_architecture
+                for model in self.models.values()
+                if model.use_lora
+            ]
+        )
+
+    def _lora_initial_num_adaptors(self) -> str:
+        return ",".join(
+            [
+                str(model.initial_num_lora_adapters)
+                for model in self.models.values()
+                if model.use_lora
+            ]
+        )
+
+    def _autoscaling_enabled_models(self) -> str:
+        return ",".join(
+            [
+                model.model_architecture
+                for model in self.models.values()
+                if model.use_autoscaling
+            ]
+        )
+
+    def _autoscaling_min_replicas(self) -> str:
+        return ",".join(
+            [
+                str(model.min_replicas)
+                for model in self.models.values()
+                if model.use_autoscaling
+            ]
+        )
+
+    def _autoscaling_max_replicas(self) -> str:
+        return ",".join(
+            [
+                str(model.max_replicas)
+                for model in self.models.values()
+                if model.use_autoscaling
+            ]
+        )
+
+    def _model_architectures(self) -> str:
+        return ",".join([model.model_architecture for model in self.models.values()])
+
+    def _tensor_parallel_degree(self) -> str:
+        return ",".join(
+            [str(model.tensor_parallel_degree) for model in self.models.values()]
+        )
+
+    def _num_replicas(self) -> str:
+        return ",".join([str(model.num_replicas) for model in self.models.values()])
+
+    def _gpu_type(self) -> str:
+        return ",".join([model.gpu_type for model in self.models.values()])
+
+    def _num_gpus(self) -> str:
+        return ",".join([str(model.num_gpus) for model in self.models.values()])
+
+    def generate_report(self) -> Dict[str, str]:
+        return {
+            TelemetryTags.LLM_SERVE_SERVE_MULTIPLE_MODELS: self._multiple_models(),
+            TelemetryTags.LLM_SERVE_SERVE_MULTIPLE_APPS: self._multiple_apps(),
+            TelemetryTags.LLM_SERVE_LORA_BASE_MODELS: self._lora_base_nodes(),
+            TelemetryTags.LLM_SERVE_INITIAL_NUM_LORA_ADAPTERS: self._lora_initial_num_adaptors(),
+            TelemetryTags.LLM_SERVE_AUTOSCALING_ENABLED_MODELS: self._autoscaling_enabled_models(),
+            TelemetryTags.LLM_SERVE_AUTOSCALING_MIN_REPLICAS: self._autoscaling_min_replicas(),
+            TelemetryTags.LLM_SERVE_AUTOSCALING_MAX_REPLICAS: self._autoscaling_max_replicas(),
+            TelemetryTags.LLM_SERVE_MODELS: self._model_architectures(),
+            TelemetryTags.LLM_SERVE_TENSOR_PARALLEL_DEGREE: self._tensor_parallel_degree(),
+            TelemetryTags.LLM_SERVE_NUM_REPLICAS: self._num_replicas(),
+            TelemetryTags.LLM_SERVE_GPU_TYPE: self._gpu_type(),
+            TelemetryTags.LLM_SERVE_NUM_GPUS: self._num_gpus(),
+        }
+
+    def record(self, model: Optional[TelemetryModel] = None) -> None:
+        """Record telemetry model."""
+        from ray._common.usage.usage_lib import TagKey
+
+        if model:
+            self.models[model.model_id_hash] = model
+
+        for key, value in self.generate_report().items():
+            try:
+                self.record_tag_func(TagKey.Value(key), value)
+            except ValueError:
+                # If the key doesn't exist in the TagKey enum, skip it.
+                continue
+
+
+def _get_or_create_telemetry_agent() -> TelemetryAgent:
+    """Get or create the detached telemetry agent.
+
+    ``get_if_exists`` makes creation atomic, so concurrent replicas converge on a
+    single actor without racing on the name.
+    """
+    return TelemetryAgent.options(
+        name=LLM_SERVE_TELEMETRY_ACTOR_NAME,
+        namespace=LLM_SERVE_TELEMETRY_NAMESPACE,
+        get_if_exists=True,
+        # Ensure the actor is created on the head node.
+        resources={HEAD_NODE_RESOURCE_NAME: 0.001},
+        # Ensure the actor is not scheduled with the existing placement group.
+        scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=None),
+    ).remote()
+
+
+def _retry_get_telemetry_agent(
+    max_retries: int = 5, base_delay: float = 0.1
+) -> TelemetryAgent:
+    """Get-or-create the telemetry agent, retrying transient failures."""
+    for attempt in range(max_retries):
+        try:
+            return _get_or_create_telemetry_agent()
+        except Exception as e:
+            logger.info(
+                "Attempt %s/%s to get telemetry agent failed", attempt + 1, max_retries
+            )
+            if attempt == max_retries - 1:
+                raise e
+            # Exponential backoff with jitter; ~3.5s total over 5 attempts.
+            time.sleep(base_delay * (2**attempt) + random.uniform(0, 0.5))
+
+
+def _push_telemetry_report(model: Optional[TelemetryModel] = None) -> None:
+    """Push telemetry report for a model."""
+    telemetry_agent = _retry_get_telemetry_agent()
+    assert telemetry_agent is not None
+    ray.get(telemetry_agent.record.remote(model))
+
+
+class HardwareUsage:
+    """Hardware usage class to report telemetry."""
+
+    def __init__(self, get_hardware_fn: Callable = get_hardware_usages_to_report):
+        self._get_hardware_fn = get_hardware_fn
+
+    def infer_gpu_from_hardware(self) -> str:
+        """Infer the GPU type from the hardware when the accelerator type on llm config is
+        not specified.
+        """
+        from ray.llm._internal.serve.core.configs.accelerators import AcceleratorType
+
+        all_accelerator_types = [t.value for t in AcceleratorType]
+        gcs_client = ray.experimental.internal_kv.internal_kv_get_gcs_client()
+        hardwares = self._get_hardware_fn(gcs_client)
+        for hardware in hardwares:
+            if hardware in all_accelerator_types:
+                return hardware
+
+        return DEFAULT_GPU_TYPE
+
+
+def push_telemetry_report_for_all_models(
+    all_models: Optional[Sequence["LLMConfig"]] = None,
+    get_lora_model_func: Callable = get_lora_model_ids,
+    get_hardware_fn: Callable = get_hardware_usages_to_report,
+):
+    """Push a telemetry report for each model. Never raises."""
+    if not all_models:
+        return
+
+    for model in all_models:
+        # Telemetry must never break the caller (e.g. engine start).
+        try:
+            _push_model_telemetry(model, get_lora_model_func, get_hardware_fn)
+        except Exception:
+            logger.exception(
+                "Failed to push telemetry for model %s",
+                getattr(model, "model_id", "<unknown>"),
+            )
+
+
+def _push_model_telemetry(
+    model: "LLMConfig",
+    get_lora_model_func: Callable,
+    get_hardware_fn: Callable,
+) -> None:
+    use_lora = (
+        model.lora_config is not None
+        and model.lora_config.dynamic_lora_loading_path is not None
+    )
+    initial_num_lora_adapters = 0
+    if use_lora:
+        lora_model_ids = get_lora_model_func(
+            dynamic_lora_loading_path=model.lora_config.dynamic_lora_loading_path,
+            base_model_id=model.model_id,
+        )
+        initial_num_lora_adapters = len(lora_model_ids)
+
+    deployment_config = model.deployment_config
+    autoscaling_config = deployment_config.get("autoscaling_config")
+    if deployment_config.get("num_replicas") == "auto":
+        # "auto" resolves to an autoscaling config; mirror LLMConfig validation
+        # since the stored deployment_config keeps the literal "auto".
+        from ray.serve._private.config import handle_num_replicas_auto
+
+        _, autoscaling_config = handle_num_replicas_auto(
+            deployment_config.get("max_ongoing_requests"), autoscaling_config
+        )
+
+    use_autoscaling = autoscaling_config is not None
+    if use_autoscaling:
+        from ray.serve.config import AutoscalingConfig
+
+        if isinstance(autoscaling_config, dict):
+            autoscaling_config = AutoscalingConfig(**autoscaling_config)
+        num_replicas = (
+            autoscaling_config.initial_replicas or autoscaling_config.min_replicas
+        )
+        min_replicas = autoscaling_config.min_replicas
+        max_replicas = autoscaling_config.max_replicas
+    else:
+        # Fixed replica count; honor the configured value (including 0),
+        # defaulting to 1 only when unset.
+        num_replicas = deployment_config.get("num_replicas")
+        if num_replicas is None:
+            num_replicas = 1
+        min_replicas = max_replicas = num_replicas
+
+    engine_config = model.get_engine_config()
+    hardware_usage = HardwareUsage(get_hardware_fn)
+
+    telemetry_model = TelemetryModel(
+        # Hash so the cleartext model name (possibly proprietary) never reaches
+        # the head-node actor; deterministic across replicas/restarts so dedup holds.
+        model_id_hash=hashlib.sha256(model.model_id.encode("utf-8")).hexdigest(),
+        model_architecture=model.model_architecture,
+        num_replicas=num_replicas,
+        use_lora=use_lora,
+        initial_num_lora_adapters=initial_num_lora_adapters,
+        use_autoscaling=use_autoscaling,
+        min_replicas=min_replicas,
+        max_replicas=max_replicas,
+        tensor_parallel_degree=engine_config.tensor_parallel_degree,
+        gpu_type=model.accelerator_type or hardware_usage.infer_gpu_from_hardware(),
+        num_gpus=engine_config.num_devices,
+    )
+    _push_telemetry_report(telemetry_model)