chore: import upstream snapshot with attribution

python/sglang/srt/observability/cpu_monitor.py

@@ -0,0 +1,31 @@
+import threading
+import time
+
+import psutil
+
+
+def start_cpu_monitor_thread(component: str, interval: float = 5.0) -> threading.Thread:
+    from prometheus_client import Counter
+
+    cpu_seconds_total = Counter(
+        name="sglang:process_cpu_seconds_total",
+        documentation="Total CPU time consumed by this process (user + system)",
+        labelnames=["component"],
+    )
+
+    def monitor():
+        process = psutil.Process()
+        last_times = process.cpu_times()
+
+        while True:
+            time.sleep(interval)
+            curr_times = process.cpu_times()
+            delta = (curr_times.user - last_times.user) + (
+                curr_times.system - last_times.system
+            )
+            cpu_seconds_total.labels(component=component).inc(delta)
+            last_times = curr_times
+
+    t = threading.Thread(target=monitor, daemon=True)
+    t.start()
+    return t

python/sglang/srt/observability/forward_pass_metrics.py

@@ -0,0 +1,221 @@
+"""
+Forward pass metrics for per-iteration scheduler telemetry.
+
+Emits per-iteration scheduling metrics over ZMQ PUB so that external
+consumers can observe scheduler behavior in real time without polling
+Prometheus.
+
+Uses msgspec.Struct for zero-copy serialization.
+
+Data flow::
+
+    Scheduler process:
+        SchedulerMetricsMixin._emit_forward_pass_metrics()
+          -> _FpmPublisherThread -> ZMQ PUB (localhost)
+
+    External consumer:
+        ZMQ SUB -> deserialize ForwardPassMetrics
+
+"""
+
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+import time
+from itertools import count
+
+import msgspec
+
+# Schema version. Must match the consumer (Dynamo's ForwardPassMetrics).
+# Bump when the schema changes incompatibly.
+FPM_VERSION: int = 1
+
+logger = logging.getLogger(__name__)
+
+
+class WelfordAccumulator:
+    """Welford's online algorithm for count / total / population-variance.
+
+    Numerically stable single-pass computation.
+    """
+
+    __slots__ = ("count", "total", "_mean", "_m2")
+
+    def __init__(self) -> None:
+        self.count = 0
+        self.total = 0
+        self._mean = 0.0
+        self._m2 = 0.0
+
+    def add(self, v: int) -> None:
+        self.count += 1
+        self.total += v
+        delta = v - self._mean
+        self._mean += delta / self.count
+        delta2 = v - self._mean
+        self._m2 += delta * delta2
+
+    def variance(self) -> float:
+        if self.count == 0:
+            return 0.0
+        return self._m2 / self.count
+
+
+class ScheduledRequestMetrics(
+    msgspec.Struct,
+    frozen=True,
+    gc=False,
+):
+    """Metrics for requests scheduled in this iteration."""
+
+    num_prefill_requests: int = 0
+    sum_prefill_tokens: int = 0
+    var_prefill_length: float = 0.0
+    sum_prefill_kv_tokens: int = 0
+    num_decode_requests: int = 0
+    sum_decode_kv_tokens: int = 0
+    var_decode_kv_tokens: float = 0.0
+
+
+class QueuedRequestMetrics(
+    msgspec.Struct,
+    frozen=True,
+    gc=False,
+):
+    """Metrics for requests waiting in the queue."""
+
+    num_prefill_requests: int = 0
+    sum_prefill_tokens: int = 0
+    var_prefill_length: float = 0.0
+    num_decode_requests: int = 0
+    sum_decode_kv_tokens: int = 0
+    var_decode_kv_tokens: float = 0.0
+
+
+class ForwardPassMetrics(
+    msgspec.Struct,
+    frozen=True,
+    gc=False,
+):
+    """Per-iteration metrics emitted by the scheduler.
+
+    One message per scheduler iteration (one per forward pass).
+    ``wall_time`` is the iteration duration in seconds.
+    An idle heartbeat (all zeros, wall_time=0) is emitted when the
+    engine transitions from active to idle.
+
+    Field order must match Dynamo's ``ForwardPassMetrics`` in
+    ``dynamo.common.forward_pass_metrics`` — msgspec uses positional
+    encoding so any mismatch silently corrupts data.
+    """
+
+    version: int = FPM_VERSION
+    worker_id: str = ""
+    dp_rank: int = 0
+    counter_id: int = 0
+    wall_time: float = 0.0
+    scheduled_requests: ScheduledRequestMetrics = ScheduledRequestMetrics()
+    queued_requests: QueuedRequestMetrics = QueuedRequestMetrics()
+
+
+_encoder = msgspec.msgpack.Encoder()
+_decoder = msgspec.msgpack.Decoder(ForwardPassMetrics)
+
+
+def encode(metrics: ForwardPassMetrics) -> bytes:
+    return _encoder.encode(metrics)
+
+
+def decode(data: bytes) -> ForwardPassMetrics:
+    return _decoder.decode(data)
+
+
+class _FpmPublisherThread:
+    """Background thread that serializes and sends ForwardPassMetrics over ZMQ.
+
+    Also emits periodic heartbeats when idle.
+    """
+
+    SHUTDOWN_TIMEOUT: float = 1.0
+    HEARTBEAT_INTERVAL: float = 1.0
+
+    def __init__(
+        self,
+        endpoint: str,
+        worker_id: str,
+        dp_rank: int,
+        max_queue_size: int = 10_000,
+    ) -> None:
+        import zmq
+
+        self._queue: queue.Queue[ForwardPassMetrics | None] = queue.Queue(
+            maxsize=max_queue_size
+        )
+        self._seq = count()
+        self._worker_id = worker_id
+        self._dp_rank = dp_rank
+
+        self._ctx = zmq.Context()
+        self._pub = self._ctx.socket(zmq.PUB)
+        self._pub.bind(endpoint)
+        self._zmq = zmq
+
+        self._running = True
+        self._thread = threading.Thread(
+            target=self._run, daemon=True, name="fpm-zmq-publisher"
+        )
+        self._thread.start()
+
+    def publish(self, metrics: ForwardPassMetrics) -> None:
+        if not self._running:
+            return
+        try:
+            self._queue.put_nowait(metrics)
+        except queue.Full:
+            pass
+
+    def shutdown(self) -> None:
+        self._running = False
+        try:
+            self._queue.put_nowait(None)
+        except queue.Full:
+            pass
+        self._thread.join(timeout=self.SHUTDOWN_TIMEOUT)
+        try:
+            self._pub.close(linger=0)
+            self._ctx.term()
+        except Exception:
+            pass
+
+    def _run(self) -> None:
+        zmq = self._zmq
+        topic = b""
+        last_publish = time.monotonic()
+
+        while self._running or not self._queue.empty():
+            try:
+                metrics = self._queue.get(timeout=self.HEARTBEAT_INTERVAL)
+                if metrics is None:
+                    break
+            except queue.Empty:
+                if time.monotonic() - last_publish >= self.HEARTBEAT_INTERVAL:
+                    metrics = ForwardPassMetrics(
+                        worker_id=self._worker_id,
+                        dp_rank=self._dp_rank,
+                    )
+                else:
+                    continue
+
+            try:
+                seq = next(self._seq)
+                metrics = msgspec.structs.replace(metrics, counter_id=seq)
+                payload = encode(metrics)
+                seq_bytes = seq.to_bytes(8, "big")
+                self._pub.send_multipart((topic, seq_bytes, payload), flags=zmq.NOBLOCK)
+                last_publish = time.monotonic()
+            except zmq.Again:
+                pass
+            except Exception:
+                logger.warning("FPM publisher send failed", exc_info=True)

python/sglang/srt/observability/func_timer.py

@@ -0,0 +1,101 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+Records the latency of some functions
+"""
+
+import asyncio
+import time
+from functools import wraps
+from typing import Any, Callable, Optional
+
+from sglang.srt.observability.utils import exponential_buckets
+
+enable_metrics = False
+
+
+def enable_func_timer():
+    # We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR`
+    from prometheus_client import Histogram
+
+    global enable_metrics, FUNC_LATENCY
+    enable_metrics = True
+
+    FUNC_LATENCY = Histogram(
+        "sglang:func_latency_seconds",
+        "Function latency in seconds",
+        # captures latency in range [50ms - ~50s]
+        buckets=exponential_buckets(start=0.05, width=1.5, length=18),
+        labelnames=["name"],
+    )
+
+
+FUNC_LATENCY = None
+
+
+def time_func_latency(
+    func: Callable = None, name: Optional[str] = None
+) -> Callable[..., Any]:
+    """
+    A decorator to observe the latency of a function's execution. Supports both sync and async functions.
+
+    NOTE: We use our own implementation of a timer decorator since prometheus_client does not support async
+    context manager yet.
+
+    Overhead: The overhead introduced here in case of an async function could likely be because of `await` introduced
+    which will return in another coroutine object creation and under heavy load could see longer wall time
+    (scheduling delays due to introduction of another awaitable).
+    """
+
+    def measure(func: Callable[..., Any]) -> Callable[..., Any]:
+        nonlocal name
+
+        name = name or func.__name__
+
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            if not enable_metrics:
+                return await func(*args, **kwargs)
+
+            metric = FUNC_LATENCY
+            start = time.monotonic()
+            ret = func(*args, **kwargs)
+            if isinstance(ret, asyncio.Future) or asyncio.iscoroutine(ret):
+                try:
+                    ret = await ret
+                finally:
+                    metric.labels(name=name).observe(time.monotonic() - start)
+            return ret
+
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            if not enable_metrics:
+                return func(*args, **kwargs)
+
+            metric = FUNC_LATENCY
+            start = time.monotonic()
+            try:
+                ret = func(*args, **kwargs)
+            finally:
+                metric.labels(name=name).observe(time.monotonic() - start)
+            return ret
+
+        if asyncio.iscoroutinefunction(func):
+            return async_wrapper
+        return sync_wrapper
+
+    if func:
+        return measure(func)
+    else:
+        return measure

python/sglang/srt/observability/label_transform.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+_PRIORITY_MIN = 0
+_PRIORITY_MAX = 31
+_LOW_PRIORITY_VALUE = "LOW"
+_HIGH_PRIORITY_VALUE = "HIGH"
+
+UNKNOWN_PRIORITY_VALUE = "UNKNOWN"
+
+
+def transform_priority(priority: Optional[int]) -> str:
+    """Transform the priority to a string for metrics reporting.
+    Limit the range to prevent high cardinality issues.
+
+    Args:
+        priority: The priority to transform.
+    Returns:
+        The transformed priority.
+    """
+    if priority is None:
+        return UNKNOWN_PRIORITY_VALUE
+    elif priority < _PRIORITY_MIN:
+        return _LOW_PRIORITY_VALUE
+    elif priority >= _PRIORITY_MAX:
+        return _HIGH_PRIORITY_VALUE
+    else:
+        return str(priority)

python/sglang/srt/observability/mooncake_trace.py

@@ -0,0 +1,68 @@
+import time
+from typing import Union
+
+from sglang.srt.observability.req_time_stats import (
+    RequestStageConfig,
+    convert_time_to_realtime_ns,
+)
+from sglang.srt.observability.trace import TraceNullContext, TraceReqContext
+
+
+class MooncakeRequestStage:
+    MOONCAKE_SEND = RequestStageConfig(
+        "mooncake_send",
+        level=1,
+    )
+    MOONCAKE_RECV = RequestStageConfig(
+        "mooncake_recv",
+        level=1,
+    )
+    MOONCAKE_WORKER_SEND = RequestStageConfig(
+        "mooncake_worker_send",
+        level=1,
+    )
+    MOONCAKE_WORKER_SEND_SESSION = RequestStageConfig(
+        "mooncake_worker_send_session",
+        level=2,
+    )
+    MOONCAKE_WORKER_RECV = RequestStageConfig(
+        "mooncake_worker_recv",
+        level=1,
+    )
+
+
+def mooncake_trace_slice(
+    trace_ctx: Union[TraceReqContext, TraceNullContext],
+    stage: RequestStageConfig,
+    start_ts: float,
+    thread_finish_flag=False,
+):
+    if trace_ctx is None:
+        return
+
+    if not trace_ctx.tracing_enable:
+        return
+
+    start_ts = convert_time_to_realtime_ns(start_ts)
+    trace_ctx.trace_slice_start(stage.stage_name, stage.level, start_ts)
+    trace_ctx.trace_slice_end(
+        stage.stage_name,
+        stage.level,
+        thread_finish_flag=thread_finish_flag,
+    )
+
+
+def mooncake_trace_func(stage: RequestStageConfig):
+    def decorator(func):
+        def wrapper(self, *args, **kwargs):
+            if self.trace_ctx is None:
+                return func(self, *args, **kwargs)
+            start_ts = convert_time_to_realtime_ns(time.perf_counter())
+            self.trace_ctx.trace_slice_start(stage.stage_name, stage.level, start_ts)
+            ret = func(self, *args, **kwargs)
+            self.trace_ctx.trace_slice_end(stage.stage_name, stage.level)
+            return ret
+
+        return wrapper
+
+    return decorator

python/sglang/srt/observability/ray_wrappers.py

@@ -0,0 +1,307 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ray-backed implementations of the prometheus_client API surface used by
+sglang's ``*MetricsCollector`` classes.
+
+The wrappers translate prometheus_client calls into ``ray.util.metrics`` so the
+metrics emitted by an embedded sglang engine flow through Ray's metric agent and
+appear on Ray's Prometheus endpoint / dashboard alongside other Ray metrics.
+
+Mirrors ``vllm/v1/metrics/ray_wrappers.py`` with two sglang-specific additions:
+
+* ``RaySummaryWrapper`` — Ray has no Summary primitive; we fall back to a
+  Histogram with conservative default boundaries. Quantile queries can be
+  approximated through ``histogram_quantile()`` in Prometheus.
+* Five collector subclasses, one per ``*MetricsCollector`` defined in
+  :mod:`sglang.srt.observability.metrics_collector`, overriding only the
+  ``_xxx_cls`` attributes that the corresponding collector actually uses.
+
+Import is lazy: the module loads in environments without Ray installed, but
+instantiating a wrapper without Ray raises a clear :class:`ImportError`.
+"""
+
+from __future__ import annotations
+
+import copy
+import re
+import time
+from typing import List, Optional
+
+try:
+    from ray import serve as ray_serve
+    from ray.util import metrics as ray_metrics
+    from ray.util.metrics import Metric
+except ImportError:  # pragma: no cover - covered by a dedicated test
+    ray_metrics = None
+    ray_serve = None
+    Metric = None  # type: ignore[assignment]
+
+from sglang.srt.observability.metrics_collector import (
+    ExpertDispatchCollector,
+    RadixCacheMetricsCollector,
+    SchedulerMetricsCollector,
+    StorageMetricsCollector,
+    TokenizerMetricsCollector,
+)
+
+
+def _get_replica_id() -> Optional[str]:
+    """Return the current Ray Serve replica ID, or ``None`` outside Serve."""
+    if ray_serve is None:
+        return None
+    try:
+        return ray_serve.get_replica_context().replica_id.unique_id
+    except ray_serve.exceptions.RayServeException:
+        return None
+
+
+class RayPrometheusMetric:
+    """Base wrapper that exposes the prometheus_client API on Ray metrics.
+
+    Subclasses populate ``self.metric`` with a ``ray.util.metrics`` instance in
+    their ``__init__``. Shared behaviour:
+
+    * A ``ReplicaId`` tag is appended to every metric and populated at
+      instantiation (and again on each ``labels()`` call) so Ray-Serve replicas
+      are distinguishable on dashboards.
+    * ``labels()`` returns a fresh copy of the wrapper with its tags bound,
+      mirroring the ``prometheus_client`` pattern and avoiding state sharing
+      between concurrent emits.
+    * Metric names are sanitised to satisfy Ray's OpenTelemetry naming rule
+      (no ``:``, no other punctuation).
+    """
+
+    _is_labeled: bool = False
+
+    def __init__(self) -> None:
+        if ray_metrics is None:
+            raise ImportError(
+                "RayPrometheusMetric requires Ray to be installed. "
+                "Install with: pip install 'ray[serve]'"
+            )
+        self.metric: Optional[Metric] = None
+        self._tags: dict = {"ReplicaId": _get_replica_id() or ""}
+
+    @staticmethod
+    def _get_tag_keys(labelnames: Optional[List[str]]) -> tuple:
+        labels = list(labelnames) if labelnames else []
+        labels.append("ReplicaId")
+        return tuple(labels)
+
+    def _build_tags(self, *labels: str, **labelskwargs: str) -> dict:
+        if labels:
+            # The trailing entry of ``_tag_keys`` is always ``ReplicaId`` which we
+            # populate ourselves; positional args fill the preceding keys only.
+            expected = len(self.metric._tag_keys) - 1
+            if len(labels) != expected:
+                raise ValueError(
+                    "Number of labels must match the number of tag keys. "
+                    f"Expected {expected}, got {len(labels)}"
+                )
+            labelskwargs.update(zip(self.metric._tag_keys, labels))
+        labelskwargs["ReplicaId"] = _get_replica_id() or ""
+        return {k: v if isinstance(v, str) else str(v) for k, v in labelskwargs.items()}
+
+    def labels(self, *labels: str, **labelskwargs: str) -> RayPrometheusMetric:
+        if self._is_labeled:
+            raise ValueError("labels() cannot be called on an already-labeled metric.")
+        clone = copy.copy(self)
+        clone._tags = self._build_tags(*labels, **labelskwargs)
+        clone._is_labeled = True
+        return clone
+
+    @staticmethod
+    def _coerce_positive_boundaries(buckets):
+        # Ray (gRPC OpenCensus / OpenTelemetry export) rejects boundaries
+        # <= 0. sglang ships several histograms whose lowest bucket is 0.0
+        # (e.g. queue_time, e2e latency). Silently drop those so we never
+        # break engine startup when the metrics backend is Ray.
+        if not buckets:
+            return []
+        return [b for b in buckets if b > 0]
+
+    @staticmethod
+    def _get_sanitized_opentelemetry_name(name: str) -> str:
+        """Replace characters Ray's OTel-backed metric name validator rejects.
+
+        Ray is migrating from OpenCensus to OpenTelemetry, whose instrument names
+        only allow ``a-zA-Z0-9_``. sglang's existing names use a ``sglang:foo``
+        prefix; converting ``:`` (and any other punctuation) to ``_`` keeps the
+        names valid without churn on the prometheus_client side.
+        """
+        return re.sub(r"[^a-zA-Z0-9_]", "_", name)
+
+
+class RayCounterWrapper(RayPrometheusMetric):
+    """``prometheus_client.Counter`` compatible wrapper."""
+
+    def __init__(
+        self,
+        name: str,
+        documentation: Optional[str] = "",
+        labelnames: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__()
+        tag_keys = self._get_tag_keys(labelnames)
+        name = self._get_sanitized_opentelemetry_name(name)
+        self.metric = ray_metrics.Counter(
+            name=name,
+            description=documentation,
+            tag_keys=tag_keys,
+        )
+
+    def inc(self, value: float = 1.0) -> None:
+        if value == 0:
+            return
+        return self.metric.inc(value, tags=self._tags)
+
+
+class RayGaugeWrapper(RayPrometheusMetric):
+    """``prometheus_client.Gauge`` compatible wrapper."""
+
+    def __init__(
+        self,
+        name: str,
+        documentation: Optional[str] = "",
+        labelnames: Optional[List[str]] = None,
+        multiprocess_mode: Optional[str] = "",
+    ) -> None:
+        # Ray aggregates per WorkerId/ReplicaId at the metric agent, so the
+        # prometheus_client multiproc modes ("mostrecent", "all", "sum") are not
+        # meaningful here. Accept and discard for API parity.
+        del multiprocess_mode
+        super().__init__()
+        tag_keys = self._get_tag_keys(labelnames)
+        name = self._get_sanitized_opentelemetry_name(name)
+        self.metric = ray_metrics.Gauge(
+            name=name,
+            description=documentation,
+            tag_keys=tag_keys,
+        )
+
+    def set(self, value: float) -> None:
+        return self.metric.set(value, tags=self._tags)
+
+    def set_to_current_time(self) -> None:
+        return self.set(time.time())
+
+
+class RayHistogramWrapper(RayPrometheusMetric):
+    """``prometheus_client.Histogram`` compatible wrapper."""
+
+    def __init__(
+        self,
+        name: str,
+        documentation: Optional[str] = "",
+        labelnames: Optional[List[str]] = None,
+        buckets: Optional[List[float]] = None,
+    ) -> None:
+        super().__init__()
+        tag_keys = self._get_tag_keys(labelnames)
+        name = self._get_sanitized_opentelemetry_name(name)
+        self.metric = ray_metrics.Histogram(
+            name=name,
+            description=documentation,
+            tag_keys=tag_keys,
+            boundaries=self._coerce_positive_boundaries(buckets),
+        )
+
+    def observe(self, value: float) -> None:
+        return self.metric.observe(value, tags=self._tags)
+
+
+class RaySummaryWrapper(RayPrometheusMetric):
+    """``prometheus_client.Summary`` compatible wrapper.
+
+    ``ray.util.metrics`` does not provide a Summary primitive. We approximate by
+    emitting through a Histogram with conservative default boundaries; quantile
+    queries can be approximated downstream via ``histogram_quantile()``.
+    """
+
+    DEFAULT_BOUNDARIES: List[float] = [
+        0.005,
+        0.01,
+        0.025,
+        0.05,
+        0.1,
+        0.25,
+        0.5,
+        1.0,
+        2.5,
+        5.0,
+        10.0,
+    ]
+
+    def __init__(
+        self,
+        name: str,
+        documentation: Optional[str] = "",
+        labelnames: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__()
+        tag_keys = self._get_tag_keys(labelnames)
+        name = self._get_sanitized_opentelemetry_name(name)
+        self.metric = ray_metrics.Histogram(
+            name=name,
+            description=documentation,
+            tag_keys=tag_keys,
+            boundaries=self._coerce_positive_boundaries(self.DEFAULT_BOUNDARIES),
+        )
+
+    def observe(self, value: float) -> None:
+        return self.metric.observe(value, tags=self._tags)
+
+
+# ---------------------------------------------------------------------------
+# Collector subclasses
+#
+# Each subclass only overrides the ``_xxx_cls`` attributes its parent actually
+# uses; the parent's ``_StatLoggerDIMixin`` defaults handle the rest.
+# ---------------------------------------------------------------------------
+
+
+class RaySchedulerMetricsCollector(SchedulerMetricsCollector):
+    """``SchedulerMetricsCollector`` that emits via Ray's metric system."""
+
+    _counter_cls = RayCounterWrapper
+    _gauge_cls = RayGaugeWrapper
+    _histogram_cls = RayHistogramWrapper
+    _summary_cls = RaySummaryWrapper
+
+
+class RayTokenizerMetricsCollector(TokenizerMetricsCollector):
+    """``TokenizerMetricsCollector`` that emits via Ray's metric system."""
+
+    _counter_cls = RayCounterWrapper
+    _histogram_cls = RayHistogramWrapper
+
+
+class RayStorageMetricsCollector(StorageMetricsCollector):
+    """``StorageMetricsCollector`` that emits via Ray's metric system."""
+
+    _counter_cls = RayCounterWrapper
+    _histogram_cls = RayHistogramWrapper
+
+
+class RayRadixCacheMetricsCollector(RadixCacheMetricsCollector):
+    """``RadixCacheMetricsCollector`` that emits via Ray's metric system."""
+
+    _counter_cls = RayCounterWrapper
+    _histogram_cls = RayHistogramWrapper
+
+
+class RayExpertDispatchCollector(ExpertDispatchCollector):
+    """``ExpertDispatchCollector`` that emits via Ray's metric system."""
+
+    _histogram_cls = RayHistogramWrapper

python/sglang/srt/observability/request_metrics_exporter.py

@@ -0,0 +1,220 @@
+import asyncio
+import dataclasses
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import List, Optional, Union
+
+from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
+from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
+from sglang.srt.server_args import ServerArgs
+
+logger = logging.getLogger(__name__)
+
+# Fields that should always be excluded from request parameters
+# because they contain non-JSON-serializable objects (e.g., ImageData, tensors)
+ALWAYS_EXCLUDE_FIELDS = {"image_data", "video_data", "audio_data", "input_embeds"}
+
+
+class RequestMetricsExporter(ABC):
+    """Abstract base class for exporting request-level performance metrics to a data destination."""
+
+    def __init__(
+        self,
+        server_args: ServerArgs,
+        obj_skip_names: Optional[set[str]],
+        out_skip_names: Optional[set[str]],
+    ):
+        self.server_args = server_args
+        self.obj_skip_names = obj_skip_names or set()
+        self.out_skip_names = out_skip_names or set()
+
+    def _format_output_data(
+        self, obj: Union[GenerateReqInput, EmbeddingReqInput], out_dict: dict
+    ) -> dict:
+        """Format request-level output data containing performance metrics. This method
+        should be called prior to writing the data record with `self.write_record()`."""
+
+        request_params = {}
+        for field in dataclasses.fields(obj):
+            field_name = field.name
+            # Skip fields in obj_skip_names or fields that are always excluded (not JSON serializable)
+            if (
+                field_name not in self.obj_skip_names
+                and field_name not in ALWAYS_EXCLUDE_FIELDS
+            ):
+                value = getattr(obj, field_name)
+                # Convert to serializable format
+                if value is not None:
+                    request_params[field_name] = value
+
+        meta_info = out_dict.get("meta_info", {})
+        filtered_out_meta_info = {
+            k: v for k, v in meta_info.items() if k not in self.out_skip_names
+        }
+
+        request_output_data = {
+            "request_parameters": json.dumps(request_params),
+            **filtered_out_meta_info,
+        }
+        return request_output_data
+
+    @abstractmethod
+    async def write_record(
+        self, obj: Union[GenerateReqInput, EmbeddingReqInput], out_dict: dict
+    ):
+        """Write a data record corresponding to a single request, containing performance metric data."""
+        pass
+
+
+class FileRequestMetricsExporter(RequestMetricsExporter):
+    """Lightweight `RequestMetricsExporter` implementation that writes records to files on disk.
+
+    Records are written to files in the directory specified by `--export-metrics-to-file-dir`
+    server launch flag. File names are of the form `"sglang-request-metrics-{hour_suffix}.log"`.
+    """
+
+    def __init__(
+        self,
+        server_args: ServerArgs,
+        obj_skip_names: Optional[set[str]],
+        out_skip_names: Optional[set[str]],
+    ):
+        super().__init__(server_args, obj_skip_names, out_skip_names)
+        self.export_dir = getattr(server_args, "export_metrics_to_file_dir")
+        os.makedirs(self.export_dir, exist_ok=True)
+
+        # File handler state management
+        self._current_file_handler = None
+        self._current_file_lock = asyncio.Lock()
+        self._current_hour_suffix = None
+
+    def _ensure_file_handler(self, hour_suffix: str):
+        """Ensure the file handler is open for the current hour suffix."""
+        if self._current_hour_suffix != hour_suffix:
+            # Close previous file handler if it exists
+            if self._current_file_handler is not None:
+                try:
+                    self._current_file_handler.close()
+                except Exception as e:
+                    logger.warning(f"Failed to close previous file handler: {e}")
+
+            # Open new file handler
+            log_filename = f"sglang-request-metrics-{hour_suffix}.log"
+            log_filepath = os.path.join(self.export_dir, log_filename)
+
+            try:
+                self._current_file_handler = open(log_filepath, "a", encoding="utf-8")
+                self._current_hour_suffix = hour_suffix
+            except Exception as e:
+                logger.error(f"Failed to open log file {log_filepath}: {e}")
+                self._current_file_handler = None
+                self._current_hour_suffix = None
+                raise
+
+    def close(self):
+        """Close the current file handler."""
+        if self._current_file_handler is not None:
+            try:
+                self._current_file_handler.close()
+            except Exception as e:
+                logger.warning(f"Failed to close file handler: {e}")
+            finally:
+                self._current_file_handler = None
+                self._current_hour_suffix = None
+
+    async def write_record(
+        self, obj: Union[GenerateReqInput, EmbeddingReqInput], out_dict: dict
+    ):
+        # Do not log health check requests, since they don't represent real user requests.
+        if isinstance(obj.rid, str) and HEALTH_CHECK_RID_PREFIX in obj.rid:
+            return
+
+        try:
+            # Get the log file path for the current time.
+            current_time = datetime.now()
+            hour_suffix = current_time.strftime("%Y%m%d_%H")
+
+            async with self._current_file_lock:
+                # Ensure correct file handler is open for current hour
+                self._ensure_file_handler(hour_suffix)
+
+                if self._current_file_handler is None:
+                    return
+
+                metrics_data = self._format_output_data(obj, out_dict)
+
+                def write_file():
+                    json.dump(metrics_data, self._current_file_handler)
+                    self._current_file_handler.write("\n")
+                    self._current_file_handler.flush()
+
+                await asyncio.to_thread(write_file)
+        except Exception as e:
+            logger.exception(f"Failed to write perf metrics to file: {e}")
+
+
+class RequestMetricsExporterManager:
+    """Manager class for creating and managing RequestMetricsExporter instances."""
+
+    def __init__(
+        self,
+        server_args: ServerArgs,
+        obj_skip_names: Optional[set[str]] = None,
+        out_skip_names: Optional[set[str]] = None,
+    ):
+        self.server_args = server_args
+        self.obj_skip_names = obj_skip_names or set()
+        self.out_skip_names = out_skip_names or set()
+        self._exporters: List[RequestMetricsExporter] = []
+        self._create_exporters()
+
+    def _create_exporters(self) -> None:
+        """Create and configure RequestMetricsExporter instances based on server args."""
+        # Create standard exporters
+        self._exporters.extend(
+            create_request_metrics_exporters(
+                self.server_args, self.obj_skip_names, self.out_skip_names
+            )
+        )
+
+        # Import additional RequestMetricsExporter from private fork if available; skip otherwise.
+        try:
+            from sglang.private.managers.request_metrics_exporter_factory import (
+                create_private_request_metrics_exporters,
+            )
+
+            self._exporters.extend(
+                create_private_request_metrics_exporters(
+                    self.server_args, self.obj_skip_names, self.out_skip_names
+                )
+            )
+        except ImportError:
+            pass
+
+    def exporter_enabled(self) -> bool:
+        """Return true if at least one RequestMetricsExporter is enabled."""
+        return len(self._exporters) > 0
+
+    async def write_record(self, obj, out_dict: dict) -> None:
+        """Write a record using all configured exporters."""
+        for exporter in self._exporters:
+            await exporter.write_record(obj, out_dict)
+
+
+def create_request_metrics_exporters(
+    server_args: ServerArgs,
+    obj_skip_names: Optional[set[str]] = None,
+    out_skip_names: Optional[set[str]] = None,
+) -> List[RequestMetricsExporter]:
+    """Create and configure `RequestMetricsExporter`s based on server args."""
+    metrics_exporters = []
+
+    if server_args.export_metrics_to_file:
+        metrics_exporters.append(
+            FileRequestMetricsExporter(server_args, obj_skip_names, out_skip_names)
+        )
+
+    return metrics_exporters

python/sglang/srt/observability/startup_func_log_and_timer.py

@@ -0,0 +1,150 @@
+"""
+Records startup latency breakdown by context using gauge metrics in seconds
+"""
+
+import logging
+import time
+from contextlib import contextmanager
+from functools import wraps
+from typing import Any, Callable, Dict, Generator, Optional
+
+logger = logging.getLogger(__name__)
+
+enable_startup_metrics = False
+STARTUP_LATENCY_SECONDS = None
+# Track maximum durations for each context
+_max_durations: Dict[str, float] = {}
+
+
+def enable_startup_timer():
+    """Initialize startup latency metrics when metrics are enabled"""
+    # We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR`
+    from prometheus_client import Gauge
+
+    global enable_startup_metrics, STARTUP_LATENCY_SECONDS
+    enable_startup_metrics = True
+
+    STARTUP_LATENCY_SECONDS = Gauge(
+        "sglang:startup_latency_breakdown_seconds_max",
+        "Startup latency breakdown in seconds by context, only records the maximum duration if the context is called multiple times.",
+        labelnames=["context"],
+        multiprocess_mode="mostrecent",
+    )
+
+
+def set_startup_metric(context: str, value: float, should_log: bool = True):
+    """Set the startup metric for a given context"""
+    if should_log:
+        logger.info(f"Setting startup metric: {context} took {value:.3f}s")
+
+    if not enable_startup_metrics:
+        return
+    current_max = _max_durations.get(context, 0.0)
+    if value > current_max:
+        _max_durations[context] = value
+        STARTUP_LATENCY_SECONDS.labels(context=context).set(value)
+
+
+def reset_startup_timers():
+    """Reset all recorded maximum durations. Useful for testing or reinitialization."""
+    global _max_durations
+    _max_durations.clear()
+
+
+def get_max_duration(context: str) -> Optional[float]:
+    """Get the maximum recorded duration for a context name."""
+    return _max_durations.get(context)
+
+
+@contextmanager
+def startup_timer(name: str, log_only: bool = False) -> Generator[None, None, None]:
+    """
+    Context manager to measure startup latency for arbitrary code blocks.
+    Only records the maximum duration if the context is called multiple times.
+
+    Usage:
+        with startup_timer("model_loading"):
+            # model loading code
+            model = load_model()
+
+        with startup_timer("memory_allocation"):
+            # memory setup code
+            allocate_memory()
+    """
+    start_time = time.monotonic()
+    try:
+        yield
+    finally:
+        duration_seconds = time.monotonic() - start_time
+
+        # Track the maximum duration for this context name
+        current_max = _max_durations.get(name, 0.0)
+        is_new_max = duration_seconds > current_max
+
+        if is_new_max:
+            _max_durations[name] = duration_seconds
+
+            # Only update Prometheus gauge if this is a new maximum
+            if enable_startup_metrics and not log_only:
+                STARTUP_LATENCY_SECONDS.labels(context=name).set(duration_seconds)
+
+        # Log with indication if this was a new max
+        logger.info(f"Startup timing: {name} took {duration_seconds:.3f}s")
+
+
+def time_startup_latency(
+    func: Callable = None, name: Optional[str] = None, log_only: bool = False
+) -> Callable[..., Any]:
+    """
+    A decorator to measure startup context latency and record it in seconds.
+    Only records the maximum duration if the context is called multiple times.
+
+    Usage:
+        @time_startup_latency
+        def load_model():
+            # model loading code
+
+        @time_startup_latency(name="custom_init")
+        def initialize_something():
+            # initialization code
+
+        @time_startup_latency(name="debug_only", log_only=True)
+        def debug_function():
+            # This will only log, not record to Prometheus
+    """
+
+    def measure(func: Callable[..., Any]) -> Callable[..., Any]:
+        nonlocal name
+        name = name or func.__name__
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            start_time = time.monotonic()
+            try:
+                result = func(*args, **kwargs)
+                return result
+            finally:
+                duration_seconds = time.monotonic() - start_time
+
+                # Track the maximum duration for this context name
+                current_max = _max_durations.get(name, 0.0)
+                is_new_max = duration_seconds > current_max
+
+                if is_new_max:
+                    _max_durations[name] = duration_seconds
+
+                    # Only update Prometheus gauge if this is a new maximum
+                    if enable_startup_metrics and not log_only:
+                        STARTUP_LATENCY_SECONDS.labels(context=name).set(
+                            duration_seconds
+                        )
+
+                # Log the timing
+                logger.info(f"Startup timing: {name} took {duration_seconds:.3f}s")
+
+        return wrapper
+
+    if func:
+        return measure(func)
+    else:
+        return measure

python/sglang/srt/observability/trace.py

@@ -0,0 +1,807 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""package for sglang requests tracing"""
+
+from __future__ import annotations
+
+import logging
+import os
+import random
+import threading
+import time
+import uuid
+from dataclasses import dataclass
+from typing import Any, Dict, List, Mapping, Optional
+
+from sglang.srt.utils import get_int_env_var
+
+logger = logging.getLogger(__name__)
+opentelemetry_imported = False
+opentelemetry_initialized = False
+_trace_context_propagator = None
+tracer: Optional[trace.Tracer] = None
+
+
+# Modules allowed to emit spans (from --trace-modules); None means no filtering.
+global_trace_modules: Optional[List[str]] = None
+
+TRACE_HEADERS = ["traceparent", "tracestate"]
+
+try:
+    from opentelemetry import context, propagate, trace
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+        OTLPSpanExporter as GRPCSpanExporter,
+    )
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter as HTTPSpanExporter,
+    )
+    from opentelemetry.sdk.environment_variables import (
+        OTEL_EXPORTER_OTLP_TRACES_PROTOCOL,
+    )
+    from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+    from opentelemetry.sdk.trace import TracerProvider, id_generator
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.trace import Status, StatusCode
+    from opentelemetry.trace.propagation.tracecontext import (
+        TraceContextTextMapPropagator,
+    )
+
+    _trace_context_propagator = TraceContextTextMapPropagator()
+
+    opentelemetry_imported = True
+except ImportError:
+
+    class id_generator:
+        class IdGenerator:
+            pass
+
+    logger.debug("opentelemetry package is not installed, tracing disabled")
+
+
+def extract_trace_headers(headers: Mapping[str, str]) -> Optional[Dict]:
+    return {h: headers[h] for h in TRACE_HEADERS if h in headers}
+
+
+def get_global_trace_level() -> int:
+    from sglang.srt.runtime_context import get_resources
+
+    resources = get_resources()
+    if resources.trace_level is None:
+        resources.trace_level = get_int_env_var("SGLANG_TRACE_LEVEL", 3)
+    return resources.trace_level
+
+
+def set_global_trace_level(level: int):
+    from sglang.srt.runtime_context import get_resources
+
+    get_resources().trace_level = level
+
+
+@dataclass
+class TraceThreadInfo:
+    host_id: str
+    pid: int
+    thread_label: str
+    tp_rank: int
+    dp_rank: int
+    pp_rank: int
+
+
+@dataclass
+class TraceEvent:
+    event_name: str
+    ts: int
+    attrs: Dict[str, Any]
+
+
+@dataclass
+class TraceSliceContext:
+    slice_name: str
+    start_time_ns: int
+    end_time_ns: Optional[int] = None
+    span: Optional[trace.span.Span] = None
+    level: int = 1
+    attrs: Optional[Dict[str, Any]] = None
+    events: Optional[List[TraceEvent]] = None
+
+
+@dataclass
+class TraceThreadContext:
+    thread_info: TraceThreadInfo
+    cur_slice_stack: Optional[List[TraceSliceContext]] = None
+    thread_span: Optional[trace.span.Span] = None
+
+
+class TraceCustomIdGenerator(id_generator.IdGenerator):
+    """
+    The default IdGenerator may produce duplicate trace IDs across multiple TP scheduler processes,
+    hence a custom IdGenerator is implemented.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.local_random = random.Random()
+        self.local_random.seed(time.time())
+
+    def generate_trace_id(self) -> int:
+        return self.local_random.getrandbits(64)
+
+    def generate_span_id(self) -> int:
+        return self.local_random.getrandbits(64)
+
+
+# global variables
+threads_info: Dict[int, TraceThreadInfo] = {}
+
+get_cur_time_ns = lambda: int(time.time() * 1e9)
+if hasattr(time, "time_ns"):
+    get_cur_time_ns = lambda: int(time.time_ns())
+
+
+def _get_host_id() -> str:
+    """
+    In distributed tracing systems, obtain a unique node identifier
+    and inject it into all subsequently generated spans
+    to prevent PID conflicts between threads on different nodes.
+    """
+    if os.path.exists("/etc/machine-id"):
+        try:
+            with open("/etc/machine-id", "r") as f:
+                return f.read().strip()
+        except:
+            pass
+
+    mac = uuid.getnode()
+    if mac != 0:
+        return uuid.UUID(int=mac).hex
+
+    return "unknown"
+
+
+# Should be called by each tracked process.
+def process_tracing_init(
+    otlp_endpoint, server_name, trace_modules: Optional[str] = None
+):
+    global opentelemetry_initialized
+    global get_cur_time_ns
+    global tracer
+    global global_trace_modules
+
+    if trace_modules is not None:
+        global_trace_modules = [
+            module.strip() for module in trace_modules.split(",") if module.strip()
+        ]
+
+    if not opentelemetry_imported:
+        opentelemetry_initialized = False
+        raise RuntimeError(
+            "opentelemetry package is not installed!!! Please not enable tracing or install opentelemetry"
+        )
+
+    try:
+        resource = Resource.create(
+            attributes={
+                SERVICE_NAME: server_name,
+            }
+        )
+        tracer_provider = TracerProvider(
+            resource=resource, id_generator=TraceCustomIdGenerator()
+        )
+
+        schedule_delay_millis = get_int_env_var(
+            "SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS", 500
+        )
+        max_export_batch_size = get_int_env_var(
+            "SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE", 64
+        )
+
+        processor = BatchSpanProcessor(
+            span_exporter=get_otlp_span_exporter(otlp_endpoint),
+            schedule_delay_millis=schedule_delay_millis,
+            max_export_batch_size=max_export_batch_size,
+        )
+        tracer_provider.add_span_processor(processor)
+        trace.set_tracer_provider(tracer_provider)
+    except Exception as e:
+        opentelemetry_initialized = False
+        raise RuntimeError(
+            f"initialize opentelemetry error:{e}. Please set correct otlp endpoint."
+        )
+
+    opentelemetry_initialized = True
+    tracer = trace.get_tracer("sglang server")
+
+
+def get_global_tracing_enabled():
+    return opentelemetry_initialized
+
+
+def get_otlp_span_exporter(endpoint):
+    protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "grpc")
+    supported_protocols = {"grpc", "http/protobuf"}
+
+    if protocol not in supported_protocols:
+        raise ValueError(
+            f"Unsupported OTLP protocol '{protocol}' configured. "
+            f"Supported protocols are: {', '.join(sorted(supported_protocols))}"
+        )
+
+    if protocol == "grpc":
+        return GRPCSpanExporter(endpoint=endpoint, insecure=True)
+    elif protocol == "http/protobuf":
+        return HTTPSpanExporter(endpoint=endpoint)
+
+
+# Should be called by each tracked thread.
+def trace_set_thread_info(
+    thread_label: str,
+    tp_rank: Optional[int] = None,
+    dp_rank: Optional[int] = None,
+    pp_rank: Optional[int] = None,
+):
+    if not opentelemetry_initialized:
+        return
+
+    pid = threading.get_native_id()
+    if pid in threads_info:
+        return
+
+    threads_info[pid] = TraceThreadInfo(
+        host_id=_get_host_id(),
+        pid=pid,
+        thread_label=thread_label,
+        tp_rank=tp_rank,
+        dp_rank=dp_rank,
+        pp_rank=pp_rank,
+    )
+
+
+class TraceReqContext:
+    def __init__(
+        self,
+        rid,
+        bootstrap_room=None,
+        role="unified",
+        module_name="",
+        external_trace_header: Optional[Dict[str, str]] = None,
+    ):
+        self.rid: str = str(rid)
+        self.trace_level = get_global_trace_level()
+        self.tracing_enable: bool = opentelemetry_initialized and self.trace_level > 0
+
+        # Filter by --trace-modules only for explicitly named modules; contexts
+        # created with the default empty module_name are always traced.
+        if (
+            module_name
+            and global_trace_modules is not None
+            and module_name not in global_trace_modules
+        ):
+            self.tracing_enable = False
+
+        if not self.tracing_enable:
+            return
+
+        self.start_time_ns: Optional[int] = None
+        self.thread_context: Optional[TraceThreadContext] = None
+        self.bootstrap_room: Optional[int] = bootstrap_room
+        self.role: str = role
+        self.module_name = module_name
+
+        # Indicates whether this instance is a replica from the main process.
+        # When True, root_span is None and only root_span_context is preserved.
+        self.is_copy: bool = False
+        self.root_span: Optional[trace.span.Span] = None
+        self.root_span_context: Optional[context.Context] = None
+        # Record the most recently completed span as the previous span for the next span to be created.
+        self.last_span_context: Optional[trace.span.SpanContext] = None
+        self.external_trace_header: Optional[Dict[str, str]] = external_trace_header
+
+        self.events_cache: List[TraceEvent] = []
+
+        self.pid: int = threading.get_native_id()
+
+    def is_tracing_enabled(self) -> bool:
+        return self.tracing_enable
+
+    def __create_thread_context(self, ts: int):
+        if self.pid not in threads_info:
+            trace_set_thread_info("unknown")
+
+        thread_info = threads_info[self.pid]
+        thread_context = TraceThreadContext(
+            thread_info=thread_info,
+            cur_slice_stack=[],
+        )
+
+        thread_name = f"{thread_info.thread_label}"
+        if thread_info.tp_rank is not None:
+            thread_name += f" [TP {thread_info.tp_rank}] "
+        if thread_info.pp_rank is not None:
+            thread_name += f" [PP {thread_info.pp_rank}] "
+        if thread_info.dp_rank is not None:
+            thread_name += f" [DP {thread_info.dp_rank}] "
+        thread_name += f"(host:{thread_info.host_id[:8]} | pid:{self.pid})"
+        thread_context.thread_span = tracer.start_span(
+            name=thread_name,
+            start_time=ts,
+            context=self.root_span_context,
+        )
+
+        rank_attrs = {}
+        if thread_info.tp_rank is not None:
+            rank_attrs["tp_rank"] = thread_info.tp_rank
+        if thread_info.pp_rank is not None:
+            rank_attrs["pp_rank"] = thread_info.pp_rank
+        if thread_info.dp_rank is not None:
+            rank_attrs["dp_rank"] = thread_info.dp_rank
+        if rank_attrs:
+            thread_context.thread_span.set_attributes(rank_attrs)
+
+        thread_context.thread_span.set_attributes(
+            {
+                "host_id": thread_info.host_id,
+                "pid": thread_info.pid,
+                "thread_label": thread_info.thread_label,
+            }
+        )
+
+        return thread_context
+
+    def __getstate__(self) -> Optional[Dict[str, Any]]:
+        if not self.tracing_enable:
+            return {"tracing_enable": False}
+
+        if not self.root_span_context:
+            return {"tracing_enable": False}
+
+        state = {
+            "tracing_enable": self.tracing_enable,
+            "rid": self.rid,
+            "bootstrap_room": self.bootstrap_room,
+            "start_time_ns": self.start_time_ns,
+            "role": self.role,
+            "trace_level": self.trace_level,
+            "module_name": self.module_name,
+            "is_copy": self.is_copy,
+            "pid": self.pid,
+            "thread_context": None,
+            "root_span": None,
+            "last_span_context": None,
+        }
+
+        carrier: dict[str, str] = {}
+        propagate.inject(carrier, self.root_span_context)
+        state["root_span_context"] = carrier
+
+        prev_span_context = self.last_span_context
+        if self.thread_context and self.thread_context.cur_slice_stack:
+            cur_slice = self.thread_context.cur_slice_stack[0]
+            if cur_slice.span:
+                prev_span_context = cur_slice.span.get_span_context()
+
+        if prev_span_context:
+            state["last_span_context"] = {
+                "span_id": prev_span_context.span_id,
+                "trace_id": prev_span_context.trace_id,
+            }
+
+        return state
+
+    def __setstate__(self, state: Dict[str, Any]):
+        self.__dict__.update(state)
+        if not opentelemetry_initialized:
+            self.tracing_enable = False
+        if not self.tracing_enable:
+            return
+
+        self.is_copy = True
+        self.pid = threading.get_native_id()
+        self.root_span_context = propagate.extract(self.root_span_context)
+        if self.last_span_context:
+            self.last_span_context = trace.span.SpanContext(
+                trace_id=self.last_span_context["trace_id"],
+                span_id=self.last_span_context["span_id"],
+                is_remote=True,
+            )
+        self.events_cache = []
+
+    def copy_for_thread(self) -> TraceReqContext:
+        """
+        Create a copy of this context for use in another thread.
+
+        The copy shares the same root_span_context but has its own thread_context.
+        This is useful for propagating trace context across threads (e.g., worker threads).
+
+        Usage:
+            # Sender (main thread)
+            trace_ctx_copy = trace_ctx.copy_for_thread()
+            queue.put(TransferKVChunk(..., trace_ctx=trace_ctx_copy))
+
+            # Receiver (worker thread)
+            kv_chunk = queue.get()
+            kv_chunk.trace_ctx.rebuild_thread_context()
+        """
+        # Fast path: not tracing
+        if not self.tracing_enable or not self.root_span_context:
+            return TraceNullContext()
+
+        # Extract prev_span_context from current thread state
+        prev_span_context = self.last_span_context
+        if self.thread_context and self.thread_context.cur_slice_stack:
+            cur_slice = self.thread_context.cur_slice_stack[0]
+            if cur_slice.span:
+                prev_span_context = cur_slice.span.get_span_context()
+
+        # Create new instance with shared state
+        copied = TraceReqContext.__new__(TraceReqContext)
+        copied.tracing_enable = self.tracing_enable
+        copied.rid = self.rid
+        copied.bootstrap_room = self.bootstrap_room
+        copied.start_time_ns = self.start_time_ns
+        copied.role = self.role
+        copied.trace_level = self.trace_level
+        copied.module_name = self.module_name
+        copied.is_copy = True  # Mark as copy
+        copied.pid = self.pid
+
+        # thread_context is None, will be rebuilt via rebuild_thread_context()
+        copied.thread_context = None
+        copied.root_span = None
+
+        # Share root_span_context (already a context, no need to serialize)
+        copied.root_span_context = self.root_span_context
+
+        # Set prev_span_context for linking spans
+        if prev_span_context:
+            copied.last_span_context = trace.span.SpanContext(
+                trace_id=prev_span_context.trace_id,
+                span_id=prev_span_context.span_id,
+                is_remote=True,
+            )
+        else:
+            copied.last_span_context = None
+
+        copied.events_cache = []
+
+        return copied
+
+    def rebuild_thread_context(self, ts: Optional[int] = None):
+        if not self.tracing_enable:
+            return
+
+        ts = ts or get_cur_time_ns()
+        self.thread_context = self.__create_thread_context(ts)
+
+    def trace_req_start(
+        self,
+        ts: Optional[int] = None,
+    ):
+        if not self.tracing_enable:
+            return
+
+        ts = ts or get_cur_time_ns()
+
+        # create req context and root span
+        self.start_time_ns = ts
+
+        external_trace_context = _trace_context_propagator.extract(
+            self.external_trace_header or {}
+        )
+
+        # Drop the worker_id added by MultiTokenizer
+        orig_rid = self.rid.split("_")[-1]
+        role = "" if self.role == "unified" else self.role
+        attrs = {"rid": orig_rid, "module": f"sglang::{self.module_name}"}
+        if self.bootstrap_room:
+            attrs["bootstrap_room"] = str(hex(self.bootstrap_room))
+        root_span = tracer.start_span(
+            name=f"{role} Req {orig_rid[:8]}",
+            start_time=ts,
+            context=external_trace_context,
+            attributes=attrs,
+        )
+
+        self.root_span = root_span
+        self.root_span_context = trace.set_span_in_context(root_span)
+
+        # create thread context and thread span
+        self.thread_context = self.__create_thread_context(ts)
+
+    def trace_req_finish(
+        self, ts: Optional[int] = None, attrs: Optional[Dict[str, Any]] = None
+    ):
+        if not self.tracing_enable:
+            return
+
+        if not self.root_span:
+            return
+
+        ts = ts or get_cur_time_ns()
+
+        # End all unclosed thread spans.
+        self.abort()
+
+        if attrs:
+            self.root_span.set_attributes(attrs)
+
+        self.root_span.end(end_time=ts)
+        self.root_span = None
+
+    def __check_fast_return(self, level=None):
+        if not self.tracing_enable:
+            return True
+
+        if not self.thread_context:
+            return True
+
+        if level and level > self.trace_level:
+            return True
+
+        return False
+
+    def trace_slice_start(
+        self,
+        name: str,
+        level: int,
+        ts: Optional[int] = None,
+    ):
+        if self.__check_fast_return(level):
+            return
+
+        ts = ts or get_cur_time_ns()
+
+        cur_slice = TraceSliceContext(
+            slice_name=name,
+            start_time_ns=ts,
+            level=level,
+            attrs={},
+            events=[],
+        )
+
+        parent_span = self.thread_context.thread_span
+        prev_span_context = None
+        if not self.thread_context.cur_slice_stack:
+            if self.last_span_context:
+                prev_span_context = self.last_span_context
+        else:
+            parent_span = self.thread_context.cur_slice_stack[-1].span
+
+        parent_span_context = trace.set_span_in_context(parent_span)
+
+        span = tracer.start_span(
+            name=cur_slice.slice_name,
+            start_time=cur_slice.start_time_ns,
+            context=parent_span_context,
+        )
+        cur_slice.span = span
+
+        if prev_span_context:
+            span.add_link(prev_span_context)
+
+        self.thread_context.cur_slice_stack.append(cur_slice)
+
+    def trace_slice_end(
+        self,
+        name: str,
+        level: int,
+        ts: Optional[int] = None,
+        attrs: Optional[Dict[str, Any]] = None,
+        thread_finish_flag: bool = False,
+    ):
+        if self.__check_fast_return(level):
+            return
+
+        if not self.thread_context.cur_slice_stack:
+            logger.warning(
+                f"No matching with the SLICE_START event {name} is required."
+            )
+            return
+
+        cur_slice = self.thread_context.cur_slice_stack[-1]
+        ts = ts or get_cur_time_ns()
+
+        # check if slice_name matching and level matching
+        # unlikely path, excepting error API usage
+        if cur_slice.slice_name != name or cur_slice.level != level:
+            logger.warning(
+                f"Slice name mismatch: {name} != {cur_slice.slice_name} or level mismatch: {level} != {cur_slice.level}"
+            )
+            self.thread_context.cur_slice_stack.pop()
+            return
+
+        span = cur_slice.span
+
+        if attrs:
+            span.set_attributes(attrs)
+
+        if self.events_cache:
+            new_events_cache = []
+            for event in self.events_cache:
+                if event.ts >= cur_slice.start_time_ns and event.ts < ts:
+                    span.add_event(
+                        name=event.event_name,
+                        timestamp=event.ts,
+                        attributes=event.attrs,
+                    )
+                else:
+                    new_events_cache.append(event)
+            self.events_cache = new_events_cache
+
+        span.end(end_time=ts)
+
+        self.thread_context.cur_slice_stack.pop()
+        # only for first level slice
+        if not self.thread_context.cur_slice_stack:
+            self.last_span_context = span.get_span_context()
+
+        if thread_finish_flag:
+            self.abort(ts)
+
+    def trace_slice(
+        self,
+        slice: TraceSliceContext,
+        thread_finish_flag: bool = False,
+    ):
+        if self.__check_fast_return(slice.level):
+            return
+
+        parent_span = self.thread_context.thread_span
+        prev_span_context = None
+        if not self.thread_context.cur_slice_stack:
+            if self.last_span_context:
+                prev_span_context = self.last_span_context
+        else:
+            parent_span = self.thread_context.cur_slice_stack[-1].span
+
+        parent_span_context = trace.set_span_in_context(parent_span)
+
+        span = tracer.start_span(
+            name=slice.slice_name,
+            start_time=slice.start_time_ns,
+            context=parent_span_context,
+        )
+
+        if prev_span_context:
+            span.add_link(prev_span_context)
+
+        if slice.attrs:
+            span.set_attributes(slice.attrs)
+
+        if slice.events:
+            for event in slice.events:
+                span.add_event(
+                    name=event.event_name, timestamp=event.ts, attributes=event.attrs
+                )
+
+        if self.events_cache:
+            new_events_cache = []
+            for event in self.events_cache:
+                if event.ts >= slice.start_time_ns and event.ts < slice.end_time_ns:
+                    span.add_event(
+                        name=event.event_name,
+                        timestamp=event.ts,
+                        attributes=event.attrs,
+                    )
+                else:
+                    new_events_cache.append(event)
+            self.events_cache = new_events_cache
+
+        span.end(end_time=slice.end_time_ns)
+
+        # only for first level slice
+        if not self.thread_context.cur_slice_stack:
+            self.last_span_context = span.get_span_context()
+
+        if thread_finish_flag:
+            self.abort(slice.end_time_ns)
+
+    # Add event to the current slice on the same thread with the same rid.
+    def trace_event(
+        self,
+        name: str,
+        level: int,
+        ts: Optional[int] = None,
+        attrs: Dict[str, Any] = None,
+    ):
+        if self.__check_fast_return(level):
+            return
+
+        ts = ts or get_cur_time_ns()
+
+        if attrs is None:
+            attrs = {}
+        self.events_cache.append(TraceEvent(name, ts, attrs))
+
+    def trace_set_root_attrs(self, attrs: Dict[str, Any]):
+        if not self.tracing_enable:
+            return
+
+        if self.root_span:
+            self.root_span.set_attributes(attrs)
+
+    def trace_set_thread_attrs(self, attrs: Dict[str, Any]):
+        if self.__check_fast_return():
+            return
+
+        if self.thread_context.thread_span:
+            self.thread_context.thread_span.set_attributes(attrs)
+
+    def abort(self, ts=None, abort_info: Optional[Dict] = None):
+        if self.__check_fast_return():
+            return
+
+        # close all slice spans (unlikely, except error API usage)
+        ts = ts or get_cur_time_ns()
+        while len(self.thread_context.cur_slice_stack) > 0:
+            if self.thread_context.cur_slice_stack[-1].span:
+                self.thread_context.cur_slice_stack[-1].span.end(end_time=ts)
+            self.thread_context.cur_slice_stack.pop()
+
+        # set abort info into thread span
+        if self.thread_context.thread_span:
+            if abort_info:
+                from sglang.srt.managers.schedule_batch import BaseFinishReason
+
+                if isinstance(abort_info, BaseFinishReason):
+                    abort_info = abort_info.to_json()
+                self.thread_context.thread_span.set_status(Status(StatusCode.ERROR))
+                self.thread_context.thread_span.set_attributes(abort_info)
+
+            if self.events_cache:
+                for event in self.events_cache:
+                    self.thread_context.thread_span.add_event(
+                        name=event.event_name,
+                        timestamp=event.ts,
+                        attributes=event.attrs,
+                    )
+                self.events_cache = []
+
+            self.thread_context.thread_span.end(end_time=ts)
+        self.thread_context = None
+
+    def __del__(self):
+        self.abort(abort_info={"reason": "have unclosed span, auto closed"})
+
+
+@dataclass
+class TraceNullContext:
+    tracing_enable: bool = False
+
+    def __getattr__(self, name):
+        return self
+
+    def __call__(self, *args, **kwargs):
+        return self
+
+
+class SpanAttributes:
+    # Attribute names copied from here to avoid version conflicts:
+    # https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md
+    GEN_AI_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens"
+    GEN_AI_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens"
+    GEN_AI_USAGE_CACHED_TOKENS = "gen_ai.usage.cached_tokens"
+    GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"
+    GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"
+    GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k"
+    GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"
+    GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
+    GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"
+    GEN_AI_REQUEST_ID = "gen_ai.request.id"
+    GEN_AI_REQUEST_N = "gen_ai.request.n"
+    GEN_AI_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
+    GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
+    GEN_AI_LATENCY_E2E = "gen_ai.latency.e2e"
+    GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL = "gen_ai.latency.time_in_model_prefill"
+    GEN_AI_LATENCY_TIME_IN_MODEL_DECODE = "gen_ai.latency.time_in_model_decode"
+    GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE = "gen_ai.latency.time_in_model_inference"

python/sglang/srt/observability/utils.py

@@ -0,0 +1,56 @@
+# Copyright 2023-2025 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for Prometheus Metrics."""
+
+import math
+from typing import List
+
+
+def two_sides_exponential_buckets(
+    middle: float, base: float, count: int
+) -> List[float]:
+    buckets = []
+    half_count = math.ceil(count / 2)
+    distance = 1
+    buckets.append(middle)
+    for i in range(half_count):
+        distance *= base
+        buckets.append(middle + distance)
+        buckets.append(max(0, middle - distance))
+    return sorted(set(buckets))
+
+
+def generate_buckets(
+    buckets_rule: List[str], default_buckets: List[float]
+) -> List[float]:
+    if not buckets_rule:
+        buckets_rule = ["default"]
+
+    assert len(buckets_rule) > 0
+    rule = buckets_rule[0]
+    if rule == "tse":
+        middle, base, count = buckets_rule[1:]
+        assert float(base) > 1.0, "Base must be greater than 1.0"
+        return two_sides_exponential_buckets(float(middle), float(base), int(count))
+    if rule == "default":
+        return sorted(set(default_buckets))
+    assert rule == "custom"
+    return sorted(set([float(x) for x in buckets_rule[1:]]))
+
+
+def exponential_buckets(start: float, width: float, length: int) -> List[float]:
+    buckets = []
+    for i in range(length):
+        buckets.append(start * (width**i))
+    return buckets