ray-project--ray/python/ray/serve/_private/tracing_utils.py

import inspect
import os
import threading
from contextvars import ContextVar, Token
from functools import wraps
from typing import Any, Callable, Dict, List, Optional

from ray._common.utils import import_attr
from ray.serve._private.constants import (
    DEFAULT_TRACING_EXPORTER_IMPORT_PATH,
    RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH,
    RAY_SERVE_TRACING_SAMPLING_RATIO,
)

try:
    from opentelemetry import trace
    from opentelemetry.context import attach, detach, get_current
    from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
    from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor
    from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio
    from opentelemetry.semconv.trace import SpanAttributes
    from opentelemetry.trace import SpanKind
    from opentelemetry.trace.propagation import set_span_in_context
    from opentelemetry.trace.propagation.tracecontext import (
        TraceContextTextMapPropagator,
    )
    from opentelemetry.trace.status import Status, StatusCode

except ImportError:
    SpanProcessor = None
    ConsoleSpanExporter = None
    SimpleSpanProcessor = None
    trace = None
    SpanKind = None
    TracerProvider = None
    TraceIdRatioBased = None
    Status = None
    StatusCode = None
    set_span_in_context = None
    TraceContextTextMapPropagator = None
    get_current = None
    attach = None
    detach = None
    SpanAttributes = None
    ParentBasedTraceIdRatio = None


TRACE_STACK: ContextVar[List[Any]] = ContextVar(
    "trace_stack"
)  # Create tracer once at module level

_tracer = None
_tracer_lock = threading.Lock()


def get_tracer():
    global _tracer
    if _tracer is None:
        with _tracer_lock:
            if _tracer is None:
                _tracer = trace.get_tracer(__name__)
    return _tracer


# Default tracing exporter needs to map to DEFAULT_TRACING_EXPORTER_IMPORT_PATH
# defined in "python/ray/serve/_private/constants.py"
def default_tracing_exporter(tracing_file_name):
    from ray.serve._private.logging_utils import get_serve_logs_dir

    serve_logs_dir = get_serve_logs_dir()
    spans_dir = os.path.join(serve_logs_dir, "spans")
    os.makedirs(spans_dir, exist_ok=True)
    spans_file = os.path.join(spans_dir, tracing_file_name)

    out_file = open(spans_file, "a")

    class FileConsoleSpanExporter(ConsoleSpanExporter):
        def shutdown(self):
            if not out_file.closed:
                out_file.flush()
                out_file.close()

    return [SimpleSpanProcessor(FileConsoleSpanExporter(out=out_file))]


class TraceContextManager:
    def __init__(
        self, trace_name, span_kind=None, trace_context: Optional[Dict[str, str]] = None
    ):
        self.span = None
        self.trace_name = trace_name
        self.span_kind = span_kind
        self.trace_context = trace_context

        self.is_tracing_enabled = is_tracing_enabled()

    def __enter__(self):
        if self.is_tracing_enabled:
            self.span_kind = self.span_kind or SpanKind.SERVER

            tracer = get_tracer()
            ctx = self.trace_context if self.trace_context else get_trace_context()

            self.span = tracer.start_span(
                self.trace_name,
                kind=self.span_kind,
                context=ctx,
            )
            if not self.span.get_span_context().trace_flags.sampled:
                return self
            new_ctx = set_span_in_context(self.span)
            set_trace_context(new_ctx)
            _append_trace_stack(self.span)
            set_span_name(self.trace_name)

        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.is_tracing_enabled and self.span is not None:
            # if exc_type is not None, we have made a explicit decision
            # to not set the span status to error. This is because
            # errors are spans internal to Ray Serve and should not
            # be reported as errors in the trace. They cause noise
            # in the trace and are not meaningful to the user.
            self.span.end()
            _pop_trace_stack()

        return False


class BatchTraceContextManager:
    """Attach/detach a tracing context around a block to scope the span of a batch."""

    def __init__(self, trace_context: Optional[object]):
        self._enabled = is_tracing_enabled() and trace_context is not None
        self._trace_context = trace_context
        self._token: Optional[Token] = None

    def __enter__(self):
        if self._enabled:
            self._token = set_trace_context(self._trace_context)

        return self

    def __exit__(self, exc_type, exc, tb):
        if self._enabled and self._token is not None:
            detach_trace_context(self._token)
        return False


def tracing_decorator_factory(
    trace_name: str, span_kind: Optional[Any] = None
) -> Callable:
    """
    Factory function to create a tracing decorator for instrumenting functions/methods
    with distributed tracing.

    Args:
        trace_name: The name of the trace.
        span_kind: The kind of span to create
            (e.g., SERVER, CLIENT). Defaults to trace.SpanKind.SERVER.

    Returns:
        Callable: A decorator function that can be used to wrap
            functions/methods with distributed tracing.

    Example Usage:
    ```python
    @tracing_decorator_factory(
        "my_trace",
        span_kind=trace.SpanKind.CLIENT,
    )
    def my_function(obj):
        # Function implementation
    ```
    """

    def tracing_decorator(func):
        if not is_tracing_enabled():
            # if tracing is not enabled, we don't want to wrap the function
            # with the tracing decorator.
            return func

        @wraps(func)
        def synchronous_wrapper(*args, **kwargs):
            with TraceContextManager(trace_name, span_kind):
                result = func(*args, **kwargs)

            return result

        @wraps(func)
        def generator_wrapper(*args, **kwargs):
            with TraceContextManager(trace_name, span_kind):
                for item in func(*args, **kwargs):
                    yield item

        @wraps(func)
        async def asynchronous_wrapper(*args, **kwargs):
            with TraceContextManager(trace_name, span_kind):
                result = await func(*args, **kwargs)
            return result

        @wraps(func)
        async def asyc_generator_wrapper(*args, **kwargs):
            with TraceContextManager(trace_name, span_kind):
                async for item in func(*args, **kwargs):
                    yield item

        is_generator = _is_generator_function(func)
        is_async = _is_async_function(func)
        if is_generator and is_async:
            return asyc_generator_wrapper
        elif is_async:
            return asynchronous_wrapper
        elif is_generator:
            return generator_wrapper
        else:
            return synchronous_wrapper

    return tracing_decorator


def setup_tracing(
    component_name: str,
    component_id: str,
    component_type: Optional["ServeComponentType"] = None,  # noqa: F821
    tracing_exporter_import_path: Optional[
        str
    ] = RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH,
    tracing_sampling_ratio: Optional[float] = RAY_SERVE_TRACING_SAMPLING_RATIO,
) -> bool:
    """
    Set up tracing for a specific Serve component.

    Args:
        component_name: The name of the component.
        component_id: The unique identifier of the component.
        component_type: The type of the component.
        tracing_exporter_import_path: Path to tracing exporter function.
        tracing_sampling_ratio: Sampling ratio for traces (0.0 to 1.0).

    Returns:
        bool: True if tracing setup is successful, False otherwise.
    """
    if tracing_exporter_import_path == "":
        return False

    # Check dependencies
    if not trace:
        raise ImportError(
            "You must `pip install opentelemetry-api` and "
            "`pip install opentelemetry-sdk` "
            "to enable tracing on Ray Serve."
        )

    from ray.serve._private.utils import get_component_file_name

    tracing_file_name = get_component_file_name(
        component_name=component_name,
        component_id=component_id,
        component_type=component_type,
        suffix="_tracing.json",
    )

    span_processors = _load_span_processors(
        tracing_exporter_import_path, tracing_file_name
    )

    # Intialize tracing
    # Sets the tracer_provider. This is only allowed once~ per execution
    # context and will log a warning if attempted multiple times.
    # use ParentBasedTraceIdRatio to respect the parent span's sampling decision
    # and sample probabilistically based on the tracing_sampling_ratio
    sampler = ParentBasedTraceIdRatio(tracing_sampling_ratio)

    trace.set_tracer_provider(TracerProvider(sampler=sampler))

    for span_processor in span_processors:
        trace.get_tracer_provider().add_span_processor(span_processor)

    return True


def create_propagated_context() -> Dict[str, str]:
    """Create context that can be used across services and processes.

    This function retrieves the current context and converts it
    into a dictionary that can be used across actors and tasks since
    it is serializable.

    Returns:
    - Trace Context Propagator (dict or None): A dictionary containing the propagated
    trace context if available, otherwise None.
    """
    trace_context = get_trace_context()
    if trace_context and TraceContextTextMapPropagator:
        ctx = {}
        TraceContextTextMapPropagator().inject(ctx, trace_context)
        return ctx

    return None


def extract_propagated_context(
    propagated_context: Optional[Dict[str, str]] = None
) -> Optional[Dict[str, str]]:
    """Extract the trace context from a Trace Context Propagator."""
    if is_tracing_enabled() and propagated_context and TraceContextTextMapPropagator:
        return TraceContextTextMapPropagator().extract(carrier=propagated_context)

    return None


def set_trace_context(trace_context: Dict[str, str]) -> Optional[Token]:
    """Set the current trace context."""
    if attach is None:
        return

    return attach(trace_context)


def detach_trace_context(token: Token):
    """Detach the current trace context corresponding to the token."""
    if detach is None:
        return

    detach(token)


def get_trace_context() -> Optional[Dict[str, str]]:
    """Retrieve the current trace context."""
    if get_current is None:
        return None

    context = get_current()
    return context if context else None


def set_span_name(name: str):
    """Set the name for the current span in context."""
    if TRACE_STACK:
        trace_stack = TRACE_STACK.get([])
        if trace_stack:
            trace_stack[-1].update_name(name)
    # this is added specifically for Datadog tracing.
    # See https://docs.datadoghq.com/tracing/guide/configuring-primary-operation/#opentracing
    set_span_attributes({"resource.name": name})


def set_rpc_span_attributes(
    system: str = "grpc",
    method: Optional[str] = None,
    status_code: Optional[str] = None,
    service: Optional[str] = None,
):
    """
    Use this function to set attributes for RPC spans.
    Only include attributes that are in the OpenTelemetry
    RPC span attributes spec https://opentelemetry.io/docs/specs/semconv/attributes-registry/rpc/.
    """
    if not is_tracing_enabled():
        return
    attributes = {
        SpanAttributes.RPC_SYSTEM: system,
        SpanAttributes.RPC_METHOD: method,
        SpanAttributes.RPC_GRPC_STATUS_CODE: status_code,
        SpanAttributes.RPC_SERVICE: service,
    }
    set_span_attributes(attributes)


def set_http_span_attributes(
    method: Optional[str] = None,
    status_code: Optional[str] = None,
    route: Optional[str] = None,
):
    """
    Use this function to set attributes for HTTP spans.
    Only include attributes that are in the OpenTelemetry
    HTTP span attributes spec https://opentelemetry.io/docs/specs/semconv/attributes-registry/http/.
    """
    if not is_tracing_enabled():
        return
    attributes = {
        SpanAttributes.HTTP_METHOD: method,
        SpanAttributes.HTTP_STATUS_CODE: status_code,
        SpanAttributes.HTTP_ROUTE: route,
    }
    set_span_attributes(attributes)


def set_span_attributes(attributes: Dict[str, Any]):
    """Set attributes for the current span in context."""
    if TRACE_STACK:
        trace_stack = TRACE_STACK.get([])
        if trace_stack:
            # filter attribute values that are None, otherwise they
            # will show up as warning logs on the console.
            attributes = {k: v for k, v in attributes.items() if v is not None}
            trace_stack[-1].set_attributes(attributes)


def set_trace_status(is_error: bool, description: str = ""):
    """Set the status for the current span in context."""
    trace_stack = TRACE_STACK.get([])
    if trace_stack:
        if is_error:
            status_code = StatusCode.ERROR
        else:
            status_code = StatusCode.OK
            description = None
        trace_stack[-1].set_status(
            Status(status_code=status_code, description=description)
        )


def set_span_exception(exc: Exception, escaped: bool = False):
    """Set the exception for the current span in context."""
    trace_stack = TRACE_STACK.get([])
    if trace_stack:
        trace_stack[-1].record_exception(exc, escaped=escaped)


def is_tracing_enabled() -> bool:
    return RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH != "" and trace is not None


def is_span_recording() -> bool:
    if TRACE_STACK:
        trace_stack = TRACE_STACK.get([])
        if trace_stack:
            return True
    return False


def _append_trace_stack(span):
    """Append span to global trace stack."""
    trace_stack = TRACE_STACK.get([])
    trace_stack.append(span)
    TRACE_STACK.set(trace_stack)


def _pop_trace_stack():
    """Pop span to global trace stack."""
    trace_stack = TRACE_STACK.get([])
    if trace_stack:
        trace_stack.pop()
        TRACE_STACK.set(trace_stack)


def _validate_tracing_exporter(func: Callable) -> None:
    """Validate that the custom tracing exporter
    is a function that takes no arguments.
    """
    if inspect.isfunction(func) is False:
        raise TypeError("Tracing exporter must be a function.")

    signature = inspect.signature(func)

    if len(signature.parameters) != 0:
        raise TypeError("Tracing exporter cannot take any arguments.")


def _validate_tracing_exporter_processors(span_processors: List[Any]):
    """Validate that the output of a custom tracing exporter
    returns type List[SpanProcessor].
    """
    if not isinstance(span_processors, list):
        raise TypeError(
            "Output of tracing exporter needs to be of type "
            f"List[SpanProcessor], but received type {type(span_processors)}."
        )

    for span_processor in span_processors:
        if not isinstance(span_processor, SpanProcessor):
            raise TypeError(
                "Output of tracing exporter needs to be of "
                "type List[SpanProcessor], "
                f"but received type {type(span_processor)}."
            )


def _load_span_processors(
    tracing_exporter_import_path: str,
    tracing_file_name: str,
):
    """Load span processors from a custome tracing
    exporter function.
    """
    tracing_exporter_def = import_attr(tracing_exporter_import_path)

    if tracing_exporter_import_path == DEFAULT_TRACING_EXPORTER_IMPORT_PATH:
        return tracing_exporter_def(tracing_file_name)
    else:
        # Validate tracing exporter function
        _validate_tracing_exporter(tracing_exporter_def)
        # Validate tracing exporter processors
        span_processors = tracing_exporter_def()
        _validate_tracing_exporter_processors(span_processors)

    return span_processors


def _is_generator_function(func):
    return inspect.isgeneratorfunction(func) or inspect.isasyncgenfunction(func)


def _is_async_function(func):
    return inspect.iscoroutinefunction(func) or inspect.isasyncgenfunction(func)