chore: import upstream snapshot with attribution

agentlightning/utils/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Microsoft. All rights reserved.

agentlightning/utils/id.py

@@ -0,0 +1,24 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import hashlib
+import uuid
+
+__all__ = ["generate_id"]
+
+
+def generate_id(length: int) -> str:
+    """Generate a random hexadecimal ID of the given length.
+
+    Args:
+        length: The desired length of the generated ID. Must be a positive integer.
+
+    Returns:
+        A random hexadecimal ID string of the given length.
+
+    Raises:
+        ValueError: If length is not a positive integer.
+    """
+    if length <= 0:
+        raise ValueError("length must be a positive integer")
+
+    return hashlib.sha1(uuid.uuid4().bytes).hexdigest()[:length]

agentlightning/utils/otel.py

@@ -0,0 +1,541 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Utilities shared for OpenTelemetry span (attributes) support."""
+
+import json
+import logging
+import traceback
+from typing import Any, Dict, List, Sequence, Type, TypeVar, Union, cast
+from warnings import filterwarnings
+
+import opentelemetry.trace as trace_api
+from agentops.sdk.exporters import OTLPSpanExporter
+from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, SpanProcessor, SynchronousMultiSpanProcessor, Tracer
+from opentelemetry.sdk.trace import TracerProvider as TracerProviderImpl
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor
+from opentelemetry.sdk.util.instrumentation import InstrumentationInfo, InstrumentationScope
+from opentelemetry.semconv.attributes import exception_attributes
+from opentelemetry.trace import get_tracer_provider as otel_get_tracer_provider
+from pydantic import TypeAdapter
+
+from agentlightning.env_var import LightningEnvVar, resolve_bool_env_var
+from agentlightning.semconv import LightningSpanAttributes, LinkAttributes, LinkPydanticModel
+from agentlightning.types import Attributes, AttributeValue, SpanLike
+from agentlightning.utils.otlp import LightningStoreOTLPExporter
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "full_qualified_name",
+    "get_tracer_provider",
+    "get_tracer",
+    "make_tag_attributes",
+    "extract_tags_from_attributes",
+    "make_link_attributes",
+    "query_linked_spans",
+    "extract_links_from_attributes",
+    "filter_attributes",
+    "filter_and_unflatten_attributes",
+    "flatten_attributes",
+    "unflatten_attributes",
+    "sanitize_attribute_value",
+    "sanitize_attributes",
+    "sanitize_list_attribute_sanity",
+    "check_attributes_sanity",
+    "format_exception_attributes",
+]
+
+T_SpanLike = TypeVar("T_SpanLike", bound=SpanLike)
+T_SpanProcessor = TypeVar("T_SpanProcessor", bound=SpanProcessor)
+
+
+def full_qualified_name(obj: type) -> str:
+    if str(obj.__module__) == "builtins":
+        return obj.__qualname__
+    return f"{obj.__module__}.{obj.__qualname__}"
+
+
+def get_tracer_provider(inspect: bool = True) -> TracerProviderImpl:
+    """Get the OpenTelemetry tracer provider configured for Agent Lightning.
+
+    Args:
+        inspect: Whether to inspect the tracer provider and log its configuration.
+            When it's on, make sure you also set the logger level to DEBUG to see the logs.
+    """
+    from agentlightning.tracer.otel import LightningSpanProcessor
+
+    if hasattr(trace_api, "_TRACER_PROVIDER") and trace_api._TRACER_PROVIDER is None:  # type: ignore[attr-defined]
+        raise RuntimeError("Tracer is not initialized. Cannot emit a meaningful span.")
+    tracer_provider = otel_get_tracer_provider()
+    if not isinstance(tracer_provider, TracerProviderImpl):
+        logger.error(
+            "Tracer provider is expected to be an instance of opentelemetry.sdk.trace.TracerProvider, found: %s",
+            full_qualified_name(type(tracer_provider)),
+        )
+        return cast(TracerProviderImpl, tracer_provider)
+
+    if not inspect:
+        return tracer_provider
+
+    emitter_debug = resolve_bool_env_var(LightningEnvVar.AGL_EMITTER_DEBUG, fallback=None)
+    logger_effective_level = logger.getEffectiveLevel()
+    if emitter_debug is True and logger_effective_level > logging.DEBUG:
+        logger.warning(
+            "Emitter debug logging is enabled but logging level is not set to DEBUG. Nothing will be logged."
+        )
+
+    if emitter_debug is None:
+        # Set to true by default if the logging level is lower than DEBUG
+        emitter_debug = logging.DEBUG >= logger_effective_level
+
+    if emitter_debug:
+        active_span_processor = tracer_provider._active_span_processor  # pyright: ignore[reportPrivateUsage]
+        processors: List[str] = []
+        active_span_processor_cls = active_span_processor.__class__.__name__
+        for processor in active_span_processor._span_processors:  # pyright: ignore[reportPrivateUsage]
+            if isinstance(processor, LightningSpanProcessor):
+                # The legacy case for tracers without OTLP support.
+                processors.append(f"{active_span_processor_cls} - {processor!r}")
+            elif isinstance(processor, (SimpleSpanProcessor, BatchSpanProcessor)):
+                processor_cls = processor.__class__.__name__
+                if isinstance(processor.span_exporter, LightningStoreOTLPExporter):
+                    # This should be the main path now.
+                    processors.append(f"{active_span_processor_cls} - {processor_cls} - {processor.span_exporter!r}")
+                elif isinstance(processor.span_exporter, OTLPSpanExporter):
+                    # You need to be careful if the code goes into this path.
+                    endpoint = processor.span_exporter._endpoint  # pyright: ignore[reportPrivateUsage]
+                    processors.append(
+                        f"{active_span_processor_cls} - {processor_cls} - "
+                        f"{processor.span_exporter.__class__.__name__}(endpoint={endpoint!r})"
+                    )
+                else:
+                    # Other cases like Console Span Exporter.
+                    processors.append(
+                        f"{active_span_processor_cls} - {processor_cls} - {processor.span_exporter.__class__.__name__}"
+                    )
+            else:
+                processors.append(f"{active_span_processor_cls} - {processor.__class__.__name__}")
+
+        logger.debug(f"Tracer provider: {tracer_provider!r}. Active span processors:")
+        for processor in processors:
+            logger.debug("  * " + processor)
+
+    return tracer_provider
+
+
+def get_span_processors(
+    tracer_provider: TracerProviderImpl, expected_type: Type[T_SpanProcessor]
+) -> List[T_SpanProcessor]:
+    """Get the span processors from the tracer provider.
+
+    Args:
+        tracer_provider: The tracer provider to get the span processors from.
+        expected_type: The type of the span processors to get.
+
+    Returns:
+        A list of span processors of the expected type.
+    """
+    processors: List[T_SpanProcessor] = []
+    for processor in tracer_provider._active_span_processor._span_processors:  # pyright: ignore[reportPrivateUsage]
+        if isinstance(processor, expected_type):
+            processors.append(processor)
+    return processors
+
+
+def get_tracer(use_active_span_processor: bool = True) -> trace_api.Tracer:
+    """Resolve the OpenTelemetry tracer configured for Agent Lightning.
+
+    Args:
+        use_active_span_processor: Whether to use the active span processor.
+
+    Returns:
+        OpenTelemetry tracer tagged with the `agentlightning` instrumentation name.
+
+    Raises:
+        RuntimeError: If OpenTelemetry was not initialized before calling this helper.
+    """
+    if hasattr(trace_api, "_TRACER_PROVIDER") and trace_api._TRACER_PROVIDER is None:  # type: ignore[attr-defined]
+        raise RuntimeError("Tracer is not initialized. Cannot emit a meaningful span.")
+
+    tracer_provider = get_tracer_provider(inspect=True)  # inspection is on by default
+
+    if use_active_span_processor:
+        return tracer_provider.get_tracer("agentlightning")
+
+    else:
+        filterwarnings(
+            "ignore",
+            message=r"You should use InstrumentationScope. Deprecated since version 1.11.1.",
+            category=DeprecationWarning,
+            module="opentelemetry.sdk.trace",
+        )
+
+        return Tracer(
+            tracer_provider.sampler,
+            tracer_provider.resource,
+            # We use an empty span processor to avoid emitting spans to the tracer
+            SynchronousMultiSpanProcessor(),
+            tracer_provider.id_generator,
+            InstrumentationInfo("agentlightning", "", ""),  # type: ignore
+            SpanLimits(),
+            InstrumentationScope(
+                "agentlightning",
+                "",
+                "",
+                {},
+            ),
+        )
+
+
+def make_tag_attributes(tags: List[str]) -> Dict[str, Any]:
+    """Convert a list of tags into flattened attributes for span tagging.
+
+    There is no syntax enforced for tags, they are just strings. For example:
+
+    ```python
+    ["gen_ai.model:gpt-4", "reward.extrinsic"]
+    ```
+    """
+    return flatten_attributes({LightningSpanAttributes.TAG.value: tags}, expand_leaf_lists=True)
+
+
+def extract_tags_from_attributes(attributes: Dict[str, Any]) -> List[str]:
+    """Extract tag attributes from flattened span attributes.
+
+    Args:
+        attributes: A dictionary of flattened span attributes.
+    """
+    maybe_tag_list = filter_and_unflatten_attributes(attributes, LightningSpanAttributes.TAG.value)
+    return TypeAdapter(List[str]).validate_python(maybe_tag_list)
+
+
+def make_link_attributes(links: Dict[str, str]) -> Dict[str, Any]:
+    """Convert a dictionary of links into flattened attributes for span linking.
+
+    Links example:
+
+    ```python
+    {
+        "gen_ai.response.id": "response-123",
+        "span_id": "abcd-efgh-ijkl",
+    }
+    ```
+    """
+    link_list: List[Dict[str, str]] = []
+    for key, value in links.items():
+        if not isinstance(value, str):  # pyright: ignore[reportUnnecessaryIsInstance]
+            raise ValueError(f"Link value must be a string, got {type(value)} for key '{key}'")
+        link_list.append({LinkAttributes.KEY_MATCH.value: key, LinkAttributes.VALUE_MATCH.value: value})
+    return flatten_attributes({LightningSpanAttributes.LINK.value: link_list}, expand_leaf_lists=True)
+
+
+def query_linked_spans(spans: Sequence[T_SpanLike], links: List[LinkPydanticModel]) -> List[T_SpanLike]:
+    """Query spans that are linked by the given link attributes.
+
+    Args:
+        spans: A sequence of spans to search.
+        links: A list of link attributes to match.
+
+    Returns:
+        A list of spans that match the given link attributes.
+    """
+    matched_spans: List[T_SpanLike] = []
+
+    for span in spans:
+        span_attributes = span.attributes or {}
+        is_match = True
+        for link in links:
+            # trace_id and span_id must be full match.
+            if link.key_match == "trace_id":
+                if isinstance(span, ReadableSpan):
+                    trace_id = trace_api.format_trace_id(span.context.trace_id) if span.context else None
+                else:
+                    trace_id = span.trace_id
+                if trace_id != link.value_match:
+                    is_match = False
+                    break
+
+            elif link.key_match == "span_id":
+                if isinstance(span, ReadableSpan):
+                    span_id = trace_api.format_span_id(span.context.span_id) if span.context else None
+                else:
+                    span_id = span.span_id
+                if span_id != link.value_match:
+                    is_match = False
+                    break
+
+            else:
+                attribute = span_attributes.get(link.key_match)
+                # attributes must also be a full match currently.
+                if attribute != link.value_match:
+                    is_match = False
+                    break
+
+        if is_match:
+            matched_spans.append(span)
+
+    return matched_spans
+
+
+def extract_links_from_attributes(attributes: Dict[str, Any]) -> List[LinkPydanticModel]:
+    """Extract link attributes from flattened span attributes.
+
+    Args:
+        attributes: A dictionary of flattened span attributes.
+    """
+    maybe_link_list = filter_and_unflatten_attributes(attributes, LightningSpanAttributes.LINK.value)
+    return TypeAdapter(List[LinkPydanticModel]).validate_python(maybe_link_list)
+
+
+def filter_attributes(attributes: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+    """Filter attributes that start with the given prefix.
+
+    The attribute must start with `prefix.` or be exactly `prefix` to be included.
+
+    Args:
+        attributes: A dictionary of span attributes.
+        prefix: The prefix to filter by.
+
+    Returns:
+        A dictionary of attributes that start with the given prefix.
+    """
+    return {k: v for k, v in attributes.items() if k.startswith(prefix + ".") or k == prefix}
+
+
+def filter_and_unflatten_attributes(attributes: Dict[str, Any], prefix: str) -> Union[Dict[str, Any], List[Any]]:
+    """Filter attributes that start with the given prefix and unflatten them.
+    The prefix will be removed during unflattening.
+
+    Args:
+        attributes: A dictionary of span attributes.
+        prefix: The prefix to filter by.
+
+    Returns:
+        A nested dictionary or list of attributes that start with the given prefix.
+    """
+    filtered_attributes = filter_attributes(attributes, prefix)
+    stripped_attributes: Dict[str, Any] = {}
+    for k, v in filtered_attributes.items():
+        if k == prefix:
+            raise ValueError(f"Cannot unflatten attribute with key exactly equal to prefix: {prefix}")
+        else:
+            stripped_key = k[len(prefix) + 1 :]  # +1 to remove the dot
+        stripped_attributes[stripped_key] = v
+    return unflatten_attributes(stripped_attributes)
+
+
+def flatten_attributes(
+    nested_data: Union[Dict[str, Any], List[Any]], *, expand_leaf_lists: bool = False
+) -> Dict[str, Any]:
+    """Flatten a nested dictionary or list into a flat dictionary with dotted keys.
+
+    This function recursively traverses dictionaries and lists, producing a flat
+    key-value mapping where nested paths are represented via dot-separated keys.
+    Lists are indexed numerically.
+
+    Example:
+
+        >>> flatten_attributes({"a": {"b": 1, "c": [2, 3]}}, expand_leaf_lists=True)
+        {"a.b": 1, "a.c.0": 2, "a.c.1": 3}
+
+    Args:
+        nested_data: A nested structure composed of dictionaries, lists, or primitive values.
+        expand_leaf_lists: Whether to expand lists composed only of primitive values.
+            When `False` (the default), lists of str/int/float/bool are treated as
+            leaf values and stored without enumerating their indices.
+
+    Returns:
+        A flat dictionary mapping dotted-string paths to primitive values.
+    """
+
+    flat: Dict[str, Any] = {}
+
+    def _primitive_type(value: Any) -> Union[type[str], type[int], type[float], type[bool]]:
+        if isinstance(value, bool):
+            return bool
+        if isinstance(value, int):
+            return int
+        if isinstance(value, float):
+            return float
+        return str
+
+    def _walk(value: Any, prefix: str = "") -> None:
+        if isinstance(value, dict):
+            for k, v in cast(Dict[Any, Any], value).items():
+                if not isinstance(k, str):
+                    raise ValueError(
+                        f"Only string keys are supported in dictionaries, got '{k}' of type {type(k)} in {prefix}"
+                    )
+                new_prefix = f"{prefix}.{k}" if prefix else k
+                _walk(v, new_prefix)
+        elif isinstance(value, list):
+            maybe_list = cast(List[Any], value)
+            is_leaf_candidate = bool(maybe_list) and all(
+                isinstance(item, (str, int, float, bool)) for item in maybe_list
+            )
+            if not expand_leaf_lists and is_leaf_candidate and prefix:
+                primitive_types = {_primitive_type(item) for item in maybe_list}
+                if len(primitive_types) == 1:
+                    flat[prefix] = maybe_list
+                    return
+                logger.warning(
+                    "List attribute '%s' contains mixed primitive types %s; expanding indexed keys instead.",
+                    prefix,
+                    primitive_types,
+                )
+
+            for idx, item in enumerate(maybe_list):
+                new_prefix = f"{prefix}.{idx}" if prefix else str(idx)
+                _walk(item, new_prefix)
+        else:
+            flat[prefix] = value
+
+    _walk(nested_data)
+    return flat
+
+
+def unflatten_attributes(flat_data: Dict[str, Any]) -> Union[Dict[str, Any], List[Any]]:
+    """Reconstruct a nested dictionary/list structure from a flat dictionary.
+
+    Keys are dot-separated paths. Segments that are digit strings will only
+    become list indices if *all* keys in that dict form a consecutive
+    0..n-1 range. Otherwise they remain dict keys.
+
+    Example:
+
+        >>> unflatten_attributes({"a.b": 1, "a.c.0": 2, "a.c.1": 3})
+        {"a": {"b": 1, "c": [2, 3]}}
+
+    Args:
+        flat_data: A dictionary whose keys are dot-separated paths and whose
+            values are primitive data elements.
+
+    Returns:
+        A nested dictionary (and lists where appropriate) corresponding to
+        the flattened structure.
+    """
+    # 1) Build a pure dict tree first (no lists yet)
+    root: Dict[str, Any] = {}
+
+    for flat_key, value in flat_data.items():
+        parts = flat_key.split(".")
+        curr: Dict[str, Any] = root
+
+        for part in parts[:-1]:
+            # Ensure intermediate node is a dict
+            if part not in curr or not isinstance(curr[part], dict):
+                curr[part] = {}
+            curr = curr[part]  # type: ignore[assignment]
+
+        curr[parts[-1]] = value
+
+    # 2) Recursively convert dicts-with-consecutive-numeric-keys into lists
+    def convert(node: Union[Dict[str, Any], List[Any]]) -> Union[Dict[str, Any], List[Any]]:
+        if isinstance(node, dict):
+            # First convert children
+            for k, v in list(node.items()):
+                node[k] = convert(v)
+
+            if not node:
+                # empty dict stays dict
+                return node
+
+            # Check if keys are all numeric strings
+            keys = list(node.keys())
+            if all(isinstance(k, str) and k.isdigit() for k in keys):  # pyright: ignore[reportUnnecessaryIsInstance]
+                indices = sorted(int(k) for k in keys)
+                # Must be exactly 0..n-1
+                if indices == list(range(len(indices))):
+                    return [node[str(i)] for i in range(len(indices))]
+
+            return node
+
+        if isinstance(node, list):  # pyright: ignore[reportUnnecessaryIsInstance]
+            return [convert(v) for v in node]
+
+        # Keep as is
+        return node
+
+    return convert(root)
+
+
+def sanitize_attribute_value(object: Any, force: bool = True) -> AttributeValue:
+    """Sanitize an attribute value to be a valid OpenTelemetry attribute value."""
+    if isinstance(object, (str, int, float, bool)):
+        return object
+
+    if isinstance(object, list):
+        try:
+            return sanitize_list_attribute_sanity(cast(List[Any], object))
+        except ValueError as exc:
+            logger.warning(f"Failed to sanitize list attribute. Fallback to JSON serialization: {exc}")
+
+    try:
+        # This include null, dict, etc.
+        serialized = json.dumps(object, default=str if force else None)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"Object must be JSON serializable, got: {type(cast(Any, object))}.") from exc
+    return serialized
+
+
+def sanitize_attributes(attributes: Dict[str, Any], force: bool = True) -> Attributes:
+    """Sanitize a dictionary of attributes to be a valid OpenTelemetry attributes.
+
+    Args:
+        attributes: A dictionary of attributes to sanitize.
+        force: Whether to force sanitization even when the value is not JSON serializable.
+    """
+    result: Attributes = {}
+    for k, v in attributes.items():
+        try:
+            result[k] = sanitize_attribute_value(v, force=force)
+        except ValueError as exc:
+            raise ValueError(f"Failed to sanitize attribute '{k}': {exc}") from exc
+    return result
+
+
+def sanitize_list_attribute_sanity(maybe_list: List[Any]) -> AttributeValue:
+    """Try to sanitize a list of attributes to be a valid OpenTelemetry attribute value.
+
+    Raise error if the list contains multiple types of primitive values.
+    """
+    if all(isinstance(item, str) for item in maybe_list):
+        return list[str](maybe_list)
+    if all(isinstance(item, bool) for item in maybe_list):
+        return list[bool](maybe_list)
+    if all(isinstance(item, (int, bool)) for item in maybe_list):
+        return [int(item) for item in maybe_list]
+    if all(isinstance(item, (float, int, bool)) for item in maybe_list):
+        return [float(item) for item in maybe_list]
+
+    list_types: List[Any] = [type(item) for item in maybe_list]
+    raise ValueError(f"List must contain only one type of primitive values, got: {set(list_types)}.")
+
+
+def check_attributes_sanity(attributes: Dict[Any, Any]) -> None:
+    """Check if a dictionary of attributes is a valid OpenTelemetry attributes."""
+    for k, v in attributes.items():
+        if not isinstance(k, str):
+            raise ValueError(f"Attribute key must be a string, got {type(k)} for key '{k}'")
+        if isinstance(v, list):
+            try:
+                sanitize_list_attribute_sanity(cast(List[Any], v))
+            except ValueError as exc:
+                raise ValueError(f"Failed to sanitize list attribute '{k}': {exc}") from exc
+        elif not isinstance(v, (str, int, float, bool)):
+            raise ValueError(
+                f"Attribute value must be a string, int, float, bool, or list of these, got {type(v)} for value '{v}'"
+            )
+
+
+def format_exception_attributes(exception: BaseException) -> Attributes:
+    """Format an exception into a dictionary of attributes."""
+    stacktrace = "".join(traceback.format_exception(type(exception), exception, exception.__traceback__))
+    span_attributes: Attributes = {
+        exception_attributes.EXCEPTION_TYPE: type(exception).__name__,
+        exception_attributes.EXCEPTION_MESSAGE: str(exception),
+        exception_attributes.EXCEPTION_ESCAPED: True,
+    }
+    if stacktrace.strip():
+        span_attributes[exception_attributes.EXCEPTION_STACKTRACE] = stacktrace
+    return span_attributes

agentlightning/utils/otlp.py

@@ -0,0 +1,475 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from __future__ import annotations
+
+import gzip
+import logging
+from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Type, TypeVar
+
+from fastapi import Request, Response
+from google.protobuf import json_format
+from google.rpc.status_pb2 import Status
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import (
+    ExportLogsServiceRequest,
+    ExportLogsServiceResponse,
+)
+from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (
+    ExportMetricsServiceRequest,
+    ExportMetricsServiceResponse,
+)
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
+    ExportTraceServiceRequest,
+    ExportTraceServiceResponse,
+)
+from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
+from opentelemetry.proto.resource.v1.resource_pb2 import Resource as ProtoResource
+from opentelemetry.proto.trace.v1.trace_pb2 import Span as ProtoSpan
+from opentelemetry.proto.trace.v1.trace_pb2 import Status as ProtoStatus
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExportResult
+from opentelemetry.util.types import AttributeValue
+
+from agentlightning.semconv import LightningResourceAttributes
+from agentlightning.types.tracer import (
+    Attributes,
+    Event,
+    Link,
+    OtelResource,
+    Span,
+    SpanContext,
+    StatusCode,
+    TraceStatus,
+    convert_timestamp,
+)
+
+PROTOBUF_CT = "application/x-protobuf"
+
+logger = logging.getLogger(__name__)
+
+
+T_request = TypeVar("T_request", ExportLogsServiceRequest, ExportMetricsServiceRequest, ExportTraceServiceRequest)
+T_response = TypeVar("T_response", ExportLogsServiceResponse, ExportMetricsServiceResponse, ExportTraceServiceResponse)
+
+
+async def handle_otlp_export(
+    request: Request,
+    request_message_cls: Type[T_request],
+    response_message_cls: Type[T_response],
+    message_callback: Optional[Callable[[T_request], Awaitable[None]]],
+    signal_name: str,
+) -> Response:
+    """
+    Generic handler for /v1/traces, /v1/metrics, /v1/logs.
+
+    Convert the OTLP Protobuf request to a JSON-like object.
+    """
+    content_type = request.headers.get("Content-Type", "").split(";")[0].strip()
+
+    if content_type != PROTOBUF_CT:
+        # For brevity we only support binary protobuf here.
+        return _bad_request_response(
+            request,
+            f"Unsupported Content-Type '{content_type}', expected '{PROTOBUF_CT}'",
+            content_type=PROTOBUF_CT,
+        )
+
+    raw_body = await request.body()
+    body = _read_body_maybe_gzip(request, raw_body)
+
+    # Empty request is allowed and should still succeed.
+    if not body:
+        req_msg = request_message_cls()
+    else:
+        req_msg = request_message_cls()
+        try:
+            req_msg.ParseFromString(body)
+        except Exception as exc:
+            return _bad_request_response(request, f"Unable to parse OTLP {signal_name} payload: {exc}")
+
+    if message_callback is not None:
+        await message_callback(req_msg)
+
+    # Build success response. Partial success field is left unset.
+    resp_msg = response_message_cls()
+
+    # Encode response in the same Content-Type as request.
+    if content_type == PROTOBUF_CT:
+        resp_bytes = resp_msg.SerializeToString()
+    else:
+        resp_bytes = json_format.MessageToJson(resp_msg).encode("utf-8")
+
+    resp_bytes, headers = _maybe_gzip_response(request, resp_bytes)
+
+    return Response(
+        content=resp_bytes,
+        media_type=content_type,
+        status_code=200,
+        headers=headers,
+    )
+
+
+async def spans_from_proto(
+    request: ExportTraceServiceRequest,
+    sequence_id_bulk_issuer: Callable[[Sequence[Tuple[str, str]]], Awaitable[Sequence[int]]],
+) -> List[Span]:
+    """Parse an OTLP proto payload into List[Span].
+
+    A store is needed here for generating a sequence ID for each span.
+    """
+    output_spans: List[Span] = []
+
+    for resource_spans in request.resource_spans:
+        # Resource-level attributes & IDs
+        resource_attrs = _kv_list_to_dict(resource_spans.resource.attributes)
+        # rollout_id, attempt_id from resource attributes when present.
+        rollout_id_resource = resource_attrs.get(LightningResourceAttributes.ROLLOUT_ID.value)
+        attempt_id_resource = resource_attrs.get(LightningResourceAttributes.ATTEMPT_ID.value)
+        # If sequence id is provided, all the spans will share the same sequence ID.
+        # unless otherwise overridden by span-level attributes.
+        sequence_id_resource = resource_attrs.get(LightningResourceAttributes.SPAN_SEQUENCE_ID.value)
+
+        otel_resource = _resource_from_proto(resource_spans.resource, getattr(resource_spans, "schema_url", ""))
+
+        # Each ScopeSpans contains multiple spans
+        for scope_spans in resource_spans.scope_spans:
+            for proto_span in scope_spans.spans:
+                trace_id_hex = _bytes_to_trace_id_hex(proto_span.trace_id)
+                span_id_hex = _bytes_to_span_id_hex(proto_span.span_id)
+                parent_id_hex = _bytes_to_span_id_hex(proto_span.parent_span_id) if proto_span.parent_span_id else None
+
+                # Status
+                status_code_str = _STATUS_CODE_MAP.get(proto_span.status.code, "UNSET")
+                status = TraceStatus(
+                    status_code=status_code_str,
+                    description=proto_span.status.message or None,
+                )
+
+                # Attributes
+                span_attrs = _kv_list_to_dict(proto_span.attributes)
+
+                # Context
+                context = SpanContext(
+                    trace_id=trace_id_hex,
+                    span_id=span_id_hex,
+                    is_remote=False,
+                    trace_state={},
+                )
+
+                # Try to get if span attributes contain something like rollout_id or attempt_id
+                # Override the resource-level attributes with the span-level attributes if present.
+                rollout_id_span = span_attrs.get(LightningResourceAttributes.ROLLOUT_ID.value)
+                attempt_id_span = span_attrs.get(LightningResourceAttributes.ATTEMPT_ID.value)
+                sequence_id_span = span_attrs.get(LightningResourceAttributes.SPAN_SEQUENCE_ID.value)
+
+                # Normalize to regular strings and ints
+                rollout_id_raw = rollout_id_span if rollout_id_span is not None else rollout_id_resource
+                attempt_id_raw = attempt_id_span if attempt_id_span is not None else attempt_id_resource
+                sequence_id_raw = sequence_id_span if sequence_id_span is not None else sequence_id_resource
+
+                rollout_id, attempt_id = _normalize_rollout_attempt_id(rollout_id_raw, attempt_id_raw)
+                sequence_id = _normalize_sequence_id(sequence_id_raw)
+
+                if rollout_id is None or attempt_id is None:
+                    logger.warning(
+                        "Both rollout_id and attempt_id must be present in resource attributes. "
+                        "Spans will not be able to log to the store because of missing IDs: rollout_id=%s, attempt_id=%s, sequence_id=%s",
+                        rollout_id,
+                        attempt_id,
+                        sequence_id,
+                    )
+                    continue
+
+                # Generate a new sequence ID if not provided
+                if sequence_id is None:
+                    current_sequence_id = -1
+                elif sequence_id < 0:
+                    logger.error(
+                        "Invalid sequence_id value in resource attributes: %r. Must be a positive integer. Regenerating one.",
+                        sequence_id,
+                    )
+                    current_sequence_id = -1
+                else:
+                    current_sequence_id = sequence_id
+
+                # Build Span
+                span = Span(
+                    rollout_id=rollout_id,
+                    attempt_id=attempt_id,
+                    sequence_id=current_sequence_id,
+                    trace_id=trace_id_hex,
+                    span_id=span_id_hex,
+                    parent_id=parent_id_hex,
+                    name=proto_span.name,
+                    status=status,
+                    attributes=span_attrs,
+                    events=_events_from_proto(proto_span),
+                    links=_links_from_proto(proto_span),
+                    start_time=convert_timestamp(proto_span.start_time_unix_nano),
+                    end_time=convert_timestamp(proto_span.end_time_unix_nano),
+                    context=context,
+                    parent=None,  # OTLP only has parent_span_id; we don't have full SpanContext
+                    resource=otel_resource,
+                )
+
+                output_spans.append(span)
+
+    # Finalize the sequence IDs
+    bulk_issue_requests = [(span.rollout_id, span.attempt_id) for span in output_spans if span.sequence_id < 0]
+    bulk_sequence_ids = await sequence_id_bulk_issuer(bulk_issue_requests)
+    for span, sequence_id in zip(
+        [span for span in output_spans if span.sequence_id < 0], bulk_sequence_ids, strict=True
+    ):
+        span.sequence_id = sequence_id
+
+    return output_spans
+
+
+class LightningStoreOTLPExporter(OTLPSpanExporter):
+    """OTLP Exporter that write to a LightningStore-compatible backend.
+
+    The backend requires two special attributes on each span:
+
+    - `agentlightning.rollout_id`: The rollout ID to associate the span with.
+    - `agentlightning.attempt_id`: The attempt ID to associate the span with.
+
+    It can optionally use the following attribute to sequence spans:
+
+    - `agentlightning.span_sequence_id`: A decimal string representing the sequence ID of the span.
+    """
+
+    _default_endpoint: Optional[str] = None
+    _rollout_id: Optional[str] = None
+    _attempt_id: Optional[str] = None
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}("
+            + f"endpoint={self.endpoint!r}, "
+            + f"rollout_id={self.rollout_id!r}, "
+            + f"attempt_id={self.attempt_id!r}, "
+            + f"should_bypass={self.should_bypass()!r})"
+        )
+
+    @property
+    def endpoint(self) -> Optional[str]:
+        """The endpoint to submit the spans to."""
+        if hasattr(self, "_endpoint"):
+            return self._endpoint
+        return None
+
+    @property
+    def rollout_id(self) -> Optional[str]:
+        """The rollout ID to submit the spans to."""
+        if hasattr(self, "_rollout_id"):
+            return self._rollout_id
+        return None
+
+    @property
+    def attempt_id(self) -> Optional[str]:
+        """The attempt ID to submit the spans to."""
+        if hasattr(self, "_attempt_id"):
+            return self._attempt_id
+        return None
+
+    def enable_store_otlp(self, endpoint: str, rollout_id: str, attempt_id: str) -> None:
+        """Enable storing OTLP data to a specific LightningStore rollout/attempt."""
+        self._rollout_id = rollout_id
+        self._attempt_id = attempt_id
+
+        self._default_endpoint = self._endpoint
+        self._endpoint = endpoint
+
+    def disable_store_otlp(self) -> None:
+        """Disable storing OTLP data to LightningStore."""
+        self._rollout_id = None
+        self._attempt_id = None
+        if self._default_endpoint is not None:
+            self._endpoint = self._default_endpoint
+
+    def should_bypass(self) -> bool:
+        """Check if the exporter should bypass the default export if rollout_id and attempt_id are not set."""
+        return True
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        if self._rollout_id is not None and self._attempt_id is not None:
+            # rollout_id and attempt_id are present in resource attributes
+            # It means that the server supports OTLP endpoint.
+            for span in spans:
+                # Override the resources so that the server knows where the request comes from.
+                span._resource = span._resource.merge(  # pyright: ignore[reportPrivateUsage]
+                    Resource.create(
+                        {
+                            LightningResourceAttributes.ROLLOUT_ID.value: self._rollout_id,
+                            LightningResourceAttributes.ATTEMPT_ID.value: self._attempt_id,
+                        }
+                    )
+                )
+            return super().export(spans)
+        elif not self.should_bypass():
+            logger.debug("Rollout ID and Attempt ID not set; using default OTLP exporter behavior.")
+            return super().export(spans)
+        else:
+            logger.debug("Rollout ID and Attempt ID not set; bypassing export.")
+            return SpanExportResult.SUCCESS
+
+
+def _read_body_maybe_gzip(request: Request, raw_body: bytes) -> bytes:
+    """
+    Decompress body if Content-Encoding: gzip; otherwise return as is.
+    """
+    encoding = request.headers.get("Content-Encoding", "").lower()
+    if encoding == "gzip":
+        return gzip.decompress(raw_body)
+    return raw_body
+
+
+def _maybe_gzip_response(request: Request, payload: bytes) -> Tuple[bytes, Dict[str, str]]:
+    """
+    If Accept-Encoding includes gzip, gzip the payload and set Content-Encoding header.
+    """
+    ae = request.headers.get("Accept-Encoding", "")
+    tokens = [token.split(";")[0].strip().lower() for token in ae.split(",") if token.strip()]
+    headers: Dict[str, str] = {}
+    if "gzip" in tokens:
+        payload = gzip.compress(payload)
+        headers["Content-Encoding"] = "gzip"
+    return payload, headers
+
+
+def _bad_request_response(request: Request, message: str, content_type: str = PROTOBUF_CT) -> Response:
+    """
+    Build a 400 response whose body is a protobuf Status message, encoded
+    in the same Content-Type as the request (OTLP/HTTP requirement).
+    """
+    status_msg = Status(message=message)
+
+    if content_type == PROTOBUF_CT:
+        body = status_msg.SerializeToString()
+    else:
+        # Fallback: JSON representation of Status.
+        body = json_format.MessageToJson(status_msg).encode("utf-8")
+
+    body, headers = _maybe_gzip_response(request, body)
+
+    return Response(
+        content=body,
+        status_code=400,
+        media_type=content_type,
+        headers=headers,
+    )
+
+
+def _normalize_rollout_attempt_id(
+    rollout_id: Optional[AttributeValue], attempt_id: Optional[AttributeValue]
+) -> Tuple[Optional[str], Optional[str]]:
+    """Normalize a rollout or attempt ID to a string."""
+    rollout_id_str = str(rollout_id) if rollout_id is not None else None
+    attempt_id_str = str(attempt_id) if attempt_id is not None else None
+    return rollout_id_str, attempt_id_str
+
+
+def _normalize_sequence_id(sequence_id: Optional[AttributeValue]) -> Optional[int]:
+    """Normalize a sequence ID to an integer."""
+    if sequence_id is None:
+        return None
+    try:
+        sequence_id_int = int(str(sequence_id))
+    except (ValueError, TypeError):
+        logger.warning(
+            "Invalid sequence_id value in resource attributes: %r. Must be an integer or string representing an integer. Assuming None.",
+            sequence_id,
+        )
+        sequence_id_int = None
+    return sequence_id_int
+
+
+def _any_value_to_python(value: AnyValue) -> Any:
+    """Convert OTLP AnyValue -> plain Python value."""
+    kind = value.WhichOneof("value")
+    if kind is None:
+        return None
+    if kind == "string_value":
+        return value.string_value
+    if kind == "bool_value":
+        return value.bool_value
+    if kind == "int_value":
+        return int(value.int_value)
+    if kind == "double_value":
+        return float(value.double_value)
+    if kind == "array_value":
+        return [_any_value_to_python(v) for v in value.array_value.values]
+    if kind == "kvlist_value":
+        # Map<string, AnyValue> -> dict
+        return {kv.key: _any_value_to_python(kv.value) for kv in value.kvlist_value.values}
+    if kind == "bytes_value":
+        # Serialize bytes as hex string to stay JSON-friendly
+        return value.bytes_value.hex()
+    return None
+
+
+def _kv_list_to_dict(kvs: Sequence[KeyValue]) -> Attributes:
+    """Convert repeated KeyValue -> Attributes dict."""
+    return {kv.key: _any_value_to_python(kv.value) for kv in kvs}
+
+
+_STATUS_CODE_MAP: Mapping[ProtoStatus.StatusCode.ValueType, StatusCode] = {
+    ProtoStatus.STATUS_CODE_UNSET: "UNSET",
+    ProtoStatus.STATUS_CODE_OK: "OK",
+    ProtoStatus.STATUS_CODE_ERROR: "ERROR",
+}
+
+
+def _bytes_to_trace_id_hex(b: bytes) -> str:
+    # OTLP uses 16-byte trace IDs; format as 32-char hex
+    if not b:
+        return "0" * 32
+    return b.hex().rjust(32, "0")
+
+
+def _bytes_to_span_id_hex(b: bytes) -> str:
+    # OTLP uses 8-byte span IDs; format as 16-char hex
+    if not b:
+        return "0" * 16
+    return b.hex().rjust(16, "0")
+
+
+def _events_from_proto(span: ProtoSpan) -> List[Event]:
+    """Event converter from OTLP ProtoSpan to List[Event]."""
+    return [
+        Event(
+            name=e.name,
+            attributes=_kv_list_to_dict(e.attributes),
+            timestamp=convert_timestamp(e.time_unix_nano),
+        )
+        for e in span.events
+    ]
+
+
+def _links_from_proto(span: ProtoSpan) -> List[Link]:
+    """Link converter from OTLP ProtoSpan to List[Link]."""
+    links: List[Link] = []
+    for link in span.links:
+        trace_id_hex = _bytes_to_trace_id_hex(link.trace_id)
+        span_id_hex = _bytes_to_span_id_hex(link.span_id)
+        ctx = SpanContext(
+            trace_id=trace_id_hex,
+            span_id=span_id_hex,
+            is_remote=False,
+            trace_state={},  # OTLP trace_state is currently a string; you can parse if needed
+        )
+        links.append(
+            Link(
+                context=ctx,
+                attributes=_kv_list_to_dict(link.attributes) or None,
+            )
+        )
+    return links
+
+
+def _resource_from_proto(resource: ProtoResource, schema_url: str = "") -> OtelResource:
+    return OtelResource(
+        attributes=_kv_list_to_dict(resource.attributes),
+        schema_url=schema_url or "",
+    )

agentlightning/utils/system_snapshot.py

@@ -0,0 +1,81 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from __future__ import annotations
+
+import platform
+import socket
+from contextlib import suppress
+from datetime import datetime
+from typing import Any, Dict, List, cast
+
+import psutil
+from gpustat import GPUStat, GPUStatCollection
+
+
+def system_snapshot(include_gpu: bool = False) -> Dict[str, Any]:
+    """Capture a snapshot of the system's hardware and software information.
+
+    Args:
+        include_gpu: Whether to include GPU information.
+
+    Returns:
+        A dictionary containing the system's hardware and software information.
+    """
+    # CPU
+    cpu = {
+        "cpu_name": platform.processor(),
+        "cpu_cores": psutil.cpu_count(logical=False),
+        "cpu_threads": psutil.cpu_count(logical=True),
+        "cpu_usage_pct": psutil.cpu_percent(0.0),
+    }
+
+    # Memory
+    vm = psutil.virtual_memory()
+    mem = {
+        "mem_used_gb": round(vm.used / (2**30), 2),
+        "mem_total_gb": round(vm.total / (2**30), 2),
+        "mem_pct": vm.percent,
+    }
+
+    # Disk
+    du = psutil.disk_usage("/")
+    disk = {
+        "disk_used_gb": round(du.used / (2**30), 2),
+        "disk_total_gb": round(du.total / (2**30), 2),
+        "disk_pct": du.percent,
+    }
+
+    # GPU (only query if explicitly requested)
+    gpus: List[Dict[str, Any]] = []
+    if include_gpu:
+        with suppress(Exception):
+            for g in GPUStatCollection.new_query().gpus:  # type: ignore
+                g = cast(GPUStat, g)
+                gpus.append(
+                    {
+                        "gpu": g.name,  # type: ignore
+                        "util_pct": g.utilization,
+                        "mem_used_mb": g.memory_used,
+                        "mem_total_mb": g.memory_total,
+                        "temp_c": g.temperature,
+                    }
+                )
+
+    # Network
+    net = psutil.net_io_counters()
+    netinfo = {
+        "bytes_sent_mb": round(net.bytes_sent / (2**20), 2),
+        "bytes_recv_mb": round(net.bytes_recv / (2**20), 2),
+    }
+
+    # OS / meta
+    return {
+        "timestamp": datetime.now().isoformat(timespec="seconds"),
+        "host": socket.gethostname(),
+        "os": platform.platform(),
+        **cpu,
+        **mem,
+        **disk,
+        **netinfo,
+        **({"gpus": gpus} if include_gpu else {}),
+    }