Files
mlflow--mlflow/tests/utils/test_databricks_tracing_utils.py
2026-07-13 13:22:34 +08:00

519 lines
20 KiB
Python

import json
import pytest
from google.protobuf.timestamp_pb2 import Timestamp
import mlflow
from mlflow.entities import (
AssessmentSource,
Expectation,
Feedback,
Trace,
TraceData,
TraceInfo,
TraceState,
)
from mlflow.entities.trace_location import (
InferenceTableLocation,
MlflowExperimentLocation,
TraceLocation,
TraceLocationType,
UCSchemaLocation,
UnityCatalog,
)
from mlflow.protos import assessments_pb2
from mlflow.protos import databricks_tracing_pb2 as pb
from mlflow.protos.assessments_pb2 import AssessmentSource as ProtoAssessmentSource
from mlflow.tracing.constant import (
TRACE_ID_V4_PREFIX,
TRACE_SCHEMA_VERSION,
TRACE_SCHEMA_VERSION_KEY,
SpanAttributeKey,
)
from mlflow.tracing.utils import TraceMetadataKey, add_size_stats_to_trace_metadata
from mlflow.utils.databricks_tracing_utils import (
assessment_to_proto,
get_trace_id_from_assessment_proto,
inference_table_location_to_proto,
mlflow_experiment_location_to_proto,
parse_uc_location,
trace_from_proto,
trace_location_from_proto,
trace_location_to_proto,
trace_to_proto,
uc_location_to_str,
uc_schema_location_from_proto,
uc_schema_location_to_proto,
)
def test_trace_location_to_proto_uc_schema():
trace_location = TraceLocation.from_databricks_uc_schema(
catalog_name="test_catalog", schema_name="test_schema"
)
proto = trace_location_to_proto(trace_location)
assert proto.type == pb.TraceLocation.TraceLocationType.UC_SCHEMA
assert proto.uc_schema.catalog_name == "test_catalog"
assert proto.uc_schema.schema_name == "test_schema"
def test_parse_uc_location():
assert parse_uc_location("catalog.schema") == ("catalog", "schema", None)
assert parse_uc_location("catalog.schema.prefix") == ("catalog", "schema", "prefix")
with pytest.raises(ValueError, match="Invalid UC location"):
parse_uc_location("a.b.c.d")
def test_uc_location_to_str():
assert uc_location_to_str("catalog", "schema") == "catalog.schema"
assert uc_location_to_str("catalog", "schema", "prefix") == "catalog.schema.prefix"
def test_trace_location_to_proto_mlflow_experiment():
trace_location = TraceLocation.from_experiment_id(experiment_id="1234")
proto = trace_location_to_proto(trace_location)
assert proto.type == pb.TraceLocation.TraceLocationType.MLFLOW_EXPERIMENT
assert proto.mlflow_experiment.experiment_id == "1234"
def test_trace_location_to_proto_inference_table():
trace_location = TraceLocation(
type=TraceLocationType.INFERENCE_TABLE,
inference_table=InferenceTableLocation(
full_table_name="test_catalog.test_schema.test_table"
),
)
proto = trace_location_to_proto(trace_location)
assert proto.type == pb.TraceLocation.TraceLocationType.INFERENCE_TABLE
assert proto.inference_table.full_table_name == "test_catalog.test_schema.test_table"
def test_uc_schema_location_to_proto():
schema_location = UCSchemaLocation(catalog_name="test_catalog", schema_name="test_schema")
proto = uc_schema_location_to_proto(schema_location)
assert proto.catalog_name == "test_catalog"
assert proto.schema_name == "test_schema"
def test_uc_schema_location_from_proto():
proto = pb.UCSchemaLocation(
catalog_name="test_catalog",
schema_name="test_schema",
otel_spans_table_name="test_spans",
otel_logs_table_name="test_logs",
)
schema_location = uc_schema_location_from_proto(proto)
assert schema_location.catalog_name == "test_catalog"
assert schema_location.schema_name == "test_schema"
assert schema_location.full_otel_spans_table_name == "test_catalog.test_schema.test_spans"
assert schema_location.full_otel_logs_table_name == "test_catalog.test_schema.test_logs"
def test_inference_table_location_to_proto():
table_location = InferenceTableLocation(full_table_name="test_catalog.test_schema.test_table")
proto = inference_table_location_to_proto(table_location)
assert proto.full_table_name == "test_catalog.test_schema.test_table"
def test_mlflow_experiment_location_to_proto():
experiment_location = MlflowExperimentLocation(experiment_id="1234")
proto = mlflow_experiment_location_to_proto(experiment_location)
assert proto.experiment_id == "1234"
def test_schema_location_to_proto():
schema_location = UCSchemaLocation(
catalog_name="test_catalog",
schema_name="test_schema",
)
schema_location._otel_spans_table_name = "test_spans"
schema_location._otel_logs_table_name = "test_logs"
proto = uc_schema_location_to_proto(schema_location)
assert proto.catalog_name == "test_catalog"
assert proto.schema_name == "test_schema"
assert proto.otel_spans_table_name == "test_spans"
assert proto.otel_logs_table_name == "test_logs"
def test_trace_location_from_proto_uc_schema():
proto = pb.TraceLocation(
type=pb.TraceLocation.TraceLocationType.UC_SCHEMA,
uc_schema=pb.UCSchemaLocation(
catalog_name="catalog",
schema_name="schema",
otel_spans_table_name="spans",
otel_logs_table_name="logs",
),
)
trace_location = trace_location_from_proto(proto)
assert trace_location.uc_schema.catalog_name == "catalog"
assert trace_location.uc_schema.schema_name == "schema"
assert trace_location.uc_schema.full_otel_spans_table_name == "catalog.schema.spans"
assert trace_location.uc_schema.full_otel_logs_table_name == "catalog.schema.logs"
def test_trace_location_from_proto_mlflow_experiment():
proto = pb.TraceLocation(
type=pb.TraceLocation.TraceLocationType.MLFLOW_EXPERIMENT,
mlflow_experiment=mlflow_experiment_location_to_proto(
MlflowExperimentLocation(experiment_id="1234")
),
)
trace_location = trace_location_from_proto(proto)
assert trace_location.type == TraceLocationType.MLFLOW_EXPERIMENT
assert trace_location.mlflow_experiment.experiment_id == "1234"
def test_trace_location_from_proto_inference_table():
proto = pb.TraceLocation(
type=pb.TraceLocation.TraceLocationType.INFERENCE_TABLE,
inference_table=inference_table_location_to_proto(
InferenceTableLocation(full_table_name="test_catalog.test_schema.test_table")
),
)
trace_location = trace_location_from_proto(proto)
assert trace_location.type == TraceLocationType.INFERENCE_TABLE
assert trace_location.inference_table.full_table_name == "test_catalog.test_schema.test_table"
def test_trace_info_to_v4_proto():
otel_trace_id = "2efb31387ff19263f92b2c0a61b0a8bc"
trace_id = f"trace:/catalog.schema/{otel_trace_id}"
trace_info = TraceInfo(
trace_id=trace_id,
trace_location=TraceLocation.from_databricks_uc_schema(
catalog_name="catalog", schema_name="schema"
),
request_time=0,
state=TraceState.OK,
request_preview="request",
response_preview="response",
client_request_id="client_request_id",
tags={"key": "value"},
)
proto_trace_info = trace_info.to_proto()
assert proto_trace_info.trace_id == otel_trace_id
assert proto_trace_info.trace_location.uc_schema.catalog_name == "catalog"
assert proto_trace_info.trace_location.uc_schema.schema_name == "schema"
assert proto_trace_info.state == 1
assert proto_trace_info.request_preview == "request"
assert proto_trace_info.response_preview == "response"
assert proto_trace_info.client_request_id == "client_request_id"
assert proto_trace_info.tags == {"key": "value"}
assert len(proto_trace_info.assessments) == 0
trace_info_from_proto = TraceInfo.from_proto(proto_trace_info)
assert trace_info_from_proto == trace_info
def test_trace_to_proto_and_from_proto():
with mlflow.start_span() as span:
otel_trace_id = span.trace_id.removeprefix("tr-")
uc_schema = "catalog.schema"
trace_id = f"trace:/{uc_schema}/{otel_trace_id}"
span.set_attribute(SpanAttributeKey.REQUEST_ID, trace_id)
mlflow_span = span.to_immutable_span()
assert mlflow_span.trace_id == trace_id
trace = Trace(
info=TraceInfo(
trace_id=trace_id,
trace_location=TraceLocation.from_databricks_uc_schema(
catalog_name="catalog", schema_name="schema"
),
request_time=0,
state=TraceState.OK,
request_preview="request",
response_preview="response",
client_request_id="client_request_id",
tags={"key": "value"},
),
data=TraceData(spans=[mlflow_span]),
)
proto_trace_v4 = trace_to_proto(trace)
assert proto_trace_v4.trace_info.trace_id == otel_trace_id
assert proto_trace_v4.trace_info.trace_location.uc_schema.catalog_name == "catalog"
assert proto_trace_v4.trace_info.trace_location.uc_schema.schema_name == "schema"
assert len(proto_trace_v4.spans) == len(trace.data.spans)
reconstructed_trace = trace_from_proto(proto_trace_v4, location_id="catalog.schema")
assert reconstructed_trace.info.trace_id == trace_id
assert reconstructed_trace.info.trace_location.uc_schema.catalog_name == "catalog"
assert reconstructed_trace.info.trace_location.uc_schema.schema_name == "schema"
assert len(reconstructed_trace.data.spans) == len(trace.data.spans)
original_span = trace.data.spans[0]
reconstructed_span = reconstructed_trace.data.spans[0]
assert reconstructed_span.name == original_span.name
assert reconstructed_span.span_id == original_span.span_id
assert reconstructed_span.trace_id == original_span.trace_id
assert reconstructed_span.inputs == original_span.inputs
assert reconstructed_span.outputs == original_span.outputs
assert reconstructed_span.get_attribute("custom") == original_span.get_attribute("custom")
def test_trace_from_proto_with_location_preserves_v4_trace_id():
with mlflow.start_span() as span:
otel_trace_id = span.trace_id.removeprefix("tr-")
uc_schema = "catalog.schema"
trace_id_v4 = f"{TRACE_ID_V4_PREFIX}{uc_schema}/{otel_trace_id}"
span.set_attribute(SpanAttributeKey.REQUEST_ID, trace_id_v4)
mlflow_span = span.to_immutable_span()
# Create trace with v4 trace ID
trace = Trace(
info=TraceInfo(
trace_id=trace_id_v4,
trace_location=TraceLocation.from_databricks_uc_schema(
catalog_name="catalog", schema_name="schema"
),
request_time=0,
state=TraceState.OK,
),
data=TraceData(spans=[mlflow_span]),
)
# Convert to proto
proto_trace = trace_to_proto(trace)
# Reconstruct with location parameter
reconstructed_trace = trace_from_proto(proto_trace, location_id=uc_schema)
# Verify that all spans have the correct v4 trace_id format
for reconstructed_span in reconstructed_trace.data.spans:
assert reconstructed_span.trace_id == trace_id_v4
assert reconstructed_span.trace_id.startswith(TRACE_ID_V4_PREFIX)
# Verify the REQUEST_ID attribute is also in v4 format
request_id = reconstructed_span.get_attribute("mlflow.traceRequestId")
assert request_id == trace_id_v4
def test_trace_info_from_proto_handles_uc_schema_location():
request_time = Timestamp()
request_time.FromMilliseconds(1234567890)
proto = pb.TraceInfo(
trace_id="test_trace_id",
trace_location=trace_location_to_proto(
TraceLocation.from_databricks_uc_schema(catalog_name="catalog", schema_name="schema")
),
request_preview="test request",
response_preview="test response",
request_time=request_time,
state=TraceState.OK.to_proto(),
trace_metadata={
TRACE_SCHEMA_VERSION_KEY: str(TRACE_SCHEMA_VERSION),
"other_key": "other_value",
},
tags={"test_tag": "test_value"},
)
trace_info = TraceInfo.from_proto(proto)
assert trace_info.trace_location.uc_schema.catalog_name == "catalog"
assert trace_info.trace_location.uc_schema.schema_name == "schema"
assert trace_info.trace_metadata[TRACE_SCHEMA_VERSION_KEY] == str(TRACE_SCHEMA_VERSION)
assert trace_info.trace_metadata["other_key"] == "other_value"
assert trace_info.tags == {"test_tag": "test_value"}
def test_add_size_stats_to_trace_metadata_for_v4_trace():
with mlflow.start_span() as span:
otel_trace_id = span.trace_id.removeprefix("tr-")
uc_schema = "catalog.schema"
trace_id = f"trace:/{uc_schema}/{otel_trace_id}"
span.set_attribute(SpanAttributeKey.REQUEST_ID, trace_id)
mlflow_span = span.to_immutable_span()
trace = Trace(
info=TraceInfo(
trace_id="test_trace_id",
trace_location=TraceLocation.from_databricks_uc_schema(
catalog_name="catalog", schema_name="schema"
),
request_time=0,
state=TraceState.OK,
request_preview="request",
response_preview="response",
client_request_id="client_request_id",
tags={"key": "value"},
),
data=TraceData(spans=[mlflow_span]),
)
add_size_stats_to_trace_metadata(trace)
assert TraceMetadataKey.SIZE_STATS in trace.info.trace_metadata
def test_assessment_to_proto():
# Test with Feedback assessment
feedback = Feedback(
name="correctness",
value=0.95,
source=AssessmentSource(source_type="LLM_JUDGE", source_id="gpt-4"),
trace_id="trace:/catalog.schema/trace123",
metadata={"model": "gpt-4", "temperature": "0.7"},
span_id="span456",
rationale="The response is accurate and complete",
overrides="old_assessment_id",
valid=False,
)
feedback.assessment_id = "assessment789"
proto_v4 = assessment_to_proto(feedback)
# Validate proto structure
assert isinstance(proto_v4, pb.Assessment)
assert proto_v4.assessment_name == "correctness"
assert proto_v4.assessment_id == "assessment789"
assert proto_v4.span_id == "span456"
assert proto_v4.rationale == "The response is accurate and complete"
assert proto_v4.overrides == "old_assessment_id"
assert proto_v4.valid is False
# Check TraceIdentifier
assert proto_v4.trace_id == "trace123"
assert proto_v4.trace_location.uc_schema.catalog_name == "catalog"
assert proto_v4.trace_location.uc_schema.schema_name == "schema"
# Check source
assert proto_v4.source.source_type == ProtoAssessmentSource.SourceType.Value("LLM_JUDGE")
assert proto_v4.source.source_id == "gpt-4"
# Check metadata
assert proto_v4.metadata["model"] == "gpt-4"
assert proto_v4.metadata["temperature"] == "0.7"
# Check feedback value
assert proto_v4.HasField("feedback")
assert proto_v4.feedback.value.number_value == 0.95
# Test with Expectation assessment
expectation = Expectation(
name="expected_answer",
value={"answer": "Paris", "confidence": 0.99},
source=AssessmentSource(source_type="HUMAN", source_id="user@example.com"),
trace_id="trace:/main.default/trace789",
metadata={"question": "What is the capital of France?"},
span_id="span111",
)
expectation.assessment_id = "exp_assessment123"
proto_v4_exp = assessment_to_proto(expectation)
assert isinstance(proto_v4_exp, pb.Assessment)
assert proto_v4_exp.assessment_name == "expected_answer"
assert proto_v4_exp.assessment_id == "exp_assessment123"
assert proto_v4_exp.span_id == "span111"
# Check TraceIdentifier for expectation
assert proto_v4_exp.trace_id == "trace789"
assert proto_v4_exp.trace_location.uc_schema.catalog_name == "main"
assert proto_v4_exp.trace_location.uc_schema.schema_name == "default"
# Check expectation value
assert proto_v4_exp.HasField("expectation")
assert proto_v4_exp.expectation.HasField("serialized_value")
assert json.loads(proto_v4_exp.expectation.serialized_value.value) == {
"answer": "Paris",
"confidence": 0.99,
}
def test_get_trace_id_from_assessment_proto():
proto = pb.Assessment(
trace_id="1234",
trace_location=trace_location_to_proto(
TraceLocation.from_databricks_uc_schema(catalog_name="catalog", schema_name="schema")
),
)
assert get_trace_id_from_assessment_proto(proto) == "trace:/catalog.schema/1234"
proto = assessments_pb2.Assessment(
trace_id="tr-123",
)
assert get_trace_id_from_assessment_proto(proto) == "tr-123"
def test_trace_location_uc_table_prefix_proto_round_trip():
location = UnityCatalog(
catalog_name="catalog",
schema_name="schema",
table_prefix="prefix",
)
location._otel_spans_table_name = "catalog.schema.prefix_otel_spans"
location._otel_logs_table_name = "catalog.schema.prefix_otel_logs"
location._annotations_table_name = "catalog.schema.prefix_otel_annotations"
trace_location = TraceLocation(type=TraceLocationType.UC_TABLE_PREFIX, uc_table_prefix=location)
proto = trace_location_to_proto(trace_location)
assert proto.type == pb.TraceLocation.TraceLocationType.UC_TABLE_PREFIX
assert proto.uc_table_prefix.catalog_name == "catalog"
assert proto.uc_table_prefix.schema_name == "schema"
assert proto.uc_table_prefix.table_prefix == "prefix"
assert proto.uc_table_prefix.spans_table_name == "catalog.schema.prefix_otel_spans"
assert proto.uc_table_prefix.logs_table_name == "catalog.schema.prefix_otel_logs"
assert proto.uc_table_prefix.annotations_table_name == "catalog.schema.prefix_otel_annotations"
reconstructed = trace_location_from_proto(proto)
assert reconstructed.type == TraceLocationType.UC_TABLE_PREFIX
uc = reconstructed.uc_table_prefix
assert uc.catalog_name == "catalog"
assert uc.schema_name == "schema"
assert uc.table_prefix == "prefix"
assert uc.full_otel_spans_table_name == "catalog.schema.prefix_otel_spans"
assert uc.full_otel_logs_table_name == "catalog.schema.prefix_otel_logs"
assert uc.full_annotations_table_name == "catalog.schema.prefix_otel_annotations"
def test_trace_info_from_proto_handles_uc_table_prefix_location():
request_time = Timestamp()
request_time.FromMilliseconds(1234567890)
proto = pb.TraceInfo(
trace_id="test_trace_id",
trace_location=trace_location_to_proto(
TraceLocation.from_databricks_uc_table_prefix(
catalog_name="catalog", schema_name="schema", table_prefix="prefix"
)
),
request_preview="test request",
response_preview="test response",
request_time=request_time,
state=TraceState.OK.to_proto(),
trace_metadata={TRACE_SCHEMA_VERSION_KEY: str(TRACE_SCHEMA_VERSION)},
)
trace_info = TraceInfo.from_proto(proto)
assert trace_info.trace_id == "trace:/catalog.schema.prefix/test_trace_id"
assert trace_info.trace_location.type == TraceLocationType.UC_TABLE_PREFIX
assert trace_info.trace_location.uc_table_prefix.catalog_name == "catalog"
assert trace_info.trace_location.uc_table_prefix.schema_name == "schema"
assert trace_info.trace_location.uc_table_prefix.table_prefix == "prefix"
def test_assessment_to_proto_uc_table_prefix():
feedback = Feedback(
name="correctness",
value=0.95,
source=AssessmentSource(source_type="LLM_JUDGE", source_id="gpt-4"),
trace_id="trace:/catalog.schema.prefix/trace123",
)
proto = assessment_to_proto(feedback)
assert proto.trace_id == "trace123"
assert proto.trace_location.type == pb.TraceLocation.TraceLocationType.UC_TABLE_PREFIX
assert proto.trace_location.uc_table_prefix.catalog_name == "catalog"
assert proto.trace_location.uc_table_prefix.schema_name == "schema"
assert proto.trace_location.uc_table_prefix.table_prefix == "prefix"
def test_get_trace_id_from_assessment_proto_uc_table_prefix():
proto = pb.Assessment(
trace_id="1234",
trace_location=trace_location_to_proto(
TraceLocation.from_databricks_uc_table_prefix(
catalog_name="catalog", schema_name="schema", table_prefix="prefix"
)
),
)
assert get_trace_id_from_assessment_proto(proto) == "trace:/catalog.schema.prefix/1234"