chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,493 @@
|
||||
import pytest
|
||||
from inline_snapshot import snapshot
|
||||
from openai import AsyncOpenAI
|
||||
from openai.types.responses import ResponseCompletedEvent
|
||||
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
|
||||
|
||||
from agents import ModelBehaviorError, ModelSettings, ModelTracing, OpenAIResponsesModel, trace
|
||||
from agents.tracing.span_data import ResponseSpanData
|
||||
from tests import fake_model
|
||||
|
||||
from .testing_processor import assert_no_spans, fetch_normalized_spans, fetch_ordered_spans
|
||||
|
||||
|
||||
class DummyTracing:
|
||||
def is_disabled(self):
|
||||
return False
|
||||
|
||||
|
||||
class DummyUsage:
|
||||
def __init__(
|
||||
self,
|
||||
input_tokens: int = 1,
|
||||
input_tokens_details: InputTokensDetails | None = None,
|
||||
output_tokens: int = 1,
|
||||
output_tokens_details: OutputTokensDetails | None = None,
|
||||
total_tokens: int = 2,
|
||||
):
|
||||
self.input_tokens = input_tokens
|
||||
self.output_tokens = output_tokens
|
||||
self.total_tokens = total_tokens
|
||||
self.input_tokens_details = (
|
||||
input_tokens_details
|
||||
if input_tokens_details
|
||||
else InputTokensDetails.model_validate({"cache_write_tokens": 0, "cached_tokens": 0})
|
||||
)
|
||||
self.output_tokens_details = (
|
||||
output_tokens_details
|
||||
if output_tokens_details
|
||||
else OutputTokensDetails(reasoning_tokens=0)
|
||||
)
|
||||
|
||||
|
||||
class DummyResponse:
|
||||
def __init__(self):
|
||||
self.id = "dummy-id"
|
||||
self.output = []
|
||||
self.usage = DummyUsage()
|
||||
|
||||
def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj(self.output),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_response_creates_trace(monkeypatch):
|
||||
with trace(workflow_name="test"):
|
||||
# Create an instance of the model
|
||||
model = OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test"))
|
||||
|
||||
# Mock _fetch_response to return a dummy response with a known id
|
||||
async def dummy_fetch_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
stream,
|
||||
prompt,
|
||||
):
|
||||
return DummyResponse()
|
||||
|
||||
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
|
||||
|
||||
# Call get_response
|
||||
await model.get_response(
|
||||
"instr",
|
||||
"input",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.ENABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert fetch_normalized_spans() == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "test",
|
||||
"children": [
|
||||
{
|
||||
"type": "response",
|
||||
"data": {
|
||||
"response_id": "dummy-id",
|
||||
"usage": {
|
||||
"requests": 1,
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_data_tracing_doesnt_set_response_id(monkeypatch):
|
||||
with trace(workflow_name="test"):
|
||||
# Create an instance of the model
|
||||
model = OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test"))
|
||||
|
||||
# Mock _fetch_response to return a dummy response with a known id
|
||||
async def dummy_fetch_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
stream,
|
||||
prompt,
|
||||
):
|
||||
return DummyResponse()
|
||||
|
||||
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
|
||||
|
||||
# Call get_response
|
||||
await model.get_response(
|
||||
"instr",
|
||||
"input",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.ENABLED_WITHOUT_DATA,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert fetch_normalized_spans() == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "test",
|
||||
"children": [
|
||||
{
|
||||
"type": "response",
|
||||
"data": {
|
||||
"usage": {
|
||||
"requests": 1,
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
}
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
[span] = fetch_ordered_spans()
|
||||
assert span.span_data.response is None
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_disable_tracing_does_not_create_span(monkeypatch):
|
||||
with trace(workflow_name="test"):
|
||||
# Create an instance of the model
|
||||
model = OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test"))
|
||||
|
||||
# Mock _fetch_response to return a dummy response with a known id
|
||||
async def dummy_fetch_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
stream,
|
||||
prompt,
|
||||
):
|
||||
return DummyResponse()
|
||||
|
||||
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
|
||||
|
||||
# Call get_response
|
||||
await model.get_response(
|
||||
"instr",
|
||||
"input",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert fetch_normalized_spans() == snapshot([{"workflow_name": "test"}])
|
||||
|
||||
assert_no_spans()
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_creates_trace(monkeypatch):
|
||||
with trace(workflow_name="test"):
|
||||
# Create an instance of the model
|
||||
model = OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test"))
|
||||
|
||||
# Define a dummy fetch function that returns an async stream with a dummy response
|
||||
async def dummy_fetch_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
stream,
|
||||
prompt,
|
||||
):
|
||||
class DummyStream:
|
||||
async def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj([], "dummy-id-123"),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
return DummyStream()
|
||||
|
||||
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
|
||||
|
||||
# Consume the stream to trigger processing of the final response
|
||||
async for _ in model.stream_response(
|
||||
"instr",
|
||||
"input",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.ENABLED,
|
||||
previous_response_id=None,
|
||||
):
|
||||
pass
|
||||
|
||||
assert fetch_normalized_spans() == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "test",
|
||||
"children": [
|
||||
{
|
||||
"type": "response",
|
||||
"data": {
|
||||
"response_id": "dummy-id-123",
|
||||
"usage": {
|
||||
"requests": 1,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("terminal_event_type", ["response.failed", "response.incomplete"])
|
||||
async def test_stream_response_failed_or_incomplete_terminal_event_creates_trace(
|
||||
monkeypatch, terminal_event_type: str
|
||||
):
|
||||
with trace(workflow_name="test"):
|
||||
model = OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test"))
|
||||
|
||||
async def dummy_fetch_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
stream,
|
||||
prompt,
|
||||
):
|
||||
class DummyTerminalEvent:
|
||||
def __init__(self):
|
||||
self.type = terminal_event_type
|
||||
self.response = fake_model.get_response_obj([], "dummy-id-terminal")
|
||||
self.sequence_number = 0
|
||||
|
||||
class DummyStream:
|
||||
async def __aiter__(self):
|
||||
yield DummyTerminalEvent()
|
||||
|
||||
return DummyStream()
|
||||
|
||||
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
|
||||
async for _ in model.stream_response(
|
||||
"instr",
|
||||
"input",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.ENABLED,
|
||||
previous_response_id=None,
|
||||
):
|
||||
pass
|
||||
|
||||
assert fetch_normalized_spans() == [
|
||||
{
|
||||
"workflow_name": "test",
|
||||
"children": [
|
||||
{
|
||||
"type": "response",
|
||||
"error": {
|
||||
"message": "Error streaming response",
|
||||
"data": {
|
||||
"error": (
|
||||
f"Responses stream ended with terminal event "
|
||||
f"`{terminal_event_type}`."
|
||||
)
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_non_data_tracing_doesnt_set_response_id(monkeypatch):
|
||||
with trace(workflow_name="test"):
|
||||
# Create an instance of the model
|
||||
model = OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test"))
|
||||
|
||||
# Define a dummy fetch function that returns an async stream with a dummy response
|
||||
async def dummy_fetch_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
stream,
|
||||
prompt,
|
||||
):
|
||||
class DummyStream:
|
||||
async def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj([], "dummy-id-123"),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
return DummyStream()
|
||||
|
||||
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
|
||||
|
||||
# Consume the stream to trigger processing of the final response
|
||||
async for _ in model.stream_response(
|
||||
"instr",
|
||||
"input",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.ENABLED_WITHOUT_DATA,
|
||||
previous_response_id=None,
|
||||
):
|
||||
pass
|
||||
|
||||
assert fetch_normalized_spans() == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "test",
|
||||
"children": [
|
||||
{
|
||||
"type": "response",
|
||||
"data": {
|
||||
"usage": {
|
||||
"requests": 1,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
}
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
[span] = fetch_ordered_spans()
|
||||
assert isinstance(span.span_data, ResponseSpanData)
|
||||
assert span.span_data.response is None
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_disabled_tracing_doesnt_create_span(monkeypatch):
|
||||
with trace(workflow_name="test"):
|
||||
# Create an instance of the model
|
||||
model = OpenAIResponsesModel(model="test-model", openai_client=AsyncOpenAI(api_key="test"))
|
||||
|
||||
# Define a dummy fetch function that returns an async stream with a dummy response
|
||||
async def dummy_fetch_response(
|
||||
system_instructions,
|
||||
input,
|
||||
model_settings,
|
||||
tools,
|
||||
output_schema,
|
||||
handoffs,
|
||||
previous_response_id,
|
||||
conversation_id,
|
||||
stream,
|
||||
prompt,
|
||||
):
|
||||
class DummyStream:
|
||||
async def __aiter__(self):
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=fake_model.get_response_obj([], "dummy-id-123"),
|
||||
sequence_number=0,
|
||||
)
|
||||
|
||||
return DummyStream()
|
||||
|
||||
monkeypatch.setattr(model, "_fetch_response", dummy_fetch_response)
|
||||
|
||||
# Consume the stream to trigger processing of the final response
|
||||
async for _ in model.stream_response(
|
||||
"instr",
|
||||
"input",
|
||||
ModelSettings(),
|
||||
[],
|
||||
None,
|
||||
[],
|
||||
ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
):
|
||||
pass
|
||||
|
||||
assert fetch_normalized_spans() == snapshot([{"workflow_name": "test"}])
|
||||
|
||||
assert_no_spans()
|
||||
Reference in New Issue
Block a user