chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import (
|
||||
OpenAIAgentRegistrationConfig,
|
||||
RunConfig,
|
||||
set_default_openai_agent_registration,
|
||||
set_default_openai_harness,
|
||||
)
|
||||
from agents.models.multi_provider import MultiProvider
|
||||
from agents.models.openai_agent_registration import (
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY,
|
||||
resolve_openai_agent_registration_config,
|
||||
resolve_openai_harness_id_for_model_provider,
|
||||
)
|
||||
from agents.models.openai_provider import OpenAIProvider
|
||||
from agents.run_internal.agent_runner_helpers import resolve_trace_settings
|
||||
from agents.tracing import agent_span, trace
|
||||
|
||||
|
||||
def test_agent_registration_config_precedence(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
set_default_openai_agent_registration(
|
||||
OpenAIAgentRegistrationConfig(harness_id="default-harness")
|
||||
)
|
||||
|
||||
try:
|
||||
resolved = resolve_openai_agent_registration_config(
|
||||
OpenAIAgentRegistrationConfig(harness_id="explicit-harness")
|
||||
)
|
||||
finally:
|
||||
set_default_openai_agent_registration(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "explicit-harness"
|
||||
|
||||
|
||||
def test_agent_registration_uses_default_before_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
set_default_openai_agent_registration(
|
||||
OpenAIAgentRegistrationConfig(harness_id="default-harness")
|
||||
)
|
||||
|
||||
try:
|
||||
resolved = resolve_openai_agent_registration_config(None)
|
||||
finally:
|
||||
set_default_openai_agent_registration(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "default-harness"
|
||||
|
||||
|
||||
def test_agent_registration_uses_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
|
||||
resolved = resolve_openai_agent_registration_config(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "env-harness"
|
||||
|
||||
|
||||
def test_set_default_openai_harness(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
set_default_openai_harness("helper-harness")
|
||||
|
||||
try:
|
||||
resolved = resolve_openai_agent_registration_config(None)
|
||||
finally:
|
||||
set_default_openai_harness(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "helper-harness"
|
||||
|
||||
|
||||
def test_agent_registration_disabled_without_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_AGENT_HARNESS_ID", raising=False)
|
||||
|
||||
assert resolve_openai_agent_registration_config(None) is None
|
||||
|
||||
|
||||
def test_agent_registration_provider_constructor_config() -> None:
|
||||
config = OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
|
||||
openai_provider = OpenAIProvider(agent_registration=config)
|
||||
multi_provider = MultiProvider(openai_agent_registration=config)
|
||||
|
||||
assert openai_provider.agent_registration is not None
|
||||
assert openai_provider.agent_registration.harness_id == "provider-harness"
|
||||
assert multi_provider.openai_provider.agent_registration is not None
|
||||
assert multi_provider.openai_provider.agent_registration.harness_id == "provider-harness"
|
||||
|
||||
|
||||
def test_harness_id_resolves_private_agent_registration() -> None:
|
||||
class Provider:
|
||||
_agent_registration = OpenAIAgentRegistrationConfig(harness_id="private-harness")
|
||||
|
||||
assert resolve_openai_harness_id_for_model_provider(Provider()) == "private-harness"
|
||||
|
||||
|
||||
def test_harness_id_is_added_to_trace_metadata() -> None:
|
||||
provider = OpenAIProvider(
|
||||
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
)
|
||||
|
||||
_, _, _, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(model_provider=provider),
|
||||
)
|
||||
|
||||
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
|
||||
|
||||
|
||||
def test_harness_id_preserves_explicit_trace_metadata() -> None:
|
||||
provider = OpenAIProvider(
|
||||
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
)
|
||||
|
||||
_, _, _, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(
|
||||
model_provider=provider,
|
||||
trace_metadata={
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
|
||||
"source": "test",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
assert metadata == {
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
|
||||
"source": "test",
|
||||
}
|
||||
|
||||
|
||||
def test_env_harness_id_is_added_to_trace_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
|
||||
_, _, _, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(),
|
||||
)
|
||||
|
||||
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "env-harness"}
|
||||
|
||||
|
||||
def test_harness_id_trace_metadata_propagates_to_spans() -> None:
|
||||
provider = OpenAIProvider(
|
||||
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
)
|
||||
workflow_name, trace_id, group_id, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(model_provider=provider),
|
||||
)
|
||||
|
||||
with trace(
|
||||
workflow_name=workflow_name,
|
||||
trace_id=trace_id,
|
||||
group_id=group_id,
|
||||
metadata=metadata,
|
||||
):
|
||||
with agent_span(name="agent") as span:
|
||||
assert span.trace_metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
|
||||
span_export = span.export()
|
||||
assert span_export is not None
|
||||
assert span_export["metadata"] == {
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
"""
|
||||
Test for Anthropic thinking blocks in conversation history.
|
||||
|
||||
This test validates the fix for issue #1704:
|
||||
- Thinking blocks are properly preserved from Anthropic responses
|
||||
- Reasoning items are stored in session but not sent back in conversation history
|
||||
- Non-reasoning models are unaffected
|
||||
- Token usage is not increased for non-reasoning scenarios
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
from openai.types.chat import ChatCompletionMessageToolCall
|
||||
from openai.types.chat.chat_completion_message_tool_call import Function
|
||||
|
||||
from agents.extensions.models.litellm_model import InternalChatCompletionMessage
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
|
||||
def create_mock_anthropic_response_with_thinking() -> InternalChatCompletionMessage:
|
||||
"""Create a mock Anthropic response with thinking blocks (like real response)."""
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather in Paris for you.",
|
||||
reasoning_content="I need to call the weather function for Paris",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "I need to call the weather function for Paris",
|
||||
"signature": "EqMDCkYIBxgCKkBAFZO8EyZwN1hiLctq0YjZnP0KeKgprr+C0PzgDv4GSggnFwrPQHIZ9A5s+paH+DrQBI1+Vnfq3mLAU5lJnoetEgzUEWx/Cv1022ieAvcaDCXdmg1XkMK0tZ8uCCIwURYAAX0uf2wFdnWt9n8whkhmy8ARQD5G2za4R8X5vTqBq8jpJ15T3c1Jcf3noKMZKooCWFVf0/W5VQqpZTgwDkqyTau7XraS+u48YlmJGSfyWMPO8snFLMZLGaGmVJgHfEI5PILhOEuX/R2cEeLuC715f51LMVuxTNzlOUV/037JV6P2ten7D66FnWU9JJMMJJov+DjMb728yQFHwHz4roBJ5ePHaaFP6mDwpqYuG/hai6pVv2TAK1IdKUui/oXrYtU+0gxb6UF2kS1bspqDuN++R8JdL7CMSU5l28pQ8TsH1TpVF4jZpsFbp1Du4rQIULFsCFFg+Edf9tPgyKZOq6xcskIjT7oylAPO37/jhdNknDq2S82PaSKtke3ViOigtM5uJfG521ZscBJQ1K3kwoI/repIdV9PatjOYdsYAQ==", # noqa: E501
|
||||
}
|
||||
],
|
||||
)
|
||||
return message
|
||||
|
||||
|
||||
def test_converter_skips_reasoning_items():
|
||||
"""
|
||||
Unit test to verify that reasoning items are skipped when converting items to messages.
|
||||
"""
|
||||
# Create test items including a reasoning item
|
||||
test_items: list[dict[str, Any]] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"id": "reasoning_123",
|
||||
"type": "reasoning",
|
||||
"summary": [{"text": "User said hello", "type": "summary_text"}],
|
||||
},
|
||||
{
|
||||
"id": "msg_123",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Hi there!"}],
|
||||
"status": "completed",
|
||||
},
|
||||
]
|
||||
|
||||
# Convert to messages
|
||||
messages = Converter.items_to_messages(test_items) # type: ignore[arg-type]
|
||||
|
||||
# Should have user message and assistant message, but no reasoning content
|
||||
assert len(messages) == 2
|
||||
assert messages[0]["role"] == "user"
|
||||
assert messages[1]["role"] == "assistant"
|
||||
|
||||
# Verify no thinking blocks in assistant message
|
||||
assistant_msg = messages[1]
|
||||
content = assistant_msg.get("content")
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
assert part.get("type") != "thinking"
|
||||
|
||||
|
||||
def test_reasoning_items_preserved_in_message_conversion():
|
||||
"""
|
||||
Test that reasoning content and thinking blocks are properly extracted
|
||||
from Anthropic responses and stored in reasoning items.
|
||||
"""
|
||||
# Create mock message with thinking blocks
|
||||
mock_message = create_mock_anthropic_response_with_thinking()
|
||||
|
||||
# Convert to output items
|
||||
output_items = Converter.message_to_output_items(mock_message)
|
||||
|
||||
# Should have reasoning item, message item, and tool call items
|
||||
reasoning_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
|
||||
]
|
||||
assert len(reasoning_items) == 1
|
||||
|
||||
reasoning_item = reasoning_items[0]
|
||||
assert reasoning_item.summary[0].text == "I need to call the weather function for Paris"
|
||||
|
||||
# Verify thinking blocks are stored if we preserve them
|
||||
if (
|
||||
hasattr(reasoning_item, "content")
|
||||
and reasoning_item.content
|
||||
and len(reasoning_item.content) > 0
|
||||
):
|
||||
thinking_block = reasoning_item.content[0]
|
||||
assert thinking_block.type == "reasoning_text"
|
||||
assert thinking_block.text == "I need to call the weather function for Paris"
|
||||
|
||||
|
||||
def test_anthropic_thinking_blocks_with_tool_calls():
|
||||
"""
|
||||
Test for models with extended thinking and interleaved thinking with tool calls.
|
||||
|
||||
This test verifies the Anthropic's API's requirements for thinking blocks
|
||||
to be the first content in assistant messages when reasoning is enabled and tool
|
||||
calls are present.
|
||||
"""
|
||||
# Create a message with reasoning, thinking blocks and tool calls
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather for you.",
|
||||
reasoning_content="The user wants weather information, I need to call the weather function",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": (
|
||||
"The user is asking about weather. "
|
||||
"Let me use the weather tool to get this information."
|
||||
),
|
||||
"signature": "TestSignature123",
|
||||
},
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": ("We should use the city Tokyo as the city."),
|
||||
"signature": "TestSignature456",
|
||||
},
|
||||
],
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCall(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
# Step 1: Convert message to output items
|
||||
output_items = Converter.message_to_output_items(message)
|
||||
|
||||
# Verify reasoning item exists and contains thinking blocks
|
||||
reasoning_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
|
||||
]
|
||||
assert len(reasoning_items) == 1, "Should have exactly two reasoning items"
|
||||
|
||||
reasoning_item = reasoning_items[0]
|
||||
|
||||
# Verify thinking text is stored in content
|
||||
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
|
||||
"Reasoning item should have content"
|
||||
)
|
||||
assert reasoning_item.content[0].type == "reasoning_text", (
|
||||
"Content should be reasoning_text type"
|
||||
)
|
||||
|
||||
# Verify signature is stored in encrypted_content
|
||||
assert hasattr(reasoning_item, "encrypted_content"), (
|
||||
"Reasoning item should have encrypted_content"
|
||||
)
|
||||
assert reasoning_item.encrypted_content == "TestSignature123\nTestSignature456", (
|
||||
"Signature should be preserved"
|
||||
)
|
||||
|
||||
# Verify tool calls are present
|
||||
tool_call_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "function_call"
|
||||
]
|
||||
assert len(tool_call_items) == 1, "Should have exactly one tool call"
|
||||
|
||||
# Step 2: Convert output items back to messages
|
||||
# Convert items to dicts for the converter (simulating serialization/deserialization)
|
||||
items_as_dicts: list[dict[str, Any]] = []
|
||||
for item in output_items:
|
||||
if hasattr(item, "model_dump"):
|
||||
items_as_dicts.append(item.model_dump())
|
||||
else:
|
||||
items_as_dicts.append(cast(dict[str, Any], item))
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
items_as_dicts, # type: ignore[arg-type]
|
||||
model="anthropic/claude-4-opus",
|
||||
preserve_thinking_blocks=True,
|
||||
)
|
||||
|
||||
# Find the assistant message with tool calls
|
||||
assistant_messages = [
|
||||
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
|
||||
]
|
||||
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
|
||||
|
||||
assistant_msg = assistant_messages[0]
|
||||
|
||||
# Content must start with thinking blocks, not text
|
||||
content = assistant_msg.get("content")
|
||||
assert content is not None, "Assistant message should have content"
|
||||
|
||||
assert isinstance(content, list) and len(content) > 0, (
|
||||
"Assistant message content should be a non-empty list"
|
||||
)
|
||||
|
||||
first_content = content[0]
|
||||
assert first_content.get("type") == "thinking", (
|
||||
f"First content must be 'thinking' type for Anthropic compatibility, "
|
||||
f"but got '{first_content.get('type')}'"
|
||||
)
|
||||
expected_thinking = (
|
||||
"The user is asking about weather. Let me use the weather tool to get this information."
|
||||
)
|
||||
assert first_content.get("thinking") == expected_thinking, (
|
||||
"Thinking content should be preserved"
|
||||
)
|
||||
# Signature should also be preserved
|
||||
assert first_content.get("signature") == "TestSignature123", (
|
||||
"Signature should be preserved in thinking block"
|
||||
)
|
||||
|
||||
second_content = content[1]
|
||||
assert second_content.get("type") == "thinking", (
|
||||
f"Second content must be 'thinking' type for Anthropic compatibility, "
|
||||
f"but got '{second_content.get('type')}'"
|
||||
)
|
||||
expected_thinking = "We should use the city Tokyo as the city."
|
||||
assert second_content.get("thinking") == expected_thinking, (
|
||||
"Thinking content should be preserved"
|
||||
)
|
||||
# Signature should also be preserved
|
||||
assert second_content.get("signature") == "TestSignature456", (
|
||||
"Signature should be preserved in thinking block"
|
||||
)
|
||||
|
||||
last_content = content[2]
|
||||
assert last_content.get("type") == "text", (
|
||||
f"First content must be 'text' type but got '{last_content.get('type')}'"
|
||||
)
|
||||
expected_text = "I'll check the weather for you."
|
||||
assert last_content.get("text") == expected_text, "Content text should be preserved"
|
||||
|
||||
# Verify tool calls are preserved
|
||||
tool_calls = assistant_msg.get("tool_calls", [])
|
||||
assert len(cast(list[Any], tool_calls)) == 1, "Tool calls should be preserved"
|
||||
assert cast(list[Any], tool_calls)[0]["function"]["name"] == "get_weather"
|
||||
|
||||
|
||||
def test_items_to_messages_preserves_positional_bool_arguments():
|
||||
"""
|
||||
Preserve positional compatibility for the released items_to_messages signature.
|
||||
"""
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather for you.",
|
||||
reasoning_content="The user wants weather information, I need to call the weather function",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": (
|
||||
"The user is asking about weather. "
|
||||
"Let me use the weather tool to get this information."
|
||||
),
|
||||
"signature": "TestSignature123",
|
||||
}
|
||||
],
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCall(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
output_items = Converter.message_to_output_items(message)
|
||||
items_as_dicts: list[dict[str, Any]] = []
|
||||
for item in output_items:
|
||||
if hasattr(item, "model_dump"):
|
||||
items_as_dicts.append(item.model_dump())
|
||||
else:
|
||||
items_as_dicts.append(cast(dict[str, Any], item))
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
items_as_dicts, # type: ignore[arg-type]
|
||||
"anthropic/claude-4-opus",
|
||||
True,
|
||||
True,
|
||||
)
|
||||
|
||||
assistant_messages = [
|
||||
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
|
||||
]
|
||||
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
|
||||
|
||||
assistant_msg = assistant_messages[0]
|
||||
content = assistant_msg.get("content")
|
||||
assert isinstance(content, list) and len(content) > 0, (
|
||||
"Positional bool arguments should still preserve thinking blocks"
|
||||
)
|
||||
assert content[0].get("type") == "thinking", (
|
||||
"The third positional argument must continue to map to preserve_thinking_blocks"
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_thinking_blocks_without_tool_calls():
|
||||
"""
|
||||
Test for models with extended thinking WITHOUT tool calls.
|
||||
|
||||
This test verifies that thinking blocks are properly attached to assistant
|
||||
messages even when there are no tool calls (fixes issue #2195).
|
||||
"""
|
||||
# Create a message with reasoning and thinking blocks but NO tool calls
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="The weather in Paris is sunny with a temperature of 22°C.",
|
||||
reasoning_content="The user wants to know about the weather in Paris.",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "Let me think about the weather in Paris.",
|
||||
"signature": "TestSignatureNoTools123",
|
||||
}
|
||||
],
|
||||
tool_calls=None, # No tool calls
|
||||
)
|
||||
|
||||
# Step 1: Convert message to output items
|
||||
output_items = Converter.message_to_output_items(message)
|
||||
|
||||
# Verify reasoning item exists and contains thinking blocks
|
||||
reasoning_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
|
||||
]
|
||||
assert len(reasoning_items) == 1, "Should have exactly one reasoning item"
|
||||
|
||||
reasoning_item = reasoning_items[0]
|
||||
|
||||
# Verify thinking text is stored in content
|
||||
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
|
||||
"Reasoning item should have content"
|
||||
)
|
||||
assert reasoning_item.content[0].type == "reasoning_text", (
|
||||
"Content should be reasoning_text type"
|
||||
)
|
||||
assert reasoning_item.content[0].text == "Let me think about the weather in Paris.", (
|
||||
"Thinking text should be preserved"
|
||||
)
|
||||
|
||||
# Verify signature is stored in encrypted_content
|
||||
assert hasattr(reasoning_item, "encrypted_content"), (
|
||||
"Reasoning item should have encrypted_content"
|
||||
)
|
||||
assert reasoning_item.encrypted_content == "TestSignatureNoTools123", (
|
||||
"Signature should be preserved"
|
||||
)
|
||||
|
||||
# Verify message item exists
|
||||
message_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "message"
|
||||
]
|
||||
assert len(message_items) == 1, "Should have exactly one message item"
|
||||
|
||||
# Step 2: Convert output items back to messages with preserve_thinking_blocks=True
|
||||
items_as_dicts: list[dict[str, Any]] = []
|
||||
for item in output_items:
|
||||
if hasattr(item, "model_dump"):
|
||||
items_as_dicts.append(item.model_dump())
|
||||
else:
|
||||
items_as_dicts.append(cast(dict[str, Any], item))
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
items_as_dicts, # type: ignore[arg-type]
|
||||
model="anthropic/claude-4-opus",
|
||||
preserve_thinking_blocks=True,
|
||||
)
|
||||
|
||||
# Should have one assistant message
|
||||
assistant_messages = [msg for msg in messages if msg.get("role") == "assistant"]
|
||||
assert len(assistant_messages) == 1, "Should have exactly one assistant message"
|
||||
|
||||
assistant_msg = assistant_messages[0]
|
||||
|
||||
# Content must start with thinking blocks even WITHOUT tool calls
|
||||
content = assistant_msg.get("content")
|
||||
assert content is not None, "Assistant message should have content"
|
||||
assert isinstance(content, list), (
|
||||
f"Assistant message content should be a list when thinking blocks are present, "
|
||||
f"but got {type(content)}"
|
||||
)
|
||||
assert len(content) >= 2, (
|
||||
f"Assistant message should have at least 2 content items "
|
||||
f"(thinking + text), got {len(content)}"
|
||||
)
|
||||
|
||||
# First content should be thinking block
|
||||
first_content = content[0]
|
||||
assert first_content.get("type") == "thinking", (
|
||||
f"First content must be 'thinking' type for Anthropic compatibility, "
|
||||
f"but got '{first_content.get('type')}'"
|
||||
)
|
||||
assert first_content.get("thinking") == "Let me think about the weather in Paris.", (
|
||||
"Thinking content should be preserved"
|
||||
)
|
||||
assert first_content.get("signature") == "TestSignatureNoTools123", (
|
||||
"Signature should be preserved in thinking block"
|
||||
)
|
||||
|
||||
# Second content should be text
|
||||
second_content = content[1]
|
||||
assert second_content.get("type") == "text", (
|
||||
f"Second content must be 'text' type, but got '{second_content.get('type')}'"
|
||||
)
|
||||
assert (
|
||||
second_content.get("text") == "The weather in Paris is sunny with a temperature of 22°C."
|
||||
), "Text content should be preserved"
|
||||
@@ -0,0 +1,904 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types as pytypes
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.chat import (
|
||||
ChatCompletion,
|
||||
ChatCompletionChunk,
|
||||
ChatCompletionMessage,
|
||||
ChatCompletionMessageFunctionToolCall,
|
||||
)
|
||||
from openai.types.chat.chat_completion import Choice
|
||||
from openai.types.chat.chat_completion_chunk import ChoiceDelta
|
||||
from openai.types.completion_usage import CompletionUsage, PromptTokensDetails
|
||||
from openai.types.responses import Response, ResponseCompletedEvent, ResponseOutputMessage
|
||||
from openai.types.responses.response_error_event import ResponseErrorEvent
|
||||
from openai.types.responses.response_failed_event import ResponseFailedEvent
|
||||
from openai.types.responses.response_incomplete_event import ResponseIncompleteEvent
|
||||
from openai.types.responses.response_output_text import ResponseOutputText
|
||||
from openai.types.responses.response_usage import (
|
||||
InputTokensDetails,
|
||||
OutputTokensDetails,
|
||||
ResponseUsage,
|
||||
)
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
Handoff,
|
||||
ModelBehaviorError,
|
||||
ModelSettings,
|
||||
ModelTracing,
|
||||
Tool,
|
||||
TResponseInputItem,
|
||||
__version__,
|
||||
)
|
||||
from agents.exceptions import UserError
|
||||
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
|
||||
from agents.models.fake_id import FAKE_RESPONSES_ID
|
||||
|
||||
|
||||
class FakeAnyLLMProvider:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
supports_responses: bool,
|
||||
chat_response: Any | None = None,
|
||||
responses_response: Any | None = None,
|
||||
) -> None:
|
||||
self.SUPPORTS_RESPONSES = supports_responses
|
||||
self.chat_response = chat_response
|
||||
self.responses_response = responses_response
|
||||
self.chat_calls: list[dict[str, Any]] = []
|
||||
self.responses_calls: list[dict[str, Any]] = []
|
||||
self.private_responses_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def acompletion(self, **kwargs: Any) -> Any:
|
||||
self.chat_calls.append(kwargs)
|
||||
return self.chat_response
|
||||
|
||||
async def aresponses(self, **kwargs: Any) -> Any:
|
||||
self.responses_calls.append(kwargs)
|
||||
return self.responses_response
|
||||
|
||||
async def _aresponses(self, params: Any, **kwargs: Any) -> Any:
|
||||
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
|
||||
return self.responses_response
|
||||
|
||||
|
||||
def _import_any_llm_module(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
provider: FakeAnyLLMProvider,
|
||||
) -> tuple[Any, list[dict[str, Any]]]:
|
||||
create_calls: list[dict[str, Any]] = []
|
||||
|
||||
class FakeAnyLLMFactory:
|
||||
@staticmethod
|
||||
def create(provider_name: str, api_key: str | None = None, api_base: str | None = None):
|
||||
create_calls.append(
|
||||
{
|
||||
"provider_name": provider_name,
|
||||
"api_key": api_key,
|
||||
"api_base": api_base,
|
||||
}
|
||||
)
|
||||
return provider
|
||||
|
||||
fake_any_llm: Any = pytypes.ModuleType("any_llm")
|
||||
fake_any_llm.AnyLLM = FakeAnyLLMFactory
|
||||
|
||||
sys.modules.pop("agents.extensions.models.any_llm_model", None)
|
||||
monkeypatch.setitem(sys.modules, "any_llm", fake_any_llm)
|
||||
|
||||
module = importlib.import_module("agents.extensions.models.any_llm_model")
|
||||
monkeypatch.setattr(module, "AnyLLM", FakeAnyLLMFactory, raising=True)
|
||||
return module, create_calls
|
||||
|
||||
|
||||
def _chat_completion(text: str) -> ChatCompletion:
|
||||
return ChatCompletion(
|
||||
id="chatcmpl_123",
|
||||
created=0,
|
||||
model="fake-model",
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(
|
||||
index=0,
|
||||
finish_reason="stop",
|
||||
message=ChatCompletionMessage(role="assistant", content=text),
|
||||
)
|
||||
],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=5,
|
||||
prompt_tokens=7,
|
||||
total_tokens=12,
|
||||
prompt_tokens_details=PromptTokensDetails.model_validate(
|
||||
{"cached_tokens": 2, "cache_write_tokens": 4}
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _responses_output(text: str) -> list[Any]:
|
||||
return [
|
||||
ResponseOutputMessage(
|
||||
id="msg_123",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
text=text,
|
||||
type="output_text",
|
||||
annotations=[],
|
||||
logprobs=[],
|
||||
)
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _response(text: str, response_id: str = "resp_123") -> Response:
|
||||
return Response(
|
||||
id=response_id,
|
||||
created_at=123,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=_responses_output(text),
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
usage=ResponseUsage(
|
||||
input_tokens=11,
|
||||
output_tokens=13,
|
||||
total_tokens=24,
|
||||
input_tokens_details=InputTokensDetails.model_validate(
|
||||
{"cache_write_tokens": 0, "cached_tokens": 0}
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _chat_completion_with_tool_call(*, thought_signature: str) -> ChatCompletion:
|
||||
return ChatCompletion(
|
||||
id="chatcmpl_tool_123",
|
||||
created=0,
|
||||
model="fake-model",
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(
|
||||
index=0,
|
||||
finish_reason="tool_calls",
|
||||
message=ChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="Calling a tool.",
|
||||
tool_calls=[
|
||||
ChatCompletionMessageFunctionToolCall.model_validate(
|
||||
{
|
||||
"id": "call_123",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"Paris"}',
|
||||
},
|
||||
"extra_content": {
|
||||
"google": {"thought_signature": thought_signature}
|
||||
},
|
||||
}
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=5,
|
||||
prompt_tokens=7,
|
||||
total_tokens=12,
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class GenericChatCompletionPayload(BaseModel):
|
||||
id: str
|
||||
created: int
|
||||
model: str
|
||||
object: str
|
||||
choices: list[Any]
|
||||
usage: Any
|
||||
|
||||
|
||||
async def _empty_chat_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
if False:
|
||||
yield ChatCompletionChunk(
|
||||
id="chunk_123",
|
||||
created=0,
|
||||
model="fake-model",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason=None)],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
|
||||
async def test_user_agent_header_any_llm_chat(override_ua: str | None, monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
||||
expected_ua = override_ua or f"Agents/Python {__version__}"
|
||||
|
||||
if override_ua is not None:
|
||||
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
|
||||
else:
|
||||
token = None
|
||||
try:
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
finally:
|
||||
if token is not None:
|
||||
HEADERS_OVERRIDE.reset(token)
|
||||
|
||||
assert provider.chat_calls[0]["extra_headers"]["User-Agent"] == expected_ua
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_chat_path_is_used_when_responses_are_unsupported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
||||
module, create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="router-key")
|
||||
response = await model.get_response(
|
||||
system_instructions="You are terse.",
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert create_calls == [
|
||||
{
|
||||
"provider_name": "openrouter",
|
||||
"api_key": "router-key",
|
||||
"api_base": None,
|
||||
}
|
||||
]
|
||||
assert len(provider.chat_calls) == 1
|
||||
assert provider.responses_calls == []
|
||||
assert provider.chat_calls[0]["model"] == "openai/gpt-5.4-mini"
|
||||
assert response.response_id is None
|
||||
assert response.output[0].content[0].text == "Hello"
|
||||
assert response.usage.input_tokens_details.cached_tokens == 2
|
||||
assert getattr(response.usage.input_tokens_details, "cache_write_tokens", None) == 4
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"chat_response",
|
||||
[
|
||||
pytest.param(_chat_completion("Hello").model_dump(), id="dict"),
|
||||
pytest.param(
|
||||
GenericChatCompletionPayload.model_validate(_chat_completion("Hello").model_dump()),
|
||||
id="basemodel",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_any_llm_chat_path_normalizes_non_stream_payloads(
|
||||
monkeypatch,
|
||||
chat_response: Any,
|
||||
) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=chat_response)
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
||||
response = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert response.response_id is None
|
||||
assert response.output[0].content[0].text == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_chat_path_preserves_gemini_tool_call_metadata(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(
|
||||
supports_responses=False,
|
||||
chat_response=_chat_completion_with_tool_call(thought_signature="sig_123"),
|
||||
)
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="gemini/gemini-2.0-flash")
|
||||
response = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
function_calls = [
|
||||
item for item in response.output if getattr(item, "type", None) == "function_call"
|
||||
]
|
||||
assert len(function_calls) == 1
|
||||
provider_data = function_calls[0].model_dump()["provider_data"]
|
||||
assert provider_data["model"] == "gemini/gemini-2.0-flash"
|
||||
assert provider_data["response_id"] == "chatcmpl_tool_123"
|
||||
assert provider_data["thought_signature"] == "sig_123"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_path_is_used_when_supported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
||||
module, create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="gpt-5.4-mini", api_key="openai-key")
|
||||
response = await model.get_response(
|
||||
system_instructions="You are terse.",
|
||||
input="hi",
|
||||
model_settings=ModelSettings(store=True),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert create_calls == [
|
||||
{
|
||||
"provider_name": "openai",
|
||||
"api_key": "openai-key",
|
||||
"api_base": None,
|
||||
}
|
||||
]
|
||||
assert provider.chat_calls == []
|
||||
assert provider.responses_calls == []
|
||||
assert len(provider.private_responses_calls) == 1
|
||||
params = provider.private_responses_calls[0]["params"]
|
||||
kwargs = provider.private_responses_calls[0]["kwargs"]
|
||||
assert params.model == "gpt-5.4-mini"
|
||||
assert params.previous_response_id == "resp_prev"
|
||||
assert params.conversation == "conv_123"
|
||||
assert kwargs["extra_headers"]["User-Agent"] == f"Agents/Python {__version__}"
|
||||
assert response.response_id == "resp_123"
|
||||
assert response.output[0].content[0].text == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_can_force_chat_completions_when_responses_are_supported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(
|
||||
supports_responses=True,
|
||||
chat_response=_chat_completion("Hello from chat"),
|
||||
responses_response=_response("Hello from responses"),
|
||||
)
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-4.1-mini", api="chat_completions")
|
||||
response = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert len(provider.chat_calls) == 1
|
||||
assert provider.responses_calls == []
|
||||
assert response.response_id is None
|
||||
assert response.output[0].content[0].text == "Hello from chat"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_forced_responses_errors_when_provider_does_not_support_it(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-4.1-mini", api="responses")
|
||||
with pytest.raises(UserError, match="does not support the Responses API"):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_stream_uses_chat_handler_when_responses_are_unsupported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_empty_chat_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
completed = ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=_response("Hello from stream"),
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
async def fake_handle_stream(response, stream, model=None):
|
||||
assert model == "openrouter/openai/gpt-5.4-mini"
|
||||
async for _chunk in stream:
|
||||
pass
|
||||
yield completed
|
||||
|
||||
monkeypatch.setattr(module.ChatCmplStreamHandler, "handle_stream", fake_handle_stream)
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
||||
events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
assert [event.type for event in events] == ["response.completed"]
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_stream_passthrough_uses_responses_when_supported(monkeypatch) -> None:
|
||||
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=_response("Hello from responses stream"),
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
assert [event.type for event in events] == ["response.completed"]
|
||||
assert provider.responses_calls == []
|
||||
assert provider.private_responses_calls[0]["params"].previous_response_id == "resp_prev"
|
||||
assert provider.private_responses_calls[0]["params"].conversation == "conv_123"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("terminal_event_type", "terminal_event_cls"),
|
||||
[
|
||||
("response.incomplete", ResponseIncompleteEvent),
|
||||
("response.failed", ResponseFailedEvent),
|
||||
],
|
||||
)
|
||||
async def test_any_llm_responses_stream_rejects_failed_terminal_events(
|
||||
monkeypatch,
|
||||
terminal_event_type: str,
|
||||
terminal_event_cls: type[Any],
|
||||
) -> None:
|
||||
async def response_stream() -> AsyncIterator[Any]:
|
||||
yield terminal_event_cls(
|
||||
type=terminal_event_type,
|
||||
response=_response("partial", response_id="resp-terminal"),
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
events = []
|
||||
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
assert len(events) == 1
|
||||
assert events[0].type == terminal_event_type
|
||||
assert events[0].response.id == "resp-terminal"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_stream_rejects_error_event(monkeypatch) -> None:
|
||||
async def response_stream() -> AsyncIterator[ResponseErrorEvent]:
|
||||
yield ResponseErrorEvent(
|
||||
type="error",
|
||||
code="invalid_request_error",
|
||||
message="bad request",
|
||||
param=None,
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
events = []
|
||||
with pytest.raises(ModelBehaviorError, match="invalid_request_error"):
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
assert len(events) == 1
|
||||
assert events[0].type == "error"
|
||||
assert events[0].code == "invalid_request_error"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_path_passes_transport_kwargs_via_private_provider_api(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(
|
||||
extra_headers={"X-Test-Header": "test"},
|
||||
extra_query={"trace": "1"},
|
||||
extra_body={"foo": "bar"},
|
||||
),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert provider.responses_calls == []
|
||||
assert len(provider.private_responses_calls) == 1
|
||||
call = provider.private_responses_calls[0]
|
||||
assert call["kwargs"]["extra_headers"]["X-Test-Header"] == "test"
|
||||
assert call["kwargs"]["extra_query"] == {"trace": "1"}
|
||||
assert call["kwargs"]["extra_body"] == {"foo": "bar"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_prompt_requests_fail_fast(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
with pytest.raises(Exception, match="prompt-managed requests"):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt={"id": "pmpt_123"},
|
||||
)
|
||||
|
||||
|
||||
def test_any_llm_responses_input_sanitizer_strips_none_fields_from_reasoning_items() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from agents.extensions.models.any_llm_model import AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
raw_input = [
|
||||
{
|
||||
"id": "rid1",
|
||||
"summary": [{"text": "why", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
||||
"status": None,
|
||||
"encrypted_content": None,
|
||||
}
|
||||
]
|
||||
|
||||
cleaned = model._sanitize_any_llm_responses_input(raw_input)
|
||||
|
||||
assert cleaned == [
|
||||
{
|
||||
"id": "rid1",
|
||||
"summary": [{"text": "why", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
||||
}
|
||||
]
|
||||
|
||||
ResponsesParams = importlib.import_module("any_llm.types.responses").ResponsesParams
|
||||
params = ResponsesParams(model="dummy", input=cleaned)
|
||||
assert isinstance(params.input, list)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_path_sanitizes_replayed_items_before_validation() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from agents.extensions.models.any_llm_model import AnyLLMModel
|
||||
|
||||
class ValidatingProvider:
|
||||
SUPPORTS_RESPONSES = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.private_responses_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def aresponses(self, **kwargs: Any) -> Any:
|
||||
raise AssertionError("public aresponses path should not be used in this test")
|
||||
|
||||
async def _aresponses(self, params: Any, **kwargs: Any) -> Response:
|
||||
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
|
||||
return _response("Hello from sanitized replay")
|
||||
|
||||
class TestAnyLLMModel(AnyLLMModel):
|
||||
def __init__(self, provider: ValidatingProvider) -> None:
|
||||
super().__init__(model="openai/gpt-5.4-mini", api="responses")
|
||||
self._provider = provider
|
||||
|
||||
def _get_provider(self) -> Any:
|
||||
return self._provider
|
||||
|
||||
provider = ValidatingProvider()
|
||||
model = TestAnyLLMModel(provider)
|
||||
tools: list[Tool] = []
|
||||
handoffs: list[Handoff[Any, Agent[Any]]] = []
|
||||
stream_flag: Literal[False] = False
|
||||
|
||||
replay_input = cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
||||
"status": None,
|
||||
"provider_data": {"model": "anthropic/fake-responses-model"},
|
||||
},
|
||||
{
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"status": None,
|
||||
"provider_data": {"model": "anthropic/fake-responses-model"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
response = await model._fetch_responses_response(
|
||||
system_instructions=None,
|
||||
input=replay_input,
|
||||
model_settings=ModelSettings(),
|
||||
tools=tools,
|
||||
output_schema=None,
|
||||
handoffs=handoffs,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
stream=stream_flag,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert response.id == "resp_123"
|
||||
assert len(provider.private_responses_calls) == 1
|
||||
params = provider.private_responses_calls[0]["params"]
|
||||
assert params.input == [
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_any_llm_provider_passes_api_override() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from agents.extensions.models.any_llm_model import AnyLLMModel
|
||||
from agents.extensions.models.any_llm_provider import AnyLLMProvider
|
||||
|
||||
provider = AnyLLMProvider(api="chat_completions")
|
||||
model = provider.get_model("openai/gpt-4.1-mini")
|
||||
|
||||
assert isinstance(model, AnyLLMModel)
|
||||
assert model.api == "chat_completions"
|
||||
|
||||
|
||||
def test_any_llm_reasoning_objects_prefer_content_attributes_over_iterable_pairs() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from any_llm.types.completion import Reasoning
|
||||
|
||||
from agents.extensions.models.any_llm_model import _extract_any_llm_reasoning_text
|
||||
|
||||
delta = pytypes.SimpleNamespace(reasoning=Reasoning(content="用户"))
|
||||
|
||||
assert _extract_any_llm_reasoning_text(delta) == "用户"
|
||||
|
||||
|
||||
def test_any_llm_split_does_not_duplicate_content_or_thinking(monkeypatch) -> None:
|
||||
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
|
||||
|
||||
Anthropic's extended thinking API rejects requests that include the same signed
|
||||
thinking block more than once, and duplicated assistant text corrupts conversation
|
||||
history. Only the first split should retain content, thinking_blocks, and
|
||||
reasoning_content; subsequent splits should carry the tool_call alone.
|
||||
"""
|
||||
provider = FakeAnyLLMProvider(supports_responses=False)
|
||||
module, _ = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="anthropic/claude-3-5-sonnet")
|
||||
messages: list[Any] = [
|
||||
{"role": "user", "content": "Search both"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Looking up both queries.",
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}],
|
||||
"reasoning_content": "internal plan",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
|
||||
]
|
||||
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assistants = [m for m in result if m.get("role") == "assistant"]
|
||||
assert len(assistants) == 2
|
||||
# First split keeps the shared fields.
|
||||
assert assistants[0].get("content") == "Looking up both queries."
|
||||
assert "thinking_blocks" in assistants[0]
|
||||
assert "reasoning_content" in assistants[0]
|
||||
# Second split must NOT duplicate them.
|
||||
assert "content" not in assistants[1]
|
||||
assert "thinking_blocks" not in assistants[1]
|
||||
assert "reasoning_content" not in assistants[1]
|
||||
# Tool calls are still split one-per-message.
|
||||
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
|
||||
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
|
||||
@@ -0,0 +1,361 @@
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
Choices,
|
||||
Function,
|
||||
Message,
|
||||
ModelResponse,
|
||||
Usage,
|
||||
)
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_deepseek_reasoning_content_preserved_in_tool_calls(monkeypatch):
|
||||
"""
|
||||
Ensure DeepSeek reasoning_content is preserved when converting items to messages.
|
||||
|
||||
DeepSeek requires reasoning_content field in assistant messages with tool_calls.
|
||||
This test verifies that reasoning content from reasoning items is correctly
|
||||
extracted and added to assistant messages during conversion.
|
||||
"""
|
||||
# Capture the messages sent to the model
|
||||
captured_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured_calls.append({"model": model, "messages": messages, **kwargs})
|
||||
|
||||
# First call: model returns reasoning_content + tool_call
|
||||
if len(captured_calls) == 1:
|
||||
tool_call = ChatCompletionMessageToolCall(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
msg = Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[tool_call],
|
||||
)
|
||||
# DeepSeek adds reasoning_content to the message
|
||||
msg.reasoning_content = "Let me think about getting the weather for Tokyo..."
|
||||
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
# Second call: model returns final response
|
||||
msg = Message(role="assistant", content="The weather in Tokyo is sunny.")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
model = LitellmModel(model="deepseek/deepseek-reasoner")
|
||||
|
||||
# First call: get the tool call response
|
||||
first_response = await model.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
input="What's the weather in Tokyo?",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[], # We'll simulate the tool response manually
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
assert len(first_response.output) >= 1
|
||||
|
||||
input_items: list[Any] = []
|
||||
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
|
||||
|
||||
for item in first_response.output:
|
||||
if hasattr(item, "model_dump"):
|
||||
input_items.append(item.model_dump())
|
||||
else:
|
||||
input_items.append(item)
|
||||
|
||||
input_items.append(
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_123",
|
||||
"output": "The weather in Tokyo is sunny.",
|
||||
}
|
||||
)
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek/deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_messages_with_tool_calls = [
|
||||
m
|
||||
for m in messages
|
||||
if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls")
|
||||
]
|
||||
|
||||
assert len(assistant_messages_with_tool_calls) > 0
|
||||
assistant_msg = assistant_messages_with_tool_calls[0]
|
||||
assert "reasoning_content" in assistant_msg
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_deepseek_reasoning_content_in_multi_turn_conversation(monkeypatch):
|
||||
"""
|
||||
Verify reasoning_content is included in assistant messages during multi-turn conversations.
|
||||
|
||||
When DeepSeek returns reasoning_content with tool_calls, subsequent API calls must
|
||||
include the reasoning_content field in the assistant message to avoid 400 errors.
|
||||
"""
|
||||
captured_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured_calls.append({"model": model, "messages": messages, **kwargs})
|
||||
|
||||
# First call: model returns reasoning_content + tool_call
|
||||
if len(captured_calls) == 1:
|
||||
tool_call = ChatCompletionMessageToolCall(
|
||||
id="call_weather_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
msg = Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[tool_call],
|
||||
)
|
||||
# DeepSeek adds reasoning_content
|
||||
msg.reasoning_content = "I need to get the weather for Tokyo first."
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
# Second call: check if reasoning_content was in the request
|
||||
# In real DeepSeek API, this would fail with 400 if reasoning_content is missing
|
||||
msg = Message(
|
||||
role="assistant", content="Based on my findings, the weather in Tokyo is sunny."
|
||||
)
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
model = LitellmModel(model="deepseek/deepseek-reasoner")
|
||||
|
||||
# First call
|
||||
first_response = await model.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
input="What's the weather in Tokyo?",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
input_items: list[Any] = []
|
||||
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
|
||||
|
||||
for item in first_response.output:
|
||||
if hasattr(item, "model_dump"):
|
||||
input_items.append(item.model_dump())
|
||||
else:
|
||||
input_items.append(item)
|
||||
|
||||
input_items.append(
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
}
|
||||
)
|
||||
|
||||
await model.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
input=input_items,
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
assert len(captured_calls) == 2
|
||||
|
||||
second_call_messages = captured_calls[1]["messages"]
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in second_call_messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
assert "reasoning_content" in assistant_with_tools
|
||||
|
||||
|
||||
def test_deepseek_reasoning_content_with_openai_chatcompletions_path():
|
||||
"""
|
||||
Verify reasoning_content works when using OpenAIChatCompletionsModel.
|
||||
|
||||
This ensures the fix works for both LiteLLM and OpenAI ChatCompletions code paths.
|
||||
"""
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
input_items: list[Any] = [
|
||||
{"role": "user", "content": "What's the weather in Paris?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [{"text": "I need to check the weather in Paris.", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": "deepseek-reasoner", "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Paris"}',
|
||||
"call_id": "call_weather_456",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": "deepseek-reasoner"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_456",
|
||||
"output": "The weather in Paris is cloudy and 15°C.",
|
||||
},
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
assert "reasoning_content" in assistant_with_tools
|
||||
# Use type: ignore since reasoning_content is a dynamic field not in OpenAI's TypedDict
|
||||
assert assistant_with_tools["reasoning_content"] == "I need to check the weather in Paris." # type: ignore[typeddict-item]
|
||||
|
||||
|
||||
def test_reasoning_content_from_other_provider_not_attached_to_deepseek():
|
||||
"""
|
||||
Verify reasoning_content from non-DeepSeek providers is NOT attached to DeepSeek messages.
|
||||
|
||||
When switching models mid-conversation (e.g., from Claude to DeepSeek), reasoning items
|
||||
that originated from Claude should not have their summaries attached as reasoning_content
|
||||
to DeepSeek assistant messages, as this would leak unrelated reasoning and may trigger
|
||||
DeepSeek 400 errors.
|
||||
"""
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
input_items: list[Any] = [
|
||||
{"role": "user", "content": "What's the weather in Paris?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [{"text": "Claude's reasoning about the weather.", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
# this one came from Claude, not DeepSeek
|
||||
"provider_data": {"model": "claude-sonnet-4-20250514", "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Paris"}',
|
||||
"call_id": "call_weather_789",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": "claude-sonnet-4-20250514"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_789",
|
||||
"output": "The weather in Paris is cloudy.",
|
||||
},
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
# reasoning_content should NOT be present since the reasoning came from Claude, not DeepSeek
|
||||
assert "reasoning_content" not in assistant_with_tools
|
||||
|
||||
|
||||
def test_reasoning_content_without_provider_data_attached_for_backward_compat():
|
||||
"""
|
||||
Verify reasoning_content from items without provider_data is attached for backward compat.
|
||||
|
||||
For older items that don't have provider_data (before provider tracking was added),
|
||||
we should still attach reasoning_content to maintain backward compatibility.
|
||||
"""
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
# Reasoning item without provider_data (older format)
|
||||
input_items: list[Any] = [
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [{"text": "Reasoning without provider info.", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
# No provider_data
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_101",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_101",
|
||||
"output": "The weather in Tokyo is sunny.",
|
||||
},
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
# reasoning_content SHOULD be present for backward compatibility
|
||||
assert "reasoning_content" in assistant_with_tools
|
||||
assert assistant_with_tools["reasoning_content"] == "Reasoning without provider info." # type: ignore[typeddict-item]
|
||||
@@ -0,0 +1,201 @@
|
||||
import os
|
||||
from typing import Literal
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from openai.types.shared.reasoning import Reasoning
|
||||
|
||||
from agents import Agent
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models import (
|
||||
get_default_model,
|
||||
get_default_model_settings,
|
||||
gpt_5_reasoning_settings_required,
|
||||
is_gpt_5_default,
|
||||
)
|
||||
|
||||
|
||||
def _gpt_5_default_settings(
|
||||
reasoning_effort: Literal["none", "low", "medium"] | None,
|
||||
) -> ModelSettings:
|
||||
if reasoning_effort is None:
|
||||
return ModelSettings(verbosity="low")
|
||||
return ModelSettings(reasoning=Reasoning(effort=reasoning_effort), verbosity="low")
|
||||
|
||||
|
||||
def test_default_model_is_gpt_5_4_mini():
|
||||
assert get_default_model() == "gpt-5.4-mini"
|
||||
assert is_gpt_5_default() is True
|
||||
assert gpt_5_reasoning_settings_required(get_default_model()) is True
|
||||
assert get_default_model_settings() == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.4"})
|
||||
def test_is_gpt_5_default_with_real_model_name():
|
||||
assert get_default_model() == "gpt-5.4"
|
||||
assert is_gpt_5_default() is True
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-4.1"})
|
||||
def test_is_gpt_5_default_returns_false_for_non_gpt_5_default_model():
|
||||
assert get_default_model() == "gpt-4.1"
|
||||
assert is_gpt_5_default() is False
|
||||
|
||||
|
||||
def test_gpt_5_reasoning_settings_required_detects_gpt_5_models_while_ignoring_chat_latest():
|
||||
assert gpt_5_reasoning_settings_required("gpt-5") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.1") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2-codex") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2-pro") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.4-pro") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.5") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5-mini") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5-nano") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5-chat-latest") is False
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.1-chat-latest") is False
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2-chat-latest") is False
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.3-chat-latest") is False
|
||||
|
||||
|
||||
def test_gpt_5_reasoning_settings_required_returns_false_for_non_gpt_5_models():
|
||||
assert gpt_5_reasoning_settings_required("gpt-4.1") is False
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_1_models():
|
||||
assert get_default_model_settings("gpt-5.1") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.1-2025-11-13") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_2_models():
|
||||
assert get_default_model_settings("gpt-5.2") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.2-2025-12-11") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_3_codex_models():
|
||||
assert get_default_model_settings("gpt-5.3-codex") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_models():
|
||||
assert get_default_model_settings("gpt-5.4") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_snapshot_families():
|
||||
assert get_default_model_settings("gpt-5.4-2026-03-05") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.4-mini-2026-03-17") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.4-nano-2026-03-17") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_mini_and_nano():
|
||||
assert get_default_model_settings("gpt-5.4-mini") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.4-nano") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_5_models():
|
||||
assert get_default_model_settings("gpt-5.5") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.5-2026-04-23") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_6_models(
|
||||
model: str,
|
||||
):
|
||||
assert get_default_model_settings(model) == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
|
||||
def test_agent_uses_gpt_5_6_model_settings_from_default_model_env(
|
||||
model: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", model.upper())
|
||||
|
||||
agent = Agent(name="test")
|
||||
|
||||
assert get_default_model() == model
|
||||
assert agent.model is None
|
||||
assert agent.model_settings == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_low_reasoning_defaults_for_base_gpt_5():
|
||||
assert get_default_model_settings("gpt-5") == _gpt_5_default_settings("low")
|
||||
assert get_default_model_settings("gpt-5-2025-08-07") == _gpt_5_default_settings("low")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_low_reasoning_defaults_for_gpt_5_2_codex():
|
||||
assert get_default_model_settings("gpt-5.2-codex") == _gpt_5_default_settings("low")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_medium_reasoning_defaults_for_gpt_5_pro_models():
|
||||
assert get_default_model_settings("gpt-5.2-pro") == _gpt_5_default_settings("medium")
|
||||
assert get_default_model_settings("gpt-5.2-pro-2025-12-11") == _gpt_5_default_settings("medium")
|
||||
assert get_default_model_settings("gpt-5.4-pro") == _gpt_5_default_settings("medium")
|
||||
assert get_default_model_settings("gpt-5.4-pro-2026-03-05") == _gpt_5_default_settings("medium")
|
||||
|
||||
|
||||
def test_get_default_model_settings_omits_reasoning_for_unconfirmed_gpt_5_variants():
|
||||
assert get_default_model_settings("gpt-5-mini") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5-mini-2025-08-07") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5-nano") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5-nano-2025-08-07") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5.1-codex") == _gpt_5_default_settings(None)
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_empty_settings_for_gpt_5_chat_latest_aliases():
|
||||
assert get_default_model_settings("gpt-5-chat-latest") == ModelSettings()
|
||||
assert get_default_model_settings("gpt-5.1-chat-latest") == ModelSettings()
|
||||
assert get_default_model_settings("gpt-5.2-chat-latest") == ModelSettings()
|
||||
assert get_default_model_settings("gpt-5.3-chat-latest") == ModelSettings()
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_empty_settings_for_non_gpt_5_models():
|
||||
assert get_default_model_settings("gpt-4.1") == ModelSettings()
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
|
||||
def test_agent_uses_gpt_5_default_model_settings():
|
||||
"""Agent should inherit GPT-5 default model settings."""
|
||||
agent = Agent(name="test")
|
||||
assert agent.model is None
|
||||
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
|
||||
assert agent.model_settings.verbosity == "low"
|
||||
|
||||
|
||||
def test_agent_uses_model_specific_settings_for_explicit_gpt_5_models():
|
||||
"""Agent should not apply the fallback model's GPT-5 settings to explicit GPT-5 models."""
|
||||
agent = Agent(name="test", model="gpt-5")
|
||||
assert agent.model == "gpt-5"
|
||||
assert agent.model_settings == get_default_model_settings("gpt-5")
|
||||
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_agent_uses_empty_settings_for_explicit_non_gpt_5_models():
|
||||
"""Agent should not apply GPT-5 defaults to explicit non-GPT-5 models."""
|
||||
agent = Agent(name="test", model="gpt-4.1")
|
||||
assert agent.model == "gpt-4.1"
|
||||
assert agent.model_settings == ModelSettings()
|
||||
|
||||
|
||||
def test_agent_clone_recomputes_implicit_settings_when_model_changes():
|
||||
"""Agent.clone should keep implicit model settings aligned with the cloned model."""
|
||||
agent = Agent(name="test", model="gpt-5")
|
||||
cloned = agent.clone(model="gpt-5.4-mini")
|
||||
assert cloned.model == "gpt-5.4-mini"
|
||||
assert cloned.model_settings == get_default_model_settings("gpt-5.4-mini")
|
||||
assert cloned.model_settings.reasoning.effort == "none" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_agent_clone_preserves_explicit_settings_when_model_changes():
|
||||
"""Agent.clone should not recompute model settings that were explicitly customized."""
|
||||
model_settings = ModelSettings(temperature=0.3)
|
||||
agent = Agent(name="test", model="gpt-5", model_settings=model_settings)
|
||||
cloned = agent.clone(model="gpt-5.4-mini")
|
||||
assert cloned.model == "gpt-5.4-mini"
|
||||
assert cloned.model_settings == model_settings
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
|
||||
def test_agent_resets_model_settings_for_non_gpt_5_models():
|
||||
"""Agent should reset default GPT-5 settings when using a non-GPT-5 model."""
|
||||
agent = Agent(name="test", model="gpt-4.1")
|
||||
assert agent.model == "gpt-4.1"
|
||||
assert agent.model_settings == ModelSettings()
|
||||
@@ -0,0 +1,354 @@
|
||||
"""Tests for the extended thinking message order bug fix in LitellmModel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
from openai.types.chat import ChatCompletionMessageParam
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
|
||||
|
||||
class TestExtendedThinkingMessageOrder:
|
||||
"""Test the _fix_tool_message_ordering method."""
|
||||
|
||||
def test_basic_reordering_tool_result_before_call(self):
|
||||
"""Test that a tool result appearing before its tool call gets reordered correctly."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "tool", "tool_call_id": "call_123", "content": "Result for call_123"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_123",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Thanks"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should reorder to: user, assistant+tool_call, tool_result, user
|
||||
assert len(result) == 4
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_123" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_123"
|
||||
assert result[3]["role"] == "user"
|
||||
|
||||
def test_consecutive_tool_calls_get_separated(self):
|
||||
"""Test that consecutive assistant messages with tool calls get properly paired with results.""" # noqa: E501
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test1", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "test2", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should pair each tool call with its result immediately
|
||||
assert len(result) == 5
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "assistant"
|
||||
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
|
||||
assert result[4]["role"] == "tool"
|
||||
assert result[4]["tool_call_id"] == "call_2"
|
||||
|
||||
def test_unmatched_tool_results_preserved(self):
|
||||
"""Test that tool results without matching tool calls are preserved."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Matched result"},
|
||||
{"role": "tool", "tool_call_id": "call_orphan", "content": "Orphaned result"},
|
||||
{"role": "user", "content": "End"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should preserve the orphaned tool result
|
||||
assert len(result) == 5
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "tool" # Orphaned result preserved
|
||||
assert result[3]["tool_call_id"] == "call_orphan"
|
||||
assert result[4]["role"] == "user"
|
||||
|
||||
def test_tool_calls_without_results_preserved(self):
|
||||
"""Test that tool calls without results are still included."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "End"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should preserve the tool call even without a result
|
||||
assert len(result) == 3
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "user"
|
||||
|
||||
def test_correctly_ordered_messages_unchanged(self):
|
||||
"""Test that correctly ordered messages remain in the same order."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Result"},
|
||||
{"role": "assistant", "content": "Done"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should remain exactly the same
|
||||
assert len(result) == 4
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "assistant"
|
||||
|
||||
def test_multiple_tool_calls_single_message(self):
|
||||
"""Test assistant message with multiple tool calls gets split properly."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test1", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "test2", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should split the multi-tool message and pair each properly
|
||||
assert len(result) == 5
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert len(result[1]["tool_calls"]) == 1 # type: ignore
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "assistant"
|
||||
assert len(result[3]["tool_calls"]) == 1 # type: ignore
|
||||
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
|
||||
assert result[4]["role"] == "tool"
|
||||
assert result[4]["tool_call_id"] == "call_2"
|
||||
|
||||
def test_split_does_not_duplicate_content_or_thinking(self):
|
||||
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
|
||||
|
||||
Anthropic's extended thinking API rejects requests that include the same signed
|
||||
thinking block more than once, and duplicated assistant text corrupts conversation
|
||||
history. Only the first split should retain content, thinking_blocks, and
|
||||
reasoning_content; subsequent splits should carry the tool_call alone.
|
||||
"""
|
||||
# Build the assistant message via cast so mypy doesn't reject the
|
||||
# extra keys (`thinking_blocks`, `reasoning_content`) which are not
|
||||
# part of the upstream ChatCompletionAssistantMessageParam TypedDict
|
||||
# but are surfaced by litellm for Anthropic extended thinking.
|
||||
assistant_msg = cast(
|
||||
ChatCompletionMessageParam,
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Looking up both queries.",
|
||||
"thinking_blocks": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}
|
||||
],
|
||||
"reasoning_content": "internal plan",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Search both"},
|
||||
assistant_msg,
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
|
||||
]
|
||||
|
||||
model = LitellmModel("claude-3-5-sonnet")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assistants = [cast(dict[str, Any], m) for m in result if m.get("role") == "assistant"]
|
||||
assert len(assistants) == 2
|
||||
# First split keeps the shared fields.
|
||||
assert assistants[0].get("content") == "Looking up both queries."
|
||||
assert "thinking_blocks" in assistants[0]
|
||||
assert "reasoning_content" in assistants[0]
|
||||
# Second split must NOT duplicate them.
|
||||
assert "content" not in assistants[1]
|
||||
assert "thinking_blocks" not in assistants[1]
|
||||
assert "reasoning_content" not in assistants[1]
|
||||
# Tool calls are still split one-per-message.
|
||||
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
|
||||
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
|
||||
|
||||
def test_empty_messages_list(self):
|
||||
"""Test that empty message list is handled correctly."""
|
||||
messages: list[ChatCompletionMessageParam] = []
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_no_tool_messages(self):
|
||||
"""Test that messages without tool calls are left unchanged."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assert result == messages
|
||||
|
||||
def test_complex_mixed_scenario(self):
|
||||
"""Test a complex scenario with various message types and orderings."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Start"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_out_of_order",
|
||||
"content": "Out of order result",
|
||||
}, # This comes before its call
|
||||
{"role": "assistant", "content": "Regular response"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_out_of_order",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_normal",
|
||||
"type": "function",
|
||||
"function": {"name": "test2", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_normal", "content": "Normal result"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_orphan",
|
||||
"content": "Orphaned result",
|
||||
}, # No matching call
|
||||
{"role": "user", "content": "End"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should reorder properly while preserving all messages
|
||||
assert len(result) == 8
|
||||
assert result[0]["role"] == "user" # Start
|
||||
assert result[1]["role"] == "assistant" # Regular response
|
||||
assert result[2]["role"] == "assistant" # call_out_of_order
|
||||
assert result[2]["tool_calls"][0]["id"] == "call_out_of_order" # type: ignore
|
||||
assert result[3]["role"] == "tool" # Out of order result (now properly paired)
|
||||
assert result[3]["tool_call_id"] == "call_out_of_order"
|
||||
assert result[4]["role"] == "assistant" # call_normal
|
||||
assert result[4]["tool_calls"][0]["id"] == "call_normal" # type: ignore
|
||||
assert result[5]["role"] == "tool" # Normal result
|
||||
assert result[5]["tool_call_id"] == "call_normal"
|
||||
assert result[6]["role"] == "tool" # Orphaned result (preserved)
|
||||
assert result[6]["tool_call_id"] == "call_orphan"
|
||||
assert result[7]["role"] == "user" # End
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Test for Gemini thought signatures in function calling.
|
||||
|
||||
Validates that thought signatures are preserved through the bidirectional roundtrip:
|
||||
- Gemini chatcmpl message → response item → back to message
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from openai.types.chat.chat_completion_message_tool_call import Function
|
||||
|
||||
from agents.extensions.models.litellm_model import InternalChatCompletionMessage, InternalToolCall
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
|
||||
def test_gemini_thought_signature_roundtrip():
|
||||
"""Test that thought signatures are preserved from Gemini responses to messages."""
|
||||
|
||||
# Create mock Gemini response with thought signature in new extra_content structure
|
||||
class MockToolCall(InternalToolCall):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Paris"}'),
|
||||
extra_content={"google": {"thought_signature": "test_signature_abc"}},
|
||||
)
|
||||
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather.",
|
||||
reasoning_content="",
|
||||
tool_calls=[MockToolCall()],
|
||||
)
|
||||
|
||||
# Step 1: Convert to items
|
||||
provider_data = {"model": "gemini/gemini-3-pro", "response_id": "gemini-response-id-123"}
|
||||
|
||||
items = Converter.message_to_output_items(message, provider_data=provider_data)
|
||||
|
||||
func_calls = [item for item in items if hasattr(item, "type") and item.type == "function_call"]
|
||||
assert len(func_calls) == 1
|
||||
|
||||
# Verify thought_signature is stored in items with our provider_data structure
|
||||
func_call_dict = func_calls[0].model_dump()
|
||||
|
||||
assert func_call_dict["provider_data"]["model"] == "gemini/gemini-3-pro"
|
||||
assert func_call_dict["provider_data"]["response_id"] == "gemini-response-id-123"
|
||||
assert func_call_dict["provider_data"]["thought_signature"] == "test_signature_abc"
|
||||
|
||||
# Step 2: Convert back to messages
|
||||
items_as_dicts = [item.model_dump() for item in items]
|
||||
messages = Converter.items_to_messages(
|
||||
[{"role": "user", "content": "test"}] + items_as_dicts,
|
||||
model="gemini/gemini-3-pro",
|
||||
)
|
||||
|
||||
# Verify thought_signature is restored in extra_content format
|
||||
assistant_msg = [msg for msg in messages if msg.get("role") == "assistant"][0]
|
||||
tool_call = assistant_msg["tool_calls"][0] # type: ignore[index, typeddict-item]
|
||||
assert tool_call["extra_content"]["google"]["thought_signature"] == "test_signature_abc"
|
||||
|
||||
|
||||
def test_gemini_multiple_tool_calls_with_thought_signatures():
|
||||
"""Test multiple tool calls each preserve their own thought signatures."""
|
||||
tool_call_1 = InternalToolCall(
|
||||
id="call_1",
|
||||
type="function",
|
||||
function=Function(name="func_a", arguments='{"x": 1}'),
|
||||
extra_content={"google": {"thought_signature": "sig_aaa"}},
|
||||
)
|
||||
tool_call_2 = InternalToolCall(
|
||||
id="call_2",
|
||||
type="function",
|
||||
function=Function(name="func_b", arguments='{"y": 2}'),
|
||||
extra_content={"google": {"thought_signature": "sig_bbb"}},
|
||||
)
|
||||
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="Calling two functions.",
|
||||
reasoning_content="",
|
||||
tool_calls=[tool_call_1, tool_call_2],
|
||||
)
|
||||
|
||||
provider_data = {"model": "gemini/gemini-3-pro"}
|
||||
items = Converter.message_to_output_items(message, provider_data=provider_data)
|
||||
|
||||
func_calls = [i for i in items if hasattr(i, "type") and i.type == "function_call"]
|
||||
assert len(func_calls) == 2
|
||||
|
||||
assert func_calls[0].model_dump()["provider_data"]["thought_signature"] == "sig_aaa"
|
||||
assert func_calls[1].model_dump()["provider_data"]["thought_signature"] == "sig_bbb"
|
||||
|
||||
|
||||
def test_gemini_thought_signature_items_to_messages():
|
||||
"""Test that items_to_messages restores extra_content from provider_data for Gemini."""
|
||||
|
||||
# Create a function call item with provider_data containing thought_signature
|
||||
func_call_item = {
|
||||
"id": "fake-id",
|
||||
"call_id": "call_restore",
|
||||
"name": "restore_func",
|
||||
"arguments": '{"test": true}',
|
||||
"type": "function_call",
|
||||
"provider_data": {
|
||||
"model": "gemini/gemini-3-pro",
|
||||
"response_id": "gemini-response-id-123",
|
||||
"thought_signature": "restored_sig_xyz",
|
||||
},
|
||||
}
|
||||
|
||||
items = [{"role": "user", "content": "test"}, func_call_item]
|
||||
messages = Converter.items_to_messages(items, model="gemini/gemini-3-pro") # type: ignore[arg-type]
|
||||
|
||||
# Find the assistant message with tool_calls
|
||||
assistant_msgs = [m for m in messages if m.get("role") == "assistant"]
|
||||
assert len(assistant_msgs) == 1
|
||||
|
||||
tool_calls: list[dict[str, Any]] = assistant_msgs[0].get("tool_calls", []) # type: ignore[assignment]
|
||||
assert len(tool_calls) == 1
|
||||
|
||||
# Verify extra_content is restored in Google format
|
||||
assert tool_calls[0]["extra_content"]["google"]["thought_signature"] == "restored_sig_xyz"
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Test for Gemini thought signatures in streaming function calls.
|
||||
|
||||
Validates that thought signatures are captured from streaming chunks
|
||||
and included in the final function call events.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.chat import ChatCompletionChunk
|
||||
from openai.types.chat.chat_completion_chunk import (
|
||||
Choice,
|
||||
ChoiceDelta,
|
||||
ChoiceDeltaToolCall,
|
||||
ChoiceDeltaToolCallFunction,
|
||||
)
|
||||
from openai.types.responses import Response
|
||||
|
||||
from agents.models.chatcmpl_stream_handler import ChatCmplStreamHandler
|
||||
|
||||
# ========== Helper Functions ==========
|
||||
|
||||
|
||||
def create_tool_call_delta(
|
||||
index: int,
|
||||
tool_call_id: str | None = None,
|
||||
function_name: str | None = None,
|
||||
arguments: str | None = None,
|
||||
provider_specific_fields: dict[str, Any] | None = None,
|
||||
extra_content: dict[str, Any] | None = None,
|
||||
) -> ChoiceDeltaToolCall:
|
||||
"""Create a tool call delta for streaming."""
|
||||
function = ChoiceDeltaToolCallFunction(
|
||||
name=function_name,
|
||||
arguments=arguments,
|
||||
)
|
||||
|
||||
delta = ChoiceDeltaToolCall(
|
||||
index=index,
|
||||
id=tool_call_id,
|
||||
type="function" if tool_call_id else None,
|
||||
function=function,
|
||||
)
|
||||
|
||||
# Add provider_specific_fields (litellm format)
|
||||
if provider_specific_fields:
|
||||
delta_any = cast(Any, delta)
|
||||
delta_any.provider_specific_fields = provider_specific_fields
|
||||
|
||||
# Add extra_content (Google chatcmpl format)
|
||||
if extra_content:
|
||||
delta_any = cast(Any, delta)
|
||||
delta_any.extra_content = extra_content
|
||||
|
||||
return delta
|
||||
|
||||
|
||||
def create_chunk(
|
||||
tool_calls: list[ChoiceDeltaToolCall] | None = None,
|
||||
content: str | None = None,
|
||||
include_usage: bool = False,
|
||||
) -> ChatCompletionChunk:
|
||||
"""Create a ChatCompletionChunk for testing."""
|
||||
delta = ChoiceDelta(
|
||||
content=content,
|
||||
role="assistant" if content or tool_calls else None,
|
||||
tool_calls=tool_calls,
|
||||
)
|
||||
|
||||
chunk = ChatCompletionChunk(
|
||||
id="chunk-id-123",
|
||||
created=1,
|
||||
model="gemini/gemini-3-pro",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=delta, finish_reason=None)],
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
chunk.usage = CompletionUsage(
|
||||
completion_tokens=10,
|
||||
prompt_tokens=5,
|
||||
total_tokens=15,
|
||||
)
|
||||
|
||||
return chunk
|
||||
|
||||
|
||||
def create_final_chunk() -> ChatCompletionChunk:
|
||||
"""Create a final chunk with finish_reason='tool_calls'."""
|
||||
return ChatCompletionChunk(
|
||||
id="chunk-id-456",
|
||||
created=1,
|
||||
model="gemini/gemini-3-pro",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="tool_calls")],
|
||||
)
|
||||
|
||||
|
||||
async def create_fake_stream(
|
||||
chunks: list[ChatCompletionChunk],
|
||||
) -> AsyncIterator[ChatCompletionChunk]:
|
||||
"""Create an async iterator from chunks."""
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
def create_mock_response() -> Response:
|
||||
"""Create a mock Response object."""
|
||||
return Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="gemini/gemini-3-pro",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
|
||||
|
||||
# ========== Tests ==========
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_captures_litellmprovider_specific_fields_thought_signature():
|
||||
"""Test streaming captures thought_signature from litellm's provider_specific_fields."""
|
||||
chunks = [
|
||||
create_chunk(
|
||||
tool_calls=[
|
||||
create_tool_call_delta(
|
||||
index=0,
|
||||
tool_call_id="call_stream_1",
|
||||
function_name="get_weather",
|
||||
provider_specific_fields={"thought_signature": "litellm_sig_123"},
|
||||
)
|
||||
]
|
||||
),
|
||||
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"city": "Tokyo"}')]),
|
||||
create_final_chunk(),
|
||||
]
|
||||
|
||||
response = create_mock_response()
|
||||
stream = create_fake_stream(chunks)
|
||||
|
||||
events = []
|
||||
async for event in ChatCmplStreamHandler.handle_stream(
|
||||
response,
|
||||
stream, # type: ignore[arg-type]
|
||||
model="gemini/gemini-3-pro",
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
# Find function call done event
|
||||
done_events = [e for e in events if e.type == "response.output_item.done"]
|
||||
func_done = [
|
||||
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
|
||||
]
|
||||
assert len(func_done) == 1
|
||||
|
||||
provider_data = func_done[0].item.model_dump().get("provider_data", {})
|
||||
assert provider_data.get("thought_signature") == "litellm_sig_123"
|
||||
assert provider_data["model"] == "gemini/gemini-3-pro"
|
||||
assert provider_data["response_id"] == "chunk-id-123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_captures_google_extra_content_thought_signature():
|
||||
"""Test streaming captures thought_signature from Google's extra_content format."""
|
||||
chunks = [
|
||||
create_chunk(
|
||||
tool_calls=[
|
||||
create_tool_call_delta(
|
||||
index=0,
|
||||
tool_call_id="call_stream_2",
|
||||
function_name="search",
|
||||
extra_content={"google": {"thought_signature": "google_sig_456"}},
|
||||
)
|
||||
]
|
||||
),
|
||||
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"query": "test"}')]),
|
||||
create_final_chunk(),
|
||||
]
|
||||
|
||||
response = create_mock_response()
|
||||
stream = create_fake_stream(chunks)
|
||||
|
||||
events = []
|
||||
async for event in ChatCmplStreamHandler.handle_stream(
|
||||
response,
|
||||
stream, # type: ignore[arg-type]
|
||||
model="gemini/gemini-3-pro",
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
done_events = [e for e in events if e.type == "response.output_item.done"]
|
||||
func_done = [
|
||||
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
|
||||
]
|
||||
assert len(func_done) == 1
|
||||
|
||||
provider_data = func_done[0].item.model_dump().get("provider_data", {})
|
||||
assert provider_data.get("thought_signature") == "google_sig_456"
|
||||
assert provider_data["model"] == "gemini/gemini-3-pro"
|
||||
assert provider_data["response_id"] == "chunk-id-123"
|
||||
@@ -0,0 +1,317 @@
|
||||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
from httpx import Headers, Response
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai import APIConnectionError
|
||||
from openai.types.chat.chat_completion import ChatCompletion, Choice
|
||||
from openai.types.chat.chat_completion_message import ChatCompletionMessage
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models._retry_runtime import provider_managed_retries_disabled
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.retry import ModelRetryAdviceRequest, ModelRetrySettings
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_kwargs_forwarded(monkeypatch):
|
||||
"""
|
||||
Test that kwargs from ModelSettings are forwarded to litellm.acompletion.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
extra_args={
|
||||
"custom_param": "custom_value",
|
||||
"seed": 42,
|
||||
"stop": ["END"],
|
||||
"logit_bias": {123: -100},
|
||||
},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
)
|
||||
|
||||
# Verify that all kwargs were passed through
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert captured["seed"] == 42
|
||||
assert captured["stop"] == ["END"]
|
||||
assert captured["logit_bias"] == {123: -100}
|
||||
|
||||
# Verify regular parameters are still passed
|
||||
assert captured["temperature"] == 0.5
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_chatcompletions_kwargs_forwarded(monkeypatch):
|
||||
"""
|
||||
Test that kwargs from ModelSettings are forwarded to OpenAI chat completions API.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class MockChatCompletions:
|
||||
async def create(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = ChatCompletionMessage(role="assistant", content="test response")
|
||||
choice = Choice(index=0, message=msg, finish_reason="stop")
|
||||
return ChatCompletion(
|
||||
id="test-id",
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
|
||||
)
|
||||
|
||||
class MockChat:
|
||||
def __init__(self):
|
||||
self.completions = MockChatCompletions()
|
||||
|
||||
class MockClient:
|
||||
def __init__(self):
|
||||
self.chat = MockChat()
|
||||
self.base_url = "https://api.openai.com/v1"
|
||||
|
||||
settings = ModelSettings(
|
||||
temperature=0.7,
|
||||
extra_args={
|
||||
"seed": 123,
|
||||
"logit_bias": {456: 10},
|
||||
"stop": ["STOP", "END"],
|
||||
"user": "test-user",
|
||||
},
|
||||
)
|
||||
|
||||
mock_client = MockClient()
|
||||
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=mock_client) # type: ignore
|
||||
|
||||
await model.get_response(
|
||||
system_instructions="Test system",
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Verify that all kwargs were passed through
|
||||
assert captured["seed"] == 123
|
||||
assert captured["logit_bias"] == {456: 10}
|
||||
assert captured["stop"] == ["STOP", "END"]
|
||||
assert captured["user"] == "test-user"
|
||||
|
||||
# Verify regular parameters are still passed
|
||||
assert captured["temperature"] == 0.7
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_kwargs_handling(monkeypatch):
|
||||
"""
|
||||
Test that empty or None kwargs are handled gracefully.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
# Test with None kwargs
|
||||
settings_none = ModelSettings(temperature=0.5, extra_args=None)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings_none,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Should work without error and include regular parameters
|
||||
assert captured["temperature"] == 0.5
|
||||
|
||||
# Test with empty dict
|
||||
captured.clear()
|
||||
settings_empty = ModelSettings(temperature=0.3, extra_args={})
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings_empty,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Should work without error and include regular parameters
|
||||
assert captured["temperature"] == 0.3
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_effort_falls_back_to_extra_args(monkeypatch):
|
||||
"""
|
||||
Ensure reasoning_effort from extra_args is promoted when reasoning settings are missing.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
|
||||
settings = ModelSettings(
|
||||
extra_args={"reasoning_effort": "none", "custom_param": "custom_value"}
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "none"
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert settings.extra_args == {"reasoning_effort": "none", "custom_param": "custom_value"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_retry_settings_do_not_leak_and_disable_provider_retries_on_runner_retry(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Runner retries should disable LiteLLM's own retries without forwarding SDK retry config."""
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
settings = ModelSettings(
|
||||
retry=ModelRetrySettings(
|
||||
max_retries=2,
|
||||
backoff={"initial_delay": 0.25, "jitter": False},
|
||||
),
|
||||
extra_args={"max_retries": 7, "num_retries": 6, "custom_param": "custom_value"},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
with provider_managed_retries_disabled(True):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
)
|
||||
|
||||
assert settings.retry is not None
|
||||
assert settings.retry.backoff is not None
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert captured["max_retries"] == 0
|
||||
assert captured["num_retries"] == 0
|
||||
assert "retry" not in captured
|
||||
|
||||
|
||||
def test_litellm_get_retry_advice_uses_response_headers() -> None:
|
||||
"""LiteLLM retry advice should expose OpenAI-compatible retry headers."""
|
||||
|
||||
model = LitellmModel(model="test-model")
|
||||
error = RateLimitError(
|
||||
message="rate limited",
|
||||
llm_provider="openai",
|
||||
model="gpt-4o-mini",
|
||||
response=Response(
|
||||
status_code=429,
|
||||
headers=Headers({"x-should-retry": "true", "retry-after-ms": "250"}),
|
||||
),
|
||||
)
|
||||
|
||||
advice = model.get_retry_advice(
|
||||
ModelRetryAdviceRequest(
|
||||
error=error,
|
||||
attempt=1,
|
||||
stream=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is True
|
||||
assert advice.retry_after == 0.25
|
||||
|
||||
|
||||
def test_litellm_get_retry_advice_keeps_stateful_transport_failures_ambiguous() -> None:
|
||||
model = LitellmModel(model="test-model")
|
||||
error = APIConnectionError(
|
||||
message="connection error",
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/responses"),
|
||||
)
|
||||
|
||||
advice = model.get_retry_advice(
|
||||
ModelRetryAdviceRequest(
|
||||
error=error,
|
||||
attempt=1,
|
||||
stream=False,
|
||||
previous_response_id="resp_prev",
|
||||
)
|
||||
)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is True
|
||||
assert advice.replay_safety is None
|
||||
@@ -0,0 +1,697 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import pytest
|
||||
from openai.types.chat.chat_completion_chunk import (
|
||||
ChatCompletionChunk,
|
||||
Choice,
|
||||
ChoiceDelta,
|
||||
ChoiceDeltaToolCall,
|
||||
ChoiceDeltaToolCallFunction,
|
||||
)
|
||||
from openai.types.completion_usage import (
|
||||
CompletionTokensDetails,
|
||||
CompletionUsage,
|
||||
PromptTokensDetails,
|
||||
)
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseCompletedEvent,
|
||||
ResponseContentPartAddedEvent,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputRefusal,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
ResponseRefusalDeltaEvent,
|
||||
)
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.extensions.models.litellm_provider import LitellmProvider
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_text_content(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that `stream_response` emits the correct sequence of events when
|
||||
streaming a simple assistant message consisting of plain text content.
|
||||
We simulate two chunks of text returned from the chat completion stream.
|
||||
"""
|
||||
# Create two chunks that will be emitted by the fake stream.
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content="He"))],
|
||||
)
|
||||
# Mark last chunk with usage so stream_response knows this is final.
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content="llo"))],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=5,
|
||||
prompt_tokens=7,
|
||||
total_tokens=12,
|
||||
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=6),
|
||||
),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
# Patch _fetch_response to inject our fake stream
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
# `_fetch_response` is expected to return a Response skeleton and the async stream
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
# We expect a response.created, then a response.output_item.added, content part added,
|
||||
# two content delta events (for "He" and "llo"), a content part done, the assistant message
|
||||
# output_item.done, and finally response.completed.
|
||||
# There should be 8 events in total.
|
||||
assert len(output_events) == 8
|
||||
# First event indicates creation.
|
||||
assert output_events[0].type == "response.created"
|
||||
# The output item added and content part added events should mark the assistant message.
|
||||
assert output_events[1].type == "response.output_item.added"
|
||||
assert output_events[2].type == "response.content_part.added"
|
||||
# Two text delta events.
|
||||
assert output_events[3].type == "response.output_text.delta"
|
||||
assert output_events[3].delta == "He"
|
||||
assert output_events[4].type == "response.output_text.delta"
|
||||
assert output_events[4].delta == "llo"
|
||||
# After streaming, the content part and item should be marked done.
|
||||
assert output_events[5].type == "response.content_part.done"
|
||||
assert output_events[6].type == "response.output_item.done"
|
||||
# Last event indicates completion of the stream.
|
||||
assert output_events[7].type == "response.completed"
|
||||
# The completed response should have one output message with full text.
|
||||
completed_resp = output_events[7].response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
|
||||
assert completed_resp.output[0].content[0].text == "Hello"
|
||||
|
||||
assert completed_resp.usage, "usage should not be None"
|
||||
assert completed_resp.usage.input_tokens == 7
|
||||
assert completed_resp.usage.output_tokens == 5
|
||||
assert completed_resp.usage.total_tokens == 12
|
||||
assert completed_resp.usage.input_tokens_details.cached_tokens == 6
|
||||
assert completed_resp.usage.output_tokens_details.reasoning_tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_refusal_content(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that when the model streams a refusal string instead of normal content,
|
||||
`stream_response` emits the appropriate sequence of events including
|
||||
`response.refusal.delta` events for each chunk of the refusal message and
|
||||
constructs a completed assistant message with a `ResponseOutputRefusal` part.
|
||||
"""
|
||||
# Simulate refusal text coming in two pieces, like content but using the `refusal`
|
||||
# field on the delta rather than `content`.
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(refusal="No"))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(refusal="Thanks"))],
|
||||
usage=CompletionUsage(completion_tokens=2, prompt_tokens=2, total_tokens=4),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
# Expect sequence similar to text: created, output_item.added, content part added,
|
||||
# two refusal delta events, content part done, output_item.done, completed.
|
||||
assert len(output_events) == 8
|
||||
assert output_events[0].type == "response.created"
|
||||
assert output_events[1].type == "response.output_item.added"
|
||||
assert output_events[2].type == "response.content_part.added"
|
||||
assert output_events[3].type == "response.refusal.delta"
|
||||
assert output_events[3].delta == "No"
|
||||
assert output_events[4].type == "response.refusal.delta"
|
||||
assert output_events[4].delta == "Thanks"
|
||||
assert output_events[5].type == "response.content_part.done"
|
||||
assert output_events[6].type == "response.output_item.done"
|
||||
assert output_events[7].type == "response.completed"
|
||||
completed_resp = output_events[7].response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
refusal_part = completed_resp.output[0].content[0]
|
||||
assert isinstance(refusal_part, ResponseOutputRefusal)
|
||||
assert refusal_part.refusal == "NoThanks"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_tool_call(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that `stream_response` emits the correct sequence of events when
|
||||
the model is streaming a function/tool call instead of plain text.
|
||||
The function call will be split across two chunks.
|
||||
"""
|
||||
# Simulate a single tool call with complete function name in first chunk
|
||||
# and arguments split across chunks (reflecting real API behavior)
|
||||
tool_call_delta1 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
id="tool-id",
|
||||
function=ChoiceDeltaToolCallFunction(name="my_func", arguments="arg1"),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta2 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
id="tool-id",
|
||||
function=ChoiceDeltaToolCallFunction(name=None, arguments="arg2"),
|
||||
type="function",
|
||||
)
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
|
||||
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
# Sequence should be: response.created, then after loop we expect function call-related events:
|
||||
# one response.output_item.added for function call, a response.function_call_arguments.delta,
|
||||
# a response.output_item.done, and finally response.completed.
|
||||
assert output_events[0].type == "response.created"
|
||||
# The next three events are about the tool call.
|
||||
assert output_events[1].type == "response.output_item.added"
|
||||
# The added item should be a ResponseFunctionToolCall.
|
||||
added_fn = output_events[1].item
|
||||
assert isinstance(added_fn, ResponseFunctionToolCall)
|
||||
assert added_fn.name == "my_func" # Name should be complete from first chunk
|
||||
assert added_fn.arguments == "" # Arguments start empty
|
||||
assert output_events[2].type == "response.function_call_arguments.delta"
|
||||
assert output_events[2].delta == "arg1" # First argument chunk
|
||||
assert output_events[3].type == "response.function_call_arguments.delta"
|
||||
assert output_events[3].delta == "arg2" # Second argument chunk
|
||||
assert output_events[4].type == "response.output_item.done"
|
||||
assert output_events[5].type == "response.completed"
|
||||
# Final function call should have complete arguments
|
||||
final_fn = output_events[4].item
|
||||
assert isinstance(final_fn, ResponseFunctionToolCall)
|
||||
assert final_fn.name == "my_func"
|
||||
assert final_fn.arguments == "arg1arg2"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_real_time_function_call_arguments(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that LiteLLM `stream_response` also emits function call arguments in real-time
|
||||
as they are received, ensuring consistent behavior across model providers.
|
||||
"""
|
||||
# Simulate realistic chunks: name first, then arguments incrementally
|
||||
tool_call_delta1 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
id="litellm-call-456",
|
||||
function=ChoiceDeltaToolCallFunction(name="generate_code", arguments=""),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta2 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
function=ChoiceDeltaToolCallFunction(arguments='{"language": "'),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta3 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
function=ChoiceDeltaToolCallFunction(arguments='python", "task": "'),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta4 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
function=ChoiceDeltaToolCallFunction(arguments='hello world"}'),
|
||||
type="function",
|
||||
)
|
||||
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
|
||||
)
|
||||
chunk3 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta3]))],
|
||||
)
|
||||
chunk4 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta4]))],
|
||||
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2, chunk3, chunk4):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# Extract events by type
|
||||
function_args_delta_events = [
|
||||
e for e in output_events if e.type == "response.function_call_arguments.delta"
|
||||
]
|
||||
output_item_added_events = [e for e in output_events if e.type == "response.output_item.added"]
|
||||
|
||||
# Verify we got real-time streaming (3 argument delta events)
|
||||
assert len(function_args_delta_events) == 3
|
||||
assert len(output_item_added_events) == 1
|
||||
|
||||
# Verify the deltas were streamed correctly
|
||||
expected_deltas = ['{"language": "', 'python", "task": "', 'hello world"}']
|
||||
for i, delta_event in enumerate(function_args_delta_events):
|
||||
assert delta_event.delta == expected_deltas[i]
|
||||
|
||||
# Verify function call metadata
|
||||
added_event = output_item_added_events[0]
|
||||
assert isinstance(added_event.item, ResponseFunctionToolCall)
|
||||
assert added_event.item.name == "generate_code"
|
||||
assert added_event.item.call_id == "litellm-call-456"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_synthesizes_refusal_on_content_filter(monkeypatch) -> None:
|
||||
"""A stream that terminates with finish_reason == "content_filter" and no
|
||||
emitted content (as Anthropic-on-Bedrock does via LiteLLM) must synthesize a
|
||||
ResponseOutputRefusal so the completed response carries an explicit refusal
|
||||
rather than an empty assistant turn.
|
||||
|
||||
Mirrors the real Bedrock chunk shape: an empty-string content delta followed
|
||||
by a terminal content_filter chunk with no content. The empty "" delta must
|
||||
not open a text content part; the synthesized refusal must be the only
|
||||
content part, at the same index in the stream and in response.completed.
|
||||
"""
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(role="assistant", content=""))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=0,
|
||||
prompt_tokens=7,
|
||||
total_tokens=7,
|
||||
),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
types = [e.type for e in output_events]
|
||||
# Coherent refusal sequence: the message + refusal part are opened, a refusal
|
||||
# delta is emitted, and the parts/message are closed before completion.
|
||||
assert "response.output_item.added" in types
|
||||
assert "response.content_part.added" in types
|
||||
assert "response.refusal.delta" in types
|
||||
assert types[-1] == "response.completed"
|
||||
assert "response.output_item.done" in types
|
||||
|
||||
# The refusal delta carries a non-empty message.
|
||||
refusal_deltas = [e for e in output_events if e.type == "response.refusal.delta"]
|
||||
assert refusal_deltas and refusal_deltas[0].delta
|
||||
|
||||
# Event coherence: the assistant message is announced exactly once, and every
|
||||
# content part that is opened is also closed.
|
||||
assert types.count("response.output_item.added") == 1
|
||||
assert types.count("response.content_part.added") == types.count("response.content_part.done")
|
||||
|
||||
# The empty "" content delta must NOT open a text content part: no text part
|
||||
# events and no output_text.delta are emitted at all.
|
||||
assert "response.output_text.delta" not in types
|
||||
added_parts = [e for e in output_events if e.type == "response.content_part.added"]
|
||||
assert len(added_parts) == 1
|
||||
assert isinstance(added_parts[0].part, ResponseOutputRefusal)
|
||||
|
||||
# The completed response contains exactly one content part: the refusal.
|
||||
completed_event = output_events[-1]
|
||||
assert isinstance(completed_event, ResponseCompletedEvent)
|
||||
completed_resp = completed_event.response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
assert len(completed_resp.output[0].content) == 1
|
||||
refusal_part = completed_resp.output[0].content[0]
|
||||
assert isinstance(refusal_part, ResponseOutputRefusal)
|
||||
assert refusal_part.refusal
|
||||
|
||||
# The refusal's streamed content_index matches its position in the completed
|
||||
# response (0), so raw-event replay and the final response stay aligned.
|
||||
assert added_parts[0].content_index == 0
|
||||
assert refusal_deltas[0].content_index == 0
|
||||
done_parts = [e for e in output_events if e.type == "response.content_part.done"]
|
||||
assert len(done_parts) == 1
|
||||
assert done_parts[0].content_index == 0
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_content_filter_does_not_clobber_text(monkeypatch) -> None:
|
||||
"""A content_filter finish_reason that arrives AFTER real text was streamed
|
||||
must not synthesize a refusal (the text stands)."""
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content="answer"))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
|
||||
usage=CompletionUsage(completion_tokens=1, prompt_tokens=7, total_tokens=8),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
assert "response.refusal.delta" not in [e.type for e in output_events]
|
||||
completed_event = output_events[-1]
|
||||
assert isinstance(completed_event, ResponseCompletedEvent)
|
||||
completed_resp = completed_event.response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
|
||||
assert completed_resp.output[0].content[0].text == "answer"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_content_filter_refusal_after_reasoning(monkeypatch) -> None:
|
||||
"""A content_filter turn preceded by reasoning must still place the
|
||||
synthesized refusal at content_index 0 of the assistant message. Reasoning
|
||||
is a *separate* output item (it shifts the message's output_index, not its
|
||||
content_index), so the refusal — the sole content part — stays at
|
||||
content_index 0 in both the stream and response.completed."""
|
||||
reasoning_delta = ChoiceDelta(role="assistant", content=None)
|
||||
# reasoning_content is a provider extra field the handler reads via hasattr.
|
||||
reasoning_delta.reasoning_content = "thinking..." # type: ignore[attr-defined]
|
||||
chunk_reasoning = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=reasoning_delta)],
|
||||
)
|
||||
chunk_empty = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content=""))],
|
||||
)
|
||||
chunk_filter = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
|
||||
usage=CompletionUsage(completion_tokens=0, prompt_tokens=7, total_tokens=7),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk_reasoning, chunk_empty, chunk_filter):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
# A reasoning item was produced as a separate output item.
|
||||
completed_event = output_events[-1]
|
||||
assert isinstance(completed_event, ResponseCompletedEvent)
|
||||
completed_resp = completed_event.response
|
||||
assert isinstance(completed_resp.output[0], ResponseReasoningItem)
|
||||
assistant_msg = completed_resp.output[1]
|
||||
assert isinstance(assistant_msg, ResponseOutputMessage)
|
||||
# The refusal is the sole content part of the assistant message, at index 0.
|
||||
assert len(assistant_msg.content) == 1
|
||||
assert isinstance(assistant_msg.content[0], ResponseOutputRefusal)
|
||||
|
||||
# The assistant message's output_index is 1 (after the reasoning item), and
|
||||
# every refusal event uses that output_index and content_index 0 — matching
|
||||
# the refusal's position in response.completed.
|
||||
added = [
|
||||
e
|
||||
for e in output_events
|
||||
if isinstance(e, ResponseContentPartAddedEvent)
|
||||
and isinstance(e.part, ResponseOutputRefusal)
|
||||
]
|
||||
deltas = [e for e in output_events if isinstance(e, ResponseRefusalDeltaEvent)]
|
||||
assert len(added) == 1
|
||||
assert added[0].content_index == 0
|
||||
assert added[0].output_index == 1
|
||||
assert deltas and all(d.content_index == 0 and d.output_index == 1 for d in deltas)
|
||||
# The empty "" delta still opens no text part.
|
||||
assert "response.output_text.delta" not in [e.type for e in output_events]
|
||||
@@ -0,0 +1,89 @@
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
async def _get_response(monkeypatch, *, finish_reason, content):
|
||||
"""Drive get_response against a mocked litellm completion and return the items."""
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
msg = Message(role="assistant", content=content)
|
||||
choice = Choices(index=0, finish_reason=finish_reason, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
model = LitellmModel(model="test-model")
|
||||
return await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_filter_finish_reason_surfaces_refusal(monkeypatch):
|
||||
"""A content-filter block (empty message, finish_reason=content_filter) must
|
||||
become an explicit ResponseOutputRefusal, not zero output items.
|
||||
|
||||
Some providers (e.g. Anthropic on Amazon Bedrock) signal a safety block only
|
||||
via ``finish_reason == "content_filter"`` with an empty message and no
|
||||
``refusal`` field; without this the turn is indistinguishable from an empty
|
||||
response and drives agent loops into fruitless retries.
|
||||
"""
|
||||
resp = await _get_response(monkeypatch, finish_reason="content_filter", content="")
|
||||
|
||||
refusals = [
|
||||
content
|
||||
for item in resp.output
|
||||
if isinstance(item, ResponseOutputMessage)
|
||||
for content in item.content
|
||||
if isinstance(content, ResponseOutputRefusal)
|
||||
]
|
||||
assert refusals, f"expected a refusal item, got: {resp.output}"
|
||||
assert refusals[0].refusal # non-empty message
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_filter_does_not_clobber_real_content(monkeypatch):
|
||||
"""A content_filter finish_reason that still carries text is left alone — we
|
||||
only synthesize a refusal when the message is genuinely empty."""
|
||||
resp = await _get_response(
|
||||
monkeypatch, finish_reason="content_filter", content="here is the answer"
|
||||
)
|
||||
|
||||
refusals = [
|
||||
content
|
||||
for item in resp.output
|
||||
if isinstance(item, ResponseOutputMessage)
|
||||
for content in item.content
|
||||
if isinstance(content, ResponseOutputRefusal)
|
||||
]
|
||||
assert not refusals, "should not synthesize a refusal when content is present"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_normal_stop_is_unaffected(monkeypatch):
|
||||
"""A normal completion is unchanged — no spurious refusal."""
|
||||
resp = await _get_response(monkeypatch, finish_reason="stop", content="all good")
|
||||
|
||||
refusals = [
|
||||
content
|
||||
for item in resp.output
|
||||
if isinstance(item, ResponseOutputMessage)
|
||||
for content in item.content
|
||||
if isinstance(content, ResponseOutputRefusal)
|
||||
]
|
||||
assert not refusals
|
||||
@@ -0,0 +1,265 @@
|
||||
import logging
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_is_forwarded(monkeypatch):
|
||||
"""
|
||||
Forward `extra_body` via LiteLLM's dedicated kwarg.
|
||||
|
||||
This ensures that provider-specific request fields stay nested under `extra_body`
|
||||
so LiteLLM can merge them into the upstream request body itself.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
temperature=0.1, extra_body={"cached_content": "some_cache", "foo": 123}
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["extra_body"] == {"cached_content": "some_cache", "foo": 123}
|
||||
assert "cached_content" not in captured
|
||||
assert "foo" not in captured
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_reasoning_effort_is_promoted(monkeypatch):
|
||||
"""
|
||||
Ensure reasoning_effort from extra_body is promoted to the top-level parameter.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
|
||||
settings = ModelSettings(
|
||||
extra_body={"reasoning_effort": "none", "cached_content": "some_cache"}
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "none"
|
||||
assert captured["extra_body"] == {"cached_content": "some_cache"}
|
||||
assert settings.extra_body == {"reasoning_effort": "none", "cached_content": "some_cache"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_effort_prefers_model_settings(monkeypatch):
|
||||
"""
|
||||
Verify explicit ModelSettings.reasoning takes precedence over extra_body entries.
|
||||
"""
|
||||
from openai.types.shared import Reasoning
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
reasoning=Reasoning(effort="low"),
|
||||
extra_body={"reasoning_effort": "high"},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# reasoning_effort is string when no summary is provided (backward compatible)
|
||||
assert captured["reasoning_effort"] == "low"
|
||||
assert "extra_body" not in captured
|
||||
assert settings.extra_body == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_reasoning_effort_overrides_extra_args(monkeypatch):
|
||||
"""
|
||||
Ensure extra_body reasoning_effort wins over extra_args when both are provided.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
|
||||
settings = ModelSettings(
|
||||
extra_body={"reasoning_effort": "none"},
|
||||
extra_args={"reasoning_effort": "low", "custom_param": "custom"},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "none"
|
||||
assert captured["custom_param"] == "custom"
|
||||
assert "extra_body" not in captured
|
||||
assert settings.extra_args == {"reasoning_effort": "low", "custom_param": "custom"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_metadata_stays_nested(monkeypatch):
|
||||
"""
|
||||
Keep extra_body metadata nested even when top-level metadata is also set.
|
||||
|
||||
LiteLLM resolves top-level metadata and extra_body separately. Flattening the nested
|
||||
metadata dict loses the caller's intended request shape for OpenAI-compatible proxies.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
metadata={"sdk": "agents"},
|
||||
extra_body={
|
||||
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
|
||||
"cached_content": "some_cache",
|
||||
},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["metadata"] == {"sdk": "agents"}
|
||||
assert captured["extra_body"] == {
|
||||
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
|
||||
"cached_content": "some_cache",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"openai/gpt-5-mini",
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
"gemini/gemini-2.5-pro",
|
||||
],
|
||||
)
|
||||
async def test_reasoning_summary_uses_scalar_effort_and_warns(
|
||||
monkeypatch, caplog: pytest.LogCaptureFixture, model_name: str
|
||||
):
|
||||
"""
|
||||
Ensure reasoning.summary does not change the LiteLLM chat-completions argument shape.
|
||||
|
||||
LitellmModel should continue to pass a scalar reasoning_effort value and warn that summary
|
||||
is ignored on this path, regardless of the provider encoded in the model string.
|
||||
"""
|
||||
from openai.types.shared import Reasoning
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
reasoning=Reasoning(effort="medium", summary="auto"),
|
||||
)
|
||||
model = LitellmModel(model=model_name)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "medium"
|
||||
warning_messages = [
|
||||
record.message
|
||||
for record in caplog.records
|
||||
if "does not forward Reasoning.summary" in record.message
|
||||
]
|
||||
assert len(warning_messages) == 1
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("litellm")
|
||||
|
||||
|
||||
def test_litellm_logging_patch_env_var_controls_application(monkeypatch):
|
||||
"""Assert the serializer patch only applies when the env var is enabled."""
|
||||
litellm_logging = importlib.import_module("litellm.litellm_core_utils.litellm_logging")
|
||||
litellm_model = importlib.import_module("agents.extensions.models.litellm_model")
|
||||
|
||||
monkeypatch.delenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", raising=False)
|
||||
litellm_logging = importlib.reload(litellm_logging)
|
||||
importlib.reload(litellm_model)
|
||||
|
||||
assert hasattr(
|
||||
litellm_logging,
|
||||
"_extract_response_obj_and_hidden_params",
|
||||
), "LiteLLM removed _extract_response_obj_and_hidden_params; revisit warning patch."
|
||||
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is False
|
||||
|
||||
monkeypatch.setenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", "true")
|
||||
litellm_logging = importlib.reload(litellm_logging)
|
||||
importlib.reload(litellm_model)
|
||||
|
||||
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is True
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import ModelSettings, ModelTracing, __version__
|
||||
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
|
||||
async def test_user_agent_header_litellm(override_ua: str | None, monkeypatch):
|
||||
called_kwargs: dict[str, Any] = {}
|
||||
expected_ua = override_ua or f"Agents/Python {__version__}"
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types as pytypes
|
||||
|
||||
litellm_fake: Any = pytypes.ModuleType("litellm")
|
||||
|
||||
class DummyMessage:
|
||||
role = "assistant"
|
||||
content = "Hello"
|
||||
tool_calls: list[Any] | None = None
|
||||
|
||||
def get(self, _key, _default=None):
|
||||
return None
|
||||
|
||||
def model_dump(self):
|
||||
return {"role": self.role, "content": self.content}
|
||||
|
||||
class Choices: # noqa: N801 - mimic litellm naming
|
||||
def __init__(self):
|
||||
self.message = DummyMessage()
|
||||
|
||||
class DummyModelResponse:
|
||||
def __init__(self):
|
||||
self.choices = [Choices()]
|
||||
|
||||
async def acompletion(**kwargs):
|
||||
nonlocal called_kwargs
|
||||
called_kwargs = kwargs
|
||||
return DummyModelResponse()
|
||||
|
||||
utils_ns = pytypes.SimpleNamespace()
|
||||
utils_ns.Choices = Choices
|
||||
utils_ns.ModelResponse = DummyModelResponse
|
||||
|
||||
litellm_types = pytypes.SimpleNamespace(
|
||||
utils=utils_ns,
|
||||
llms=pytypes.SimpleNamespace(openai=pytypes.SimpleNamespace(ChatCompletionAnnotation=dict)),
|
||||
)
|
||||
litellm_fake.acompletion = acompletion
|
||||
litellm_fake.types = litellm_types
|
||||
|
||||
monkeypatch.setitem(sys.modules, "litellm", litellm_fake)
|
||||
|
||||
litellm_mod = importlib.import_module("agents.extensions.models.litellm_model")
|
||||
monkeypatch.setattr(litellm_mod, "litellm", litellm_fake, raising=True)
|
||||
LitellmModel = litellm_mod.LitellmModel
|
||||
|
||||
model = LitellmModel(model="gpt-4")
|
||||
|
||||
if override_ua is not None:
|
||||
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
|
||||
else:
|
||||
token = None
|
||||
try:
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
finally:
|
||||
if token is not None:
|
||||
HEADERS_OVERRIDE.reset(token)
|
||||
|
||||
assert "extra_headers" in called_kwargs
|
||||
assert called_kwargs["extra_headers"]["User-Agent"] == expected_ua
|
||||
@@ -0,0 +1,209 @@
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
MultiProvider,
|
||||
OpenAIResponsesModel,
|
||||
OpenAIResponsesWSModel,
|
||||
RunConfig,
|
||||
UserError,
|
||||
)
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.models.multi_provider import MultiProviderMap
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.run_internal.run_loop import get_model
|
||||
|
||||
|
||||
def test_no_prefix_is_openai():
|
||||
agent = Agent(model="gpt-4o", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, OpenAIResponsesModel)
|
||||
|
||||
|
||||
def test_openai_prefix_is_openai():
|
||||
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, OpenAIResponsesModel)
|
||||
|
||||
|
||||
def test_litellm_prefix_is_litellm():
|
||||
agent = Agent(model="litellm/foo/bar", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, LitellmModel)
|
||||
|
||||
|
||||
def test_any_llm_prefix_uses_any_llm_provider(monkeypatch):
|
||||
import sys
|
||||
import types as pytypes
|
||||
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeAnyLLMModel:
|
||||
pass
|
||||
|
||||
class FakeAnyLLMProvider:
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return FakeAnyLLMModel()
|
||||
|
||||
fake_module: Any = pytypes.ModuleType("agents.extensions.models.any_llm_provider")
|
||||
fake_module.AnyLLMProvider = FakeAnyLLMProvider
|
||||
monkeypatch.setitem(sys.modules, "agents.extensions.models.any_llm_provider", fake_module)
|
||||
|
||||
agent = Agent(model="any-llm/openrouter/openai/gpt-5.4-mini", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, FakeAnyLLMModel)
|
||||
assert captured_model["value"] == "openrouter/openai/gpt-5.4-mini"
|
||||
|
||||
|
||||
def test_no_prefix_can_use_openai_responses_websocket():
|
||||
agent = Agent(model="gpt-4o", instructions="", name="test")
|
||||
model = get_model(
|
||||
agent,
|
||||
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
|
||||
)
|
||||
assert isinstance(model, OpenAIResponsesWSModel)
|
||||
|
||||
|
||||
def test_openai_prefix_can_use_openai_responses_websocket():
|
||||
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
|
||||
model = get_model(
|
||||
agent,
|
||||
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
|
||||
)
|
||||
assert isinstance(model, OpenAIResponsesWSModel)
|
||||
|
||||
|
||||
def test_multi_provider_passes_websocket_base_url_to_openai_provider(monkeypatch):
|
||||
captured_kwargs = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
|
||||
def get_model(self, model_name):
|
||||
raise AssertionError("This test only verifies constructor passthrough.")
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
MultiProvider(openai_websocket_base_url="wss://proxy.example.test/v1")
|
||||
assert captured_kwargs["websocket_base_url"] == "wss://proxy.example.test/v1"
|
||||
|
||||
|
||||
def test_multi_provider_forwards_openai_buffer_streamed_tool_calls_to_chat_model():
|
||||
provider = MultiProvider(
|
||||
openai_client=cast(Any, object()),
|
||||
openai_use_responses=False,
|
||||
openai_buffer_streamed_tool_calls=True,
|
||||
)
|
||||
|
||||
model = provider.get_model("gpt-4o")
|
||||
|
||||
assert isinstance(model, OpenAIChatCompletionsModel)
|
||||
assert model._buffer_streamed_tool_calls is True
|
||||
|
||||
|
||||
def test_openai_prefix_defaults_to_alias_mode(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider = MultiProvider()
|
||||
provider.get_model("openai/gpt-4o")
|
||||
assert captured_model["value"] == "gpt-4o"
|
||||
|
||||
|
||||
def test_openai_prefix_can_be_preserved_as_literal_model_id(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider = MultiProvider(openai_prefix_mode="model_id")
|
||||
provider.get_model("openai/gpt-4o")
|
||||
assert captured_model["value"] == "openai/gpt-4o"
|
||||
|
||||
|
||||
def test_unknown_prefix_defaults_to_error():
|
||||
provider = MultiProvider()
|
||||
|
||||
with pytest.raises(UserError, match="Unknown prefix: openrouter"):
|
||||
provider.get_model("openrouter/openai/gpt-4o")
|
||||
|
||||
|
||||
def test_unknown_prefix_can_be_preserved_for_openai_compatible_model_ids(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
captured_result: dict[str, Any] = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
fake_model = object()
|
||||
captured_result["value"] = fake_model
|
||||
return fake_model
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider = MultiProvider(unknown_prefix_mode="model_id")
|
||||
result = provider.get_model("openrouter/openai/gpt-4o")
|
||||
assert result is captured_result["value"]
|
||||
assert captured_model["value"] == "openrouter/openai/gpt-4o"
|
||||
|
||||
|
||||
def test_provider_map_entries_override_openai_prefix_mode(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeCustomProvider:
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return object()
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
raise AssertionError("Expected the explicit provider_map entry to win.")
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider_map = MultiProviderMap()
|
||||
provider_map.add_provider("openai", cast(Any, FakeCustomProvider()))
|
||||
|
||||
provider = MultiProvider(
|
||||
provider_map=provider_map,
|
||||
openai_prefix_mode="model_id",
|
||||
)
|
||||
provider.get_model("openai/gpt-4o")
|
||||
assert captured_model["value"] == "gpt-4o"
|
||||
|
||||
|
||||
def test_multi_provider_rejects_invalid_prefix_modes():
|
||||
bad_openai_prefix_mode: Any = "invalid"
|
||||
bad_unknown_prefix_mode: Any = "invalid"
|
||||
|
||||
with pytest.raises(UserError, match="openai_prefix_mode"):
|
||||
MultiProvider(openai_prefix_mode=bad_openai_prefix_mode)
|
||||
|
||||
with pytest.raises(UserError, match="unknown_prefix_mode"):
|
||||
MultiProvider(unknown_prefix_mode=bad_unknown_prefix_mode)
|
||||
@@ -0,0 +1,186 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Iterator
|
||||
from typing import Any, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import omit
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
|
||||
from agents import (
|
||||
ModelSettings,
|
||||
ModelTracing,
|
||||
OpenAIChatCompletionsModel,
|
||||
OpenAIResponsesModel,
|
||||
generation_span,
|
||||
)
|
||||
from agents.models import (
|
||||
openai_chatcompletions as chat_module,
|
||||
openai_responses as responses_module,
|
||||
)
|
||||
|
||||
|
||||
class _SingleUseIterable:
|
||||
"""Helper iterable that raises if iterated more than once."""
|
||||
|
||||
def __init__(self, values: list[object]) -> None:
|
||||
self._values = list(values)
|
||||
self.iterations = 0
|
||||
|
||||
def __iter__(self) -> Iterator[object]:
|
||||
if self.iterations:
|
||||
raise RuntimeError("Iterable should have been materialized exactly once.")
|
||||
self.iterations += 1
|
||||
yield from self._values
|
||||
|
||||
|
||||
def _force_materialization(value: object) -> None:
|
||||
if isinstance(value, dict):
|
||||
for nested in value.values():
|
||||
_force_materialization(nested)
|
||||
elif isinstance(value, list):
|
||||
for nested in value:
|
||||
_force_materialization(nested)
|
||||
elif isinstance(value, Iterable) and not isinstance(value, str | bytes | bytearray):
|
||||
list(value)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completions_materializes_iterator_payload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
message_iter = _SingleUseIterable([{"type": "text", "text": "hi"}])
|
||||
tool_iter = _SingleUseIterable([{"type": "string"}])
|
||||
|
||||
chat_converter = cast(Any, chat_module).Converter
|
||||
|
||||
monkeypatch.setattr(
|
||||
chat_converter,
|
||||
"items_to_messages",
|
||||
classmethod(lambda _cls, _input, **kwargs: [{"role": "user", "content": message_iter}]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chat_converter,
|
||||
"tool_to_openai",
|
||||
classmethod(
|
||||
lambda _cls, _tool: {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "dummy",
|
||||
"parameters": {"properties": tool_iter},
|
||||
},
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
captured_kwargs: dict[str, Any] = {}
|
||||
|
||||
class DummyCompletions:
|
||||
async def create(self, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
_force_materialization(kwargs["messages"])
|
||||
if kwargs["tools"] is not omit:
|
||||
_force_materialization(kwargs["tools"])
|
||||
return ChatCompletion(
|
||||
id="dummy-id",
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=None,
|
||||
)
|
||||
|
||||
class DummyClient:
|
||||
def __init__(self) -> None:
|
||||
self.chat = type("_Chat", (), {"completions": DummyCompletions()})()
|
||||
self.base_url = httpx.URL("http://example.test")
|
||||
|
||||
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=DummyClient()) # type: ignore[arg-type]
|
||||
|
||||
with generation_span(disabled=True) as span:
|
||||
await cast(Any, model)._fetch_response(
|
||||
system_instructions=None,
|
||||
input="ignored",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[object()],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
span=span,
|
||||
tracing=ModelTracing.DISABLED,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
assert message_iter.iterations == 1
|
||||
assert tool_iter.iterations == 1
|
||||
assert isinstance(captured_kwargs["messages"][0]["content"], list)
|
||||
assert isinstance(captured_kwargs["tools"][0]["function"]["parameters"]["properties"], list)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_materializes_iterator_payload(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
input_iter = _SingleUseIterable([{"type": "input_text", "text": "hello"}])
|
||||
tool_iter = _SingleUseIterable([{"type": "string"}])
|
||||
|
||||
responses_item_helpers = cast(Any, responses_module).ItemHelpers
|
||||
responses_converter = cast(Any, responses_module).Converter
|
||||
|
||||
monkeypatch.setattr(
|
||||
responses_item_helpers,
|
||||
"input_to_new_input_list",
|
||||
classmethod(lambda _cls, _input: [{"role": "user", "content": input_iter}]),
|
||||
)
|
||||
|
||||
converted_tools = responses_module.ConvertedTools(
|
||||
tools=[
|
||||
cast(
|
||||
Any,
|
||||
{
|
||||
"type": "function",
|
||||
"name": "dummy",
|
||||
"parameters": {"properties": tool_iter},
|
||||
},
|
||||
)
|
||||
],
|
||||
includes=[],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
responses_converter,
|
||||
"convert_tools",
|
||||
classmethod(lambda _cls, _tools, _handoffs, **_kwargs: converted_tools),
|
||||
)
|
||||
|
||||
captured_kwargs: dict[str, Any] = {}
|
||||
|
||||
class DummyResponses:
|
||||
async def create(self, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
_force_materialization(kwargs["input"])
|
||||
_force_materialization(kwargs["tools"])
|
||||
return object()
|
||||
|
||||
class DummyClient:
|
||||
def __init__(self) -> None:
|
||||
self.responses = DummyResponses()
|
||||
|
||||
model = OpenAIResponsesModel(model="gpt-4.1", openai_client=DummyClient()) # type: ignore[arg-type]
|
||||
|
||||
await cast(Any, model)._fetch_response(
|
||||
system_instructions=None,
|
||||
input="ignored",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
stream=False,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert input_iter.iterations == 1
|
||||
assert tool_iter.iterations == 1
|
||||
assert isinstance(captured_kwargs["input"][0]["content"], list)
|
||||
assert isinstance(captured_kwargs["tools"][0]["parameters"]["properties"], list)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,756 @@
|
||||
# Copyright (c) OpenAI
|
||||
#
|
||||
# Licensed under the MIT License.
|
||||
# See LICENSE file in the project root for full license information.
|
||||
|
||||
"""
|
||||
Unit tests for the internal `Converter` class defined in
|
||||
`agents.models.openai_chatcompletions`. The converter is responsible for
|
||||
translating between internal "item" structures (e.g., `ResponseOutputMessage`
|
||||
and related types from `openai.types.responses`) and the ChatCompletion message
|
||||
structures defined by the OpenAI client library.
|
||||
|
||||
These tests exercise both conversion directions:
|
||||
|
||||
- `Converter.message_to_output_items` turns a `ChatCompletionMessage` (as
|
||||
returned by the OpenAI API) into a list of `ResponseOutputItem` instances.
|
||||
|
||||
- `Converter.items_to_messages` takes in either a simple string prompt, or a
|
||||
list of input/output items such as `ResponseOutputMessage` and
|
||||
`ResponseFunctionToolCallParam` dicts, and constructs a list of
|
||||
`ChatCompletionMessageParam` dicts suitable for sending back to the API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import pytest
|
||||
from openai import omit
|
||||
from openai.types.chat import ChatCompletionMessage, ChatCompletionMessageFunctionToolCall
|
||||
from openai.types.chat.chat_completion_message_custom_tool_call import (
|
||||
ChatCompletionMessageCustomToolCall,
|
||||
Custom,
|
||||
)
|
||||
from openai.types.chat.chat_completion_message_tool_call import Function
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseFunctionToolCallParam,
|
||||
ResponseInputAudioParam,
|
||||
ResponseInputTextParam,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputRefusal,
|
||||
ResponseOutputText,
|
||||
)
|
||||
from openai.types.responses.response_input_item_param import FunctionCallOutput
|
||||
|
||||
from agents.agent_output import AgentOutputSchema
|
||||
from agents.exceptions import UserError
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
from agents.models.fake_id import FAKE_RESPONSES_ID
|
||||
|
||||
|
||||
def test_message_to_output_items_with_text_only():
|
||||
"""
|
||||
Make sure a simple ChatCompletionMessage with string content is converted
|
||||
into a single ResponseOutputMessage containing one ResponseOutputText.
|
||||
"""
|
||||
msg = ChatCompletionMessage(role="assistant", content="Hello")
|
||||
items = Converter.message_to_output_items(msg)
|
||||
# Expect exactly one output item (the message)
|
||||
assert len(items) == 1
|
||||
message_item = cast(ResponseOutputMessage, items[0])
|
||||
assert message_item.id == FAKE_RESPONSES_ID
|
||||
assert message_item.role == "assistant"
|
||||
assert message_item.type == "message"
|
||||
assert message_item.status == "completed"
|
||||
# Message content should have exactly one text part with the same text.
|
||||
assert len(message_item.content) == 1
|
||||
text_part = cast(ResponseOutputText, message_item.content[0])
|
||||
assert text_part.type == "output_text"
|
||||
assert text_part.text == "Hello"
|
||||
|
||||
|
||||
def test_message_to_output_items_with_refusal():
|
||||
"""
|
||||
Make sure a message with a refusal string produces a ResponseOutputMessage
|
||||
with a ResponseOutputRefusal content part.
|
||||
"""
|
||||
msg = ChatCompletionMessage(role="assistant", refusal="I'm sorry")
|
||||
items = Converter.message_to_output_items(msg)
|
||||
assert len(items) == 1
|
||||
message_item = cast(ResponseOutputMessage, items[0])
|
||||
assert len(message_item.content) == 1
|
||||
refusal_part = cast(ResponseOutputRefusal, message_item.content[0])
|
||||
assert refusal_part.type == "refusal"
|
||||
assert refusal_part.refusal == "I'm sorry"
|
||||
|
||||
|
||||
def test_message_to_output_items_with_tool_call():
|
||||
"""
|
||||
If the ChatCompletionMessage contains one or more tool_calls, they should
|
||||
be reflected as separate `ResponseFunctionToolCall` items appended after
|
||||
the message item.
|
||||
"""
|
||||
tool_call = ChatCompletionMessageFunctionToolCall(
|
||||
id="tool1",
|
||||
type="function",
|
||||
function=Function(name="myfn", arguments='{"x":1}'),
|
||||
)
|
||||
msg = ChatCompletionMessage(role="assistant", content="Hi", tool_calls=[tool_call])
|
||||
items = Converter.message_to_output_items(msg)
|
||||
# Should produce a message item followed by one function tool call item
|
||||
assert len(items) == 2
|
||||
message_item = cast(ResponseOutputMessage, items[0])
|
||||
assert isinstance(message_item, ResponseOutputMessage)
|
||||
fn_call_item = cast(ResponseFunctionToolCall, items[1])
|
||||
assert fn_call_item.id == FAKE_RESPONSES_ID
|
||||
assert fn_call_item.call_id == tool_call.id
|
||||
assert fn_call_item.name == tool_call.function.name
|
||||
assert fn_call_item.arguments == tool_call.function.arguments
|
||||
assert fn_call_item.type == "function_call"
|
||||
|
||||
|
||||
def test_message_to_output_items_with_custom_tool_call_keeps_default_compatibility():
|
||||
"""Custom tool calls should keep the default Chat Completions behavior."""
|
||||
tool_call = ChatCompletionMessageCustomToolCall(
|
||||
id="tool1",
|
||||
type="custom",
|
||||
custom=Custom(name="raw_tool", input="payload"),
|
||||
)
|
||||
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
|
||||
|
||||
assert Converter.message_to_output_items(msg) == []
|
||||
|
||||
|
||||
def test_message_to_output_items_with_custom_tool_call_raises_in_strict_mode():
|
||||
"""Strict validation should fail explicitly instead of dropping custom tool calls."""
|
||||
tool_call = ChatCompletionMessageCustomToolCall(
|
||||
id="tool1",
|
||||
type="custom",
|
||||
custom=Custom(name="raw_tool", input="payload"),
|
||||
)
|
||||
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
|
||||
|
||||
with pytest.raises(UserError, match="Custom tool calls are not supported"):
|
||||
Converter.message_to_output_items(msg, strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_message_to_output_items_with_mixed_custom_tool_call_raises_in_strict_mode():
|
||||
"""Strict validation should not partially hide an unsupported custom tool call."""
|
||||
function_tool_call = ChatCompletionMessageFunctionToolCall(
|
||||
id="function-tool",
|
||||
type="function",
|
||||
function=Function(name="myfn", arguments='{"x":1}'),
|
||||
)
|
||||
custom_tool_call = ChatCompletionMessageCustomToolCall(
|
||||
id="custom-tool",
|
||||
type="custom",
|
||||
custom=Custom(name="raw_tool", input="payload"),
|
||||
)
|
||||
msg = ChatCompletionMessage(
|
||||
role="assistant",
|
||||
tool_calls=[function_tool_call, custom_tool_call],
|
||||
)
|
||||
|
||||
with pytest.raises(UserError, match="Custom tool calls are not supported"):
|
||||
Converter.message_to_output_items(msg, strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_items_to_messages_with_string_user_content():
|
||||
"""
|
||||
A simple string as the items argument should be converted into a user
|
||||
message param dict with the same content.
|
||||
"""
|
||||
result = Converter.items_to_messages("Ask me anything")
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 1
|
||||
msg = result[0]
|
||||
assert msg["role"] == "user"
|
||||
assert msg["content"] == "Ask me anything"
|
||||
|
||||
|
||||
def test_items_to_messages_with_easy_input_message():
|
||||
"""
|
||||
Given an easy input message dict (just role/content), the converter should
|
||||
produce the appropriate ChatCompletionMessageParam with the same content.
|
||||
"""
|
||||
items: list[TResponseInputItem] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "How are you?",
|
||||
}
|
||||
]
|
||||
messages = Converter.items_to_messages(items)
|
||||
assert len(messages) == 1
|
||||
out = messages[0]
|
||||
assert out["role"] == "user"
|
||||
# For simple string inputs, the converter returns the content as a bare string
|
||||
assert out["content"] == "How are you?"
|
||||
|
||||
|
||||
def test_items_to_messages_accepts_raw_chat_completions_user_content_parts():
|
||||
"""
|
||||
Raw Chat Completions content parts should be accepted as aliases for the SDK's
|
||||
canonical input content shapes.
|
||||
"""
|
||||
items: list[TResponseInputItem] = [
|
||||
# Cast the fixture because mypy cannot infer this raw chat-style dict as a specific
|
||||
# member of the TResponseInputItem TypedDict union on its own.
|
||||
cast(
|
||||
TResponseInputItem,
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.png",
|
||||
"detail": "high",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(items)
|
||||
|
||||
assert len(messages) == 1
|
||||
message = messages[0]
|
||||
assert message["role"] == "user"
|
||||
assert message["content"] == [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.png",
|
||||
"detail": "high",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_items_to_messages_with_output_message_and_function_call():
|
||||
"""
|
||||
Given a sequence of one ResponseOutputMessageParam followed by a
|
||||
ResponseFunctionToolCallParam, the converter should produce a single
|
||||
ChatCompletionAssistantMessageParam that includes both the assistant's
|
||||
textual content and a populated `tool_calls` reflecting the function call.
|
||||
"""
|
||||
# Construct output message param dict with two content parts.
|
||||
output_text: ResponseOutputText = ResponseOutputText(
|
||||
text="Part 1",
|
||||
type="output_text",
|
||||
annotations=[],
|
||||
logprobs=[],
|
||||
)
|
||||
refusal: ResponseOutputRefusal = ResponseOutputRefusal(
|
||||
refusal="won't do that",
|
||||
type="refusal",
|
||||
)
|
||||
resp_msg: ResponseOutputMessage = ResponseOutputMessage(
|
||||
id="42",
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[output_text, refusal],
|
||||
)
|
||||
# Construct a function call item dict (as if returned from model)
|
||||
func_item: ResponseFunctionToolCallParam = {
|
||||
"id": "99",
|
||||
"call_id": "abc",
|
||||
"name": "math",
|
||||
"arguments": "{}",
|
||||
"type": "function_call",
|
||||
}
|
||||
items: list[TResponseInputItem] = [
|
||||
resp_msg.model_dump(), # type:ignore
|
||||
func_item,
|
||||
]
|
||||
messages = Converter.items_to_messages(items)
|
||||
# Should return a single assistant message
|
||||
assert len(messages) == 1
|
||||
assistant = messages[0]
|
||||
assert assistant["role"] == "assistant"
|
||||
# Content combines text portions of the output message
|
||||
assert "content" in assistant
|
||||
assert assistant["content"] == "Part 1"
|
||||
# Refusal in output message should be represented in assistant message
|
||||
assert "refusal" in assistant
|
||||
assert assistant["refusal"] == refusal.refusal
|
||||
# Tool calls list should contain one ChatCompletionMessageFunctionToolCall dict
|
||||
tool_calls = assistant.get("tool_calls")
|
||||
assert isinstance(tool_calls, list)
|
||||
assert len(tool_calls) == 1
|
||||
tool_call = tool_calls[0]
|
||||
assert tool_call["type"] == "function"
|
||||
assert tool_call["function"]["name"] == "math"
|
||||
assert tool_call["function"]["arguments"] == "{}"
|
||||
|
||||
|
||||
def test_convert_tool_choice_handles_standard_and_named_options() -> None:
|
||||
"""
|
||||
The `Converter.convert_tool_choice` method should return the omit sentinel
|
||||
if no choice is provided, pass through values like "auto", "required",
|
||||
or "none" unchanged, and translate any other string into a function
|
||||
selection dict.
|
||||
"""
|
||||
assert Converter.convert_tool_choice(None) is omit
|
||||
assert Converter.convert_tool_choice("auto") == "auto"
|
||||
assert Converter.convert_tool_choice("required") == "required"
|
||||
assert Converter.convert_tool_choice("none") == "none"
|
||||
tool_choice_dict = Converter.convert_tool_choice("mytool")
|
||||
assert isinstance(tool_choice_dict, dict)
|
||||
assert tool_choice_dict["type"] == "function"
|
||||
assert tool_choice_dict["function"]["name"] == "mytool"
|
||||
|
||||
|
||||
def test_convert_tool_choice_allows_tool_search_as_named_function_for_chat_models() -> None:
|
||||
tool_choice_dict = Converter.convert_tool_choice("tool_search")
|
||||
assert isinstance(tool_choice_dict, dict)
|
||||
assert tool_choice_dict["type"] == "function"
|
||||
assert tool_choice_dict["function"]["name"] == "tool_search"
|
||||
|
||||
|
||||
def test_convert_response_format_returns_not_given_for_plain_text_and_dict_for_schemas() -> None:
|
||||
"""
|
||||
The `Converter.convert_response_format` method should return the omit sentinel
|
||||
when no output schema is provided or if the output schema indicates
|
||||
plain text. For structured output schemas, it should return a dict
|
||||
with type `json_schema` and include the generated JSON schema and
|
||||
strict flag from the provided `AgentOutputSchema`.
|
||||
"""
|
||||
# when output is plain text (schema None or output_type str), do not include response_format
|
||||
assert Converter.convert_response_format(None) is omit
|
||||
assert Converter.convert_response_format(AgentOutputSchema(str)) is omit
|
||||
# For e.g. integer output, we expect a response_format dict
|
||||
schema = AgentOutputSchema(int)
|
||||
resp_format = Converter.convert_response_format(schema)
|
||||
assert isinstance(resp_format, dict)
|
||||
assert resp_format["type"] == "json_schema"
|
||||
assert resp_format["json_schema"]["name"] == "final_output"
|
||||
assert "strict" in resp_format["json_schema"]
|
||||
assert resp_format["json_schema"]["strict"] == schema.is_strict_json_schema()
|
||||
assert "schema" in resp_format["json_schema"]
|
||||
assert resp_format["json_schema"]["schema"] == schema.json_schema()
|
||||
|
||||
|
||||
def test_items_to_messages_with_function_output_item():
|
||||
"""
|
||||
A function call output item should be converted into a tool role message
|
||||
dict with the appropriate tool_call_id and content.
|
||||
"""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": '{"foo": "bar"}',
|
||||
}
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == func_output_item["output"]
|
||||
|
||||
|
||||
def test_items_to_messages_with_non_text_only_function_output_uses_placeholder_by_default(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
"""Default conversion should keep running without sending an empty tool message."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == "[tool output omitted]"
|
||||
assert "Replacing the tool output with a placeholder" in caplog.text
|
||||
|
||||
|
||||
def test_items_to_messages_with_non_text_only_function_output_raises_in_strict_mode():
|
||||
"""Strict validation should fail explicitly instead of silently losing the output."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
|
||||
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_items_to_messages_with_empty_function_output_uses_placeholder_by_default(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
"""Default conversion should not send an empty tool message."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [],
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == "[tool output omitted]"
|
||||
assert "Replacing the tool output with a placeholder" in caplog.text
|
||||
|
||||
|
||||
def test_items_to_messages_with_empty_function_output_raises_in_strict_mode():
|
||||
"""Strict validation should fail explicitly instead of sending empty output."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [],
|
||||
}
|
||||
|
||||
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
|
||||
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_items_to_messages_with_mixed_function_output_keeps_text_by_default(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
"""Default conversion should preserve text parts and omit unsupported non-text parts."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{"type": "input_text", "text": "visible text"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == [{"type": "text", "text": "visible text"}]
|
||||
assert "tool output omitted" not in caplog.text
|
||||
|
||||
|
||||
def test_items_to_messages_can_preserve_non_text_function_output() -> None:
|
||||
"""Compatible providers can opt in to preserving non-text tool output."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
[func_output_item],
|
||||
preserve_tool_output_all_content=True,
|
||||
)
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/image.png", "detail": "auto"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_extract_all_and_text_content_for_strings_and_lists():
|
||||
"""
|
||||
The converter provides helpers for extracting user-supplied message content
|
||||
either as a simple string or as a list of `input_text` dictionaries.
|
||||
When passed a bare string, both `extract_all_content` and
|
||||
`extract_text_content` should return the string unchanged.
|
||||
When passed a list of input dictionaries, `extract_all_content` should
|
||||
produce a list of `ChatCompletionContentPart` dicts, and `extract_text_content`
|
||||
should filter to only the textual parts.
|
||||
"""
|
||||
prompt = "just text"
|
||||
assert Converter.extract_all_content(prompt) == prompt
|
||||
assert Converter.extract_text_content(prompt) == prompt
|
||||
text1: ResponseInputTextParam = {"type": "input_text", "text": "one"}
|
||||
text2: ResponseInputTextParam = {"type": "input_text", "text": "two"}
|
||||
all_parts = Converter.extract_all_content([text1, text2])
|
||||
assert isinstance(all_parts, list)
|
||||
assert len(all_parts) == 2
|
||||
assert all_parts[0]["type"] == "text" and all_parts[0]["text"] == "one"
|
||||
assert all_parts[1]["type"] == "text" and all_parts[1]["text"] == "two"
|
||||
text_parts = Converter.extract_text_content([text1, text2])
|
||||
assert isinstance(text_parts, list)
|
||||
assert all(p["type"] == "text" for p in text_parts)
|
||||
assert [p["text"] for p in text_parts] == ["one", "two"]
|
||||
|
||||
|
||||
def test_extract_all_content_handles_input_audio():
|
||||
"""
|
||||
input_audio entries should translate into ChatCompletion input_audio parts.
|
||||
"""
|
||||
audio: ResponseInputAudioParam = {
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AAA=", "format": "wav"},
|
||||
}
|
||||
parts = Converter.extract_all_content([audio])
|
||||
assert isinstance(parts, list)
|
||||
assert parts == [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AAA=", "format": "wav"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_extract_all_content_preserves_prompt_cache_breakpoints() -> None:
|
||||
breakpoint = {"mode": "explicit"}
|
||||
content: list[dict[str, Any]] = [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "one",
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AAA=", "format": "wav"},
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
{
|
||||
"type": "input_file",
|
||||
"file_data": "data:text/plain;base64,SGVsbG8=",
|
||||
"filename": "hello.txt",
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
]
|
||||
|
||||
parts = Converter.extract_all_content(content)
|
||||
|
||||
assert isinstance(parts, list)
|
||||
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint] * 4
|
||||
|
||||
|
||||
def test_raw_chat_content_aliases_preserve_prompt_cache_breakpoints() -> None:
|
||||
breakpoint = {"mode": "explicit"}
|
||||
|
||||
parts = Converter.extract_all_content(
|
||||
cast(
|
||||
list[dict[str, Any]],
|
||||
[
|
||||
{"type": "text", "text": "one", "prompt_cache_breakpoint": breakpoint},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/image.png"},
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(parts, list)
|
||||
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint, breakpoint]
|
||||
|
||||
|
||||
def test_extract_all_content_rejects_invalid_input_audio():
|
||||
"""
|
||||
input_audio requires both data and format fields to be present.
|
||||
"""
|
||||
audio_missing_data = cast(
|
||||
ResponseInputAudioParam,
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"format": "wav"},
|
||||
},
|
||||
)
|
||||
with pytest.raises(UserError):
|
||||
Converter.extract_all_content([audio_missing_data])
|
||||
|
||||
|
||||
def test_items_to_messages_handles_system_and_developer_roles():
|
||||
"""
|
||||
Roles other than `user` (e.g. `system` and `developer`) need to be
|
||||
converted appropriately whether provided as simple dicts or as full
|
||||
`message` typed dicts.
|
||||
"""
|
||||
sys_items: list[TResponseInputItem] = [{"role": "system", "content": "setup"}]
|
||||
sys_msgs = Converter.items_to_messages(sys_items)
|
||||
assert len(sys_msgs) == 1
|
||||
assert sys_msgs[0]["role"] == "system"
|
||||
assert sys_msgs[0]["content"] == "setup"
|
||||
dev_items: list[TResponseInputItem] = [{"role": "developer", "content": "debug"}]
|
||||
dev_msgs = Converter.items_to_messages(dev_items)
|
||||
assert len(dev_msgs) == 1
|
||||
assert dev_msgs[0]["role"] == "developer"
|
||||
assert dev_msgs[0]["content"] == "debug"
|
||||
|
||||
|
||||
def test_maybe_input_message_allows_message_typed_dict():
|
||||
"""
|
||||
The `Converter.maybe_input_message` should recognize a dict with
|
||||
"type": "message" and a supported role as an input message. Ensure
|
||||
that such dicts are passed through by `items_to_messages`.
|
||||
"""
|
||||
# Construct a dict with the proper required keys for a ResponseInputParam.Message
|
||||
message_dict: TResponseInputItem = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": "hi",
|
||||
}
|
||||
assert Converter.maybe_input_message(message_dict) is not None
|
||||
# items_to_messages should process this correctly
|
||||
msgs = Converter.items_to_messages([message_dict])
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0]["role"] == "user"
|
||||
assert msgs[0]["content"] == "hi"
|
||||
|
||||
|
||||
def test_tool_call_conversion():
|
||||
"""
|
||||
Test that tool calls are converted correctly.
|
||||
"""
|
||||
function_call = ResponseFunctionToolCallParam(
|
||||
id="tool1",
|
||||
call_id="abc",
|
||||
name="math",
|
||||
arguments="{}",
|
||||
type="function_call",
|
||||
)
|
||||
|
||||
messages = Converter.items_to_messages([function_call])
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "assistant"
|
||||
assert tool_msg.get("content") is None
|
||||
|
||||
# Verify the content key exists in the message even when it is None.
|
||||
# This is for Chat Completions API compatibility.
|
||||
assert "content" in tool_msg, "content key should be present in assistant message"
|
||||
|
||||
tool_calls = list(tool_msg.get("tool_calls", []))
|
||||
assert len(tool_calls) == 1
|
||||
|
||||
tool_call = tool_calls[0]
|
||||
assert tool_call["id"] == function_call["call_id"]
|
||||
assert tool_call["function"]["name"] == function_call["name"] # type: ignore
|
||||
assert tool_call["function"]["arguments"] == function_call["arguments"] # type: ignore
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["user", "system", "developer"])
|
||||
def test_input_message_with_all_roles(role: str):
|
||||
"""
|
||||
The `Converter.maybe_input_message` should recognize a dict with
|
||||
"type": "message" and a supported role as an input message. Ensure
|
||||
that such dicts are passed through by `items_to_messages`.
|
||||
"""
|
||||
# Construct a dict with the proper required keys for a ResponseInputParam.Message
|
||||
casted_role = cast(Literal["user", "system", "developer"], role)
|
||||
message_dict: TResponseInputItem = {
|
||||
"type": "message",
|
||||
"role": casted_role,
|
||||
"content": "hi",
|
||||
}
|
||||
assert Converter.maybe_input_message(message_dict) is not None
|
||||
# items_to_messages should process this correctly
|
||||
msgs = Converter.items_to_messages([message_dict])
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0]["role"] == casted_role
|
||||
assert msgs[0]["content"] == "hi"
|
||||
|
||||
|
||||
def test_item_reference_errors():
|
||||
"""
|
||||
Test that item references are converted correctly.
|
||||
"""
|
||||
with pytest.raises(UserError):
|
||||
Converter.items_to_messages(
|
||||
[
|
||||
{
|
||||
"type": "item_reference",
|
||||
"id": "item1",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TestObject:
|
||||
pass
|
||||
|
||||
|
||||
def test_unknown_object_errors():
|
||||
"""
|
||||
Test that unknown objects are converted correctly.
|
||||
"""
|
||||
with pytest.raises(UserError, match="Unhandled item type or structure"):
|
||||
# Purposely ignore the type error
|
||||
Converter.items_to_messages([TestObject()]) # type: ignore
|
||||
|
||||
|
||||
def test_assistant_messages_in_history():
|
||||
"""
|
||||
Test that assistant messages are added to the history.
|
||||
"""
|
||||
messages = Converter.items_to_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Hello?",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What was my Name?",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert messages == [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hello?"},
|
||||
{"role": "user", "content": "What was my Name?"},
|
||||
]
|
||||
assert len(messages) == 3
|
||||
assert messages[0]["role"] == "user"
|
||||
assert messages[0]["content"] == "Hello"
|
||||
assert messages[1]["role"] == "assistant"
|
||||
assert messages[1]["content"] == "Hello?"
|
||||
assert messages[2]["role"] == "user"
|
||||
assert messages[2]["content"] == "What was my Name?"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.models.openai_client_utils import (
|
||||
is_official_openai_base_url,
|
||||
is_official_openai_client,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://api.openai.com",
|
||||
"https://api.openai.com/v1/",
|
||||
],
|
||||
)
|
||||
def test_official_openai_base_url_matches_exact_host(base_url: str) -> None:
|
||||
assert is_official_openai_base_url(base_url) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://api.openai.com.evil/v1/",
|
||||
"https://api.openai.com.proxy.local/v1/",
|
||||
"http://api.openai.com/v1/",
|
||||
"https://custom.example.test/v1/",
|
||||
],
|
||||
)
|
||||
def test_official_openai_base_url_rejects_non_openai_hosts(base_url: str) -> None:
|
||||
assert is_official_openai_base_url(base_url) is False
|
||||
|
||||
|
||||
def test_official_openai_websocket_base_url_matches_exact_host() -> None:
|
||||
assert is_official_openai_base_url("wss://api.openai.com/v1/", websocket=True) is True
|
||||
assert (
|
||||
is_official_openai_base_url("wss://api.openai.com.proxy.local/v1/", websocket=True) is False
|
||||
)
|
||||
|
||||
|
||||
def test_official_openai_client_rejects_client_without_base_url() -> None:
|
||||
assert is_official_openai_client(object()) is False # type: ignore[arg-type]
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,178 @@
|
||||
"""Unit tests for the low-level helpers in :mod:`agents.models._openai_retry`.
|
||||
|
||||
These exercise the header-parsing, status-extraction, and error-code helpers
|
||||
directly, plus a few public ``get_openai_retry_advice`` branches that the broader
|
||||
behavioral suite in ``test_model_retry.py`` does not reach.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import format_datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from agents.models._openai_retry import get_openai_retry_advice
|
||||
from agents.models._retry_runtime import (
|
||||
get_error_code as _get_error_code,
|
||||
get_error_header as _get_header_value,
|
||||
get_retry_after,
|
||||
get_status_code as _get_status_code,
|
||||
header_lookup as _header_lookup,
|
||||
parse_retry_after_ms as _parse_retry_after_ms,
|
||||
parse_retry_after_value as _parse_retry_after,
|
||||
)
|
||||
from agents.retry import ModelRetryAdviceRequest
|
||||
from agents.run_internal.model_retry import _normalize_retry_error
|
||||
|
||||
|
||||
class _HeaderError(Exception):
|
||||
"""Error that exposes headers through a plain attribute rather than a response."""
|
||||
|
||||
def __init__(self, message: str, *, headers: dict[str, str] | None = None) -> None:
|
||||
super().__init__(message)
|
||||
if headers is not None:
|
||||
self.headers = headers
|
||||
|
||||
|
||||
def _make_request(error: Exception, **kwargs: object) -> ModelRetryAdviceRequest:
|
||||
return ModelRetryAdviceRequest(error=error, attempt=1, stream=False, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_header_lookup_plain_mapping_matches_case_insensitively() -> None:
|
||||
headers = {"Retry-After": "5", "X-Other": "ignored"}
|
||||
assert _header_lookup(headers, "retry-after") == "5"
|
||||
assert _header_lookup(headers, "missing") is None
|
||||
|
||||
|
||||
def test_header_lookup_httpx_headers() -> None:
|
||||
headers = httpx.Headers({"retry-after": "7"})
|
||||
assert _header_lookup(headers, "retry-after") == "7"
|
||||
assert _header_lookup(None, "retry-after") is None
|
||||
|
||||
|
||||
def test_get_header_value_reads_response_headers_attr() -> None:
|
||||
class _Err(Exception):
|
||||
response_headers = {"retry-after": "3"}
|
||||
|
||||
assert _get_header_value(_Err("boom"), "retry-after") == "3"
|
||||
|
||||
|
||||
def test_parse_retry_after_ms_invalid_returns_none() -> None:
|
||||
assert _parse_retry_after_ms(None) is None
|
||||
assert _parse_retry_after_ms("not-a-number") is None
|
||||
assert _parse_retry_after_ms("-100") is None
|
||||
assert _parse_retry_after_ms("1500") == 1.5
|
||||
|
||||
|
||||
def test_parse_retry_after_numeric_and_http_date() -> None:
|
||||
assert _parse_retry_after(None) is None
|
||||
assert _parse_retry_after("2") == 2.0
|
||||
assert _parse_retry_after("-1") is None
|
||||
|
||||
future = datetime.now(timezone.utc) + timedelta(seconds=120)
|
||||
parsed = _parse_retry_after(format_datetime(future))
|
||||
assert parsed is not None and parsed > 0
|
||||
|
||||
assert _parse_retry_after("definitely not a date") is None
|
||||
|
||||
|
||||
def test_get_retry_after_preserves_outer_exception_precedence() -> None:
|
||||
outer = _HeaderError("wrapped", headers={"retry-after": "2"})
|
||||
outer.__cause__ = _HeaderError("provider", headers={"retry-after-ms": "1500"})
|
||||
|
||||
assert get_retry_after(outer) == 2.0
|
||||
|
||||
|
||||
def test_get_status_code_from_status_code_and_status_attrs() -> None:
|
||||
class _StatusCode(Exception):
|
||||
status_code = 503
|
||||
|
||||
class _Status(Exception):
|
||||
status = 504
|
||||
|
||||
assert _get_status_code(_StatusCode("a")) == 503
|
||||
assert _get_status_code(_Status("b")) == 504
|
||||
assert _get_status_code(Exception("none")) is None
|
||||
|
||||
|
||||
def test_get_error_code_from_body_mapping() -> None:
|
||||
class _NestedBody(Exception):
|
||||
body = {"error": {"code": "rate_limit_exceeded"}}
|
||||
|
||||
class _TopLevelBody(Exception):
|
||||
body = {"code": "server_error"}
|
||||
|
||||
assert _get_error_code(_NestedBody("a")) == "rate_limit_exceeded"
|
||||
assert _get_error_code(_TopLevelBody("b")) == "server_error"
|
||||
assert _get_error_code(Exception("none")) is None
|
||||
|
||||
|
||||
def test_provider_and_runner_retry_normalization_share_metadata() -> None:
|
||||
class _RetryableError(Exception):
|
||||
status_code = 429
|
||||
request_id = "req_test"
|
||||
body = {"error": {"code": "rate_limit_exceeded"}}
|
||||
headers = {"retry-after-ms": "1500"}
|
||||
|
||||
class _WrapperError(Exception):
|
||||
headers = {"x-other": "ignored"}
|
||||
|
||||
error = _WrapperError("wrapped")
|
||||
error.__cause__ = _RetryableError("slow down")
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
runner_normalized = _normalize_retry_error(error, None)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.normalized is not None
|
||||
assert advice.normalized.status_code == runner_normalized.status_code
|
||||
assert advice.normalized.error_code == runner_normalized.error_code
|
||||
assert advice.normalized.request_id == runner_normalized.request_id
|
||||
assert advice.normalized.retry_after == runner_normalized.retry_after
|
||||
assert runner_normalized.retry_after == 1.5
|
||||
|
||||
|
||||
def test_advice_unsafe_to_replay() -> None:
|
||||
error = Exception("cannot replay")
|
||||
error.unsafe_to_replay = True # type: ignore[attr-defined]
|
||||
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is False
|
||||
assert advice.replay_safety == "unsafe"
|
||||
|
||||
|
||||
def test_advice_websocket_request_is_unsafe() -> None:
|
||||
message = (
|
||||
"The request may have been accepted, so the SDK will not automatically "
|
||||
"retry this websocket request."
|
||||
)
|
||||
advice = get_openai_retry_advice(_make_request(Exception(message)))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is False
|
||||
assert advice.replay_safety == "unsafe"
|
||||
|
||||
|
||||
def test_advice_respects_x_should_retry_false() -> None:
|
||||
error = _HeaderError("nope", headers={"x-should-retry": "false"})
|
||||
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is False
|
||||
|
||||
|
||||
def test_advice_returns_retry_after_only_when_no_other_signal() -> None:
|
||||
# A 400 with no x-should-retry header and no network/timeout signal would not
|
||||
# normally retry, but a retry-after header still yields advice carrying the delay.
|
||||
error = _HeaderError("slow down", headers={"retry-after": "2"})
|
||||
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.retry_after == 2.0
|
||||
# This branch only conveys the server-provided delay; it does not assert a
|
||||
# retry decision, so ``suggested`` keeps its unset default.
|
||||
assert advice.suggested is None
|
||||
@@ -0,0 +1,454 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
|
||||
from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta
|
||||
from openai.types.completion_usage import (
|
||||
CompletionTokensDetails,
|
||||
CompletionUsage,
|
||||
PromptTokensDetails,
|
||||
)
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
)
|
||||
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.models.openai_provider import OpenAIProvider
|
||||
|
||||
|
||||
# Helper functions to create test objects consistently
|
||||
def create_content_delta(content: str) -> dict[str, Any]:
|
||||
"""Create a delta dictionary with regular content"""
|
||||
return {"content": content, "role": None, "function_call": None, "tool_calls": None}
|
||||
|
||||
|
||||
def create_reasoning_delta(content: str) -> dict[str, Any]:
|
||||
"""Create a delta dictionary with reasoning content. The Only difference is reasoning_content"""
|
||||
return {
|
||||
"content": None,
|
||||
"role": None,
|
||||
"function_call": None,
|
||||
"tool_calls": None,
|
||||
"reasoning_content": content,
|
||||
}
|
||||
|
||||
|
||||
def create_chunk(delta: dict[str, Any], include_usage: bool = False) -> ChatCompletionChunk:
|
||||
"""Create a ChatCompletionChunk with the given delta"""
|
||||
# Create a ChoiceDelta object from the dictionary
|
||||
delta_obj = ChoiceDelta(
|
||||
content=delta.get("content"),
|
||||
role=delta.get("role"),
|
||||
function_call=delta.get("function_call"),
|
||||
tool_calls=delta.get("tool_calls"),
|
||||
)
|
||||
|
||||
# Add reasoning_content attribute dynamically if present in the delta
|
||||
if "reasoning_content" in delta:
|
||||
# Use direct assignment for the reasoning_content attribute
|
||||
delta_obj_any = cast(Any, delta_obj)
|
||||
delta_obj_any.reasoning_content = delta["reasoning_content"]
|
||||
|
||||
# Create the chunk
|
||||
chunk = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="deepseek is usually expected",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=delta_obj)],
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
chunk.usage = CompletionUsage(
|
||||
completion_tokens=4,
|
||||
prompt_tokens=2,
|
||||
total_tokens=6,
|
||||
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
|
||||
)
|
||||
|
||||
return chunk
|
||||
|
||||
|
||||
async def create_fake_stream(
|
||||
chunks: list[ChatCompletionChunk],
|
||||
) -> AsyncIterator[ChatCompletionChunk]:
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_reasoning_content(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that when a model streams reasoning content,
|
||||
`stream_response` emits the appropriate sequence of events including
|
||||
`response.reasoning_summary_text.delta` events for each chunk of the reasoning content and
|
||||
constructs a completed response with a `ResponseReasoningItem` part.
|
||||
"""
|
||||
# Create test chunks
|
||||
chunks = [
|
||||
# Reasoning content chunks
|
||||
create_chunk(create_reasoning_delta("Let me think")),
|
||||
create_chunk(create_reasoning_delta(" about this")),
|
||||
# Regular content chunks
|
||||
create_chunk(create_content_delta("The answer")),
|
||||
create_chunk(create_content_delta(" is 42"), include_usage=True),
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# verify reasoning content events were emitted
|
||||
reasoning_delta_events = [
|
||||
e for e in output_events if e.type == "response.reasoning_summary_text.delta"
|
||||
]
|
||||
assert len(reasoning_delta_events) == 2
|
||||
assert reasoning_delta_events[0].delta == "Let me think"
|
||||
assert reasoning_delta_events[1].delta == " about this"
|
||||
|
||||
reasoning_done_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.reasoning_summary_part.done"
|
||||
)
|
||||
first_text_delta_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.output_text.delta"
|
||||
)
|
||||
assert reasoning_done_index < first_text_delta_index
|
||||
|
||||
# verify regular content events were emitted
|
||||
content_delta_events = [e for e in output_events if e.type == "response.output_text.delta"]
|
||||
assert len(content_delta_events) == 2
|
||||
assert content_delta_events[0].delta == "The answer"
|
||||
assert content_delta_events[1].delta == " is 42"
|
||||
|
||||
assistant_message_index_events = []
|
||||
for event in output_events:
|
||||
event_any = cast(Any, event)
|
||||
if event.type in {"response.output_item.added", "response.output_item.done"}:
|
||||
if event_any.item.type == "message":
|
||||
assistant_message_index_events.append(event_any)
|
||||
elif event.type in {
|
||||
"response.content_part.added",
|
||||
"response.output_text.delta",
|
||||
"response.content_part.done",
|
||||
}:
|
||||
assistant_message_index_events.append(event_any)
|
||||
|
||||
assert assistant_message_index_events
|
||||
for event in assistant_message_index_events:
|
||||
assert event.output_index == 1
|
||||
assert type(event.output_index) is int
|
||||
|
||||
# verify the final response contains both types of content
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
assert len(response_event.response.output) == 2
|
||||
|
||||
# first item should be reasoning
|
||||
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
|
||||
assert response_event.response.output[0].summary[0].text == "Let me think about this"
|
||||
|
||||
# second item should be message with text
|
||||
assert isinstance(response_event.response.output[1], ResponseOutputMessage)
|
||||
assert isinstance(response_event.response.output[1].content[0], ResponseOutputText)
|
||||
assert response_event.response.output[1].content[0].text == "The answer is 42"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_keeps_reasoning_item_open_across_interleaved_text(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
chunks = [
|
||||
create_chunk(create_reasoning_delta("Let me think")),
|
||||
create_chunk(create_content_delta("The answer")),
|
||||
create_chunk(create_reasoning_delta(" more carefully")),
|
||||
create_chunk(create_content_delta(" is 42"), include_usage=True),
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
reasoning_part_added_events = [
|
||||
event for event in output_events if event.type == "response.reasoning_summary_part.added"
|
||||
]
|
||||
assert [event.summary_index for event in reasoning_part_added_events] == [0, 1]
|
||||
|
||||
reasoning_part_done_events = [
|
||||
event for event in output_events if event.type == "response.reasoning_summary_part.done"
|
||||
]
|
||||
assert [event.summary_index for event in reasoning_part_done_events] == [0, 1]
|
||||
|
||||
first_reasoning_done_index = output_events.index(reasoning_part_done_events[0])
|
||||
first_text_delta_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.output_text.delta"
|
||||
)
|
||||
second_reasoning_delta_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.reasoning_summary_text.delta" and event.summary_index == 1
|
||||
)
|
||||
reasoning_item_done_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.output_item.done" and event.item.type == "reasoning"
|
||||
)
|
||||
|
||||
assert first_reasoning_done_index < first_text_delta_index
|
||||
assert second_reasoning_delta_index > first_text_delta_index
|
||||
assert reasoning_item_done_index > second_reasoning_delta_index
|
||||
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
|
||||
assert [summary.text for summary in response_event.response.output[0].summary] == [
|
||||
"Let me think",
|
||||
" more carefully",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_response_with_reasoning_content(monkeypatch) -> None:
|
||||
"""
|
||||
Test that when a model returns reasoning content in addition to regular content,
|
||||
`get_response` properly includes both in the response output.
|
||||
"""
|
||||
# create a message with reasoning content
|
||||
msg = ChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="The answer is 42",
|
||||
)
|
||||
# Use dynamic attribute for reasoning_content
|
||||
# We need to cast to Any to avoid mypy errors since reasoning_content is not a defined attribute
|
||||
msg_with_reasoning = cast(Any, msg)
|
||||
msg_with_reasoning.reasoning_content = "Let me think about this question carefully"
|
||||
|
||||
# create a choice with the message
|
||||
mock_choice = {
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"message": msg_with_reasoning,
|
||||
"delta": None,
|
||||
}
|
||||
|
||||
chat = ChatCompletion(
|
||||
id="resp-id",
|
||||
created=0,
|
||||
model="deepseek is expected",
|
||||
object="chat.completion",
|
||||
choices=[mock_choice], # type: ignore[list-item]
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=10,
|
||||
prompt_tokens=5,
|
||||
total_tokens=15,
|
||||
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=6),
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
|
||||
),
|
||||
)
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
return chat
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
resp = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
# should have produced a reasoning item and a message with text content
|
||||
assert len(resp.output) == 2
|
||||
|
||||
# first output should be the reasoning item
|
||||
assert isinstance(resp.output[0], ResponseReasoningItem)
|
||||
assert resp.output[0].summary[0].text == "Let me think about this question carefully"
|
||||
|
||||
# second output should be the message with text content
|
||||
assert isinstance(resp.output[1], ResponseOutputMessage)
|
||||
assert isinstance(resp.output[1].content[0], ResponseOutputText)
|
||||
assert resp.output[1].content[0].text == "The answer is 42"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_preserves_usage_from_earlier_chunk(monkeypatch) -> None:
|
||||
"""
|
||||
Test that when an earlier chunk has usage data and later chunks don't,
|
||||
the usage from the earlier chunk is preserved in the final response.
|
||||
This handles cases where some providers (e.g., LiteLLM) may not include
|
||||
usage in every chunk.
|
||||
"""
|
||||
# Create test chunks where first chunk has usage, last chunk doesn't
|
||||
chunks = [
|
||||
create_chunk(create_content_delta("Hello"), include_usage=True), # Has usage
|
||||
create_chunk(create_content_delta("")), # No usage (usage=None)
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# Verify the final response preserves usage from the first chunk
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
assert response_event.response.usage is not None
|
||||
assert response_event.response.usage.input_tokens == 2
|
||||
assert response_event.response.usage.output_tokens == 4
|
||||
assert response_event.response.usage.total_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_with_empty_reasoning_content(monkeypatch) -> None:
|
||||
"""
|
||||
Test that when a model streams empty reasoning content,
|
||||
the response still processes correctly without errors.
|
||||
"""
|
||||
# create test chunks with empty reasoning content
|
||||
chunks = [
|
||||
create_chunk(create_reasoning_delta("")),
|
||||
create_chunk(create_content_delta("The answer is 42"), include_usage=True),
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# verify the final response contains the content
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
|
||||
# should only have the message, not an empty reasoning item
|
||||
assert len(response_event.response.output) == 1
|
||||
assert isinstance(response_event.response.output[0], ResponseOutputMessage)
|
||||
assert isinstance(response_event.response.output[0].content[0], ResponseOutputText)
|
||||
assert response_event.response.output[0].content[0].text == "The answer is 42"
|
||||
@@ -0,0 +1,403 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai.types.chat.chat_completion import ChatCompletion, Choice
|
||||
from openai.types.chat.chat_completion_message import ChatCompletionMessage
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.models.reasoning_content_replay import ReasoningContentReplayContext
|
||||
|
||||
REASONING_CONTENT_MODEL_A = "reasoning-content-model-a"
|
||||
REASONING_CONTENT_MODEL_B = "reasoning-content-model-b"
|
||||
# The converter currently keys Anthropic thinking-block reconstruction off the model name,
|
||||
# so this test model keeps the "anthropic" substring while staying otherwise generic.
|
||||
REASONING_CONTENT_MODEL_C = "reasoning-content-model-c-anthropic"
|
||||
|
||||
|
||||
def _second_turn_input_items(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_message(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "I'll call the weather tool now.",
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
}
|
||||
],
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_file_search(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "Find notes about Tokyo weather."},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should search the knowledge base first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"id": "__fake_file_search_id__",
|
||||
"queries": ["Tokyo weather"],
|
||||
"status": "completed",
|
||||
"type": "file_search_call",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_message_then_reasoning(
|
||||
model_name: str,
|
||||
) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "I'll call the weather tool now.",
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
}
|
||||
],
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_thinking_blocks(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": [
|
||||
{
|
||||
"type": "reasoning_text",
|
||||
"text": "First, I need to inspect the request.",
|
||||
}
|
||||
],
|
||||
"encrypted_content": "test-signature",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _assistant_with_tool_calls(messages: list[Any]) -> dict[str, Any]:
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
return msg
|
||||
raise AssertionError("Expected an assistant message with tool_calls.")
|
||||
|
||||
|
||||
def test_converter_keeps_default_reasoning_replay_behavior_for_non_default_model() -> None:
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert "reasoning_content" not in assistant
|
||||
|
||||
|
||||
def test_converter_preserves_reasoning_content_across_output_message_with_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_message(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["content"] == "I'll call the weather tool now."
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
|
||||
|
||||
def test_converter_replays_reasoning_content_when_reasoning_follows_message_with_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_message_then_reasoning(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["content"] == "I'll call the weather tool now."
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
|
||||
|
||||
def test_converter_replays_reasoning_content_for_file_search_call_with_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_file_search(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["reasoning_content"] == "I should search the knowledge base first."
|
||||
assert assistant["tool_calls"][0]["function"]["name"] == "file_search_call"
|
||||
|
||||
|
||||
def test_converter_replays_reasoning_content_with_thinking_blocks_and_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_thinking_blocks(REASONING_CONTENT_MODEL_C),
|
||||
model=REASONING_CONTENT_MODEL_C,
|
||||
preserve_thinking_blocks=True,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
assert assistant["content"][0]["type"] == "thinking"
|
||||
assert assistant["content"][0]["thinking"] == "First, I need to inspect the request."
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_chatcompletions_hook_can_enable_reasoning_content_replay() -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
contexts: list[ReasoningContentReplayContext] = []
|
||||
|
||||
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
|
||||
contexts.append(context)
|
||||
return context.model == REASONING_CONTENT_MODEL_B
|
||||
|
||||
class MockChatCompletions:
|
||||
async def create(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = ChatCompletionMessage(role="assistant", content="done")
|
||||
choice = Choice(index=0, message=msg, finish_reason="stop")
|
||||
return ChatCompletion(
|
||||
id="test-id",
|
||||
created=0,
|
||||
model=REASONING_CONTENT_MODEL_B,
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
|
||||
)
|
||||
|
||||
class MockChat:
|
||||
def __init__(self):
|
||||
self.completions = MockChatCompletions()
|
||||
|
||||
class MockClient:
|
||||
def __init__(self):
|
||||
self.chat = MockChat()
|
||||
self.base_url = httpx.URL("https://example.com/v1/")
|
||||
|
||||
model = OpenAIChatCompletionsModel(
|
||||
model=REASONING_CONTENT_MODEL_B,
|
||||
openai_client=cast(Any, MockClient()),
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
assert len(contexts) == 1
|
||||
assert contexts[0].model == REASONING_CONTENT_MODEL_B
|
||||
assert contexts[0].base_url == "https://example.com/v1"
|
||||
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_hook_can_enable_reasoning_content_replay(monkeypatch) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
contexts: list[ReasoningContentReplayContext] = []
|
||||
|
||||
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
|
||||
contexts.append(context)
|
||||
return context.model == REASONING_CONTENT_MODEL_B
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured["messages"] = messages
|
||||
msg = Message(role="assistant", content="done")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
model = LitellmModel(
|
||||
model=REASONING_CONTENT_MODEL_B,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
assert len(contexts) == 1
|
||||
assert contexts[0].model == REASONING_CONTENT_MODEL_B
|
||||
assert contexts[0].base_url is None
|
||||
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
|
||||
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.models.fake_id import FAKE_RESPONSES_ID
|
||||
from agents.models.openai_responses import OpenAIResponsesModel
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model() -> OpenAIResponsesModel:
|
||||
"""Create a model instance for testing."""
|
||||
mock_client = MagicMock()
|
||||
return OpenAIResponsesModel(model="gpt-5", openai_client=mock_client)
|
||||
|
||||
|
||||
class TestRemoveOpenAIResponsesAPIIncompatibleFields:
|
||||
"""Tests for _remove_openai_responses_api_incompatible_fields method."""
|
||||
|
||||
def test_returns_unchanged_when_no_provider_data(self, model: OpenAIResponsesModel):
|
||||
"""When no items have provider_data, the input should be returned unchanged."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "hello"},
|
||||
{"type": "function_call", "call_id": "call_123", "name": "test"},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert result is list_input # Same object reference.
|
||||
|
||||
def test_removes_reasoning_items_with_provider_data(self, model: OpenAIResponsesModel):
|
||||
"""Reasoning items with provider_data should be completely removed."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "hello"},
|
||||
{"type": "reasoning", "provider_data": {"model": "gemini/gemini-3"}},
|
||||
{"type": "function_call", "call_id": "call_123"},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0] == {"type": "message", "content": "hello"}
|
||||
assert result[1] == {"type": "function_call", "call_id": "call_123"}
|
||||
|
||||
def test_keeps_reasoning_items_without_provider_data(self, model: OpenAIResponsesModel):
|
||||
"""Reasoning items without provider_data should be kept."""
|
||||
list_input = [
|
||||
{"type": "reasoning", "summary": []},
|
||||
{"type": "message", "content": "hello", "provider_data": {"foo": "bar"}},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0] == {"type": "reasoning", "summary": []}
|
||||
assert result[1] == {"type": "message", "content": "hello"}
|
||||
|
||||
def test_removes_provider_data_from_all_items(self, model: OpenAIResponsesModel):
|
||||
"""provider_data field should be removed from all dict items."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "hello", "provider_data": {"model": "gemini/gemini-3"}},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_123",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 2
|
||||
assert "provider_data" not in result[0]
|
||||
assert "provider_data" not in result[1]
|
||||
|
||||
def test_removes_fake_responses_id(self, model: OpenAIResponsesModel):
|
||||
"""Items with id equal to FAKE_RESPONSES_ID should have their id removed."""
|
||||
list_input = [
|
||||
{
|
||||
"type": "message",
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"content": "hello",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 1
|
||||
assert "id" not in result[0]
|
||||
assert result[0]["content"] == "hello"
|
||||
|
||||
def test_preserves_real_ids(self, model: OpenAIResponsesModel):
|
||||
"""Real IDs (not FAKE_RESPONSES_ID) should be preserved."""
|
||||
list_input = [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_real123",
|
||||
"content": "hello",
|
||||
"provider_data": {},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert result[0]["id"] == "msg_real123"
|
||||
|
||||
def test_handles_empty_list(self, model: OpenAIResponsesModel):
|
||||
"""Empty list should be returned unchanged."""
|
||||
list_input: list[dict[str, Any]] = []
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_combined_scenario(self, model: OpenAIResponsesModel):
|
||||
"""Test a realistic scenario with multiple items needing different processing."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "user input"},
|
||||
{"type": "reasoning", "summary": [], "provider_data": {"model": "gemini/gemini-3"}},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_abc_123",
|
||||
"name": "get_weather",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_abc_123",
|
||||
"output": '{"temp": 72}',
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"content": "The weather is 72F",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
# Should have 4 items (reasoning with provider_data removed).
|
||||
assert len(result) == 4
|
||||
|
||||
# First item unchanged (no provider_data).
|
||||
assert result[0] == {"type": "message", "content": "user input"}
|
||||
|
||||
# Function call: __thought__ suffix removed, provider_data removed.
|
||||
assert result[1]["type"] == "function_call"
|
||||
assert result[1]["call_id"] == "call_abc_123"
|
||||
assert "provider_data" not in result[1]
|
||||
|
||||
# Function call output: __thought__ suffix removed, provider_data removed.
|
||||
assert result[2]["type"] == "function_call_output"
|
||||
assert result[2]["call_id"] == "call_abc_123"
|
||||
|
||||
# Last message: fake id removed, provider_data removed.
|
||||
assert result[3]["type"] == "message"
|
||||
assert result[3]["content"] == "The weather is 72F"
|
||||
assert "id" not in result[3]
|
||||
assert "provider_data" not in result[3]
|
||||
@@ -0,0 +1,149 @@
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import Agent, responses_websocket_session
|
||||
from agents.models.multi_provider import MultiProvider
|
||||
from agents.models.openai_provider import OpenAIProvider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_builds_shared_run_config():
|
||||
async with responses_websocket_session() as ws:
|
||||
assert isinstance(ws.provider, OpenAIProvider)
|
||||
assert ws.provider._use_responses is True
|
||||
assert ws.provider._use_responses_websocket is True
|
||||
assert isinstance(ws.run_config.model_provider, MultiProvider)
|
||||
assert ws.run_config.model_provider.openai_provider is ws.provider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_preserves_openai_prefix_routing(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_get_model(model_name):
|
||||
captured["model_name"] = model_name
|
||||
return sentinel
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
|
||||
|
||||
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["model_name"] == "gpt-4.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_can_preserve_openai_prefix_model_ids(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_get_model(model_name):
|
||||
captured["model_name"] = model_name
|
||||
return sentinel
|
||||
|
||||
async with responses_websocket_session(openai_prefix_mode="model_id") as ws:
|
||||
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
|
||||
|
||||
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["model_name"] == "openai/gpt-4.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_can_preserve_unknown_prefix_model_ids(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_get_model(model_name):
|
||||
captured["model_name"] = model_name
|
||||
return sentinel
|
||||
|
||||
async with responses_websocket_session(unknown_prefix_mode="model_id") as ws:
|
||||
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
|
||||
|
||||
result = ws.run_config.model_provider.get_model("openrouter/openai/gpt-4.1")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["model_name"] == "openrouter/openai/gpt-4.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_run_streamed_injects_run_config(monkeypatch):
|
||||
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
|
||||
captured = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_run_streamed(starting_agent, input, **kwargs):
|
||||
captured["starting_agent"] = starting_agent
|
||||
captured["input"] = input
|
||||
captured["kwargs"] = kwargs
|
||||
return sentinel
|
||||
|
||||
ws_module = importlib.import_module("agents.responses_websocket_session")
|
||||
monkeypatch.setattr(ws_module.Runner, "run_streamed", fake_run_streamed)
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
result = ws.run_streamed(agent, "hello")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["starting_agent"] is agent
|
||||
assert captured["input"] == "hello"
|
||||
assert captured["kwargs"]["run_config"] is ws.run_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_run_injects_run_config(monkeypatch):
|
||||
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
|
||||
captured = {}
|
||||
sentinel = object()
|
||||
|
||||
async def fake_run(starting_agent, input, **kwargs):
|
||||
captured["starting_agent"] = starting_agent
|
||||
captured["input"] = input
|
||||
captured["kwargs"] = kwargs
|
||||
return sentinel
|
||||
|
||||
ws_module = importlib.import_module("agents.responses_websocket_session")
|
||||
monkeypatch.setattr(ws_module.Runner, "run", fake_run)
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
result = await ws.run(agent, "hello")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["starting_agent"] is agent
|
||||
assert captured["input"] == "hello"
|
||||
assert captured["kwargs"]["run_config"] is ws.run_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_rejects_run_config_override():
|
||||
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
with pytest.raises(ValueError, match="run_config"):
|
||||
ws.run_streamed(agent, "hello", run_config=object())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_context_manager_closes_provider(monkeypatch):
|
||||
close_calls: list[OpenAIProvider] = []
|
||||
|
||||
async def fake_aclose(self):
|
||||
close_calls.append(self)
|
||||
|
||||
monkeypatch.setattr(OpenAIProvider, "aclose", fake_aclose)
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
provider = ws.provider
|
||||
|
||||
assert close_calls == [provider]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_does_not_expose_run_sync():
|
||||
async with responses_websocket_session() as ws:
|
||||
assert not hasattr(ws, "run_sync")
|
||||
@@ -0,0 +1,32 @@
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models._trace import model_config_for_trace, sanitize_url_for_trace
|
||||
|
||||
|
||||
def test_sanitize_url_for_trace_strips_auth_query_and_fragment() -> None:
|
||||
assert (
|
||||
sanitize_url_for_trace("https://user:pass@example.com/v1?api-key=secret#fragment")
|
||||
== "https://example.com/v1"
|
||||
)
|
||||
assert sanitize_url_for_trace("https://example.com/v1?token=secret") == "https://example.com/v1"
|
||||
|
||||
|
||||
def test_model_config_for_trace_sanitizes_base_url_and_omits_request_extras() -> None:
|
||||
config = model_config_for_trace(
|
||||
ModelSettings(
|
||||
temperature=0.5,
|
||||
extra_headers={"Authorization": "Bearer provider-token"},
|
||||
extra_query={"api-key": "query-token"},
|
||||
extra_body={"secret": "body-token"},
|
||||
extra_args={"api_key": "arg-token"},
|
||||
),
|
||||
base_url="https://user:pass@example.com/v1?api-key=secret#fragment",
|
||||
extra_config={"model_impl": "test-model"},
|
||||
)
|
||||
|
||||
assert config["temperature"] == 0.5
|
||||
assert config["base_url"] == "https://example.com/v1"
|
||||
assert config["model_impl"] == "test-model"
|
||||
assert "extra_headers" not in config
|
||||
assert "extra_query" not in config
|
||||
assert "extra_body" not in config
|
||||
assert "extra_args" not in config
|
||||
Reference in New Issue
Block a user