chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:39:17 +08:00
commit 4ed4e9ff99
1368 changed files with 334957 additions and 0 deletions
View File
+168
View File
@@ -0,0 +1,168 @@
from __future__ import annotations
import pytest
from agents import (
OpenAIAgentRegistrationConfig,
RunConfig,
set_default_openai_agent_registration,
set_default_openai_harness,
)
from agents.models.multi_provider import MultiProvider
from agents.models.openai_agent_registration import (
OPENAI_HARNESS_ID_TRACE_METADATA_KEY,
resolve_openai_agent_registration_config,
resolve_openai_harness_id_for_model_provider,
)
from agents.models.openai_provider import OpenAIProvider
from agents.run_internal.agent_runner_helpers import resolve_trace_settings
from agents.tracing import agent_span, trace
def test_agent_registration_config_precedence(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
set_default_openai_agent_registration(
OpenAIAgentRegistrationConfig(harness_id="default-harness")
)
try:
resolved = resolve_openai_agent_registration_config(
OpenAIAgentRegistrationConfig(harness_id="explicit-harness")
)
finally:
set_default_openai_agent_registration(None)
assert resolved is not None
assert resolved.harness_id == "explicit-harness"
def test_agent_registration_uses_default_before_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
set_default_openai_agent_registration(
OpenAIAgentRegistrationConfig(harness_id="default-harness")
)
try:
resolved = resolve_openai_agent_registration_config(None)
finally:
set_default_openai_agent_registration(None)
assert resolved is not None
assert resolved.harness_id == "default-harness"
def test_agent_registration_uses_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
resolved = resolve_openai_agent_registration_config(None)
assert resolved is not None
assert resolved.harness_id == "env-harness"
def test_set_default_openai_harness(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
set_default_openai_harness("helper-harness")
try:
resolved = resolve_openai_agent_registration_config(None)
finally:
set_default_openai_harness(None)
assert resolved is not None
assert resolved.harness_id == "helper-harness"
def test_agent_registration_disabled_without_config(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("OPENAI_AGENT_HARNESS_ID", raising=False)
assert resolve_openai_agent_registration_config(None) is None
def test_agent_registration_provider_constructor_config() -> None:
config = OpenAIAgentRegistrationConfig(harness_id="provider-harness")
openai_provider = OpenAIProvider(agent_registration=config)
multi_provider = MultiProvider(openai_agent_registration=config)
assert openai_provider.agent_registration is not None
assert openai_provider.agent_registration.harness_id == "provider-harness"
assert multi_provider.openai_provider.agent_registration is not None
assert multi_provider.openai_provider.agent_registration.harness_id == "provider-harness"
def test_harness_id_resolves_private_agent_registration() -> None:
class Provider:
_agent_registration = OpenAIAgentRegistrationConfig(harness_id="private-harness")
assert resolve_openai_harness_id_for_model_provider(Provider()) == "private-harness"
def test_harness_id_is_added_to_trace_metadata() -> None:
provider = OpenAIProvider(
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
)
_, _, _, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(model_provider=provider),
)
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
def test_harness_id_preserves_explicit_trace_metadata() -> None:
provider = OpenAIProvider(
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
)
_, _, _, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(
model_provider=provider,
trace_metadata={
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
"source": "test",
},
),
)
assert metadata == {
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
"source": "test",
}
def test_env_harness_id_is_added_to_trace_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
_, _, _, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(),
)
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "env-harness"}
def test_harness_id_trace_metadata_propagates_to_spans() -> None:
provider = OpenAIProvider(
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
)
workflow_name, trace_id, group_id, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(model_provider=provider),
)
with trace(
workflow_name=workflow_name,
trace_id=trace_id,
group_id=group_id,
metadata=metadata,
):
with agent_span(name="agent") as span:
assert span.trace_metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
span_export = span.export()
assert span_export is not None
assert span_export["metadata"] == {
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"
}
@@ -0,0 +1,418 @@
"""
Test for Anthropic thinking blocks in conversation history.
This test validates the fix for issue #1704:
- Thinking blocks are properly preserved from Anthropic responses
- Reasoning items are stored in session but not sent back in conversation history
- Non-reasoning models are unaffected
- Token usage is not increased for non-reasoning scenarios
"""
from __future__ import annotations
from typing import Any, cast
from openai.types.chat import ChatCompletionMessageToolCall
from openai.types.chat.chat_completion_message_tool_call import Function
from agents.extensions.models.litellm_model import InternalChatCompletionMessage
from agents.models.chatcmpl_converter import Converter
def create_mock_anthropic_response_with_thinking() -> InternalChatCompletionMessage:
"""Create a mock Anthropic response with thinking blocks (like real response)."""
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather in Paris for you.",
reasoning_content="I need to call the weather function for Paris",
thinking_blocks=[
{
"type": "thinking",
"thinking": "I need to call the weather function for Paris",
"signature": "EqMDCkYIBxgCKkBAFZO8EyZwN1hiLctq0YjZnP0KeKgprr+C0PzgDv4GSggnFwrPQHIZ9A5s+paH+DrQBI1+Vnfq3mLAU5lJnoetEgzUEWx/Cv1022ieAvcaDCXdmg1XkMK0tZ8uCCIwURYAAX0uf2wFdnWt9n8whkhmy8ARQD5G2za4R8X5vTqBq8jpJ15T3c1Jcf3noKMZKooCWFVf0/W5VQqpZTgwDkqyTau7XraS+u48YlmJGSfyWMPO8snFLMZLGaGmVJgHfEI5PILhOEuX/R2cEeLuC715f51LMVuxTNzlOUV/037JV6P2ten7D66FnWU9JJMMJJov+DjMb728yQFHwHz4roBJ5ePHaaFP6mDwpqYuG/hai6pVv2TAK1IdKUui/oXrYtU+0gxb6UF2kS1bspqDuN++R8JdL7CMSU5l28pQ8TsH1TpVF4jZpsFbp1Du4rQIULFsCFFg+Edf9tPgyKZOq6xcskIjT7oylAPO37/jhdNknDq2S82PaSKtke3ViOigtM5uJfG521ZscBJQ1K3kwoI/repIdV9PatjOYdsYAQ==", # noqa: E501
}
],
)
return message
def test_converter_skips_reasoning_items():
"""
Unit test to verify that reasoning items are skipped when converting items to messages.
"""
# Create test items including a reasoning item
test_items: list[dict[str, Any]] = [
{"role": "user", "content": "Hello"},
{
"id": "reasoning_123",
"type": "reasoning",
"summary": [{"text": "User said hello", "type": "summary_text"}],
},
{
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Hi there!"}],
"status": "completed",
},
]
# Convert to messages
messages = Converter.items_to_messages(test_items) # type: ignore[arg-type]
# Should have user message and assistant message, but no reasoning content
assert len(messages) == 2
assert messages[0]["role"] == "user"
assert messages[1]["role"] == "assistant"
# Verify no thinking blocks in assistant message
assistant_msg = messages[1]
content = assistant_msg.get("content")
if isinstance(content, list):
for part in content:
assert part.get("type") != "thinking"
def test_reasoning_items_preserved_in_message_conversion():
"""
Test that reasoning content and thinking blocks are properly extracted
from Anthropic responses and stored in reasoning items.
"""
# Create mock message with thinking blocks
mock_message = create_mock_anthropic_response_with_thinking()
# Convert to output items
output_items = Converter.message_to_output_items(mock_message)
# Should have reasoning item, message item, and tool call items
reasoning_items = [
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
]
assert len(reasoning_items) == 1
reasoning_item = reasoning_items[0]
assert reasoning_item.summary[0].text == "I need to call the weather function for Paris"
# Verify thinking blocks are stored if we preserve them
if (
hasattr(reasoning_item, "content")
and reasoning_item.content
and len(reasoning_item.content) > 0
):
thinking_block = reasoning_item.content[0]
assert thinking_block.type == "reasoning_text"
assert thinking_block.text == "I need to call the weather function for Paris"
def test_anthropic_thinking_blocks_with_tool_calls():
"""
Test for models with extended thinking and interleaved thinking with tool calls.
This test verifies the Anthropic's API's requirements for thinking blocks
to be the first content in assistant messages when reasoning is enabled and tool
calls are present.
"""
# Create a message with reasoning, thinking blocks and tool calls
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather for you.",
reasoning_content="The user wants weather information, I need to call the weather function",
thinking_blocks=[
{
"type": "thinking",
"thinking": (
"The user is asking about weather. "
"Let me use the weather tool to get this information."
),
"signature": "TestSignature123",
},
{
"type": "thinking",
"thinking": ("We should use the city Tokyo as the city."),
"signature": "TestSignature456",
},
],
tool_calls=[
ChatCompletionMessageToolCall(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
],
)
# Step 1: Convert message to output items
output_items = Converter.message_to_output_items(message)
# Verify reasoning item exists and contains thinking blocks
reasoning_items = [
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
]
assert len(reasoning_items) == 1, "Should have exactly two reasoning items"
reasoning_item = reasoning_items[0]
# Verify thinking text is stored in content
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
"Reasoning item should have content"
)
assert reasoning_item.content[0].type == "reasoning_text", (
"Content should be reasoning_text type"
)
# Verify signature is stored in encrypted_content
assert hasattr(reasoning_item, "encrypted_content"), (
"Reasoning item should have encrypted_content"
)
assert reasoning_item.encrypted_content == "TestSignature123\nTestSignature456", (
"Signature should be preserved"
)
# Verify tool calls are present
tool_call_items = [
item for item in output_items if hasattr(item, "type") and item.type == "function_call"
]
assert len(tool_call_items) == 1, "Should have exactly one tool call"
# Step 2: Convert output items back to messages
# Convert items to dicts for the converter (simulating serialization/deserialization)
items_as_dicts: list[dict[str, Any]] = []
for item in output_items:
if hasattr(item, "model_dump"):
items_as_dicts.append(item.model_dump())
else:
items_as_dicts.append(cast(dict[str, Any], item))
messages = Converter.items_to_messages(
items_as_dicts, # type: ignore[arg-type]
model="anthropic/claude-4-opus",
preserve_thinking_blocks=True,
)
# Find the assistant message with tool calls
assistant_messages = [
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
]
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
assistant_msg = assistant_messages[0]
# Content must start with thinking blocks, not text
content = assistant_msg.get("content")
assert content is not None, "Assistant message should have content"
assert isinstance(content, list) and len(content) > 0, (
"Assistant message content should be a non-empty list"
)
first_content = content[0]
assert first_content.get("type") == "thinking", (
f"First content must be 'thinking' type for Anthropic compatibility, "
f"but got '{first_content.get('type')}'"
)
expected_thinking = (
"The user is asking about weather. Let me use the weather tool to get this information."
)
assert first_content.get("thinking") == expected_thinking, (
"Thinking content should be preserved"
)
# Signature should also be preserved
assert first_content.get("signature") == "TestSignature123", (
"Signature should be preserved in thinking block"
)
second_content = content[1]
assert second_content.get("type") == "thinking", (
f"Second content must be 'thinking' type for Anthropic compatibility, "
f"but got '{second_content.get('type')}'"
)
expected_thinking = "We should use the city Tokyo as the city."
assert second_content.get("thinking") == expected_thinking, (
"Thinking content should be preserved"
)
# Signature should also be preserved
assert second_content.get("signature") == "TestSignature456", (
"Signature should be preserved in thinking block"
)
last_content = content[2]
assert last_content.get("type") == "text", (
f"First content must be 'text' type but got '{last_content.get('type')}'"
)
expected_text = "I'll check the weather for you."
assert last_content.get("text") == expected_text, "Content text should be preserved"
# Verify tool calls are preserved
tool_calls = assistant_msg.get("tool_calls", [])
assert len(cast(list[Any], tool_calls)) == 1, "Tool calls should be preserved"
assert cast(list[Any], tool_calls)[0]["function"]["name"] == "get_weather"
def test_items_to_messages_preserves_positional_bool_arguments():
"""
Preserve positional compatibility for the released items_to_messages signature.
"""
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather for you.",
reasoning_content="The user wants weather information, I need to call the weather function",
thinking_blocks=[
{
"type": "thinking",
"thinking": (
"The user is asking about weather. "
"Let me use the weather tool to get this information."
),
"signature": "TestSignature123",
}
],
tool_calls=[
ChatCompletionMessageToolCall(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
],
)
output_items = Converter.message_to_output_items(message)
items_as_dicts: list[dict[str, Any]] = []
for item in output_items:
if hasattr(item, "model_dump"):
items_as_dicts.append(item.model_dump())
else:
items_as_dicts.append(cast(dict[str, Any], item))
messages = Converter.items_to_messages(
items_as_dicts, # type: ignore[arg-type]
"anthropic/claude-4-opus",
True,
True,
)
assistant_messages = [
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
]
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
assistant_msg = assistant_messages[0]
content = assistant_msg.get("content")
assert isinstance(content, list) and len(content) > 0, (
"Positional bool arguments should still preserve thinking blocks"
)
assert content[0].get("type") == "thinking", (
"The third positional argument must continue to map to preserve_thinking_blocks"
)
def test_anthropic_thinking_blocks_without_tool_calls():
"""
Test for models with extended thinking WITHOUT tool calls.
This test verifies that thinking blocks are properly attached to assistant
messages even when there are no tool calls (fixes issue #2195).
"""
# Create a message with reasoning and thinking blocks but NO tool calls
message = InternalChatCompletionMessage(
role="assistant",
content="The weather in Paris is sunny with a temperature of 22°C.",
reasoning_content="The user wants to know about the weather in Paris.",
thinking_blocks=[
{
"type": "thinking",
"thinking": "Let me think about the weather in Paris.",
"signature": "TestSignatureNoTools123",
}
],
tool_calls=None, # No tool calls
)
# Step 1: Convert message to output items
output_items = Converter.message_to_output_items(message)
# Verify reasoning item exists and contains thinking blocks
reasoning_items = [
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
]
assert len(reasoning_items) == 1, "Should have exactly one reasoning item"
reasoning_item = reasoning_items[0]
# Verify thinking text is stored in content
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
"Reasoning item should have content"
)
assert reasoning_item.content[0].type == "reasoning_text", (
"Content should be reasoning_text type"
)
assert reasoning_item.content[0].text == "Let me think about the weather in Paris.", (
"Thinking text should be preserved"
)
# Verify signature is stored in encrypted_content
assert hasattr(reasoning_item, "encrypted_content"), (
"Reasoning item should have encrypted_content"
)
assert reasoning_item.encrypted_content == "TestSignatureNoTools123", (
"Signature should be preserved"
)
# Verify message item exists
message_items = [
item for item in output_items if hasattr(item, "type") and item.type == "message"
]
assert len(message_items) == 1, "Should have exactly one message item"
# Step 2: Convert output items back to messages with preserve_thinking_blocks=True
items_as_dicts: list[dict[str, Any]] = []
for item in output_items:
if hasattr(item, "model_dump"):
items_as_dicts.append(item.model_dump())
else:
items_as_dicts.append(cast(dict[str, Any], item))
messages = Converter.items_to_messages(
items_as_dicts, # type: ignore[arg-type]
model="anthropic/claude-4-opus",
preserve_thinking_blocks=True,
)
# Should have one assistant message
assistant_messages = [msg for msg in messages if msg.get("role") == "assistant"]
assert len(assistant_messages) == 1, "Should have exactly one assistant message"
assistant_msg = assistant_messages[0]
# Content must start with thinking blocks even WITHOUT tool calls
content = assistant_msg.get("content")
assert content is not None, "Assistant message should have content"
assert isinstance(content, list), (
f"Assistant message content should be a list when thinking blocks are present, "
f"but got {type(content)}"
)
assert len(content) >= 2, (
f"Assistant message should have at least 2 content items "
f"(thinking + text), got {len(content)}"
)
# First content should be thinking block
first_content = content[0]
assert first_content.get("type") == "thinking", (
f"First content must be 'thinking' type for Anthropic compatibility, "
f"but got '{first_content.get('type')}'"
)
assert first_content.get("thinking") == "Let me think about the weather in Paris.", (
"Thinking content should be preserved"
)
assert first_content.get("signature") == "TestSignatureNoTools123", (
"Signature should be preserved in thinking block"
)
# Second content should be text
second_content = content[1]
assert second_content.get("type") == "text", (
f"Second content must be 'text' type, but got '{second_content.get('type')}'"
)
assert (
second_content.get("text") == "The weather in Paris is sunny with a temperature of 22°C."
), "Text content should be preserved"
+904
View File
@@ -0,0 +1,904 @@
from __future__ import annotations
import importlib
import sys
import types as pytypes
from collections.abc import AsyncIterator
from typing import Any, Literal, cast
import pytest
from openai.types.chat import (
ChatCompletion,
ChatCompletionChunk,
ChatCompletionMessage,
ChatCompletionMessageFunctionToolCall,
)
from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from openai.types.completion_usage import CompletionUsage, PromptTokensDetails
from openai.types.responses import Response, ResponseCompletedEvent, ResponseOutputMessage
from openai.types.responses.response_error_event import ResponseErrorEvent
from openai.types.responses.response_failed_event import ResponseFailedEvent
from openai.types.responses.response_incomplete_event import ResponseIncompleteEvent
from openai.types.responses.response_output_text import ResponseOutputText
from openai.types.responses.response_usage import (
InputTokensDetails,
OutputTokensDetails,
ResponseUsage,
)
from pydantic import BaseModel
from agents import (
Agent,
Handoff,
ModelBehaviorError,
ModelSettings,
ModelTracing,
Tool,
TResponseInputItem,
__version__,
)
from agents.exceptions import UserError
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
from agents.models.fake_id import FAKE_RESPONSES_ID
class FakeAnyLLMProvider:
def __init__(
self,
*,
supports_responses: bool,
chat_response: Any | None = None,
responses_response: Any | None = None,
) -> None:
self.SUPPORTS_RESPONSES = supports_responses
self.chat_response = chat_response
self.responses_response = responses_response
self.chat_calls: list[dict[str, Any]] = []
self.responses_calls: list[dict[str, Any]] = []
self.private_responses_calls: list[dict[str, Any]] = []
async def acompletion(self, **kwargs: Any) -> Any:
self.chat_calls.append(kwargs)
return self.chat_response
async def aresponses(self, **kwargs: Any) -> Any:
self.responses_calls.append(kwargs)
return self.responses_response
async def _aresponses(self, params: Any, **kwargs: Any) -> Any:
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
return self.responses_response
def _import_any_llm_module(
monkeypatch: pytest.MonkeyPatch,
provider: FakeAnyLLMProvider,
) -> tuple[Any, list[dict[str, Any]]]:
create_calls: list[dict[str, Any]] = []
class FakeAnyLLMFactory:
@staticmethod
def create(provider_name: str, api_key: str | None = None, api_base: str | None = None):
create_calls.append(
{
"provider_name": provider_name,
"api_key": api_key,
"api_base": api_base,
}
)
return provider
fake_any_llm: Any = pytypes.ModuleType("any_llm")
fake_any_llm.AnyLLM = FakeAnyLLMFactory
sys.modules.pop("agents.extensions.models.any_llm_model", None)
monkeypatch.setitem(sys.modules, "any_llm", fake_any_llm)
module = importlib.import_module("agents.extensions.models.any_llm_model")
monkeypatch.setattr(module, "AnyLLM", FakeAnyLLMFactory, raising=True)
return module, create_calls
def _chat_completion(text: str) -> ChatCompletion:
return ChatCompletion(
id="chatcmpl_123",
created=0,
model="fake-model",
object="chat.completion",
choices=[
Choice(
index=0,
finish_reason="stop",
message=ChatCompletionMessage(role="assistant", content=text),
)
],
usage=CompletionUsage(
completion_tokens=5,
prompt_tokens=7,
total_tokens=12,
prompt_tokens_details=PromptTokensDetails.model_validate(
{"cached_tokens": 2, "cache_write_tokens": 4}
),
),
)
def _responses_output(text: str) -> list[Any]:
return [
ResponseOutputMessage(
id="msg_123",
role="assistant",
status="completed",
type="message",
content=[
ResponseOutputText(
text=text,
type="output_text",
annotations=[],
logprobs=[],
)
],
)
]
def _response(text: str, response_id: str = "resp_123") -> Response:
return Response(
id=response_id,
created_at=123,
model="fake-model",
object="response",
output=_responses_output(text),
tool_choice="none",
tools=[],
parallel_tool_calls=False,
usage=ResponseUsage(
input_tokens=11,
output_tokens=13,
total_tokens=24,
input_tokens_details=InputTokensDetails.model_validate(
{"cache_write_tokens": 0, "cached_tokens": 0}
),
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
),
)
def _chat_completion_with_tool_call(*, thought_signature: str) -> ChatCompletion:
return ChatCompletion(
id="chatcmpl_tool_123",
created=0,
model="fake-model",
object="chat.completion",
choices=[
Choice(
index=0,
finish_reason="tool_calls",
message=ChatCompletionMessage(
role="assistant",
content="Calling a tool.",
tool_calls=[
ChatCompletionMessageFunctionToolCall.model_validate(
{
"id": "call_123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city":"Paris"}',
},
"extra_content": {
"google": {"thought_signature": thought_signature}
},
}
)
],
),
)
],
usage=CompletionUsage(
completion_tokens=5,
prompt_tokens=7,
total_tokens=12,
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
),
)
class GenericChatCompletionPayload(BaseModel):
id: str
created: int
model: str
object: str
choices: list[Any]
usage: Any
async def _empty_chat_stream() -> AsyncIterator[ChatCompletionChunk]:
if False:
yield ChatCompletionChunk(
id="chunk_123",
created=0,
model="fake-model",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason=None)],
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
async def test_user_agent_header_any_llm_chat(override_ua: str | None, monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
expected_ua = override_ua or f"Agents/Python {__version__}"
if override_ua is not None:
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
else:
token = None
try:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
finally:
if token is not None:
HEADERS_OVERRIDE.reset(token)
assert provider.chat_calls[0]["extra_headers"]["User-Agent"] == expected_ua
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_chat_path_is_used_when_responses_are_unsupported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
module, create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="router-key")
response = await model.get_response(
system_instructions="You are terse.",
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
assert create_calls == [
{
"provider_name": "openrouter",
"api_key": "router-key",
"api_base": None,
}
]
assert len(provider.chat_calls) == 1
assert provider.responses_calls == []
assert provider.chat_calls[0]["model"] == "openai/gpt-5.4-mini"
assert response.response_id is None
assert response.output[0].content[0].text == "Hello"
assert response.usage.input_tokens_details.cached_tokens == 2
assert getattr(response.usage.input_tokens_details, "cache_write_tokens", None) == 4
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
"chat_response",
[
pytest.param(_chat_completion("Hello").model_dump(), id="dict"),
pytest.param(
GenericChatCompletionPayload.model_validate(_chat_completion("Hello").model_dump()),
id="basemodel",
),
],
)
async def test_any_llm_chat_path_normalizes_non_stream_payloads(
monkeypatch,
chat_response: Any,
) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=chat_response)
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
assert response.response_id is None
assert response.output[0].content[0].text == "Hello"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_chat_path_preserves_gemini_tool_call_metadata(monkeypatch) -> None:
provider = FakeAnyLLMProvider(
supports_responses=False,
chat_response=_chat_completion_with_tool_call(thought_signature="sig_123"),
)
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="gemini/gemini-2.0-flash")
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
function_calls = [
item for item in response.output if getattr(item, "type", None) == "function_call"
]
assert len(function_calls) == 1
provider_data = function_calls[0].model_dump()["provider_data"]
assert provider_data["model"] == "gemini/gemini-2.0-flash"
assert provider_data["response_id"] == "chatcmpl_tool_123"
assert provider_data["thought_signature"] == "sig_123"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_path_is_used_when_supported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
module, create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="gpt-5.4-mini", api_key="openai-key")
response = await model.get_response(
system_instructions="You are terse.",
input="hi",
model_settings=ModelSettings(store=True),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
assert create_calls == [
{
"provider_name": "openai",
"api_key": "openai-key",
"api_base": None,
}
]
assert provider.chat_calls == []
assert provider.responses_calls == []
assert len(provider.private_responses_calls) == 1
params = provider.private_responses_calls[0]["params"]
kwargs = provider.private_responses_calls[0]["kwargs"]
assert params.model == "gpt-5.4-mini"
assert params.previous_response_id == "resp_prev"
assert params.conversation == "conv_123"
assert kwargs["extra_headers"]["User-Agent"] == f"Agents/Python {__version__}"
assert response.response_id == "resp_123"
assert response.output[0].content[0].text == "Hello"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_can_force_chat_completions_when_responses_are_supported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(
supports_responses=True,
chat_response=_chat_completion("Hello from chat"),
responses_response=_response("Hello from responses"),
)
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-4.1-mini", api="chat_completions")
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
assert len(provider.chat_calls) == 1
assert provider.responses_calls == []
assert response.response_id is None
assert response.output[0].content[0].text == "Hello from chat"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_forced_responses_errors_when_provider_does_not_support_it(
monkeypatch,
) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-4.1-mini", api="responses")
with pytest.raises(UserError, match="does not support the Responses API"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_stream_uses_chat_handler_when_responses_are_unsupported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_empty_chat_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
completed = ResponseCompletedEvent(
type="response.completed",
response=_response("Hello from stream"),
sequence_number=1,
)
async def fake_handle_stream(response, stream, model=None):
assert model == "openrouter/openai/gpt-5.4-mini"
async for _chunk in stream:
pass
yield completed
monkeypatch.setattr(module.ChatCmplStreamHandler, "handle_stream", fake_handle_stream)
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
events = [
event
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
]
assert [event.type for event in events] == ["response.completed"]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_stream_passthrough_uses_responses_when_supported(monkeypatch) -> None:
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
yield ResponseCompletedEvent(
type="response.completed",
response=_response("Hello from responses stream"),
sequence_number=1,
)
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
events = [
event
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
]
assert [event.type for event in events] == ["response.completed"]
assert provider.responses_calls == []
assert provider.private_responses_calls[0]["params"].previous_response_id == "resp_prev"
assert provider.private_responses_calls[0]["params"].conversation == "conv_123"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
("terminal_event_type", "terminal_event_cls"),
[
("response.incomplete", ResponseIncompleteEvent),
("response.failed", ResponseFailedEvent),
],
)
async def test_any_llm_responses_stream_rejects_failed_terminal_events(
monkeypatch,
terminal_event_type: str,
terminal_event_cls: type[Any],
) -> None:
async def response_stream() -> AsyncIterator[Any]:
yield terminal_event_cls(
type=terminal_event_type,
response=_response("partial", response_id="resp-terminal"),
sequence_number=1,
)
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
events = []
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
events.append(event)
assert len(events) == 1
assert events[0].type == terminal_event_type
assert events[0].response.id == "resp-terminal"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_stream_rejects_error_event(monkeypatch) -> None:
async def response_stream() -> AsyncIterator[ResponseErrorEvent]:
yield ResponseErrorEvent(
type="error",
code="invalid_request_error",
message="bad request",
param=None,
sequence_number=1,
)
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
events = []
with pytest.raises(ModelBehaviorError, match="invalid_request_error"):
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
events.append(event)
assert len(events) == 1
assert events[0].type == "error"
assert events[0].code == "invalid_request_error"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_path_passes_transport_kwargs_via_private_provider_api(
monkeypatch,
) -> None:
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(
extra_headers={"X-Test-Header": "test"},
extra_query={"trace": "1"},
extra_body={"foo": "bar"},
),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
assert provider.responses_calls == []
assert len(provider.private_responses_calls) == 1
call = provider.private_responses_calls[0]
assert call["kwargs"]["extra_headers"]["X-Test-Header"] == "test"
assert call["kwargs"]["extra_query"] == {"trace": "1"}
assert call["kwargs"]["extra_body"] == {"foo": "bar"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_prompt_requests_fail_fast(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
with pytest.raises(Exception, match="prompt-managed requests"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt={"id": "pmpt_123"},
)
def test_any_llm_responses_input_sanitizer_strips_none_fields_from_reasoning_items() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from agents.extensions.models.any_llm_model import AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
raw_input = [
{
"id": "rid1",
"summary": [{"text": "why", "type": "summary_text"}],
"type": "reasoning",
"content": [{"type": "reasoning_text", "text": "thinking"}],
"status": None,
"encrypted_content": None,
}
]
cleaned = model._sanitize_any_llm_responses_input(raw_input)
assert cleaned == [
{
"id": "rid1",
"summary": [{"text": "why", "type": "summary_text"}],
"type": "reasoning",
"content": [{"type": "reasoning_text", "text": "thinking"}],
}
]
ResponsesParams = importlib.import_module("any_llm.types.responses").ResponsesParams
params = ResponsesParams(model="dummy", input=cleaned)
assert isinstance(params.input, list)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_path_sanitizes_replayed_items_before_validation() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from agents.extensions.models.any_llm_model import AnyLLMModel
class ValidatingProvider:
SUPPORTS_RESPONSES = True
def __init__(self) -> None:
self.private_responses_calls: list[dict[str, Any]] = []
async def aresponses(self, **kwargs: Any) -> Any:
raise AssertionError("public aresponses path should not be used in this test")
async def _aresponses(self, params: Any, **kwargs: Any) -> Response:
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
return _response("Hello from sanitized replay")
class TestAnyLLMModel(AnyLLMModel):
def __init__(self, provider: ValidatingProvider) -> None:
super().__init__(model="openai/gpt-5.4-mini", api="responses")
self._provider = provider
def _get_provider(self) -> Any:
return self._provider
provider = ValidatingProvider()
model = TestAnyLLMModel(provider)
tools: list[Tool] = []
handoffs: list[Handoff[Any, Agent[Any]]] = []
stream_flag: Literal[False] = False
replay_input = cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": FAKE_RESPONSES_ID,
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": [{"type": "reasoning_text", "text": "thinking"}],
"status": None,
"provider_data": {"model": "anthropic/fake-responses-model"},
},
{
"id": FAKE_RESPONSES_ID,
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"status": None,
"provider_data": {"model": "anthropic/fake-responses-model"},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
response = await model._fetch_responses_response(
system_instructions=None,
input=replay_input,
model_settings=ModelSettings(),
tools=tools,
output_schema=None,
handoffs=handoffs,
previous_response_id=None,
conversation_id=None,
stream=stream_flag,
prompt=None,
)
assert response.id == "resp_123"
assert len(provider.private_responses_calls) == 1
params = provider.private_responses_calls[0]["params"]
assert params.input == [
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
]
def test_any_llm_provider_passes_api_override() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from agents.extensions.models.any_llm_model import AnyLLMModel
from agents.extensions.models.any_llm_provider import AnyLLMProvider
provider = AnyLLMProvider(api="chat_completions")
model = provider.get_model("openai/gpt-4.1-mini")
assert isinstance(model, AnyLLMModel)
assert model.api == "chat_completions"
def test_any_llm_reasoning_objects_prefer_content_attributes_over_iterable_pairs() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from any_llm.types.completion import Reasoning
from agents.extensions.models.any_llm_model import _extract_any_llm_reasoning_text
delta = pytypes.SimpleNamespace(reasoning=Reasoning(content="用户"))
assert _extract_any_llm_reasoning_text(delta) == "用户"
def test_any_llm_split_does_not_duplicate_content_or_thinking(monkeypatch) -> None:
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
Anthropic's extended thinking API rejects requests that include the same signed
thinking block more than once, and duplicated assistant text corrupts conversation
history. Only the first split should retain content, thinking_blocks, and
reasoning_content; subsequent splits should carry the tool_call alone.
"""
provider = FakeAnyLLMProvider(supports_responses=False)
module, _ = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="anthropic/claude-3-5-sonnet")
messages: list[Any] = [
{"role": "user", "content": "Search both"},
{
"role": "assistant",
"content": "Looking up both queries.",
"thinking_blocks": [{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}],
"reasoning_content": "internal plan",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
]
result = model._fix_tool_message_ordering(messages)
assistants = [m for m in result if m.get("role") == "assistant"]
assert len(assistants) == 2
# First split keeps the shared fields.
assert assistants[0].get("content") == "Looking up both queries."
assert "thinking_blocks" in assistants[0]
assert "reasoning_content" in assistants[0]
# Second split must NOT duplicate them.
assert "content" not in assistants[1]
assert "thinking_blocks" not in assistants[1]
assert "reasoning_content" not in assistants[1]
# Tool calls are still split one-per-message.
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
@@ -0,0 +1,361 @@
from typing import Any
import litellm
import pytest
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
Function,
Message,
ModelResponse,
Usage,
)
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models.chatcmpl_converter import Converter
from agents.models.interface import ModelTracing
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_deepseek_reasoning_content_preserved_in_tool_calls(monkeypatch):
"""
Ensure DeepSeek reasoning_content is preserved when converting items to messages.
DeepSeek requires reasoning_content field in assistant messages with tool_calls.
This test verifies that reasoning content from reasoning items is correctly
extracted and added to assistant messages during conversion.
"""
# Capture the messages sent to the model
captured_calls: list[dict[str, Any]] = []
async def fake_acompletion(model, messages=None, **kwargs):
captured_calls.append({"model": model, "messages": messages, **kwargs})
# First call: model returns reasoning_content + tool_call
if len(captured_calls) == 1:
tool_call = ChatCompletionMessageToolCall(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
msg = Message(
role="assistant",
content=None,
tool_calls=[tool_call],
)
# DeepSeek adds reasoning_content to the message
msg.reasoning_content = "Let me think about getting the weather for Tokyo..."
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
# Second call: model returns final response
msg = Message(role="assistant", content="The weather in Tokyo is sunny.")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(model="deepseek/deepseek-reasoner")
# First call: get the tool call response
first_response = await model.get_response(
system_instructions="You are a helpful assistant.",
input="What's the weather in Tokyo?",
model_settings=ModelSettings(),
tools=[], # We'll simulate the tool response manually
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert len(first_response.output) >= 1
input_items: list[Any] = []
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
for item in first_response.output:
if hasattr(item, "model_dump"):
input_items.append(item.model_dump())
else:
input_items.append(item)
input_items.append(
{
"type": "function_call_output",
"call_id": "call_123",
"output": "The weather in Tokyo is sunny.",
}
)
messages = Converter.items_to_messages(
input_items,
model="deepseek/deepseek-reasoner",
)
assistant_messages_with_tool_calls = [
m
for m in messages
if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls")
]
assert len(assistant_messages_with_tool_calls) > 0
assistant_msg = assistant_messages_with_tool_calls[0]
assert "reasoning_content" in assistant_msg
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_deepseek_reasoning_content_in_multi_turn_conversation(monkeypatch):
"""
Verify reasoning_content is included in assistant messages during multi-turn conversations.
When DeepSeek returns reasoning_content with tool_calls, subsequent API calls must
include the reasoning_content field in the assistant message to avoid 400 errors.
"""
captured_calls: list[dict[str, Any]] = []
async def fake_acompletion(model, messages=None, **kwargs):
captured_calls.append({"model": model, "messages": messages, **kwargs})
# First call: model returns reasoning_content + tool_call
if len(captured_calls) == 1:
tool_call = ChatCompletionMessageToolCall(
id="call_weather_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
msg = Message(
role="assistant",
content=None,
tool_calls=[tool_call],
)
# DeepSeek adds reasoning_content
msg.reasoning_content = "I need to get the weather for Tokyo first."
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
# Second call: check if reasoning_content was in the request
# In real DeepSeek API, this would fail with 400 if reasoning_content is missing
msg = Message(
role="assistant", content="Based on my findings, the weather in Tokyo is sunny."
)
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(model="deepseek/deepseek-reasoner")
# First call
first_response = await model.get_response(
system_instructions="You are a helpful assistant.",
input="What's the weather in Tokyo?",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
input_items: list[Any] = []
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
for item in first_response.output:
if hasattr(item, "model_dump"):
input_items.append(item.model_dump())
else:
input_items.append(item)
input_items.append(
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
}
)
await model.get_response(
system_instructions="You are a helpful assistant.",
input=input_items,
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert len(captured_calls) == 2
second_call_messages = captured_calls[1]["messages"]
assistant_with_tools = None
for msg in second_call_messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
assert "reasoning_content" in assistant_with_tools
def test_deepseek_reasoning_content_with_openai_chatcompletions_path():
"""
Verify reasoning_content works when using OpenAIChatCompletionsModel.
This ensures the fix works for both LiteLLM and OpenAI ChatCompletions code paths.
"""
from agents.models.chatcmpl_converter import Converter
input_items: list[Any] = [
{"role": "user", "content": "What's the weather in Paris?"},
{
"id": "__fake_id__",
"summary": [{"text": "I need to check the weather in Paris.", "type": "summary_text"}],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": "deepseek-reasoner", "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Paris"}',
"call_id": "call_weather_456",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": "deepseek-reasoner"},
},
{
"type": "function_call_output",
"call_id": "call_weather_456",
"output": "The weather in Paris is cloudy and 15°C.",
},
]
messages = Converter.items_to_messages(
input_items,
model="deepseek-reasoner",
)
assistant_with_tools = None
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
assert "reasoning_content" in assistant_with_tools
# Use type: ignore since reasoning_content is a dynamic field not in OpenAI's TypedDict
assert assistant_with_tools["reasoning_content"] == "I need to check the weather in Paris." # type: ignore[typeddict-item]
def test_reasoning_content_from_other_provider_not_attached_to_deepseek():
"""
Verify reasoning_content from non-DeepSeek providers is NOT attached to DeepSeek messages.
When switching models mid-conversation (e.g., from Claude to DeepSeek), reasoning items
that originated from Claude should not have their summaries attached as reasoning_content
to DeepSeek assistant messages, as this would leak unrelated reasoning and may trigger
DeepSeek 400 errors.
"""
from agents.models.chatcmpl_converter import Converter
input_items: list[Any] = [
{"role": "user", "content": "What's the weather in Paris?"},
{
"id": "__fake_id__",
"summary": [{"text": "Claude's reasoning about the weather.", "type": "summary_text"}],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
# this one came from Claude, not DeepSeek
"provider_data": {"model": "claude-sonnet-4-20250514", "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Paris"}',
"call_id": "call_weather_789",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": "claude-sonnet-4-20250514"},
},
{
"type": "function_call_output",
"call_id": "call_weather_789",
"output": "The weather in Paris is cloudy.",
},
]
messages = Converter.items_to_messages(
input_items,
model="deepseek-reasoner",
)
assistant_with_tools = None
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
# reasoning_content should NOT be present since the reasoning came from Claude, not DeepSeek
assert "reasoning_content" not in assistant_with_tools
def test_reasoning_content_without_provider_data_attached_for_backward_compat():
"""
Verify reasoning_content from items without provider_data is attached for backward compat.
For older items that don't have provider_data (before provider tracking was added),
we should still attach reasoning_content to maintain backward compatibility.
"""
from agents.models.chatcmpl_converter import Converter
# Reasoning item without provider_data (older format)
input_items: list[Any] = [
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [{"text": "Reasoning without provider info.", "type": "summary_text"}],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
# No provider_data
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_101",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
},
{
"type": "function_call_output",
"call_id": "call_weather_101",
"output": "The weather in Tokyo is sunny.",
},
]
messages = Converter.items_to_messages(
input_items,
model="deepseek-reasoner",
)
assistant_with_tools = None
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
# reasoning_content SHOULD be present for backward compatibility
assert "reasoning_content" in assistant_with_tools
assert assistant_with_tools["reasoning_content"] == "Reasoning without provider info." # type: ignore[typeddict-item]
+201
View File
@@ -0,0 +1,201 @@
import os
from typing import Literal
from unittest.mock import patch
import pytest
from openai.types.shared.reasoning import Reasoning
from agents import Agent
from agents.model_settings import ModelSettings
from agents.models import (
get_default_model,
get_default_model_settings,
gpt_5_reasoning_settings_required,
is_gpt_5_default,
)
def _gpt_5_default_settings(
reasoning_effort: Literal["none", "low", "medium"] | None,
) -> ModelSettings:
if reasoning_effort is None:
return ModelSettings(verbosity="low")
return ModelSettings(reasoning=Reasoning(effort=reasoning_effort), verbosity="low")
def test_default_model_is_gpt_5_4_mini():
assert get_default_model() == "gpt-5.4-mini"
assert is_gpt_5_default() is True
assert gpt_5_reasoning_settings_required(get_default_model()) is True
assert get_default_model_settings() == _gpt_5_default_settings("none")
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.4"})
def test_is_gpt_5_default_with_real_model_name():
assert get_default_model() == "gpt-5.4"
assert is_gpt_5_default() is True
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-4.1"})
def test_is_gpt_5_default_returns_false_for_non_gpt_5_default_model():
assert get_default_model() == "gpt-4.1"
assert is_gpt_5_default() is False
def test_gpt_5_reasoning_settings_required_detects_gpt_5_models_while_ignoring_chat_latest():
assert gpt_5_reasoning_settings_required("gpt-5") is True
assert gpt_5_reasoning_settings_required("gpt-5.1") is True
assert gpt_5_reasoning_settings_required("gpt-5.2") is True
assert gpt_5_reasoning_settings_required("gpt-5.2-codex") is True
assert gpt_5_reasoning_settings_required("gpt-5.2-pro") is True
assert gpt_5_reasoning_settings_required("gpt-5.4-pro") is True
assert gpt_5_reasoning_settings_required("gpt-5.5") is True
assert gpt_5_reasoning_settings_required("gpt-5-mini") is True
assert gpt_5_reasoning_settings_required("gpt-5-nano") is True
assert gpt_5_reasoning_settings_required("gpt-5-chat-latest") is False
assert gpt_5_reasoning_settings_required("gpt-5.1-chat-latest") is False
assert gpt_5_reasoning_settings_required("gpt-5.2-chat-latest") is False
assert gpt_5_reasoning_settings_required("gpt-5.3-chat-latest") is False
def test_gpt_5_reasoning_settings_required_returns_false_for_non_gpt_5_models():
assert gpt_5_reasoning_settings_required("gpt-4.1") is False
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_1_models():
assert get_default_model_settings("gpt-5.1") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.1-2025-11-13") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_2_models():
assert get_default_model_settings("gpt-5.2") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.2-2025-12-11") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_3_codex_models():
assert get_default_model_settings("gpt-5.3-codex") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_models():
assert get_default_model_settings("gpt-5.4") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_snapshot_families():
assert get_default_model_settings("gpt-5.4-2026-03-05") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.4-mini-2026-03-17") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.4-nano-2026-03-17") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_mini_and_nano():
assert get_default_model_settings("gpt-5.4-mini") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.4-nano") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_5_models():
assert get_default_model_settings("gpt-5.5") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.5-2026-04-23") == _gpt_5_default_settings("none")
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_6_models(
model: str,
):
assert get_default_model_settings(model) == _gpt_5_default_settings("none")
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
def test_agent_uses_gpt_5_6_model_settings_from_default_model_env(
model: str, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", model.upper())
agent = Agent(name="test")
assert get_default_model() == model
assert agent.model is None
assert agent.model_settings == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_low_reasoning_defaults_for_base_gpt_5():
assert get_default_model_settings("gpt-5") == _gpt_5_default_settings("low")
assert get_default_model_settings("gpt-5-2025-08-07") == _gpt_5_default_settings("low")
def test_get_default_model_settings_returns_low_reasoning_defaults_for_gpt_5_2_codex():
assert get_default_model_settings("gpt-5.2-codex") == _gpt_5_default_settings("low")
def test_get_default_model_settings_returns_medium_reasoning_defaults_for_gpt_5_pro_models():
assert get_default_model_settings("gpt-5.2-pro") == _gpt_5_default_settings("medium")
assert get_default_model_settings("gpt-5.2-pro-2025-12-11") == _gpt_5_default_settings("medium")
assert get_default_model_settings("gpt-5.4-pro") == _gpt_5_default_settings("medium")
assert get_default_model_settings("gpt-5.4-pro-2026-03-05") == _gpt_5_default_settings("medium")
def test_get_default_model_settings_omits_reasoning_for_unconfirmed_gpt_5_variants():
assert get_default_model_settings("gpt-5-mini") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5-mini-2025-08-07") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5-nano") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5-nano-2025-08-07") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5.1-codex") == _gpt_5_default_settings(None)
def test_get_default_model_settings_returns_empty_settings_for_gpt_5_chat_latest_aliases():
assert get_default_model_settings("gpt-5-chat-latest") == ModelSettings()
assert get_default_model_settings("gpt-5.1-chat-latest") == ModelSettings()
assert get_default_model_settings("gpt-5.2-chat-latest") == ModelSettings()
assert get_default_model_settings("gpt-5.3-chat-latest") == ModelSettings()
def test_get_default_model_settings_returns_empty_settings_for_non_gpt_5_models():
assert get_default_model_settings("gpt-4.1") == ModelSettings()
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
def test_agent_uses_gpt_5_default_model_settings():
"""Agent should inherit GPT-5 default model settings."""
agent = Agent(name="test")
assert agent.model is None
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
assert agent.model_settings.verbosity == "low"
def test_agent_uses_model_specific_settings_for_explicit_gpt_5_models():
"""Agent should not apply the fallback model's GPT-5 settings to explicit GPT-5 models."""
agent = Agent(name="test", model="gpt-5")
assert agent.model == "gpt-5"
assert agent.model_settings == get_default_model_settings("gpt-5")
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
def test_agent_uses_empty_settings_for_explicit_non_gpt_5_models():
"""Agent should not apply GPT-5 defaults to explicit non-GPT-5 models."""
agent = Agent(name="test", model="gpt-4.1")
assert agent.model == "gpt-4.1"
assert agent.model_settings == ModelSettings()
def test_agent_clone_recomputes_implicit_settings_when_model_changes():
"""Agent.clone should keep implicit model settings aligned with the cloned model."""
agent = Agent(name="test", model="gpt-5")
cloned = agent.clone(model="gpt-5.4-mini")
assert cloned.model == "gpt-5.4-mini"
assert cloned.model_settings == get_default_model_settings("gpt-5.4-mini")
assert cloned.model_settings.reasoning.effort == "none" # type: ignore[union-attr]
def test_agent_clone_preserves_explicit_settings_when_model_changes():
"""Agent.clone should not recompute model settings that were explicitly customized."""
model_settings = ModelSettings(temperature=0.3)
agent = Agent(name="test", model="gpt-5", model_settings=model_settings)
cloned = agent.clone(model="gpt-5.4-mini")
assert cloned.model == "gpt-5.4-mini"
assert cloned.model_settings == model_settings
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
def test_agent_resets_model_settings_for_non_gpt_5_models():
"""Agent should reset default GPT-5 settings when using a non-GPT-5 model."""
agent = Agent(name="test", model="gpt-4.1")
assert agent.model == "gpt-4.1"
assert agent.model_settings == ModelSettings()
@@ -0,0 +1,354 @@
"""Tests for the extended thinking message order bug fix in LitellmModel."""
from __future__ import annotations
from typing import Any, cast
from openai.types.chat import ChatCompletionMessageParam
from agents.extensions.models.litellm_model import LitellmModel
class TestExtendedThinkingMessageOrder:
"""Test the _fix_tool_message_ordering method."""
def test_basic_reordering_tool_result_before_call(self):
"""Test that a tool result appearing before its tool call gets reordered correctly."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{"role": "tool", "tool_call_id": "call_123", "content": "Result for call_123"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_123",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "user", "content": "Thanks"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should reorder to: user, assistant+tool_call, tool_result, user
assert len(result) == 4
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_123" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_123"
assert result[3]["role"] == "user"
def test_consecutive_tool_calls_get_separated(self):
"""Test that consecutive assistant messages with tool calls get properly paired with results.""" # noqa: E501
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test1", "arguments": "{}"},
}
],
},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_2",
"type": "function",
"function": {"name": "test2", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should pair each tool call with its result immediately
assert len(result) == 5
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "assistant"
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
assert result[4]["role"] == "tool"
assert result[4]["tool_call_id"] == "call_2"
def test_unmatched_tool_results_preserved(self):
"""Test that tool results without matching tool calls are preserved."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Matched result"},
{"role": "tool", "tool_call_id": "call_orphan", "content": "Orphaned result"},
{"role": "user", "content": "End"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should preserve the orphaned tool result
assert len(result) == 5
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "tool" # Orphaned result preserved
assert result[3]["tool_call_id"] == "call_orphan"
assert result[4]["role"] == "user"
def test_tool_calls_without_results_preserved(self):
"""Test that tool calls without results are still included."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "user", "content": "End"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should preserve the tool call even without a result
assert len(result) == 3
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "user"
def test_correctly_ordered_messages_unchanged(self):
"""Test that correctly ordered messages remain in the same order."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Result"},
{"role": "assistant", "content": "Done"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should remain exactly the same
assert len(result) == 4
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "assistant"
def test_multiple_tool_calls_single_message(self):
"""Test assistant message with multiple tool calls gets split properly."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test1", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "test2", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should split the multi-tool message and pair each properly
assert len(result) == 5
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert len(result[1]["tool_calls"]) == 1 # type: ignore
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "assistant"
assert len(result[3]["tool_calls"]) == 1 # type: ignore
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
assert result[4]["role"] == "tool"
assert result[4]["tool_call_id"] == "call_2"
def test_split_does_not_duplicate_content_or_thinking(self):
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
Anthropic's extended thinking API rejects requests that include the same signed
thinking block more than once, and duplicated assistant text corrupts conversation
history. Only the first split should retain content, thinking_blocks, and
reasoning_content; subsequent splits should carry the tool_call alone.
"""
# Build the assistant message via cast so mypy doesn't reject the
# extra keys (`thinking_blocks`, `reasoning_content`) which are not
# part of the upstream ChatCompletionAssistantMessageParam TypedDict
# but are surfaced by litellm for Anthropic extended thinking.
assistant_msg = cast(
ChatCompletionMessageParam,
{
"role": "assistant",
"content": "Looking up both queries.",
"thinking_blocks": [
{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}
],
"reasoning_content": "internal plan",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
],
},
)
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Search both"},
assistant_msg,
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
]
model = LitellmModel("claude-3-5-sonnet")
result = model._fix_tool_message_ordering(messages)
assistants = [cast(dict[str, Any], m) for m in result if m.get("role") == "assistant"]
assert len(assistants) == 2
# First split keeps the shared fields.
assert assistants[0].get("content") == "Looking up both queries."
assert "thinking_blocks" in assistants[0]
assert "reasoning_content" in assistants[0]
# Second split must NOT duplicate them.
assert "content" not in assistants[1]
assert "thinking_blocks" not in assistants[1]
assert "reasoning_content" not in assistants[1]
# Tool calls are still split one-per-message.
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
def test_empty_messages_list(self):
"""Test that empty message list is handled correctly."""
messages: list[ChatCompletionMessageParam] = []
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
assert result == []
def test_no_tool_messages(self):
"""Test that messages without tool calls are left unchanged."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there"},
{"role": "user", "content": "How are you?"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
assert result == messages
def test_complex_mixed_scenario(self):
"""Test a complex scenario with various message types and orderings."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Start"},
{
"role": "tool",
"tool_call_id": "call_out_of_order",
"content": "Out of order result",
}, # This comes before its call
{"role": "assistant", "content": "Regular response"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_out_of_order",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_normal",
"type": "function",
"function": {"name": "test2", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_normal", "content": "Normal result"},
{
"role": "tool",
"tool_call_id": "call_orphan",
"content": "Orphaned result",
}, # No matching call
{"role": "user", "content": "End"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should reorder properly while preserving all messages
assert len(result) == 8
assert result[0]["role"] == "user" # Start
assert result[1]["role"] == "assistant" # Regular response
assert result[2]["role"] == "assistant" # call_out_of_order
assert result[2]["tool_calls"][0]["id"] == "call_out_of_order" # type: ignore
assert result[3]["role"] == "tool" # Out of order result (now properly paired)
assert result[3]["tool_call_id"] == "call_out_of_order"
assert result[4]["role"] == "assistant" # call_normal
assert result[4]["tool_calls"][0]["id"] == "call_normal" # type: ignore
assert result[5]["role"] == "tool" # Normal result
assert result[5]["tool_call_id"] == "call_normal"
assert result[6]["role"] == "tool" # Orphaned result (preserved)
assert result[6]["tool_call_id"] == "call_orphan"
assert result[7]["role"] == "user" # End
@@ -0,0 +1,126 @@
"""
Test for Gemini thought signatures in function calling.
Validates that thought signatures are preserved through the bidirectional roundtrip:
- Gemini chatcmpl message → response item → back to message
"""
from __future__ import annotations
from typing import Any
from openai.types.chat.chat_completion_message_tool_call import Function
from agents.extensions.models.litellm_model import InternalChatCompletionMessage, InternalToolCall
from agents.models.chatcmpl_converter import Converter
def test_gemini_thought_signature_roundtrip():
"""Test that thought signatures are preserved from Gemini responses to messages."""
# Create mock Gemini response with thought signature in new extra_content structure
class MockToolCall(InternalToolCall):
def __init__(self):
super().__init__(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Paris"}'),
extra_content={"google": {"thought_signature": "test_signature_abc"}},
)
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather.",
reasoning_content="",
tool_calls=[MockToolCall()],
)
# Step 1: Convert to items
provider_data = {"model": "gemini/gemini-3-pro", "response_id": "gemini-response-id-123"}
items = Converter.message_to_output_items(message, provider_data=provider_data)
func_calls = [item for item in items if hasattr(item, "type") and item.type == "function_call"]
assert len(func_calls) == 1
# Verify thought_signature is stored in items with our provider_data structure
func_call_dict = func_calls[0].model_dump()
assert func_call_dict["provider_data"]["model"] == "gemini/gemini-3-pro"
assert func_call_dict["provider_data"]["response_id"] == "gemini-response-id-123"
assert func_call_dict["provider_data"]["thought_signature"] == "test_signature_abc"
# Step 2: Convert back to messages
items_as_dicts = [item.model_dump() for item in items]
messages = Converter.items_to_messages(
[{"role": "user", "content": "test"}] + items_as_dicts,
model="gemini/gemini-3-pro",
)
# Verify thought_signature is restored in extra_content format
assistant_msg = [msg for msg in messages if msg.get("role") == "assistant"][0]
tool_call = assistant_msg["tool_calls"][0] # type: ignore[index, typeddict-item]
assert tool_call["extra_content"]["google"]["thought_signature"] == "test_signature_abc"
def test_gemini_multiple_tool_calls_with_thought_signatures():
"""Test multiple tool calls each preserve their own thought signatures."""
tool_call_1 = InternalToolCall(
id="call_1",
type="function",
function=Function(name="func_a", arguments='{"x": 1}'),
extra_content={"google": {"thought_signature": "sig_aaa"}},
)
tool_call_2 = InternalToolCall(
id="call_2",
type="function",
function=Function(name="func_b", arguments='{"y": 2}'),
extra_content={"google": {"thought_signature": "sig_bbb"}},
)
message = InternalChatCompletionMessage(
role="assistant",
content="Calling two functions.",
reasoning_content="",
tool_calls=[tool_call_1, tool_call_2],
)
provider_data = {"model": "gemini/gemini-3-pro"}
items = Converter.message_to_output_items(message, provider_data=provider_data)
func_calls = [i for i in items if hasattr(i, "type") and i.type == "function_call"]
assert len(func_calls) == 2
assert func_calls[0].model_dump()["provider_data"]["thought_signature"] == "sig_aaa"
assert func_calls[1].model_dump()["provider_data"]["thought_signature"] == "sig_bbb"
def test_gemini_thought_signature_items_to_messages():
"""Test that items_to_messages restores extra_content from provider_data for Gemini."""
# Create a function call item with provider_data containing thought_signature
func_call_item = {
"id": "fake-id",
"call_id": "call_restore",
"name": "restore_func",
"arguments": '{"test": true}',
"type": "function_call",
"provider_data": {
"model": "gemini/gemini-3-pro",
"response_id": "gemini-response-id-123",
"thought_signature": "restored_sig_xyz",
},
}
items = [{"role": "user", "content": "test"}, func_call_item]
messages = Converter.items_to_messages(items, model="gemini/gemini-3-pro") # type: ignore[arg-type]
# Find the assistant message with tool_calls
assistant_msgs = [m for m in messages if m.get("role") == "assistant"]
assert len(assistant_msgs) == 1
tool_calls: list[dict[str, Any]] = assistant_msgs[0].get("tool_calls", []) # type: ignore[assignment]
assert len(tool_calls) == 1
# Verify extra_content is restored in Google format
assert tool_calls[0]["extra_content"]["google"]["thought_signature"] == "restored_sig_xyz"
@@ -0,0 +1,210 @@
"""
Test for Gemini thought signatures in streaming function calls.
Validates that thought signatures are captured from streaming chunks
and included in the final function call events.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any, cast
import pytest
from openai.types.chat import ChatCompletionChunk
from openai.types.chat.chat_completion_chunk import (
Choice,
ChoiceDelta,
ChoiceDeltaToolCall,
ChoiceDeltaToolCallFunction,
)
from openai.types.responses import Response
from agents.models.chatcmpl_stream_handler import ChatCmplStreamHandler
# ========== Helper Functions ==========
def create_tool_call_delta(
index: int,
tool_call_id: str | None = None,
function_name: str | None = None,
arguments: str | None = None,
provider_specific_fields: dict[str, Any] | None = None,
extra_content: dict[str, Any] | None = None,
) -> ChoiceDeltaToolCall:
"""Create a tool call delta for streaming."""
function = ChoiceDeltaToolCallFunction(
name=function_name,
arguments=arguments,
)
delta = ChoiceDeltaToolCall(
index=index,
id=tool_call_id,
type="function" if tool_call_id else None,
function=function,
)
# Add provider_specific_fields (litellm format)
if provider_specific_fields:
delta_any = cast(Any, delta)
delta_any.provider_specific_fields = provider_specific_fields
# Add extra_content (Google chatcmpl format)
if extra_content:
delta_any = cast(Any, delta)
delta_any.extra_content = extra_content
return delta
def create_chunk(
tool_calls: list[ChoiceDeltaToolCall] | None = None,
content: str | None = None,
include_usage: bool = False,
) -> ChatCompletionChunk:
"""Create a ChatCompletionChunk for testing."""
delta = ChoiceDelta(
content=content,
role="assistant" if content or tool_calls else None,
tool_calls=tool_calls,
)
chunk = ChatCompletionChunk(
id="chunk-id-123",
created=1,
model="gemini/gemini-3-pro",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=delta, finish_reason=None)],
)
if include_usage:
from openai.types.completion_usage import CompletionUsage
chunk.usage = CompletionUsage(
completion_tokens=10,
prompt_tokens=5,
total_tokens=15,
)
return chunk
def create_final_chunk() -> ChatCompletionChunk:
"""Create a final chunk with finish_reason='tool_calls'."""
return ChatCompletionChunk(
id="chunk-id-456",
created=1,
model="gemini/gemini-3-pro",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="tool_calls")],
)
async def create_fake_stream(
chunks: list[ChatCompletionChunk],
) -> AsyncIterator[ChatCompletionChunk]:
"""Create an async iterator from chunks."""
for chunk in chunks:
yield chunk
def create_mock_response() -> Response:
"""Create a mock Response object."""
return Response(
id="resp-id",
created_at=0,
model="gemini/gemini-3-pro",
object="response",
output=[],
tool_choice="auto",
tools=[],
parallel_tool_calls=False,
)
# ========== Tests ==========
@pytest.mark.asyncio
async def test_stream_captures_litellmprovider_specific_fields_thought_signature():
"""Test streaming captures thought_signature from litellm's provider_specific_fields."""
chunks = [
create_chunk(
tool_calls=[
create_tool_call_delta(
index=0,
tool_call_id="call_stream_1",
function_name="get_weather",
provider_specific_fields={"thought_signature": "litellm_sig_123"},
)
]
),
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"city": "Tokyo"}')]),
create_final_chunk(),
]
response = create_mock_response()
stream = create_fake_stream(chunks)
events = []
async for event in ChatCmplStreamHandler.handle_stream(
response,
stream, # type: ignore[arg-type]
model="gemini/gemini-3-pro",
):
events.append(event)
# Find function call done event
done_events = [e for e in events if e.type == "response.output_item.done"]
func_done = [
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
]
assert len(func_done) == 1
provider_data = func_done[0].item.model_dump().get("provider_data", {})
assert provider_data.get("thought_signature") == "litellm_sig_123"
assert provider_data["model"] == "gemini/gemini-3-pro"
assert provider_data["response_id"] == "chunk-id-123"
@pytest.mark.asyncio
async def test_stream_captures_google_extra_content_thought_signature():
"""Test streaming captures thought_signature from Google's extra_content format."""
chunks = [
create_chunk(
tool_calls=[
create_tool_call_delta(
index=0,
tool_call_id="call_stream_2",
function_name="search",
extra_content={"google": {"thought_signature": "google_sig_456"}},
)
]
),
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"query": "test"}')]),
create_final_chunk(),
]
response = create_mock_response()
stream = create_fake_stream(chunks)
events = []
async for event in ChatCmplStreamHandler.handle_stream(
response,
stream, # type: ignore[arg-type]
model="gemini/gemini-3-pro",
):
events.append(event)
done_events = [e for e in events if e.type == "response.output_item.done"]
func_done = [
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
]
assert len(func_done) == 1
provider_data = func_done[0].item.model_dump().get("provider_data", {})
assert provider_data.get("thought_signature") == "google_sig_456"
assert provider_data["model"] == "gemini/gemini-3-pro"
assert provider_data["response_id"] == "chunk-id-123"
+317
View File
@@ -0,0 +1,317 @@
import httpx
import litellm
import pytest
from httpx import Headers, Response
from litellm.exceptions import RateLimitError
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from openai import APIConnectionError
from openai.types.chat.chat_completion import ChatCompletion, Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.completion_usage import CompletionUsage
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models._retry_runtime import provider_managed_retries_disabled
from agents.models.interface import ModelTracing
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.retry import ModelRetryAdviceRequest, ModelRetrySettings
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_litellm_kwargs_forwarded(monkeypatch):
"""
Test that kwargs from ModelSettings are forwarded to litellm.acompletion.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
temperature=0.5,
extra_args={
"custom_param": "custom_value",
"seed": 42,
"stop": ["END"],
"logit_bias": {123: -100},
},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
)
# Verify that all kwargs were passed through
assert captured["custom_param"] == "custom_value"
assert captured["seed"] == 42
assert captured["stop"] == ["END"]
assert captured["logit_bias"] == {123: -100}
# Verify regular parameters are still passed
assert captured["temperature"] == 0.5
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_openai_chatcompletions_kwargs_forwarded(monkeypatch):
"""
Test that kwargs from ModelSettings are forwarded to OpenAI chat completions API.
"""
captured: dict[str, object] = {}
class MockChatCompletions:
async def create(self, **kwargs):
captured.update(kwargs)
msg = ChatCompletionMessage(role="assistant", content="test response")
choice = Choice(index=0, message=msg, finish_reason="stop")
return ChatCompletion(
id="test-id",
created=0,
model="gpt-4",
object="chat.completion",
choices=[choice],
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
)
class MockChat:
def __init__(self):
self.completions = MockChatCompletions()
class MockClient:
def __init__(self):
self.chat = MockChat()
self.base_url = "https://api.openai.com/v1"
settings = ModelSettings(
temperature=0.7,
extra_args={
"seed": 123,
"logit_bias": {456: 10},
"stop": ["STOP", "END"],
"user": "test-user",
},
)
mock_client = MockClient()
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=mock_client) # type: ignore
await model.get_response(
system_instructions="Test system",
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# Verify that all kwargs were passed through
assert captured["seed"] == 123
assert captured["logit_bias"] == {456: 10}
assert captured["stop"] == ["STOP", "END"]
assert captured["user"] == "test-user"
# Verify regular parameters are still passed
assert captured["temperature"] == 0.7
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_empty_kwargs_handling(monkeypatch):
"""
Test that empty or None kwargs are handled gracefully.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# Test with None kwargs
settings_none = ModelSettings(temperature=0.5, extra_args=None)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings_none,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# Should work without error and include regular parameters
assert captured["temperature"] == 0.5
# Test with empty dict
captured.clear()
settings_empty = ModelSettings(temperature=0.3, extra_args={})
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings_empty,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# Should work without error and include regular parameters
assert captured["temperature"] == 0.3
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_reasoning_effort_falls_back_to_extra_args(monkeypatch):
"""
Ensure reasoning_effort from extra_args is promoted when reasoning settings are missing.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
settings = ModelSettings(
extra_args={"reasoning_effort": "none", "custom_param": "custom_value"}
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "none"
assert captured["custom_param"] == "custom_value"
assert settings.extra_args == {"reasoning_effort": "none", "custom_param": "custom_value"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_litellm_retry_settings_do_not_leak_and_disable_provider_retries_on_runner_retry(
monkeypatch,
):
"""Runner retries should disable LiteLLM's own retries without forwarding SDK retry config."""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
retry=ModelRetrySettings(
max_retries=2,
backoff={"initial_delay": 0.25, "jitter": False},
),
extra_args={"max_retries": 7, "num_retries": 6, "custom_param": "custom_value"},
)
model = LitellmModel(model="test-model")
with provider_managed_retries_disabled(True):
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
)
assert settings.retry is not None
assert settings.retry.backoff is not None
assert captured["custom_param"] == "custom_value"
assert captured["max_retries"] == 0
assert captured["num_retries"] == 0
assert "retry" not in captured
def test_litellm_get_retry_advice_uses_response_headers() -> None:
"""LiteLLM retry advice should expose OpenAI-compatible retry headers."""
model = LitellmModel(model="test-model")
error = RateLimitError(
message="rate limited",
llm_provider="openai",
model="gpt-4o-mini",
response=Response(
status_code=429,
headers=Headers({"x-should-retry": "true", "retry-after-ms": "250"}),
),
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.retry_after == 0.25
def test_litellm_get_retry_advice_keeps_stateful_transport_failures_ambiguous() -> None:
model = LitellmModel(model="test-model")
error = APIConnectionError(
message="connection error",
request=httpx.Request("POST", "https://api.openai.com/v1/responses"),
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@@ -0,0 +1,697 @@
from collections.abc import AsyncIterator
import pytest
from openai.types.chat.chat_completion_chunk import (
ChatCompletionChunk,
Choice,
ChoiceDelta,
ChoiceDeltaToolCall,
ChoiceDeltaToolCallFunction,
)
from openai.types.completion_usage import (
CompletionTokensDetails,
CompletionUsage,
PromptTokensDetails,
)
from openai.types.responses import (
Response,
ResponseCompletedEvent,
ResponseContentPartAddedEvent,
ResponseFunctionToolCall,
ResponseOutputMessage,
ResponseOutputRefusal,
ResponseOutputText,
ResponseReasoningItem,
ResponseRefusalDeltaEvent,
)
from agents.extensions.models.litellm_model import LitellmModel
from agents.extensions.models.litellm_provider import LitellmProvider
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_text_content(monkeypatch) -> None:
"""
Validate that `stream_response` emits the correct sequence of events when
streaming a simple assistant message consisting of plain text content.
We simulate two chunks of text returned from the chat completion stream.
"""
# Create two chunks that will be emitted by the fake stream.
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content="He"))],
)
# Mark last chunk with usage so stream_response knows this is final.
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content="llo"))],
usage=CompletionUsage(
completion_tokens=5,
prompt_tokens=7,
total_tokens=12,
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
prompt_tokens_details=PromptTokensDetails(cached_tokens=6),
),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
# Patch _fetch_response to inject our fake stream
async def patched_fetch_response(self, *args, **kwargs):
# `_fetch_response` is expected to return a Response skeleton and the async stream
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# We expect a response.created, then a response.output_item.added, content part added,
# two content delta events (for "He" and "llo"), a content part done, the assistant message
# output_item.done, and finally response.completed.
# There should be 8 events in total.
assert len(output_events) == 8
# First event indicates creation.
assert output_events[0].type == "response.created"
# The output item added and content part added events should mark the assistant message.
assert output_events[1].type == "response.output_item.added"
assert output_events[2].type == "response.content_part.added"
# Two text delta events.
assert output_events[3].type == "response.output_text.delta"
assert output_events[3].delta == "He"
assert output_events[4].type == "response.output_text.delta"
assert output_events[4].delta == "llo"
# After streaming, the content part and item should be marked done.
assert output_events[5].type == "response.content_part.done"
assert output_events[6].type == "response.output_item.done"
# Last event indicates completion of the stream.
assert output_events[7].type == "response.completed"
# The completed response should have one output message with full text.
completed_resp = output_events[7].response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
assert completed_resp.output[0].content[0].text == "Hello"
assert completed_resp.usage, "usage should not be None"
assert completed_resp.usage.input_tokens == 7
assert completed_resp.usage.output_tokens == 5
assert completed_resp.usage.total_tokens == 12
assert completed_resp.usage.input_tokens_details.cached_tokens == 6
assert completed_resp.usage.output_tokens_details.reasoning_tokens == 2
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_refusal_content(monkeypatch) -> None:
"""
Validate that when the model streams a refusal string instead of normal content,
`stream_response` emits the appropriate sequence of events including
`response.refusal.delta` events for each chunk of the refusal message and
constructs a completed assistant message with a `ResponseOutputRefusal` part.
"""
# Simulate refusal text coming in two pieces, like content but using the `refusal`
# field on the delta rather than `content`.
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(refusal="No"))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(refusal="Thanks"))],
usage=CompletionUsage(completion_tokens=2, prompt_tokens=2, total_tokens=4),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Expect sequence similar to text: created, output_item.added, content part added,
# two refusal delta events, content part done, output_item.done, completed.
assert len(output_events) == 8
assert output_events[0].type == "response.created"
assert output_events[1].type == "response.output_item.added"
assert output_events[2].type == "response.content_part.added"
assert output_events[3].type == "response.refusal.delta"
assert output_events[3].delta == "No"
assert output_events[4].type == "response.refusal.delta"
assert output_events[4].delta == "Thanks"
assert output_events[5].type == "response.content_part.done"
assert output_events[6].type == "response.output_item.done"
assert output_events[7].type == "response.completed"
completed_resp = output_events[7].response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
refusal_part = completed_resp.output[0].content[0]
assert isinstance(refusal_part, ResponseOutputRefusal)
assert refusal_part.refusal == "NoThanks"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_tool_call(monkeypatch) -> None:
"""
Validate that `stream_response` emits the correct sequence of events when
the model is streaming a function/tool call instead of plain text.
The function call will be split across two chunks.
"""
# Simulate a single tool call with complete function name in first chunk
# and arguments split across chunks (reflecting real API behavior)
tool_call_delta1 = ChoiceDeltaToolCall(
index=0,
id="tool-id",
function=ChoiceDeltaToolCallFunction(name="my_func", arguments="arg1"),
type="function",
)
tool_call_delta2 = ChoiceDeltaToolCall(
index=0,
id="tool-id",
function=ChoiceDeltaToolCallFunction(name=None, arguments="arg2"),
type="function",
)
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Sequence should be: response.created, then after loop we expect function call-related events:
# one response.output_item.added for function call, a response.function_call_arguments.delta,
# a response.output_item.done, and finally response.completed.
assert output_events[0].type == "response.created"
# The next three events are about the tool call.
assert output_events[1].type == "response.output_item.added"
# The added item should be a ResponseFunctionToolCall.
added_fn = output_events[1].item
assert isinstance(added_fn, ResponseFunctionToolCall)
assert added_fn.name == "my_func" # Name should be complete from first chunk
assert added_fn.arguments == "" # Arguments start empty
assert output_events[2].type == "response.function_call_arguments.delta"
assert output_events[2].delta == "arg1" # First argument chunk
assert output_events[3].type == "response.function_call_arguments.delta"
assert output_events[3].delta == "arg2" # Second argument chunk
assert output_events[4].type == "response.output_item.done"
assert output_events[5].type == "response.completed"
# Final function call should have complete arguments
final_fn = output_events[4].item
assert isinstance(final_fn, ResponseFunctionToolCall)
assert final_fn.name == "my_func"
assert final_fn.arguments == "arg1arg2"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_real_time_function_call_arguments(monkeypatch) -> None:
"""
Validate that LiteLLM `stream_response` also emits function call arguments in real-time
as they are received, ensuring consistent behavior across model providers.
"""
# Simulate realistic chunks: name first, then arguments incrementally
tool_call_delta1 = ChoiceDeltaToolCall(
index=0,
id="litellm-call-456",
function=ChoiceDeltaToolCallFunction(name="generate_code", arguments=""),
type="function",
)
tool_call_delta2 = ChoiceDeltaToolCall(
index=0,
function=ChoiceDeltaToolCallFunction(arguments='{"language": "'),
type="function",
)
tool_call_delta3 = ChoiceDeltaToolCall(
index=0,
function=ChoiceDeltaToolCallFunction(arguments='python", "task": "'),
type="function",
)
tool_call_delta4 = ChoiceDeltaToolCall(
index=0,
function=ChoiceDeltaToolCallFunction(arguments='hello world"}'),
type="function",
)
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
)
chunk3 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta3]))],
)
chunk4 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta4]))],
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2, chunk3, chunk4):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Extract events by type
function_args_delta_events = [
e for e in output_events if e.type == "response.function_call_arguments.delta"
]
output_item_added_events = [e for e in output_events if e.type == "response.output_item.added"]
# Verify we got real-time streaming (3 argument delta events)
assert len(function_args_delta_events) == 3
assert len(output_item_added_events) == 1
# Verify the deltas were streamed correctly
expected_deltas = ['{"language": "', 'python", "task": "', 'hello world"}']
for i, delta_event in enumerate(function_args_delta_events):
assert delta_event.delta == expected_deltas[i]
# Verify function call metadata
added_event = output_item_added_events[0]
assert isinstance(added_event.item, ResponseFunctionToolCall)
assert added_event.item.name == "generate_code"
assert added_event.item.call_id == "litellm-call-456"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_synthesizes_refusal_on_content_filter(monkeypatch) -> None:
"""A stream that terminates with finish_reason == "content_filter" and no
emitted content (as Anthropic-on-Bedrock does via LiteLLM) must synthesize a
ResponseOutputRefusal so the completed response carries an explicit refusal
rather than an empty assistant turn.
Mirrors the real Bedrock chunk shape: an empty-string content delta followed
by a terminal content_filter chunk with no content. The empty "" delta must
not open a text content part; the synthesized refusal must be the only
content part, at the same index in the stream and in response.completed.
"""
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(role="assistant", content=""))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
usage=CompletionUsage(
completion_tokens=0,
prompt_tokens=7,
total_tokens=7,
),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
types = [e.type for e in output_events]
# Coherent refusal sequence: the message + refusal part are opened, a refusal
# delta is emitted, and the parts/message are closed before completion.
assert "response.output_item.added" in types
assert "response.content_part.added" in types
assert "response.refusal.delta" in types
assert types[-1] == "response.completed"
assert "response.output_item.done" in types
# The refusal delta carries a non-empty message.
refusal_deltas = [e for e in output_events if e.type == "response.refusal.delta"]
assert refusal_deltas and refusal_deltas[0].delta
# Event coherence: the assistant message is announced exactly once, and every
# content part that is opened is also closed.
assert types.count("response.output_item.added") == 1
assert types.count("response.content_part.added") == types.count("response.content_part.done")
# The empty "" content delta must NOT open a text content part: no text part
# events and no output_text.delta are emitted at all.
assert "response.output_text.delta" not in types
added_parts = [e for e in output_events if e.type == "response.content_part.added"]
assert len(added_parts) == 1
assert isinstance(added_parts[0].part, ResponseOutputRefusal)
# The completed response contains exactly one content part: the refusal.
completed_event = output_events[-1]
assert isinstance(completed_event, ResponseCompletedEvent)
completed_resp = completed_event.response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
assert len(completed_resp.output[0].content) == 1
refusal_part = completed_resp.output[0].content[0]
assert isinstance(refusal_part, ResponseOutputRefusal)
assert refusal_part.refusal
# The refusal's streamed content_index matches its position in the completed
# response (0), so raw-event replay and the final response stay aligned.
assert added_parts[0].content_index == 0
assert refusal_deltas[0].content_index == 0
done_parts = [e for e in output_events if e.type == "response.content_part.done"]
assert len(done_parts) == 1
assert done_parts[0].content_index == 0
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_content_filter_does_not_clobber_text(monkeypatch) -> None:
"""A content_filter finish_reason that arrives AFTER real text was streamed
must not synthesize a refusal (the text stands)."""
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content="answer"))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
usage=CompletionUsage(completion_tokens=1, prompt_tokens=7, total_tokens=8),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = [
event
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
]
assert "response.refusal.delta" not in [e.type for e in output_events]
completed_event = output_events[-1]
assert isinstance(completed_event, ResponseCompletedEvent)
completed_resp = completed_event.response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
assert completed_resp.output[0].content[0].text == "answer"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_content_filter_refusal_after_reasoning(monkeypatch) -> None:
"""A content_filter turn preceded by reasoning must still place the
synthesized refusal at content_index 0 of the assistant message. Reasoning
is a *separate* output item (it shifts the message's output_index, not its
content_index), so the refusal — the sole content part — stays at
content_index 0 in both the stream and response.completed."""
reasoning_delta = ChoiceDelta(role="assistant", content=None)
# reasoning_content is a provider extra field the handler reads via hasattr.
reasoning_delta.reasoning_content = "thinking..." # type: ignore[attr-defined]
chunk_reasoning = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=reasoning_delta)],
)
chunk_empty = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content=""))],
)
chunk_filter = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
usage=CompletionUsage(completion_tokens=0, prompt_tokens=7, total_tokens=7),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk_reasoning, chunk_empty, chunk_filter):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = [
event
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
]
# A reasoning item was produced as a separate output item.
completed_event = output_events[-1]
assert isinstance(completed_event, ResponseCompletedEvent)
completed_resp = completed_event.response
assert isinstance(completed_resp.output[0], ResponseReasoningItem)
assistant_msg = completed_resp.output[1]
assert isinstance(assistant_msg, ResponseOutputMessage)
# The refusal is the sole content part of the assistant message, at index 0.
assert len(assistant_msg.content) == 1
assert isinstance(assistant_msg.content[0], ResponseOutputRefusal)
# The assistant message's output_index is 1 (after the reasoning item), and
# every refusal event uses that output_index and content_index 0 — matching
# the refusal's position in response.completed.
added = [
e
for e in output_events
if isinstance(e, ResponseContentPartAddedEvent)
and isinstance(e.part, ResponseOutputRefusal)
]
deltas = [e for e in output_events if isinstance(e, ResponseRefusalDeltaEvent)]
assert len(added) == 1
assert added[0].content_index == 0
assert added[0].output_index == 1
assert deltas and all(d.content_index == 0 and d.output_index == 1 for d in deltas)
# The empty "" delta still opens no text part.
assert "response.output_text.delta" not in [e.type for e in output_events]
@@ -0,0 +1,89 @@
import litellm
import pytest
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
async def _get_response(monkeypatch, *, finish_reason, content):
"""Drive get_response against a mocked litellm completion and return the items."""
async def fake_acompletion(model, messages=None, **kwargs):
msg = Message(role="assistant", content=content)
choice = Choices(index=0, finish_reason=finish_reason, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(model="test-model")
return await model.get_response(
system_instructions=None,
input=[],
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_content_filter_finish_reason_surfaces_refusal(monkeypatch):
"""A content-filter block (empty message, finish_reason=content_filter) must
become an explicit ResponseOutputRefusal, not zero output items.
Some providers (e.g. Anthropic on Amazon Bedrock) signal a safety block only
via ``finish_reason == "content_filter"`` with an empty message and no
``refusal`` field; without this the turn is indistinguishable from an empty
response and drives agent loops into fruitless retries.
"""
resp = await _get_response(monkeypatch, finish_reason="content_filter", content="")
refusals = [
content
for item in resp.output
if isinstance(item, ResponseOutputMessage)
for content in item.content
if isinstance(content, ResponseOutputRefusal)
]
assert refusals, f"expected a refusal item, got: {resp.output}"
assert refusals[0].refusal # non-empty message
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_content_filter_does_not_clobber_real_content(monkeypatch):
"""A content_filter finish_reason that still carries text is left alone — we
only synthesize a refusal when the message is genuinely empty."""
resp = await _get_response(
monkeypatch, finish_reason="content_filter", content="here is the answer"
)
refusals = [
content
for item in resp.output
if isinstance(item, ResponseOutputMessage)
for content in item.content
if isinstance(content, ResponseOutputRefusal)
]
assert not refusals, "should not synthesize a refusal when content is present"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_normal_stop_is_unaffected(monkeypatch):
"""A normal completion is unchanged — no spurious refusal."""
resp = await _get_response(monkeypatch, finish_reason="stop", content="all good")
refusals = [
content
for item in resp.output
if isinstance(item, ResponseOutputMessage)
for content in item.content
if isinstance(content, ResponseOutputRefusal)
]
assert not refusals
+265
View File
@@ -0,0 +1,265 @@
import logging
import litellm
import pytest
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_is_forwarded(monkeypatch):
"""
Forward `extra_body` via LiteLLM's dedicated kwarg.
This ensures that provider-specific request fields stay nested under `extra_body`
so LiteLLM can merge them into the upstream request body itself.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
temperature=0.1, extra_body={"cached_content": "some_cache", "foo": 123}
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["extra_body"] == {"cached_content": "some_cache", "foo": 123}
assert "cached_content" not in captured
assert "foo" not in captured
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_reasoning_effort_is_promoted(monkeypatch):
"""
Ensure reasoning_effort from extra_body is promoted to the top-level parameter.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
settings = ModelSettings(
extra_body={"reasoning_effort": "none", "cached_content": "some_cache"}
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "none"
assert captured["extra_body"] == {"cached_content": "some_cache"}
assert settings.extra_body == {"reasoning_effort": "none", "cached_content": "some_cache"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_reasoning_effort_prefers_model_settings(monkeypatch):
"""
Verify explicit ModelSettings.reasoning takes precedence over extra_body entries.
"""
from openai.types.shared import Reasoning
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
reasoning=Reasoning(effort="low"),
extra_body={"reasoning_effort": "high"},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# reasoning_effort is string when no summary is provided (backward compatible)
assert captured["reasoning_effort"] == "low"
assert "extra_body" not in captured
assert settings.extra_body == {"reasoning_effort": "high"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_reasoning_effort_overrides_extra_args(monkeypatch):
"""
Ensure extra_body reasoning_effort wins over extra_args when both are provided.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
settings = ModelSettings(
extra_body={"reasoning_effort": "none"},
extra_args={"reasoning_effort": "low", "custom_param": "custom"},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "none"
assert captured["custom_param"] == "custom"
assert "extra_body" not in captured
assert settings.extra_args == {"reasoning_effort": "low", "custom_param": "custom"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_metadata_stays_nested(monkeypatch):
"""
Keep extra_body metadata nested even when top-level metadata is also set.
LiteLLM resolves top-level metadata and extra_body separately. Flattening the nested
metadata dict loses the caller's intended request shape for OpenAI-compatible proxies.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
metadata={"sdk": "agents"},
extra_body={
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
"cached_content": "some_cache",
},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["metadata"] == {"sdk": "agents"}
assert captured["extra_body"] == {
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
"cached_content": "some_cache",
}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[
"openai/gpt-5-mini",
"anthropic/claude-sonnet-4-5",
"gemini/gemini-2.5-pro",
],
)
async def test_reasoning_summary_uses_scalar_effort_and_warns(
monkeypatch, caplog: pytest.LogCaptureFixture, model_name: str
):
"""
Ensure reasoning.summary does not change the LiteLLM chat-completions argument shape.
LitellmModel should continue to pass a scalar reasoning_effort value and warn that summary
is ignored on this path, regardless of the provider encoded in the model string.
"""
from openai.types.shared import Reasoning
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
reasoning=Reasoning(effort="medium", summary="auto"),
)
model = LitellmModel(model=model_name)
with caplog.at_level(logging.WARNING, logger="openai.agents"):
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "medium"
warning_messages = [
record.message
for record in caplog.records
if "does not forward Reasoning.summary" in record.message
]
assert len(warning_messages) == 1
@@ -0,0 +1,29 @@
from __future__ import annotations
import importlib
import pytest
pytest.importorskip("litellm")
def test_litellm_logging_patch_env_var_controls_application(monkeypatch):
"""Assert the serializer patch only applies when the env var is enabled."""
litellm_logging = importlib.import_module("litellm.litellm_core_utils.litellm_logging")
litellm_model = importlib.import_module("agents.extensions.models.litellm_model")
monkeypatch.delenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", raising=False)
litellm_logging = importlib.reload(litellm_logging)
importlib.reload(litellm_model)
assert hasattr(
litellm_logging,
"_extract_response_obj_and_hidden_params",
), "LiteLLM removed _extract_response_obj_and_hidden_params; revisit warning patch."
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is False
monkeypatch.setenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", "true")
litellm_logging = importlib.reload(litellm_logging)
importlib.reload(litellm_model)
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is True
+89
View File
@@ -0,0 +1,89 @@
from __future__ import annotations
from typing import Any
import pytest
from agents import ModelSettings, ModelTracing, __version__
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
async def test_user_agent_header_litellm(override_ua: str | None, monkeypatch):
called_kwargs: dict[str, Any] = {}
expected_ua = override_ua or f"Agents/Python {__version__}"
import importlib
import sys
import types as pytypes
litellm_fake: Any = pytypes.ModuleType("litellm")
class DummyMessage:
role = "assistant"
content = "Hello"
tool_calls: list[Any] | None = None
def get(self, _key, _default=None):
return None
def model_dump(self):
return {"role": self.role, "content": self.content}
class Choices: # noqa: N801 - mimic litellm naming
def __init__(self):
self.message = DummyMessage()
class DummyModelResponse:
def __init__(self):
self.choices = [Choices()]
async def acompletion(**kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return DummyModelResponse()
utils_ns = pytypes.SimpleNamespace()
utils_ns.Choices = Choices
utils_ns.ModelResponse = DummyModelResponse
litellm_types = pytypes.SimpleNamespace(
utils=utils_ns,
llms=pytypes.SimpleNamespace(openai=pytypes.SimpleNamespace(ChatCompletionAnnotation=dict)),
)
litellm_fake.acompletion = acompletion
litellm_fake.types = litellm_types
monkeypatch.setitem(sys.modules, "litellm", litellm_fake)
litellm_mod = importlib.import_module("agents.extensions.models.litellm_model")
monkeypatch.setattr(litellm_mod, "litellm", litellm_fake, raising=True)
LitellmModel = litellm_mod.LitellmModel
model = LitellmModel(model="gpt-4")
if override_ua is not None:
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
else:
token = None
try:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
finally:
if token is not None:
HEADERS_OVERRIDE.reset(token)
assert "extra_headers" in called_kwargs
assert called_kwargs["extra_headers"]["User-Agent"] == expected_ua
+209
View File
@@ -0,0 +1,209 @@
from typing import Any, cast
import pytest
from agents import (
Agent,
MultiProvider,
OpenAIResponsesModel,
OpenAIResponsesWSModel,
RunConfig,
UserError,
)
from agents.extensions.models.litellm_model import LitellmModel
from agents.models.multi_provider import MultiProviderMap
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.run_internal.run_loop import get_model
def test_no_prefix_is_openai():
agent = Agent(model="gpt-4o", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, OpenAIResponsesModel)
def test_openai_prefix_is_openai():
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, OpenAIResponsesModel)
def test_litellm_prefix_is_litellm():
agent = Agent(model="litellm/foo/bar", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, LitellmModel)
def test_any_llm_prefix_uses_any_llm_provider(monkeypatch):
import sys
import types as pytypes
captured_model: dict[str, Any] = {}
class FakeAnyLLMModel:
pass
class FakeAnyLLMProvider:
def get_model(self, model_name):
captured_model["value"] = model_name
return FakeAnyLLMModel()
fake_module: Any = pytypes.ModuleType("agents.extensions.models.any_llm_provider")
fake_module.AnyLLMProvider = FakeAnyLLMProvider
monkeypatch.setitem(sys.modules, "agents.extensions.models.any_llm_provider", fake_module)
agent = Agent(model="any-llm/openrouter/openai/gpt-5.4-mini", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, FakeAnyLLMModel)
assert captured_model["value"] == "openrouter/openai/gpt-5.4-mini"
def test_no_prefix_can_use_openai_responses_websocket():
agent = Agent(model="gpt-4o", instructions="", name="test")
model = get_model(
agent,
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
)
assert isinstance(model, OpenAIResponsesWSModel)
def test_openai_prefix_can_use_openai_responses_websocket():
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
model = get_model(
agent,
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
)
assert isinstance(model, OpenAIResponsesWSModel)
def test_multi_provider_passes_websocket_base_url_to_openai_provider(monkeypatch):
captured_kwargs = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
captured_kwargs.update(kwargs)
def get_model(self, model_name):
raise AssertionError("This test only verifies constructor passthrough.")
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
MultiProvider(openai_websocket_base_url="wss://proxy.example.test/v1")
assert captured_kwargs["websocket_base_url"] == "wss://proxy.example.test/v1"
def test_multi_provider_forwards_openai_buffer_streamed_tool_calls_to_chat_model():
provider = MultiProvider(
openai_client=cast(Any, object()),
openai_use_responses=False,
openai_buffer_streamed_tool_calls=True,
)
model = provider.get_model("gpt-4o")
assert isinstance(model, OpenAIChatCompletionsModel)
assert model._buffer_streamed_tool_calls is True
def test_openai_prefix_defaults_to_alias_mode(monkeypatch):
captured_model: dict[str, Any] = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
captured_model["value"] = model_name
return object()
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider = MultiProvider()
provider.get_model("openai/gpt-4o")
assert captured_model["value"] == "gpt-4o"
def test_openai_prefix_can_be_preserved_as_literal_model_id(monkeypatch):
captured_model: dict[str, Any] = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
captured_model["value"] = model_name
return object()
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider = MultiProvider(openai_prefix_mode="model_id")
provider.get_model("openai/gpt-4o")
assert captured_model["value"] == "openai/gpt-4o"
def test_unknown_prefix_defaults_to_error():
provider = MultiProvider()
with pytest.raises(UserError, match="Unknown prefix: openrouter"):
provider.get_model("openrouter/openai/gpt-4o")
def test_unknown_prefix_can_be_preserved_for_openai_compatible_model_ids(monkeypatch):
captured_model: dict[str, Any] = {}
captured_result: dict[str, Any] = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
captured_model["value"] = model_name
fake_model = object()
captured_result["value"] = fake_model
return fake_model
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider = MultiProvider(unknown_prefix_mode="model_id")
result = provider.get_model("openrouter/openai/gpt-4o")
assert result is captured_result["value"]
assert captured_model["value"] == "openrouter/openai/gpt-4o"
def test_provider_map_entries_override_openai_prefix_mode(monkeypatch):
captured_model: dict[str, Any] = {}
class FakeCustomProvider:
def get_model(self, model_name):
captured_model["value"] = model_name
return object()
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
raise AssertionError("Expected the explicit provider_map entry to win.")
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider_map = MultiProviderMap()
provider_map.add_provider("openai", cast(Any, FakeCustomProvider()))
provider = MultiProvider(
provider_map=provider_map,
openai_prefix_mode="model_id",
)
provider.get_model("openai/gpt-4o")
assert captured_model["value"] == "gpt-4o"
def test_multi_provider_rejects_invalid_prefix_modes():
bad_openai_prefix_mode: Any = "invalid"
bad_unknown_prefix_mode: Any = "invalid"
with pytest.raises(UserError, match="openai_prefix_mode"):
MultiProvider(openai_prefix_mode=bad_openai_prefix_mode)
with pytest.raises(UserError, match="unknown_prefix_mode"):
MultiProvider(unknown_prefix_mode=bad_unknown_prefix_mode)
@@ -0,0 +1,186 @@
from __future__ import annotations
from collections.abc import Iterable, Iterator
from typing import Any, cast
import httpx
import pytest
from openai import omit
from openai.types.chat.chat_completion import ChatCompletion
from agents import (
ModelSettings,
ModelTracing,
OpenAIChatCompletionsModel,
OpenAIResponsesModel,
generation_span,
)
from agents.models import (
openai_chatcompletions as chat_module,
openai_responses as responses_module,
)
class _SingleUseIterable:
"""Helper iterable that raises if iterated more than once."""
def __init__(self, values: list[object]) -> None:
self._values = list(values)
self.iterations = 0
def __iter__(self) -> Iterator[object]:
if self.iterations:
raise RuntimeError("Iterable should have been materialized exactly once.")
self.iterations += 1
yield from self._values
def _force_materialization(value: object) -> None:
if isinstance(value, dict):
for nested in value.values():
_force_materialization(nested)
elif isinstance(value, list):
for nested in value:
_force_materialization(nested)
elif isinstance(value, Iterable) and not isinstance(value, str | bytes | bytearray):
list(value)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_chat_completions_materializes_iterator_payload(
monkeypatch: pytest.MonkeyPatch,
) -> None:
message_iter = _SingleUseIterable([{"type": "text", "text": "hi"}])
tool_iter = _SingleUseIterable([{"type": "string"}])
chat_converter = cast(Any, chat_module).Converter
monkeypatch.setattr(
chat_converter,
"items_to_messages",
classmethod(lambda _cls, _input, **kwargs: [{"role": "user", "content": message_iter}]),
)
monkeypatch.setattr(
chat_converter,
"tool_to_openai",
classmethod(
lambda _cls, _tool: {
"type": "function",
"function": {
"name": "dummy",
"parameters": {"properties": tool_iter},
},
}
),
)
captured_kwargs: dict[str, Any] = {}
class DummyCompletions:
async def create(self, **kwargs):
captured_kwargs.update(kwargs)
_force_materialization(kwargs["messages"])
if kwargs["tools"] is not omit:
_force_materialization(kwargs["tools"])
return ChatCompletion(
id="dummy-id",
created=0,
model="gpt-4",
object="chat.completion",
choices=[],
usage=None,
)
class DummyClient:
def __init__(self) -> None:
self.chat = type("_Chat", (), {"completions": DummyCompletions()})()
self.base_url = httpx.URL("http://example.test")
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=DummyClient()) # type: ignore[arg-type]
with generation_span(disabled=True) as span:
await cast(Any, model)._fetch_response(
system_instructions=None,
input="ignored",
model_settings=ModelSettings(),
tools=[object()],
output_schema=None,
handoffs=[],
span=span,
tracing=ModelTracing.DISABLED,
stream=False,
)
assert message_iter.iterations == 1
assert tool_iter.iterations == 1
assert isinstance(captured_kwargs["messages"][0]["content"], list)
assert isinstance(captured_kwargs["tools"][0]["function"]["parameters"]["properties"], list)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_responses_materializes_iterator_payload(monkeypatch: pytest.MonkeyPatch) -> None:
input_iter = _SingleUseIterable([{"type": "input_text", "text": "hello"}])
tool_iter = _SingleUseIterable([{"type": "string"}])
responses_item_helpers = cast(Any, responses_module).ItemHelpers
responses_converter = cast(Any, responses_module).Converter
monkeypatch.setattr(
responses_item_helpers,
"input_to_new_input_list",
classmethod(lambda _cls, _input: [{"role": "user", "content": input_iter}]),
)
converted_tools = responses_module.ConvertedTools(
tools=[
cast(
Any,
{
"type": "function",
"name": "dummy",
"parameters": {"properties": tool_iter},
},
)
],
includes=[],
)
monkeypatch.setattr(
responses_converter,
"convert_tools",
classmethod(lambda _cls, _tools, _handoffs, **_kwargs: converted_tools),
)
captured_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
captured_kwargs.update(kwargs)
_force_materialization(kwargs["input"])
_force_materialization(kwargs["tools"])
return object()
class DummyClient:
def __init__(self) -> None:
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4.1", openai_client=DummyClient()) # type: ignore[arg-type]
await cast(Any, model)._fetch_response(
system_instructions=None,
input="ignored",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert input_iter.iterations == 1
assert tool_iter.iterations == 1
assert isinstance(captured_kwargs["input"][0]["content"], list)
assert isinstance(captured_kwargs["tools"][0]["parameters"]["properties"], list)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,756 @@
# Copyright (c) OpenAI
#
# Licensed under the MIT License.
# See LICENSE file in the project root for full license information.
"""
Unit tests for the internal `Converter` class defined in
`agents.models.openai_chatcompletions`. The converter is responsible for
translating between internal "item" structures (e.g., `ResponseOutputMessage`
and related types from `openai.types.responses`) and the ChatCompletion message
structures defined by the OpenAI client library.
These tests exercise both conversion directions:
- `Converter.message_to_output_items` turns a `ChatCompletionMessage` (as
returned by the OpenAI API) into a list of `ResponseOutputItem` instances.
- `Converter.items_to_messages` takes in either a simple string prompt, or a
list of input/output items such as `ResponseOutputMessage` and
`ResponseFunctionToolCallParam` dicts, and constructs a list of
`ChatCompletionMessageParam` dicts suitable for sending back to the API.
"""
from __future__ import annotations
import logging
from typing import Any, Literal, cast
import pytest
from openai import omit
from openai.types.chat import ChatCompletionMessage, ChatCompletionMessageFunctionToolCall
from openai.types.chat.chat_completion_message_custom_tool_call import (
ChatCompletionMessageCustomToolCall,
Custom,
)
from openai.types.chat.chat_completion_message_tool_call import Function
from openai.types.responses import (
ResponseFunctionToolCall,
ResponseFunctionToolCallParam,
ResponseInputAudioParam,
ResponseInputTextParam,
ResponseOutputMessage,
ResponseOutputRefusal,
ResponseOutputText,
)
from openai.types.responses.response_input_item_param import FunctionCallOutput
from agents.agent_output import AgentOutputSchema
from agents.exceptions import UserError
from agents.items import TResponseInputItem
from agents.models.chatcmpl_converter import Converter
from agents.models.fake_id import FAKE_RESPONSES_ID
def test_message_to_output_items_with_text_only():
"""
Make sure a simple ChatCompletionMessage with string content is converted
into a single ResponseOutputMessage containing one ResponseOutputText.
"""
msg = ChatCompletionMessage(role="assistant", content="Hello")
items = Converter.message_to_output_items(msg)
# Expect exactly one output item (the message)
assert len(items) == 1
message_item = cast(ResponseOutputMessage, items[0])
assert message_item.id == FAKE_RESPONSES_ID
assert message_item.role == "assistant"
assert message_item.type == "message"
assert message_item.status == "completed"
# Message content should have exactly one text part with the same text.
assert len(message_item.content) == 1
text_part = cast(ResponseOutputText, message_item.content[0])
assert text_part.type == "output_text"
assert text_part.text == "Hello"
def test_message_to_output_items_with_refusal():
"""
Make sure a message with a refusal string produces a ResponseOutputMessage
with a ResponseOutputRefusal content part.
"""
msg = ChatCompletionMessage(role="assistant", refusal="I'm sorry")
items = Converter.message_to_output_items(msg)
assert len(items) == 1
message_item = cast(ResponseOutputMessage, items[0])
assert len(message_item.content) == 1
refusal_part = cast(ResponseOutputRefusal, message_item.content[0])
assert refusal_part.type == "refusal"
assert refusal_part.refusal == "I'm sorry"
def test_message_to_output_items_with_tool_call():
"""
If the ChatCompletionMessage contains one or more tool_calls, they should
be reflected as separate `ResponseFunctionToolCall` items appended after
the message item.
"""
tool_call = ChatCompletionMessageFunctionToolCall(
id="tool1",
type="function",
function=Function(name="myfn", arguments='{"x":1}'),
)
msg = ChatCompletionMessage(role="assistant", content="Hi", tool_calls=[tool_call])
items = Converter.message_to_output_items(msg)
# Should produce a message item followed by one function tool call item
assert len(items) == 2
message_item = cast(ResponseOutputMessage, items[0])
assert isinstance(message_item, ResponseOutputMessage)
fn_call_item = cast(ResponseFunctionToolCall, items[1])
assert fn_call_item.id == FAKE_RESPONSES_ID
assert fn_call_item.call_id == tool_call.id
assert fn_call_item.name == tool_call.function.name
assert fn_call_item.arguments == tool_call.function.arguments
assert fn_call_item.type == "function_call"
def test_message_to_output_items_with_custom_tool_call_keeps_default_compatibility():
"""Custom tool calls should keep the default Chat Completions behavior."""
tool_call = ChatCompletionMessageCustomToolCall(
id="tool1",
type="custom",
custom=Custom(name="raw_tool", input="payload"),
)
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
assert Converter.message_to_output_items(msg) == []
def test_message_to_output_items_with_custom_tool_call_raises_in_strict_mode():
"""Strict validation should fail explicitly instead of dropping custom tool calls."""
tool_call = ChatCompletionMessageCustomToolCall(
id="tool1",
type="custom",
custom=Custom(name="raw_tool", input="payload"),
)
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
with pytest.raises(UserError, match="Custom tool calls are not supported"):
Converter.message_to_output_items(msg, strict_feature_validation=True)
def test_message_to_output_items_with_mixed_custom_tool_call_raises_in_strict_mode():
"""Strict validation should not partially hide an unsupported custom tool call."""
function_tool_call = ChatCompletionMessageFunctionToolCall(
id="function-tool",
type="function",
function=Function(name="myfn", arguments='{"x":1}'),
)
custom_tool_call = ChatCompletionMessageCustomToolCall(
id="custom-tool",
type="custom",
custom=Custom(name="raw_tool", input="payload"),
)
msg = ChatCompletionMessage(
role="assistant",
tool_calls=[function_tool_call, custom_tool_call],
)
with pytest.raises(UserError, match="Custom tool calls are not supported"):
Converter.message_to_output_items(msg, strict_feature_validation=True)
def test_items_to_messages_with_string_user_content():
"""
A simple string as the items argument should be converted into a user
message param dict with the same content.
"""
result = Converter.items_to_messages("Ask me anything")
assert isinstance(result, list)
assert len(result) == 1
msg = result[0]
assert msg["role"] == "user"
assert msg["content"] == "Ask me anything"
def test_items_to_messages_with_easy_input_message():
"""
Given an easy input message dict (just role/content), the converter should
produce the appropriate ChatCompletionMessageParam with the same content.
"""
items: list[TResponseInputItem] = [
{
"role": "user",
"content": "How are you?",
}
]
messages = Converter.items_to_messages(items)
assert len(messages) == 1
out = messages[0]
assert out["role"] == "user"
# For simple string inputs, the converter returns the content as a bare string
assert out["content"] == "How are you?"
def test_items_to_messages_accepts_raw_chat_completions_user_content_parts():
"""
Raw Chat Completions content parts should be accepted as aliases for the SDK's
canonical input content shapes.
"""
items: list[TResponseInputItem] = [
# Cast the fixture because mypy cannot infer this raw chat-style dict as a specific
# member of the TResponseInputItem TypedDict union on its own.
cast(
TResponseInputItem,
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {
"url": "https://example.com/image.png",
"detail": "high",
},
},
],
},
)
]
messages = Converter.items_to_messages(items)
assert len(messages) == 1
message = messages[0]
assert message["role"] == "user"
assert message["content"] == [
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {
"url": "https://example.com/image.png",
"detail": "high",
},
},
]
def test_items_to_messages_with_output_message_and_function_call():
"""
Given a sequence of one ResponseOutputMessageParam followed by a
ResponseFunctionToolCallParam, the converter should produce a single
ChatCompletionAssistantMessageParam that includes both the assistant's
textual content and a populated `tool_calls` reflecting the function call.
"""
# Construct output message param dict with two content parts.
output_text: ResponseOutputText = ResponseOutputText(
text="Part 1",
type="output_text",
annotations=[],
logprobs=[],
)
refusal: ResponseOutputRefusal = ResponseOutputRefusal(
refusal="won't do that",
type="refusal",
)
resp_msg: ResponseOutputMessage = ResponseOutputMessage(
id="42",
type="message",
role="assistant",
status="completed",
content=[output_text, refusal],
)
# Construct a function call item dict (as if returned from model)
func_item: ResponseFunctionToolCallParam = {
"id": "99",
"call_id": "abc",
"name": "math",
"arguments": "{}",
"type": "function_call",
}
items: list[TResponseInputItem] = [
resp_msg.model_dump(), # type:ignore
func_item,
]
messages = Converter.items_to_messages(items)
# Should return a single assistant message
assert len(messages) == 1
assistant = messages[0]
assert assistant["role"] == "assistant"
# Content combines text portions of the output message
assert "content" in assistant
assert assistant["content"] == "Part 1"
# Refusal in output message should be represented in assistant message
assert "refusal" in assistant
assert assistant["refusal"] == refusal.refusal
# Tool calls list should contain one ChatCompletionMessageFunctionToolCall dict
tool_calls = assistant.get("tool_calls")
assert isinstance(tool_calls, list)
assert len(tool_calls) == 1
tool_call = tool_calls[0]
assert tool_call["type"] == "function"
assert tool_call["function"]["name"] == "math"
assert tool_call["function"]["arguments"] == "{}"
def test_convert_tool_choice_handles_standard_and_named_options() -> None:
"""
The `Converter.convert_tool_choice` method should return the omit sentinel
if no choice is provided, pass through values like "auto", "required",
or "none" unchanged, and translate any other string into a function
selection dict.
"""
assert Converter.convert_tool_choice(None) is omit
assert Converter.convert_tool_choice("auto") == "auto"
assert Converter.convert_tool_choice("required") == "required"
assert Converter.convert_tool_choice("none") == "none"
tool_choice_dict = Converter.convert_tool_choice("mytool")
assert isinstance(tool_choice_dict, dict)
assert tool_choice_dict["type"] == "function"
assert tool_choice_dict["function"]["name"] == "mytool"
def test_convert_tool_choice_allows_tool_search_as_named_function_for_chat_models() -> None:
tool_choice_dict = Converter.convert_tool_choice("tool_search")
assert isinstance(tool_choice_dict, dict)
assert tool_choice_dict["type"] == "function"
assert tool_choice_dict["function"]["name"] == "tool_search"
def test_convert_response_format_returns_not_given_for_plain_text_and_dict_for_schemas() -> None:
"""
The `Converter.convert_response_format` method should return the omit sentinel
when no output schema is provided or if the output schema indicates
plain text. For structured output schemas, it should return a dict
with type `json_schema` and include the generated JSON schema and
strict flag from the provided `AgentOutputSchema`.
"""
# when output is plain text (schema None or output_type str), do not include response_format
assert Converter.convert_response_format(None) is omit
assert Converter.convert_response_format(AgentOutputSchema(str)) is omit
# For e.g. integer output, we expect a response_format dict
schema = AgentOutputSchema(int)
resp_format = Converter.convert_response_format(schema)
assert isinstance(resp_format, dict)
assert resp_format["type"] == "json_schema"
assert resp_format["json_schema"]["name"] == "final_output"
assert "strict" in resp_format["json_schema"]
assert resp_format["json_schema"]["strict"] == schema.is_strict_json_schema()
assert "schema" in resp_format["json_schema"]
assert resp_format["json_schema"]["schema"] == schema.json_schema()
def test_items_to_messages_with_function_output_item():
"""
A function call output item should be converted into a tool role message
dict with the appropriate tool_call_id and content.
"""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": '{"foo": "bar"}',
}
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == func_output_item["output"]
def test_items_to_messages_with_non_text_only_function_output_uses_placeholder_by_default(
caplog: pytest.LogCaptureFixture,
):
"""Default conversion should keep running without sending an empty tool message."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/image.png",
}
],
}
with caplog.at_level(logging.WARNING, logger="openai.agents"):
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == "[tool output omitted]"
assert "Replacing the tool output with a placeholder" in caplog.text
def test_items_to_messages_with_non_text_only_function_output_raises_in_strict_mode():
"""Strict validation should fail explicitly instead of silently losing the output."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/image.png",
}
],
}
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
def test_items_to_messages_with_empty_function_output_uses_placeholder_by_default(
caplog: pytest.LogCaptureFixture,
):
"""Default conversion should not send an empty tool message."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [],
}
with caplog.at_level(logging.WARNING, logger="openai.agents"):
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == "[tool output omitted]"
assert "Replacing the tool output with a placeholder" in caplog.text
def test_items_to_messages_with_empty_function_output_raises_in_strict_mode():
"""Strict validation should fail explicitly instead of sending empty output."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [],
}
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
def test_items_to_messages_with_mixed_function_output_keeps_text_by_default(
caplog: pytest.LogCaptureFixture,
):
"""Default conversion should preserve text parts and omit unsupported non-text parts."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{"type": "input_text", "text": "visible text"},
{
"type": "input_image",
"image_url": "https://example.com/image.png",
},
],
}
with caplog.at_level(logging.WARNING, logger="openai.agents"):
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == [{"type": "text", "text": "visible text"}]
assert "tool output omitted" not in caplog.text
def test_items_to_messages_can_preserve_non_text_function_output() -> None:
"""Compatible providers can opt in to preserving non-text tool output."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/image.png",
}
],
}
messages = Converter.items_to_messages(
[func_output_item],
preserve_tool_output_all_content=True,
)
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == [
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.png", "detail": "auto"},
}
]
def test_extract_all_and_text_content_for_strings_and_lists():
"""
The converter provides helpers for extracting user-supplied message content
either as a simple string or as a list of `input_text` dictionaries.
When passed a bare string, both `extract_all_content` and
`extract_text_content` should return the string unchanged.
When passed a list of input dictionaries, `extract_all_content` should
produce a list of `ChatCompletionContentPart` dicts, and `extract_text_content`
should filter to only the textual parts.
"""
prompt = "just text"
assert Converter.extract_all_content(prompt) == prompt
assert Converter.extract_text_content(prompt) == prompt
text1: ResponseInputTextParam = {"type": "input_text", "text": "one"}
text2: ResponseInputTextParam = {"type": "input_text", "text": "two"}
all_parts = Converter.extract_all_content([text1, text2])
assert isinstance(all_parts, list)
assert len(all_parts) == 2
assert all_parts[0]["type"] == "text" and all_parts[0]["text"] == "one"
assert all_parts[1]["type"] == "text" and all_parts[1]["text"] == "two"
text_parts = Converter.extract_text_content([text1, text2])
assert isinstance(text_parts, list)
assert all(p["type"] == "text" for p in text_parts)
assert [p["text"] for p in text_parts] == ["one", "two"]
def test_extract_all_content_handles_input_audio():
"""
input_audio entries should translate into ChatCompletion input_audio parts.
"""
audio: ResponseInputAudioParam = {
"type": "input_audio",
"input_audio": {"data": "AAA=", "format": "wav"},
}
parts = Converter.extract_all_content([audio])
assert isinstance(parts, list)
assert parts == [
{
"type": "input_audio",
"input_audio": {"data": "AAA=", "format": "wav"},
}
]
def test_extract_all_content_preserves_prompt_cache_breakpoints() -> None:
breakpoint = {"mode": "explicit"}
content: list[dict[str, Any]] = [
{
"type": "input_text",
"text": "one",
"prompt_cache_breakpoint": breakpoint,
},
{
"type": "input_image",
"image_url": "https://example.com/image.png",
"prompt_cache_breakpoint": breakpoint,
},
{
"type": "input_audio",
"input_audio": {"data": "AAA=", "format": "wav"},
"prompt_cache_breakpoint": breakpoint,
},
{
"type": "input_file",
"file_data": "data:text/plain;base64,SGVsbG8=",
"filename": "hello.txt",
"prompt_cache_breakpoint": breakpoint,
},
]
parts = Converter.extract_all_content(content)
assert isinstance(parts, list)
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint] * 4
def test_raw_chat_content_aliases_preserve_prompt_cache_breakpoints() -> None:
breakpoint = {"mode": "explicit"}
parts = Converter.extract_all_content(
cast(
list[dict[str, Any]],
[
{"type": "text", "text": "one", "prompt_cache_breakpoint": breakpoint},
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.png"},
"prompt_cache_breakpoint": breakpoint,
},
],
)
)
assert isinstance(parts, list)
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint, breakpoint]
def test_extract_all_content_rejects_invalid_input_audio():
"""
input_audio requires both data and format fields to be present.
"""
audio_missing_data = cast(
ResponseInputAudioParam,
{
"type": "input_audio",
"input_audio": {"format": "wav"},
},
)
with pytest.raises(UserError):
Converter.extract_all_content([audio_missing_data])
def test_items_to_messages_handles_system_and_developer_roles():
"""
Roles other than `user` (e.g. `system` and `developer`) need to be
converted appropriately whether provided as simple dicts or as full
`message` typed dicts.
"""
sys_items: list[TResponseInputItem] = [{"role": "system", "content": "setup"}]
sys_msgs = Converter.items_to_messages(sys_items)
assert len(sys_msgs) == 1
assert sys_msgs[0]["role"] == "system"
assert sys_msgs[0]["content"] == "setup"
dev_items: list[TResponseInputItem] = [{"role": "developer", "content": "debug"}]
dev_msgs = Converter.items_to_messages(dev_items)
assert len(dev_msgs) == 1
assert dev_msgs[0]["role"] == "developer"
assert dev_msgs[0]["content"] == "debug"
def test_maybe_input_message_allows_message_typed_dict():
"""
The `Converter.maybe_input_message` should recognize a dict with
"type": "message" and a supported role as an input message. Ensure
that such dicts are passed through by `items_to_messages`.
"""
# Construct a dict with the proper required keys for a ResponseInputParam.Message
message_dict: TResponseInputItem = {
"type": "message",
"role": "user",
"content": "hi",
}
assert Converter.maybe_input_message(message_dict) is not None
# items_to_messages should process this correctly
msgs = Converter.items_to_messages([message_dict])
assert len(msgs) == 1
assert msgs[0]["role"] == "user"
assert msgs[0]["content"] == "hi"
def test_tool_call_conversion():
"""
Test that tool calls are converted correctly.
"""
function_call = ResponseFunctionToolCallParam(
id="tool1",
call_id="abc",
name="math",
arguments="{}",
type="function_call",
)
messages = Converter.items_to_messages([function_call])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "assistant"
assert tool_msg.get("content") is None
# Verify the content key exists in the message even when it is None.
# This is for Chat Completions API compatibility.
assert "content" in tool_msg, "content key should be present in assistant message"
tool_calls = list(tool_msg.get("tool_calls", []))
assert len(tool_calls) == 1
tool_call = tool_calls[0]
assert tool_call["id"] == function_call["call_id"]
assert tool_call["function"]["name"] == function_call["name"] # type: ignore
assert tool_call["function"]["arguments"] == function_call["arguments"] # type: ignore
@pytest.mark.parametrize("role", ["user", "system", "developer"])
def test_input_message_with_all_roles(role: str):
"""
The `Converter.maybe_input_message` should recognize a dict with
"type": "message" and a supported role as an input message. Ensure
that such dicts are passed through by `items_to_messages`.
"""
# Construct a dict with the proper required keys for a ResponseInputParam.Message
casted_role = cast(Literal["user", "system", "developer"], role)
message_dict: TResponseInputItem = {
"type": "message",
"role": casted_role,
"content": "hi",
}
assert Converter.maybe_input_message(message_dict) is not None
# items_to_messages should process this correctly
msgs = Converter.items_to_messages([message_dict])
assert len(msgs) == 1
assert msgs[0]["role"] == casted_role
assert msgs[0]["content"] == "hi"
def test_item_reference_errors():
"""
Test that item references are converted correctly.
"""
with pytest.raises(UserError):
Converter.items_to_messages(
[
{
"type": "item_reference",
"id": "item1",
}
]
)
class TestObject:
pass
def test_unknown_object_errors():
"""
Test that unknown objects are converted correctly.
"""
with pytest.raises(UserError, match="Unhandled item type or structure"):
# Purposely ignore the type error
Converter.items_to_messages([TestObject()]) # type: ignore
def test_assistant_messages_in_history():
"""
Test that assistant messages are added to the history.
"""
messages = Converter.items_to_messages(
[
{
"role": "user",
"content": "Hello",
},
{
"role": "assistant",
"content": "Hello?",
},
{
"role": "user",
"content": "What was my Name?",
},
]
)
assert messages == [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hello?"},
{"role": "user", "content": "What was my Name?"},
]
assert len(messages) == 3
assert messages[0]["role"] == "user"
assert messages[0]["content"] == "Hello"
assert messages[1]["role"] == "assistant"
assert messages[1]["content"] == "Hello?"
assert messages[2]["role"] == "user"
assert messages[2]["content"] == "What was my Name?"
File diff suppressed because it is too large Load Diff
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
import pytest
from agents.models.openai_client_utils import (
is_official_openai_base_url,
is_official_openai_client,
)
@pytest.mark.parametrize(
"base_url",
[
"https://api.openai.com",
"https://api.openai.com/v1/",
],
)
def test_official_openai_base_url_matches_exact_host(base_url: str) -> None:
assert is_official_openai_base_url(base_url) is True
@pytest.mark.parametrize(
"base_url",
[
"https://api.openai.com.evil/v1/",
"https://api.openai.com.proxy.local/v1/",
"http://api.openai.com/v1/",
"https://custom.example.test/v1/",
],
)
def test_official_openai_base_url_rejects_non_openai_hosts(base_url: str) -> None:
assert is_official_openai_base_url(base_url) is False
def test_official_openai_websocket_base_url_matches_exact_host() -> None:
assert is_official_openai_base_url("wss://api.openai.com/v1/", websocket=True) is True
assert (
is_official_openai_base_url("wss://api.openai.com.proxy.local/v1/", websocket=True) is False
)
def test_official_openai_client_rejects_client_without_base_url() -> None:
assert is_official_openai_client(object()) is False # type: ignore[arg-type]
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+178
View File
@@ -0,0 +1,178 @@
"""Unit tests for the low-level helpers in :mod:`agents.models._openai_retry`.
These exercise the header-parsing, status-extraction, and error-code helpers
directly, plus a few public ``get_openai_retry_advice`` branches that the broader
behavioral suite in ``test_model_retry.py`` does not reach.
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from email.utils import format_datetime
import httpx
from agents.models._openai_retry import get_openai_retry_advice
from agents.models._retry_runtime import (
get_error_code as _get_error_code,
get_error_header as _get_header_value,
get_retry_after,
get_status_code as _get_status_code,
header_lookup as _header_lookup,
parse_retry_after_ms as _parse_retry_after_ms,
parse_retry_after_value as _parse_retry_after,
)
from agents.retry import ModelRetryAdviceRequest
from agents.run_internal.model_retry import _normalize_retry_error
class _HeaderError(Exception):
"""Error that exposes headers through a plain attribute rather than a response."""
def __init__(self, message: str, *, headers: dict[str, str] | None = None) -> None:
super().__init__(message)
if headers is not None:
self.headers = headers
def _make_request(error: Exception, **kwargs: object) -> ModelRetryAdviceRequest:
return ModelRetryAdviceRequest(error=error, attempt=1, stream=False, **kwargs) # type: ignore[arg-type]
def test_header_lookup_plain_mapping_matches_case_insensitively() -> None:
headers = {"Retry-After": "5", "X-Other": "ignored"}
assert _header_lookup(headers, "retry-after") == "5"
assert _header_lookup(headers, "missing") is None
def test_header_lookup_httpx_headers() -> None:
headers = httpx.Headers({"retry-after": "7"})
assert _header_lookup(headers, "retry-after") == "7"
assert _header_lookup(None, "retry-after") is None
def test_get_header_value_reads_response_headers_attr() -> None:
class _Err(Exception):
response_headers = {"retry-after": "3"}
assert _get_header_value(_Err("boom"), "retry-after") == "3"
def test_parse_retry_after_ms_invalid_returns_none() -> None:
assert _parse_retry_after_ms(None) is None
assert _parse_retry_after_ms("not-a-number") is None
assert _parse_retry_after_ms("-100") is None
assert _parse_retry_after_ms("1500") == 1.5
def test_parse_retry_after_numeric_and_http_date() -> None:
assert _parse_retry_after(None) is None
assert _parse_retry_after("2") == 2.0
assert _parse_retry_after("-1") is None
future = datetime.now(timezone.utc) + timedelta(seconds=120)
parsed = _parse_retry_after(format_datetime(future))
assert parsed is not None and parsed > 0
assert _parse_retry_after("definitely not a date") is None
def test_get_retry_after_preserves_outer_exception_precedence() -> None:
outer = _HeaderError("wrapped", headers={"retry-after": "2"})
outer.__cause__ = _HeaderError("provider", headers={"retry-after-ms": "1500"})
assert get_retry_after(outer) == 2.0
def test_get_status_code_from_status_code_and_status_attrs() -> None:
class _StatusCode(Exception):
status_code = 503
class _Status(Exception):
status = 504
assert _get_status_code(_StatusCode("a")) == 503
assert _get_status_code(_Status("b")) == 504
assert _get_status_code(Exception("none")) is None
def test_get_error_code_from_body_mapping() -> None:
class _NestedBody(Exception):
body = {"error": {"code": "rate_limit_exceeded"}}
class _TopLevelBody(Exception):
body = {"code": "server_error"}
assert _get_error_code(_NestedBody("a")) == "rate_limit_exceeded"
assert _get_error_code(_TopLevelBody("b")) == "server_error"
assert _get_error_code(Exception("none")) is None
def test_provider_and_runner_retry_normalization_share_metadata() -> None:
class _RetryableError(Exception):
status_code = 429
request_id = "req_test"
body = {"error": {"code": "rate_limit_exceeded"}}
headers = {"retry-after-ms": "1500"}
class _WrapperError(Exception):
headers = {"x-other": "ignored"}
error = _WrapperError("wrapped")
error.__cause__ = _RetryableError("slow down")
advice = get_openai_retry_advice(_make_request(error))
runner_normalized = _normalize_retry_error(error, None)
assert advice is not None
assert advice.normalized is not None
assert advice.normalized.status_code == runner_normalized.status_code
assert advice.normalized.error_code == runner_normalized.error_code
assert advice.normalized.request_id == runner_normalized.request_id
assert advice.normalized.retry_after == runner_normalized.retry_after
assert runner_normalized.retry_after == 1.5
def test_advice_unsafe_to_replay() -> None:
error = Exception("cannot replay")
error.unsafe_to_replay = True # type: ignore[attr-defined]
advice = get_openai_retry_advice(_make_request(error))
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
def test_advice_websocket_request_is_unsafe() -> None:
message = (
"The request may have been accepted, so the SDK will not automatically "
"retry this websocket request."
)
advice = get_openai_retry_advice(_make_request(Exception(message)))
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
def test_advice_respects_x_should_retry_false() -> None:
error = _HeaderError("nope", headers={"x-should-retry": "false"})
advice = get_openai_retry_advice(_make_request(error))
assert advice is not None
assert advice.suggested is False
def test_advice_returns_retry_after_only_when_no_other_signal() -> None:
# A 400 with no x-should-retry header and no network/timeout signal would not
# normally retry, but a retry-after header still yields advice carrying the delay.
error = _HeaderError("slow down", headers={"retry-after": "2"})
advice = get_openai_retry_advice(_make_request(error))
assert advice is not None
assert advice.retry_after == 2.0
# This branch only conveys the server-provided delay; it does not assert a
# retry decision, so ``suggested`` keeps its unset default.
assert advice.suggested is None
+454
View File
@@ -0,0 +1,454 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any, cast
import pytest
from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta
from openai.types.completion_usage import (
CompletionTokensDetails,
CompletionUsage,
PromptTokensDetails,
)
from openai.types.responses import (
Response,
ResponseOutputMessage,
ResponseOutputText,
ResponseReasoningItem,
)
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.models.openai_provider import OpenAIProvider
# Helper functions to create test objects consistently
def create_content_delta(content: str) -> dict[str, Any]:
"""Create a delta dictionary with regular content"""
return {"content": content, "role": None, "function_call": None, "tool_calls": None}
def create_reasoning_delta(content: str) -> dict[str, Any]:
"""Create a delta dictionary with reasoning content. The Only difference is reasoning_content"""
return {
"content": None,
"role": None,
"function_call": None,
"tool_calls": None,
"reasoning_content": content,
}
def create_chunk(delta: dict[str, Any], include_usage: bool = False) -> ChatCompletionChunk:
"""Create a ChatCompletionChunk with the given delta"""
# Create a ChoiceDelta object from the dictionary
delta_obj = ChoiceDelta(
content=delta.get("content"),
role=delta.get("role"),
function_call=delta.get("function_call"),
tool_calls=delta.get("tool_calls"),
)
# Add reasoning_content attribute dynamically if present in the delta
if "reasoning_content" in delta:
# Use direct assignment for the reasoning_content attribute
delta_obj_any = cast(Any, delta_obj)
delta_obj_any.reasoning_content = delta["reasoning_content"]
# Create the chunk
chunk = ChatCompletionChunk(
id="chunk-id",
created=1,
model="deepseek is usually expected",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=delta_obj)],
)
if include_usage:
chunk.usage = CompletionUsage(
completion_tokens=4,
prompt_tokens=2,
total_tokens=6,
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
)
return chunk
async def create_fake_stream(
chunks: list[ChatCompletionChunk],
) -> AsyncIterator[ChatCompletionChunk]:
for chunk in chunks:
yield chunk
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_reasoning_content(monkeypatch) -> None:
"""
Validate that when a model streams reasoning content,
`stream_response` emits the appropriate sequence of events including
`response.reasoning_summary_text.delta` events for each chunk of the reasoning content and
constructs a completed response with a `ResponseReasoningItem` part.
"""
# Create test chunks
chunks = [
# Reasoning content chunks
create_chunk(create_reasoning_delta("Let me think")),
create_chunk(create_reasoning_delta(" about this")),
# Regular content chunks
create_chunk(create_content_delta("The answer")),
create_chunk(create_content_delta(" is 42"), include_usage=True),
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# verify reasoning content events were emitted
reasoning_delta_events = [
e for e in output_events if e.type == "response.reasoning_summary_text.delta"
]
assert len(reasoning_delta_events) == 2
assert reasoning_delta_events[0].delta == "Let me think"
assert reasoning_delta_events[1].delta == " about this"
reasoning_done_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.reasoning_summary_part.done"
)
first_text_delta_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.output_text.delta"
)
assert reasoning_done_index < first_text_delta_index
# verify regular content events were emitted
content_delta_events = [e for e in output_events if e.type == "response.output_text.delta"]
assert len(content_delta_events) == 2
assert content_delta_events[0].delta == "The answer"
assert content_delta_events[1].delta == " is 42"
assistant_message_index_events = []
for event in output_events:
event_any = cast(Any, event)
if event.type in {"response.output_item.added", "response.output_item.done"}:
if event_any.item.type == "message":
assistant_message_index_events.append(event_any)
elif event.type in {
"response.content_part.added",
"response.output_text.delta",
"response.content_part.done",
}:
assistant_message_index_events.append(event_any)
assert assistant_message_index_events
for event in assistant_message_index_events:
assert event.output_index == 1
assert type(event.output_index) is int
# verify the final response contains both types of content
response_event = output_events[-1]
assert response_event.type == "response.completed"
assert len(response_event.response.output) == 2
# first item should be reasoning
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
assert response_event.response.output[0].summary[0].text == "Let me think about this"
# second item should be message with text
assert isinstance(response_event.response.output[1], ResponseOutputMessage)
assert isinstance(response_event.response.output[1].content[0], ResponseOutputText)
assert response_event.response.output[1].content[0].text == "The answer is 42"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_keeps_reasoning_item_open_across_interleaved_text(
monkeypatch,
) -> None:
chunks = [
create_chunk(create_reasoning_delta("Let me think")),
create_chunk(create_content_delta("The answer")),
create_chunk(create_reasoning_delta(" more carefully")),
create_chunk(create_content_delta(" is 42"), include_usage=True),
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
reasoning_part_added_events = [
event for event in output_events if event.type == "response.reasoning_summary_part.added"
]
assert [event.summary_index for event in reasoning_part_added_events] == [0, 1]
reasoning_part_done_events = [
event for event in output_events if event.type == "response.reasoning_summary_part.done"
]
assert [event.summary_index for event in reasoning_part_done_events] == [0, 1]
first_reasoning_done_index = output_events.index(reasoning_part_done_events[0])
first_text_delta_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.output_text.delta"
)
second_reasoning_delta_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.reasoning_summary_text.delta" and event.summary_index == 1
)
reasoning_item_done_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.output_item.done" and event.item.type == "reasoning"
)
assert first_reasoning_done_index < first_text_delta_index
assert second_reasoning_delta_index > first_text_delta_index
assert reasoning_item_done_index > second_reasoning_delta_index
response_event = output_events[-1]
assert response_event.type == "response.completed"
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
assert [summary.text for summary in response_event.response.output[0].summary] == [
"Let me think",
" more carefully",
]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_get_response_with_reasoning_content(monkeypatch) -> None:
"""
Test that when a model returns reasoning content in addition to regular content,
`get_response` properly includes both in the response output.
"""
# create a message with reasoning content
msg = ChatCompletionMessage(
role="assistant",
content="The answer is 42",
)
# Use dynamic attribute for reasoning_content
# We need to cast to Any to avoid mypy errors since reasoning_content is not a defined attribute
msg_with_reasoning = cast(Any, msg)
msg_with_reasoning.reasoning_content = "Let me think about this question carefully"
# create a choice with the message
mock_choice = {
"index": 0,
"finish_reason": "stop",
"message": msg_with_reasoning,
"delta": None,
}
chat = ChatCompletion(
id="resp-id",
created=0,
model="deepseek is expected",
object="chat.completion",
choices=[mock_choice], # type: ignore[list-item]
usage=CompletionUsage(
completion_tokens=10,
prompt_tokens=5,
total_tokens=15,
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=6),
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
),
)
async def patched_fetch_response(self, *args, **kwargs):
return chat
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
resp = await model.get_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
# should have produced a reasoning item and a message with text content
assert len(resp.output) == 2
# first output should be the reasoning item
assert isinstance(resp.output[0], ResponseReasoningItem)
assert resp.output[0].summary[0].text == "Let me think about this question carefully"
# second output should be the message with text content
assert isinstance(resp.output[1], ResponseOutputMessage)
assert isinstance(resp.output[1].content[0], ResponseOutputText)
assert resp.output[1].content[0].text == "The answer is 42"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_preserves_usage_from_earlier_chunk(monkeypatch) -> None:
"""
Test that when an earlier chunk has usage data and later chunks don't,
the usage from the earlier chunk is preserved in the final response.
This handles cases where some providers (e.g., LiteLLM) may not include
usage in every chunk.
"""
# Create test chunks where first chunk has usage, last chunk doesn't
chunks = [
create_chunk(create_content_delta("Hello"), include_usage=True), # Has usage
create_chunk(create_content_delta("")), # No usage (usage=None)
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Verify the final response preserves usage from the first chunk
response_event = output_events[-1]
assert response_event.type == "response.completed"
assert response_event.response.usage is not None
assert response_event.response.usage.input_tokens == 2
assert response_event.response.usage.output_tokens == 4
assert response_event.response.usage.total_tokens == 6
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_with_empty_reasoning_content(monkeypatch) -> None:
"""
Test that when a model streams empty reasoning content,
the response still processes correctly without errors.
"""
# create test chunks with empty reasoning content
chunks = [
create_chunk(create_reasoning_delta("")),
create_chunk(create_content_delta("The answer is 42"), include_usage=True),
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# verify the final response contains the content
response_event = output_events[-1]
assert response_event.type == "response.completed"
# should only have the message, not an empty reasoning item
assert len(response_event.response.output) == 1
assert isinstance(response_event.response.output[0], ResponseOutputMessage)
assert isinstance(response_event.response.output[0].content[0], ResponseOutputText)
assert response_event.response.output[0].content[0].text == "The answer is 42"
@@ -0,0 +1,403 @@
from __future__ import annotations
from typing import Any, cast
import httpx
import litellm
import pytest
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from openai.types.chat.chat_completion import ChatCompletion, Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.completion_usage import CompletionUsage
from agents.extensions.models.litellm_model import LitellmModel
from agents.items import TResponseInputItem
from agents.model_settings import ModelSettings
from agents.models.chatcmpl_converter import Converter
from agents.models.interface import ModelTracing
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.models.reasoning_content_replay import ReasoningContentReplayContext
REASONING_CONTENT_MODEL_A = "reasoning-content-model-a"
REASONING_CONTENT_MODEL_B = "reasoning-content-model-b"
# The converter currently keys Anthropic thinking-block reconstruction off the model name,
# so this test model keeps the "anthropic" substring while staying otherwise generic.
REASONING_CONTENT_MODEL_C = "reasoning-content-model-c-anthropic"
def _second_turn_input_items(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _second_turn_input_items_with_message(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"id": "__fake_id__",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [
{
"type": "output_text",
"text": "I'll call the weather tool now.",
"annotations": [],
"logprobs": [],
}
],
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _second_turn_input_items_with_file_search(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "Find notes about Tokyo weather."},
{
"id": "__fake_id__",
"summary": [
{"text": "I should search the knowledge base first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"id": "__fake_file_search_id__",
"queries": ["Tokyo weather"],
"status": "completed",
"type": "file_search_call",
},
],
)
def _second_turn_input_items_with_message_then_reasoning(
model_name: str,
) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [
{
"type": "output_text",
"text": "I'll call the weather tool now.",
"annotations": [],
"logprobs": [],
}
],
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _second_turn_input_items_with_thinking_blocks(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": [
{
"type": "reasoning_text",
"text": "First, I need to inspect the request.",
}
],
"encrypted_content": "test-signature",
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _assistant_with_tool_calls(messages: list[Any]) -> dict[str, Any]:
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
return msg
raise AssertionError("Expected an assistant message with tool_calls.")
def test_converter_keeps_default_reasoning_replay_behavior_for_non_default_model() -> None:
messages = Converter.items_to_messages(
_second_turn_input_items(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
)
assistant = _assistant_with_tool_calls(messages)
assert "reasoning_content" not in assistant
def test_converter_preserves_reasoning_content_across_output_message_with_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_message(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["content"] == "I'll call the weather tool now."
assert assistant["reasoning_content"] == "I should call the weather tool first."
def test_converter_replays_reasoning_content_when_reasoning_follows_message_with_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_message_then_reasoning(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["content"] == "I'll call the weather tool now."
assert assistant["reasoning_content"] == "I should call the weather tool first."
def test_converter_replays_reasoning_content_for_file_search_call_with_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_file_search(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["reasoning_content"] == "I should search the knowledge base first."
assert assistant["tool_calls"][0]["function"]["name"] == "file_search_call"
def test_converter_replays_reasoning_content_with_thinking_blocks_and_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_thinking_blocks(REASONING_CONTENT_MODEL_C),
model=REASONING_CONTENT_MODEL_C,
preserve_thinking_blocks=True,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["reasoning_content"] == "I should call the weather tool first."
assert assistant["content"][0]["type"] == "thinking"
assert assistant["content"][0]["thinking"] == "First, I need to inspect the request."
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_openai_chatcompletions_hook_can_enable_reasoning_content_replay() -> None:
captured: dict[str, Any] = {}
contexts: list[ReasoningContentReplayContext] = []
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
contexts.append(context)
return context.model == REASONING_CONTENT_MODEL_B
class MockChatCompletions:
async def create(self, **kwargs):
captured.update(kwargs)
msg = ChatCompletionMessage(role="assistant", content="done")
choice = Choice(index=0, message=msg, finish_reason="stop")
return ChatCompletion(
id="test-id",
created=0,
model=REASONING_CONTENT_MODEL_B,
object="chat.completion",
choices=[choice],
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
)
class MockChat:
def __init__(self):
self.completions = MockChatCompletions()
class MockClient:
def __init__(self):
self.chat = MockChat()
self.base_url = httpx.URL("https://example.com/v1/")
model = OpenAIChatCompletionsModel(
model=REASONING_CONTENT_MODEL_B,
openai_client=cast(Any, MockClient()),
should_replay_reasoning_content=should_replay_reasoning_content,
)
await model.get_response(
system_instructions=None,
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
assert assistant["reasoning_content"] == "I should call the weather tool first."
assert len(contexts) == 1
assert contexts[0].model == REASONING_CONTENT_MODEL_B
assert contexts[0].base_url == "https://example.com/v1"
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_litellm_hook_can_enable_reasoning_content_replay(monkeypatch) -> None:
captured: dict[str, Any] = {}
contexts: list[ReasoningContentReplayContext] = []
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
contexts.append(context)
return context.model == REASONING_CONTENT_MODEL_B
async def fake_acompletion(model, messages=None, **kwargs):
captured["messages"] = messages
msg = Message(role="assistant", content="done")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(
model=REASONING_CONTENT_MODEL_B,
should_replay_reasoning_content=should_replay_reasoning_content,
)
await model.get_response(
system_instructions=None,
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
assert assistant["reasoning_content"] == "I should call the weather tool first."
assert len(contexts) == 1
assert contexts[0].model == REASONING_CONTENT_MODEL_B
assert contexts[0].base_url is None
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
@@ -0,0 +1,162 @@
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
import pytest
from agents.models.fake_id import FAKE_RESPONSES_ID
from agents.models.openai_responses import OpenAIResponsesModel
@pytest.fixture
def model() -> OpenAIResponsesModel:
"""Create a model instance for testing."""
mock_client = MagicMock()
return OpenAIResponsesModel(model="gpt-5", openai_client=mock_client)
class TestRemoveOpenAIResponsesAPIIncompatibleFields:
"""Tests for _remove_openai_responses_api_incompatible_fields method."""
def test_returns_unchanged_when_no_provider_data(self, model: OpenAIResponsesModel):
"""When no items have provider_data, the input should be returned unchanged."""
list_input = [
{"type": "message", "content": "hello"},
{"type": "function_call", "call_id": "call_123", "name": "test"},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert result is list_input # Same object reference.
def test_removes_reasoning_items_with_provider_data(self, model: OpenAIResponsesModel):
"""Reasoning items with provider_data should be completely removed."""
list_input = [
{"type": "message", "content": "hello"},
{"type": "reasoning", "provider_data": {"model": "gemini/gemini-3"}},
{"type": "function_call", "call_id": "call_123"},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 2
assert result[0] == {"type": "message", "content": "hello"}
assert result[1] == {"type": "function_call", "call_id": "call_123"}
def test_keeps_reasoning_items_without_provider_data(self, model: OpenAIResponsesModel):
"""Reasoning items without provider_data should be kept."""
list_input = [
{"type": "reasoning", "summary": []},
{"type": "message", "content": "hello", "provider_data": {"foo": "bar"}},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 2
assert result[0] == {"type": "reasoning", "summary": []}
assert result[1] == {"type": "message", "content": "hello"}
def test_removes_provider_data_from_all_items(self, model: OpenAIResponsesModel):
"""provider_data field should be removed from all dict items."""
list_input = [
{"type": "message", "content": "hello", "provider_data": {"model": "gemini/gemini-3"}},
{
"type": "function_call",
"call_id": "call_123",
"provider_data": {"model": "gemini/gemini-3"},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 2
assert "provider_data" not in result[0]
assert "provider_data" not in result[1]
def test_removes_fake_responses_id(self, model: OpenAIResponsesModel):
"""Items with id equal to FAKE_RESPONSES_ID should have their id removed."""
list_input = [
{
"type": "message",
"id": FAKE_RESPONSES_ID,
"content": "hello",
"provider_data": {"model": "gemini/gemini-3"},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 1
assert "id" not in result[0]
assert result[0]["content"] == "hello"
def test_preserves_real_ids(self, model: OpenAIResponsesModel):
"""Real IDs (not FAKE_RESPONSES_ID) should be preserved."""
list_input = [
{
"type": "message",
"id": "msg_real123",
"content": "hello",
"provider_data": {},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert result[0]["id"] == "msg_real123"
def test_handles_empty_list(self, model: OpenAIResponsesModel):
"""Empty list should be returned unchanged."""
list_input: list[dict[str, Any]] = []
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert result == []
def test_combined_scenario(self, model: OpenAIResponsesModel):
"""Test a realistic scenario with multiple items needing different processing."""
list_input = [
{"type": "message", "content": "user input"},
{"type": "reasoning", "summary": [], "provider_data": {"model": "gemini/gemini-3"}},
{
"type": "function_call",
"call_id": "call_abc_123",
"name": "get_weather",
"provider_data": {"model": "gemini/gemini-3"},
},
{
"type": "function_call_output",
"call_id": "call_abc_123",
"output": '{"temp": 72}',
},
{
"type": "message",
"id": FAKE_RESPONSES_ID,
"content": "The weather is 72F",
"provider_data": {"model": "gemini/gemini-3"},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
# Should have 4 items (reasoning with provider_data removed).
assert len(result) == 4
# First item unchanged (no provider_data).
assert result[0] == {"type": "message", "content": "user input"}
# Function call: __thought__ suffix removed, provider_data removed.
assert result[1]["type"] == "function_call"
assert result[1]["call_id"] == "call_abc_123"
assert "provider_data" not in result[1]
# Function call output: __thought__ suffix removed, provider_data removed.
assert result[2]["type"] == "function_call_output"
assert result[2]["call_id"] == "call_abc_123"
# Last message: fake id removed, provider_data removed.
assert result[3]["type"] == "message"
assert result[3]["content"] == "The weather is 72F"
assert "id" not in result[3]
assert "provider_data" not in result[3]
@@ -0,0 +1,149 @@
import importlib
import pytest
from agents import Agent, responses_websocket_session
from agents.models.multi_provider import MultiProvider
from agents.models.openai_provider import OpenAIProvider
@pytest.mark.asyncio
async def test_responses_websocket_session_builds_shared_run_config():
async with responses_websocket_session() as ws:
assert isinstance(ws.provider, OpenAIProvider)
assert ws.provider._use_responses is True
assert ws.provider._use_responses_websocket is True
assert isinstance(ws.run_config.model_provider, MultiProvider)
assert ws.run_config.model_provider.openai_provider is ws.provider
@pytest.mark.asyncio
async def test_responses_websocket_session_preserves_openai_prefix_routing(monkeypatch):
captured: dict[str, object] = {}
sentinel = object()
def fake_get_model(model_name):
captured["model_name"] = model_name
return sentinel
async with responses_websocket_session() as ws:
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
assert result is sentinel
assert captured["model_name"] == "gpt-4.1"
@pytest.mark.asyncio
async def test_responses_websocket_session_can_preserve_openai_prefix_model_ids(monkeypatch):
captured: dict[str, object] = {}
sentinel = object()
def fake_get_model(model_name):
captured["model_name"] = model_name
return sentinel
async with responses_websocket_session(openai_prefix_mode="model_id") as ws:
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
assert result is sentinel
assert captured["model_name"] == "openai/gpt-4.1"
@pytest.mark.asyncio
async def test_responses_websocket_session_can_preserve_unknown_prefix_model_ids(monkeypatch):
captured: dict[str, object] = {}
sentinel = object()
def fake_get_model(model_name):
captured["model_name"] = model_name
return sentinel
async with responses_websocket_session(unknown_prefix_mode="model_id") as ws:
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
result = ws.run_config.model_provider.get_model("openrouter/openai/gpt-4.1")
assert result is sentinel
assert captured["model_name"] == "openrouter/openai/gpt-4.1"
@pytest.mark.asyncio
async def test_responses_websocket_session_run_streamed_injects_run_config(monkeypatch):
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
captured = {}
sentinel = object()
def fake_run_streamed(starting_agent, input, **kwargs):
captured["starting_agent"] = starting_agent
captured["input"] = input
captured["kwargs"] = kwargs
return sentinel
ws_module = importlib.import_module("agents.responses_websocket_session")
monkeypatch.setattr(ws_module.Runner, "run_streamed", fake_run_streamed)
async with responses_websocket_session() as ws:
result = ws.run_streamed(agent, "hello")
assert result is sentinel
assert captured["starting_agent"] is agent
assert captured["input"] == "hello"
assert captured["kwargs"]["run_config"] is ws.run_config
@pytest.mark.asyncio
async def test_responses_websocket_session_run_injects_run_config(monkeypatch):
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
captured = {}
sentinel = object()
async def fake_run(starting_agent, input, **kwargs):
captured["starting_agent"] = starting_agent
captured["input"] = input
captured["kwargs"] = kwargs
return sentinel
ws_module = importlib.import_module("agents.responses_websocket_session")
monkeypatch.setattr(ws_module.Runner, "run", fake_run)
async with responses_websocket_session() as ws:
result = await ws.run(agent, "hello")
assert result is sentinel
assert captured["starting_agent"] is agent
assert captured["input"] == "hello"
assert captured["kwargs"]["run_config"] is ws.run_config
@pytest.mark.asyncio
async def test_responses_websocket_session_rejects_run_config_override():
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
async with responses_websocket_session() as ws:
with pytest.raises(ValueError, match="run_config"):
ws.run_streamed(agent, "hello", run_config=object())
@pytest.mark.asyncio
async def test_responses_websocket_session_context_manager_closes_provider(monkeypatch):
close_calls: list[OpenAIProvider] = []
async def fake_aclose(self):
close_calls.append(self)
monkeypatch.setattr(OpenAIProvider, "aclose", fake_aclose)
async with responses_websocket_session() as ws:
provider = ws.provider
assert close_calls == [provider]
@pytest.mark.asyncio
async def test_responses_websocket_session_does_not_expose_run_sync():
async with responses_websocket_session() as ws:
assert not hasattr(ws, "run_sync")
+32
View File
@@ -0,0 +1,32 @@
from agents.model_settings import ModelSettings
from agents.models._trace import model_config_for_trace, sanitize_url_for_trace
def test_sanitize_url_for_trace_strips_auth_query_and_fragment() -> None:
assert (
sanitize_url_for_trace("https://user:pass@example.com/v1?api-key=secret#fragment")
== "https://example.com/v1"
)
assert sanitize_url_for_trace("https://example.com/v1?token=secret") == "https://example.com/v1"
def test_model_config_for_trace_sanitizes_base_url_and_omits_request_extras() -> None:
config = model_config_for_trace(
ModelSettings(
temperature=0.5,
extra_headers={"Authorization": "Bearer provider-token"},
extra_query={"api-key": "query-token"},
extra_body={"secret": "body-token"},
extra_args={"api_key": "arg-token"},
),
base_url="https://user:pass@example.com/v1?api-key=secret#fragment",
extra_config={"model_impl": "test-model"},
)
assert config["temperature"] == 0.5
assert config["base_url"] == "https://example.com/v1"
assert config["model_impl"] == "test-model"
assert "extra_headers" not in config
assert "extra_query" not in config
assert "extra_body" not in config
assert "extra_args" not in config