chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,317 @@
|
||||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
from httpx import Headers, Response
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai import APIConnectionError
|
||||
from openai.types.chat.chat_completion import ChatCompletion, Choice
|
||||
from openai.types.chat.chat_completion_message import ChatCompletionMessage
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models._retry_runtime import provider_managed_retries_disabled
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.retry import ModelRetryAdviceRequest, ModelRetrySettings
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_kwargs_forwarded(monkeypatch):
|
||||
"""
|
||||
Test that kwargs from ModelSettings are forwarded to litellm.acompletion.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
extra_args={
|
||||
"custom_param": "custom_value",
|
||||
"seed": 42,
|
||||
"stop": ["END"],
|
||||
"logit_bias": {123: -100},
|
||||
},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
)
|
||||
|
||||
# Verify that all kwargs were passed through
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert captured["seed"] == 42
|
||||
assert captured["stop"] == ["END"]
|
||||
assert captured["logit_bias"] == {123: -100}
|
||||
|
||||
# Verify regular parameters are still passed
|
||||
assert captured["temperature"] == 0.5
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_chatcompletions_kwargs_forwarded(monkeypatch):
|
||||
"""
|
||||
Test that kwargs from ModelSettings are forwarded to OpenAI chat completions API.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class MockChatCompletions:
|
||||
async def create(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = ChatCompletionMessage(role="assistant", content="test response")
|
||||
choice = Choice(index=0, message=msg, finish_reason="stop")
|
||||
return ChatCompletion(
|
||||
id="test-id",
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
|
||||
)
|
||||
|
||||
class MockChat:
|
||||
def __init__(self):
|
||||
self.completions = MockChatCompletions()
|
||||
|
||||
class MockClient:
|
||||
def __init__(self):
|
||||
self.chat = MockChat()
|
||||
self.base_url = "https://api.openai.com/v1"
|
||||
|
||||
settings = ModelSettings(
|
||||
temperature=0.7,
|
||||
extra_args={
|
||||
"seed": 123,
|
||||
"logit_bias": {456: 10},
|
||||
"stop": ["STOP", "END"],
|
||||
"user": "test-user",
|
||||
},
|
||||
)
|
||||
|
||||
mock_client = MockClient()
|
||||
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=mock_client) # type: ignore
|
||||
|
||||
await model.get_response(
|
||||
system_instructions="Test system",
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Verify that all kwargs were passed through
|
||||
assert captured["seed"] == 123
|
||||
assert captured["logit_bias"] == {456: 10}
|
||||
assert captured["stop"] == ["STOP", "END"]
|
||||
assert captured["user"] == "test-user"
|
||||
|
||||
# Verify regular parameters are still passed
|
||||
assert captured["temperature"] == 0.7
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_kwargs_handling(monkeypatch):
|
||||
"""
|
||||
Test that empty or None kwargs are handled gracefully.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
# Test with None kwargs
|
||||
settings_none = ModelSettings(temperature=0.5, extra_args=None)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings_none,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Should work without error and include regular parameters
|
||||
assert captured["temperature"] == 0.5
|
||||
|
||||
# Test with empty dict
|
||||
captured.clear()
|
||||
settings_empty = ModelSettings(temperature=0.3, extra_args={})
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings_empty,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Should work without error and include regular parameters
|
||||
assert captured["temperature"] == 0.3
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_effort_falls_back_to_extra_args(monkeypatch):
|
||||
"""
|
||||
Ensure reasoning_effort from extra_args is promoted when reasoning settings are missing.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
|
||||
settings = ModelSettings(
|
||||
extra_args={"reasoning_effort": "none", "custom_param": "custom_value"}
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "none"
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert settings.extra_args == {"reasoning_effort": "none", "custom_param": "custom_value"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_retry_settings_do_not_leak_and_disable_provider_retries_on_runner_retry(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Runner retries should disable LiteLLM's own retries without forwarding SDK retry config."""
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
settings = ModelSettings(
|
||||
retry=ModelRetrySettings(
|
||||
max_retries=2,
|
||||
backoff={"initial_delay": 0.25, "jitter": False},
|
||||
),
|
||||
extra_args={"max_retries": 7, "num_retries": 6, "custom_param": "custom_value"},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
with provider_managed_retries_disabled(True):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
)
|
||||
|
||||
assert settings.retry is not None
|
||||
assert settings.retry.backoff is not None
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert captured["max_retries"] == 0
|
||||
assert captured["num_retries"] == 0
|
||||
assert "retry" not in captured
|
||||
|
||||
|
||||
def test_litellm_get_retry_advice_uses_response_headers() -> None:
|
||||
"""LiteLLM retry advice should expose OpenAI-compatible retry headers."""
|
||||
|
||||
model = LitellmModel(model="test-model")
|
||||
error = RateLimitError(
|
||||
message="rate limited",
|
||||
llm_provider="openai",
|
||||
model="gpt-4o-mini",
|
||||
response=Response(
|
||||
status_code=429,
|
||||
headers=Headers({"x-should-retry": "true", "retry-after-ms": "250"}),
|
||||
),
|
||||
)
|
||||
|
||||
advice = model.get_retry_advice(
|
||||
ModelRetryAdviceRequest(
|
||||
error=error,
|
||||
attempt=1,
|
||||
stream=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is True
|
||||
assert advice.retry_after == 0.25
|
||||
|
||||
|
||||
def test_litellm_get_retry_advice_keeps_stateful_transport_failures_ambiguous() -> None:
|
||||
model = LitellmModel(model="test-model")
|
||||
error = APIConnectionError(
|
||||
message="connection error",
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/responses"),
|
||||
)
|
||||
|
||||
advice = model.get_retry_advice(
|
||||
ModelRetryAdviceRequest(
|
||||
error=error,
|
||||
attempt=1,
|
||||
stream=False,
|
||||
previous_response_id="resp_prev",
|
||||
)
|
||||
)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is True
|
||||
assert advice.replay_safety is None
|
||||
Reference in New Issue
Block a user