chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:39:17 +08:00
commit 4ed4e9ff99
1368 changed files with 334957 additions and 0 deletions
+27
View File
@@ -0,0 +1,27 @@
# Tests
Before running any tests, make sure you have `uv` installed (and ideally run `make sync` after).
## Running tests
```
make tests
```
`make tests` runs the shard-safe suite in parallel and then runs tests marked `serial` in a separate serial pass.
## Snapshots
We use [inline-snapshots](https://15r10nk.github.io/inline-snapshot/latest/) for some tests. If your code adds new snapshot tests or breaks existing ones, you can fix/create them. After fixing/creating snapshots, run `make tests` again to verify the tests pass.
### Fixing snapshots
```
make snapshots-fix
```
### Creating snapshots
```
make snapshots-create
```
View File
+108
View File
@@ -0,0 +1,108 @@
from __future__ import annotations
import shlex
from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import PurePosixPath
@dataclass(frozen=True)
class FakeResolveWorkspaceResult:
exit_code: int
stdout: str = ""
stderr: str = ""
def resolve_fake_workspace_path(
command: str | Sequence[str],
*,
symlinks: dict[str, str],
home_dir: str,
) -> FakeResolveWorkspaceResult | None:
tokens = shlex.split(command) if isinstance(command, str) else list(command)
helper_index = next(
(
index
for index, token in enumerate(tokens)
if token.startswith("/tmp/openai-agents/bin/resolve-workspace-path-")
),
None,
)
if helper_index is None or len(tokens) < helper_index + 4:
return None
root = _resolve_fake_path(tokens[helper_index + 1], symlinks=symlinks, home_dir=home_dir)
candidate = _resolve_fake_path(tokens[helper_index + 2], symlinks=symlinks, home_dir=home_dir)
for_write = tokens[helper_index + 3]
grant_tokens = tokens[helper_index + 4 :]
if _fake_path_is_under(candidate, root):
return FakeResolveWorkspaceResult(exit_code=0, stdout=candidate.as_posix())
best_grant: tuple[PurePosixPath, str, str] | None = None
for index in range(0, len(grant_tokens), 2):
grant_original = grant_tokens[index]
read_only = grant_tokens[index + 1]
grant_root = _resolve_fake_path(grant_original, symlinks=symlinks, home_dir=home_dir)
if not _fake_path_is_under(candidate, grant_root):
continue
if best_grant is None or len(grant_root.parts) > len(best_grant[0].parts):
best_grant = (grant_root, grant_original, read_only)
if best_grant is not None:
_grant_root, grant_original, read_only = best_grant
if for_write == "1" and read_only == "1":
return FakeResolveWorkspaceResult(
exit_code=114,
stderr=(
f"read-only extra path grant: {grant_original}\n"
f"resolved path: {candidate.as_posix()}\n"
),
)
return FakeResolveWorkspaceResult(exit_code=0, stdout=candidate.as_posix())
return FakeResolveWorkspaceResult(
exit_code=111,
stderr=f"workspace escape: {candidate.as_posix()}\n",
)
def _resolve_fake_path(
raw_path: str,
*,
symlinks: dict[str, str],
home_dir: str,
depth: int = 0,
) -> PurePosixPath:
if depth > 64:
raise RuntimeError(f"symlink resolution depth exceeded: {raw_path}")
path = PurePosixPath(raw_path)
if not path.is_absolute():
path = PurePosixPath(home_dir) / path
parts = path.parts
current = PurePosixPath("/")
for index, part in enumerate(parts[1:], start=1):
current = current / part
target = symlinks.get(current.as_posix())
if target is None:
continue
target_path = PurePosixPath(target)
if not target_path.is_absolute():
target_path = current.parent / target_path
for remaining in parts[index + 1 :]:
target_path /= remaining
return _resolve_fake_path(
target_path.as_posix(),
symlinks=symlinks,
home_dir=home_dir,
depth=depth + 1,
)
return path
def _fake_path_is_under(path: PurePosixPath, root: PurePosixPath) -> bool:
return path == root or root in path.parents
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
import sys
import pytest
from agents.models import _openai_shared
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.models.openai_responses import OpenAIResponsesModel
from agents.run import set_default_agent_runner
from agents.tracing.provider import DefaultTraceProvider
from agents.tracing.setup import set_trace_provider
from .testing_processor import SPAN_PROCESSOR_TESTING
collect_ignore: list[str] = []
if sys.platform == "win32":
collect_ignore.extend(
[
"test_example_workflows.py",
"test_run_state.py",
"sandbox/capabilities/test_filesystem_capability.py",
"sandbox/integration_tests/test_runner_pause_resume.py",
"sandbox/test_client_options.py",
"sandbox/test_exposed_ports.py",
"sandbox/test_extract.py",
"sandbox/test_memory.py",
"sandbox/test_runtime.py",
"sandbox/test_session_manager.py",
"sandbox/test_session_sinks.py",
"sandbox/test_snapshot.py",
"sandbox/test_unix_local.py",
]
)
# This fixture will run once before any tests are executed
@pytest.fixture(scope="session", autouse=True)
def setup_span_processor():
provider = DefaultTraceProvider()
provider.set_processors([SPAN_PROCESSOR_TESTING])
set_trace_provider(provider)
yield
provider.shutdown()
# Ensure a default OpenAI API key is present for tests that construct clients
# without explicitly configuring a key/client. Tests that need no key use
# monkeypatch.delenv("OPENAI_API_KEY", ...) to remove it locally.
@pytest.fixture(scope="session", autouse=True)
def ensure_openai_api_key():
import os
if not os.environ.get("OPENAI_API_KEY"):
os.environ["OPENAI_API_KEY"] = "test_key"
# This fixture will run before each test
@pytest.fixture(autouse=True)
def clear_span_processor():
SPAN_PROCESSOR_TESTING.force_flush()
SPAN_PROCESSOR_TESTING.shutdown()
SPAN_PROCESSOR_TESTING.clear()
# This fixture will run before each test
@pytest.fixture(autouse=True)
def clear_openai_settings():
_openai_shared._default_openai_key = None
_openai_shared._default_openai_client = None
_openai_shared._use_responses_by_default = True
_openai_shared.set_default_openai_responses_transport("http")
@pytest.fixture(autouse=True)
def clear_default_runner():
set_default_agent_runner(None)
@pytest.fixture(autouse=True)
def disable_real_model_clients(monkeypatch, request):
# If the test is marked to allow the method call, don't override it.
if request.node.get_closest_marker("allow_call_model_methods"):
return
def failing_version(*args, **kwargs):
pytest.fail("Real models should not be used in tests!")
monkeypatch.setattr(OpenAIResponsesModel, "get_response", failing_version)
monkeypatch.setattr(OpenAIResponsesModel, "stream_response", failing_version)
monkeypatch.setattr(OpenAIChatCompletionsModel, "get_response", failing_version)
monkeypatch.setattr(OpenAIChatCompletionsModel, "stream_response", failing_version)
@@ -0,0 +1,757 @@
from __future__ import annotations
import asyncio
import importlib
import inspect
import json
import os
from dataclasses import fields
from pathlib import Path
from typing import Any, cast
import pytest
from agents.exceptions import UserError
from agents.extensions.experimental.codex import Usage
from agents.extensions.experimental.codex.codex import Codex, _normalize_env
from agents.extensions.experimental.codex.codex_options import CodexOptions, coerce_codex_options
from agents.extensions.experimental.codex.exec import CodexExec
from agents.extensions.experimental.codex.output_schema_file import (
OutputSchemaFile,
create_output_schema_file,
)
from agents.extensions.experimental.codex.thread import Thread, _normalize_input
from agents.extensions.experimental.codex.thread_options import ThreadOptions, coerce_thread_options
from agents.extensions.experimental.codex.turn_options import TurnOptions
exec_module = importlib.import_module("agents.extensions.experimental.codex.exec")
thread_module = importlib.import_module("agents.extensions.experimental.codex.thread")
output_schema_module = importlib.import_module(
"agents.extensions.experimental.codex.output_schema_file"
)
class FakeStdin:
def __init__(self) -> None:
self.buffer = b""
self.closed = False
def write(self, data: bytes) -> None:
self.buffer += data
async def drain(self) -> None:
return None
def close(self) -> None:
self.closed = True
class FakeStdout:
def __init__(self, lines: list[str]) -> None:
self._lines = [line.encode("utf-8") for line in lines]
async def readline(self) -> bytes:
if not self._lines:
return b""
return self._lines.pop(0)
class FakeStderr:
def __init__(self, chunks: list[bytes]) -> None:
self._chunks = list(chunks)
async def read(self, _size: int) -> bytes:
if not self._chunks:
return b""
return self._chunks.pop(0)
class FakeProcess:
def __init__(
self,
stdout_lines: list[str],
stderr_chunks: list[bytes] | None = None,
*,
returncode: int | None = 0,
stdin_present: bool = True,
stdout_present: bool = True,
stderr_present: bool = True,
) -> None:
self.stdin = FakeStdin() if stdin_present else None
self.stdout = FakeStdout(stdout_lines) if stdout_present else None
self.stderr = FakeStderr(stderr_chunks or []) if stderr_present else None
self.returncode = returncode
self.killed = False
self.terminated = False
async def wait(self) -> None:
if self.returncode is None:
self.returncode = 0
def kill(self) -> None:
self.killed = True
def terminate(self) -> None:
self.terminated = True
class FakeExec:
def __init__(self, events: list[Any], delay: float = 0.0) -> None:
self.events = events
self.delay = delay
self.last_args: Any = None
async def run(self, args: Any):
self.last_args = args
for event in self.events:
if self.delay:
await asyncio.sleep(self.delay)
payload = event if isinstance(event, str) else json.dumps(event)
yield payload
def test_output_schema_file_none_schema() -> None:
result = create_output_schema_file(None)
assert result.schema_path is None
result.cleanup()
def test_output_schema_file_rejects_non_object() -> None:
with pytest.raises(UserError, match="output_schema must be a plain JSON object"):
create_output_schema_file(cast(Any, ["not", "an", "object"]))
def test_output_schema_file_creates_and_cleans() -> None:
schema = {"type": "object", "properties": {"foo": {"type": "string"}}}
result = create_output_schema_file(schema)
assert result.schema_path is not None
with open(result.schema_path, encoding="utf-8") as handle:
assert json.load(handle) == schema
result.cleanup()
assert not os.path.exists(result.schema_path)
def test_output_schema_file_cleanup_swallows_rmtree_errors(
monkeypatch: pytest.MonkeyPatch,
) -> None:
schema = {"type": "object"}
called = False
def bad_rmtree(_path: str, ignore_errors: bool = True) -> None:
nonlocal called
called = True
raise OSError("boom")
monkeypatch.setattr(output_schema_module.shutil, "rmtree", bad_rmtree)
result = create_output_schema_file(schema)
result.cleanup()
assert called is True
def test_output_schema_file_cleanup_on_write_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
schema = {"type": "object"}
cleanup_called = False
def bad_dump(*_args: Any, **_kwargs: Any) -> None:
raise RuntimeError("boom")
def fake_rmtree(_path: str, ignore_errors: bool = True) -> None:
nonlocal cleanup_called
cleanup_called = True
monkeypatch.setattr(output_schema_module.json, "dump", bad_dump)
monkeypatch.setattr(output_schema_module.shutil, "rmtree", fake_rmtree)
with pytest.raises(RuntimeError, match="boom"):
create_output_schema_file(schema)
assert cleanup_called is True
def test_normalize_input_merges_text_and_images() -> None:
prompt, images = _normalize_input(
[
{"type": "text", "text": "first"},
{"type": "local_image", "path": "/tmp/a.png"},
{"type": "text", "text": "second"},
{"type": "local_image", "path": ""},
]
)
assert prompt == "first\n\nsecond"
assert images == ["/tmp/a.png"]
def test_normalize_env_stringifies_values() -> None:
env = _normalize_env(CodexOptions(env=cast(dict[str, str], {"FOO": 1, 2: "bar"})))
assert env == {"FOO": "1", "2": "bar"}
def test_coerce_codex_options_rejects_unknown_fields() -> None:
with pytest.raises(UserError, match="Unknown CodexOptions field"):
coerce_codex_options({"unknown": "value"})
def test_coerce_thread_options_rejects_unknown_fields() -> None:
with pytest.raises(UserError, match="Unknown ThreadOptions field"):
coerce_thread_options({"unknown": "value"})
def test_coerce_thread_options_rejects_non_mapping() -> None:
with pytest.raises(UserError, match="ThreadOptions must be a ThreadOptions or a mapping"):
coerce_thread_options(cast(Any, ["model", "gpt"]))
def test_codex_start_and_resume_thread() -> None:
codex = Codex(CodexOptions(codex_path_override="/bin/codex"))
thread = codex.start_thread({"model": "gpt"})
assert thread.id is None
resumed = codex.resume_thread("thread-1", {"model": "gpt"})
assert resumed.id == "thread-1"
def test_codex_init_accepts_mapping_options() -> None:
codex = Codex({"codex_path_override": "/bin/codex"})
assert codex._exec._executable_path == "/bin/codex"
def test_codex_init_accepts_kwargs() -> None:
codex = Codex(codex_path_override="/bin/codex", base_url="https://example.com")
assert codex._exec._executable_path == "/bin/codex"
assert codex._options.base_url == "https://example.com"
def test_codex_init_accepts_stream_limit_kwarg() -> None:
codex = Codex(codex_path_override="/bin/codex", codex_subprocess_stream_limit_bytes=123456)
assert codex._exec._subprocess_stream_limit_bytes == 123456
def test_codex_init_rejects_options_and_kwargs() -> None:
with pytest.raises(UserError, match="Codex options must be provided"):
Codex( # type: ignore[call-overload]
cast(Any, CodexOptions()), codex_path_override="/bin/codex"
)
def test_codex_init_kw_matches_codex_options() -> None:
signature = inspect.signature(Codex.__init__)
kw_only = [
param.name
for param in signature.parameters.values()
if param.kind == inspect.Parameter.KEYWORD_ONLY
]
option_fields = [field.name for field in fields(CodexOptions)]
assert kw_only == option_fields
def test_codex_exec_stream_limit_uses_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv(exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR, "131072")
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
assert exec_client._subprocess_stream_limit_bytes == 131072
def test_codex_exec_stream_limit_explicit_overrides_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv(exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR, "262144")
exec_client = exec_module.CodexExec(
executable_path="/bin/codex",
subprocess_stream_limit_bytes=524288,
)
assert exec_client._subprocess_stream_limit_bytes == 524288
def test_codex_exec_stream_limit_rejects_invalid_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv(exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR, "not-a-number")
with pytest.raises(UserError, match=exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR):
_ = exec_module.CodexExec(executable_path="/bin/codex")
def test_codex_exec_stream_limit_rejects_out_of_range_value() -> None:
with pytest.raises(UserError, match="must be between"):
_ = exec_module.CodexExec(
executable_path="/bin/codex",
subprocess_stream_limit_bytes=1024,
)
@pytest.mark.asyncio
async def test_codex_exec_run_builds_command_args_and_env(monkeypatch: pytest.MonkeyPatch) -> None:
captured: dict[str, Any] = {}
process = FakeProcess(stdout_lines=["line-1\n", "line-2\n"])
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
captured["args"] = args
captured["kwargs"] = kwargs
return process
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
exec_client = exec_module.CodexExec(executable_path="/bin/codex", env={"FOO": "bar"})
args = exec_module.CodexExecArgs(
input="hello",
base_url="https://example.com",
api_key="api-key",
thread_id="thread-123",
images=["/tmp/img.png"],
model="gpt-4.1-mini",
sandbox_mode="read-only",
working_directory="/work",
additional_directories=["/extra-a", "/extra-b"],
skip_git_repo_check=True,
output_schema_file="/tmp/schema.json",
model_reasoning_effort="high",
network_access_enabled=True,
web_search_mode="live",
approval_policy="on-request",
)
output = [line async for line in exec_client.run(args)]
assert output == ["line-1", "line-2"]
assert process.stdin is not None
assert process.stdin.buffer == b"hello"
assert process.stdin.closed is True
assert captured["args"][0] == "/bin/codex"
assert list(captured["args"][1:]) == [
"exec",
"--experimental-json",
"--model",
"gpt-4.1-mini",
"--sandbox",
"read-only",
"--cd",
"/work",
"--add-dir",
"/extra-a",
"--add-dir",
"/extra-b",
"--skip-git-repo-check",
"--output-schema",
"/tmp/schema.json",
"--config",
'model_reasoning_effort="high"',
"--config",
"sandbox_workspace_write.network_access=true",
"--config",
'web_search="live"',
"--config",
'approval_policy="on-request"',
"resume",
"thread-123",
"--image",
"/tmp/img.png",
"-",
]
env = captured["kwargs"]["env"]
assert env["FOO"] == "bar"
assert env[exec_module._INTERNAL_ORIGINATOR_ENV] == exec_module._TYPESCRIPT_SDK_ORIGINATOR
assert env["OPENAI_BASE_URL"] == "https://example.com"
assert env["CODEX_API_KEY"] == "api-key"
@pytest.mark.asyncio
async def test_codex_exec_run_handles_large_single_line_events(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: dict[str, Any] = {}
large_payload = "x" * (2**16 + 1)
class StreamReaderProcess:
def __init__(self, *, line: str, limit: int) -> None:
self.stdin = FakeStdin()
self.stdout = asyncio.StreamReader(limit=limit)
self.stdout.feed_data(f"{line}\n".encode())
self.stdout.feed_eof()
self.stderr = FakeStderr([])
self.returncode: int | None = 0
self.killed = False
self.terminated = False
async def wait(self) -> None:
if self.returncode is None:
self.returncode = 0
def kill(self) -> None:
self.killed = True
def terminate(self) -> None:
self.terminated = True
async def fake_create_subprocess_exec(*_args: Any, **kwargs: Any) -> StreamReaderProcess:
captured["kwargs"] = kwargs
return StreamReaderProcess(line=large_payload, limit=kwargs["limit"])
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
output = [line async for line in exec_client.run(exec_module.CodexExecArgs(input="hello"))]
assert output == [large_payload]
assert captured["kwargs"]["limit"] == exec_module._DEFAULT_SUBPROCESS_STREAM_LIMIT_BYTES
@pytest.mark.asyncio
@pytest.mark.parametrize(
("enabled", "expected_config"),
[
(True, 'web_search="live"'),
(False, 'web_search="disabled"'),
],
)
async def test_codex_exec_run_web_search_enabled_flags(
monkeypatch: pytest.MonkeyPatch, enabled: bool, expected_config: str
) -> None:
captured: dict[str, Any] = {}
process = FakeProcess(stdout_lines=[])
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
captured["args"] = args
return process
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
args = exec_module.CodexExecArgs(input="hello", web_search_enabled=enabled)
_ = [line async for line in exec_client.run(args)]
command_args = list(captured["args"][1:])
assert "--config" in command_args
assert expected_config in command_args
@pytest.mark.asyncio
async def test_codex_exec_run_raises_on_non_zero_exit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
process = FakeProcess(stdout_lines=[], stderr_chunks=[b"bad"], returncode=2)
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
return process
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
args = exec_module.CodexExecArgs(input="hello")
with pytest.raises(RuntimeError, match="exited with code 2"):
async for _ in exec_client.run(args):
pass
@pytest.mark.asyncio
async def test_codex_exec_run_raises_without_stdin(monkeypatch: pytest.MonkeyPatch) -> None:
process = FakeProcess(stdout_lines=[], stdin_present=False)
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
return process
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
args = exec_module.CodexExecArgs(input="hello")
with pytest.raises(RuntimeError, match="no stdin"):
async for _ in exec_client.run(args):
pass
assert process.killed is True
@pytest.mark.asyncio
async def test_codex_exec_run_raises_without_stdout(monkeypatch: pytest.MonkeyPatch) -> None:
process = FakeProcess(stdout_lines=[], stdout_present=False)
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
return process
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
args = exec_module.CodexExecArgs(input="hello")
with pytest.raises(RuntimeError, match="no stdout"):
async for _ in exec_client.run(args):
pass
assert process.killed is True
@pytest.mark.asyncio
async def test_watch_signal_terminates_process() -> None:
signal = asyncio.Event()
process = FakeProcess(stdout_lines=[], returncode=None)
task = asyncio.create_task(exec_module._watch_signal(signal, process))
signal.set()
await task
assert process.terminated is True
@pytest.mark.parametrize(
("system", "arch", "expected"),
[
("linux", "x86_64", "x86_64-unknown-linux-musl"),
("linux", "aarch64", "aarch64-unknown-linux-musl"),
("darwin", "x86_64", "x86_64-apple-darwin"),
("darwin", "arm64", "aarch64-apple-darwin"),
("win32", "x86_64", "x86_64-pc-windows-msvc"),
("win32", "arm64", "aarch64-pc-windows-msvc"),
],
)
def test_platform_target_triple_mapping(
monkeypatch: pytest.MonkeyPatch, system: str, arch: str, expected: str
) -> None:
monkeypatch.setattr(exec_module.sys, "platform", system)
monkeypatch.setattr(exec_module.platform, "machine", lambda: arch)
assert exec_module._platform_target_triple() == expected
def test_platform_target_triple_unsupported(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(exec_module.sys, "platform", "solaris")
monkeypatch.setattr(exec_module.platform, "machine", lambda: "sparc")
with pytest.raises(RuntimeError, match="Unsupported platform"):
exec_module._platform_target_triple()
def test_find_codex_path_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("CODEX_PATH", "/custom/codex")
assert exec_module.find_codex_path() == "/custom/codex"
def test_find_codex_path_uses_shutil_which(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CODEX_PATH", raising=False)
monkeypatch.setattr(exec_module.shutil, "which", lambda _name: "/usr/local/bin/codex")
assert exec_module.find_codex_path() == "/usr/local/bin/codex"
def test_find_codex_path_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CODEX_PATH", raising=False)
monkeypatch.setattr(exec_module.shutil, "which", lambda _name: None)
monkeypatch.setattr(exec_module, "_platform_target_triple", lambda: "dummy-triple")
monkeypatch.setattr(exec_module.sys, "platform", "linux")
result = exec_module.find_codex_path()
expected_root = (
Path(cast(str, exec_module.__file__)).resolve().parent.parent.parent
/ "vendor"
/ "dummy-triple"
/ "codex"
/ "codex"
)
assert result == str(expected_root)
@pytest.mark.asyncio
async def test_thread_run_streamed_passes_options_and_updates_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
events = [
{"type": "thread.started", "thread_id": "thread-42"},
{
"type": "turn.completed",
"usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1},
},
]
fake_exec = FakeExec(events)
options = CodexOptions(base_url="https://example.com", api_key="api-key")
thread_options = ThreadOptions(
model="gpt-4.1-mini",
sandbox_mode="read-only",
working_directory="/work",
skip_git_repo_check=True,
model_reasoning_effort="low",
network_access_enabled=False,
web_search_mode="cached",
approval_policy="on-request",
additional_directories=["/extra"],
)
thread = Thread(
exec_client=cast(CodexExec, fake_exec),
options=options,
thread_options=thread_options,
)
cleanup_called = False
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
nonlocal cleanup_called
def cleanup() -> None:
nonlocal cleanup_called
cleanup_called = True
return OutputSchemaFile(schema_path="/tmp/schema.json", cleanup=cleanup)
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
streamed = await thread.run_streamed(
[
{"type": "text", "text": "hello"},
{"type": "local_image", "path": "/tmp/a.png"},
],
TurnOptions(output_schema={"type": "object"}),
)
collected = [event async for event in streamed.events]
assert collected[0].type == "thread.started"
assert thread.id == "thread-42"
assert cleanup_called is True
assert fake_exec.last_args is not None
assert fake_exec.last_args.output_schema_file == "/tmp/schema.json"
assert fake_exec.last_args.model == "gpt-4.1-mini"
assert fake_exec.last_args.sandbox_mode == "read-only"
assert fake_exec.last_args.working_directory == "/work"
assert fake_exec.last_args.skip_git_repo_check is True
assert fake_exec.last_args.model_reasoning_effort == "low"
assert fake_exec.last_args.network_access_enabled is False
assert fake_exec.last_args.web_search_mode == "cached"
assert fake_exec.last_args.approval_policy == "on-request"
assert fake_exec.last_args.additional_directories == ["/extra"]
assert fake_exec.last_args.images == ["/tmp/a.png"]
@pytest.mark.asyncio
async def test_thread_run_aggregates_items_and_usage() -> None:
events = [
{"type": "thread.started", "thread_id": "thread-1"},
{
"type": "item.completed",
"item": {"id": "agent-1", "type": "agent_message", "text": "done"},
},
{
"type": "turn.completed",
"usage": {"input_tokens": 2, "cached_input_tokens": 1, "output_tokens": 3},
},
]
thread = Thread(
exec_client=cast(CodexExec, FakeExec(events)),
options=CodexOptions(),
thread_options=ThreadOptions(),
)
result = await thread.run("hello")
assert result.final_response == "done"
assert result.usage == Usage(
input_tokens=2,
cached_input_tokens=1,
output_tokens=3,
)
assert len(result.items) == 1
@pytest.mark.asyncio
async def test_thread_run_raises_on_failure() -> None:
events = [
{"type": "turn.failed", "error": {"message": "boom"}},
]
thread = Thread(
exec_client=cast(CodexExec, FakeExec(events)),
options=CodexOptions(),
thread_options=ThreadOptions(),
)
with pytest.raises(RuntimeError, match="boom"):
await thread.run("hello")
@pytest.mark.asyncio
async def test_thread_run_raises_on_stream_error() -> None:
events = [
{"type": "error", "message": "boom"},
]
thread = Thread(
exec_client=cast(CodexExec, FakeExec(events)),
options=CodexOptions(),
thread_options=ThreadOptions(),
)
with pytest.raises(RuntimeError, match="Codex stream error: boom"):
await thread.run("hello")
@pytest.mark.asyncio
async def test_thread_run_streamed_raises_on_parse_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
events = ["not-json"]
fake_exec = FakeExec(events)
thread = Thread(
exec_client=cast(CodexExec, fake_exec),
options=CodexOptions(),
thread_options=ThreadOptions(),
)
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
return OutputSchemaFile(schema_path=None, cleanup=lambda: None)
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
streamed = await thread.run_streamed("hello")
with pytest.raises(RuntimeError, match="Failed to parse event"):
async for _ in streamed.events:
pass
@pytest.mark.asyncio
async def test_thread_run_streamed_idle_timeout_sets_signal(
monkeypatch: pytest.MonkeyPatch,
) -> None:
events = [
{
"type": "turn.completed",
"usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1},
}
]
fake_exec = FakeExec(events, delay=0.2)
thread = Thread(
exec_client=cast(CodexExec, fake_exec),
options=CodexOptions(),
thread_options=ThreadOptions(),
)
signal = asyncio.Event()
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
return OutputSchemaFile(schema_path=None, cleanup=lambda: None)
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
with pytest.raises(RuntimeError, match="Codex stream idle for"):
async for _ in thread._run_streamed_internal(
"hello", TurnOptions(signal=signal, idle_timeout_seconds=0.01)
):
pass
assert signal.is_set() is True
@pytest.mark.asyncio
async def test_thread_run_streamed_idle_timeout_creates_signal(
monkeypatch: pytest.MonkeyPatch,
) -> None:
events = [
{
"type": "turn.completed",
"usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1},
}
]
fake_exec = FakeExec(events, delay=0.2)
thread = Thread(
exec_client=cast(CodexExec, fake_exec),
options=CodexOptions(),
thread_options=ThreadOptions(),
)
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
return OutputSchemaFile(schema_path=None, cleanup=lambda: None)
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
with pytest.raises(RuntimeError, match="Codex stream idle for"):
async for _ in thread._run_streamed_internal(
"hello", TurnOptions(idle_timeout_seconds=0.01)
):
pass
assert fake_exec.last_args is not None
assert fake_exec.last_args.signal is not None
assert fake_exec.last_args.signal.is_set() is True
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,45 @@
from __future__ import annotations
import pytest
from agents.extensions.experimental.codex.items import AgentMessageItem, TodoItem, TodoListItem
def test_dict_like_supports_mapping_access_for_dataclass_fields() -> None:
item = AgentMessageItem(id="item-1", text="hello")
assert item["id"] == "item-1"
assert item["text"] == "hello"
assert item["type"] == "agent_message"
assert item.get("text") == "hello"
assert item.get("missing", "fallback") == "fallback"
assert "id" in item
assert "missing" not in item
assert object() not in item
assert list(item.keys()) == ["id", "text", "type"]
def test_dict_like_raises_key_error_for_unknown_fields() -> None:
item = AgentMessageItem(id="item-1", text="hello")
with pytest.raises(KeyError, match="missing"):
_ = item["missing"]
def test_dict_like_as_dict_recursively_converts_nested_dataclasses() -> None:
item = TodoListItem(
id="todo-list-1",
items=[
TodoItem(text="write tests", completed=True),
TodoItem(text="run tests", completed=False),
],
)
assert item.as_dict() == {
"id": "todo-list-1",
"items": [
{"text": "write tests", "completed": True},
{"text": "run tests", "completed": False},
],
"type": "todo_list",
}
@@ -0,0 +1,179 @@
from __future__ import annotations
import json
import os
from typing import Any
import pytest
from openai import AsyncOpenAI
from agents import Agent, ModelSettings, RunConfig, Runner, function_tool
from agents.extensions.experimental.hosted_multi_agent import (
HostedMultiAgentConfig,
OpenAIHostedMultiAgentModel,
get_hosted_agent_metadata,
)
from agents.tool_context import ToolContext
pytestmark = [
pytest.mark.allow_call_model_methods,
pytest.mark.skipif(
os.environ.get("OPENAI_RUN_LIVE_HOSTED_MULTI_AGENT_TESTS") != "1",
reason="Set OPENAI_RUN_LIVE_HOSTED_MULTI_AGENT_TESTS=1 to run live beta tests.",
),
]
_PROPOSALS = {
"alpha": {"estimated_weeks": 6, "risk": "medium"},
"beta": {"estimated_weeks": 8, "risk": "low"},
}
def _tool_output(arguments: str) -> str:
proposal = json.loads(arguments)["proposal"]
return json.dumps(_PROPOSALS[proposal], sort_keys=True)
async def _run_direct_baseline(client: AsyncOpenAI) -> tuple[str, set[str], set[str]]:
beta = getattr(client, "beta", None)
responses = getattr(beta, "responses", None)
connect = getattr(responses, "connect", None)
if not callable(connect):
pytest.fail("The installed openai package does not provide client.beta.responses.connect.")
tools = [
{
"type": "function",
"name": "get_proposal",
"description": "Return deterministic details for one proposal.",
"parameters": {
"type": "object",
"properties": {"proposal": {"type": "string", "enum": ["alpha", "beta"]}},
"required": ["proposal"],
"additionalProperties": False,
},
"strict": True,
}
]
callers: set[str] = set()
call_ids: set[str] = set()
completed_response: Any | None = None
response_id: str | None = None
pending_injections = 0
async with connect(
extra_headers={"OpenAI-Beta": "responses_multi_agent=v1"},
max_retries=0,
) as connection:
await connection.send(
{
"type": "response.create",
"model": "gpt-5.6-sol",
"input": [{"role": "user", "content": "Compare proposal alpha and proposal beta."}],
"instructions": (
"Create two subagents. Assign proposal alpha to one and proposal beta to the "
"other. Each subagent must call get_proposal, then the root must synthesize."
),
"tools": tools,
"store": True,
"multi_agent": {"enabled": True, "max_concurrent_subagents": 2},
}
)
async for event in connection:
if event.type == "response.created":
response_id = event.response.id
elif event.type == "response.output_item.done" and event.item.type == "function_call":
if response_id is None:
pytest.fail("Direct baseline received a function call before response.created.")
call_ids.add(event.item.call_id)
agent = getattr(event.item, "agent", None)
callers.add(getattr(agent, "agent_name", "/root"))
pending_injections += 1
await connection.send(
{
"type": "response.inject",
"response_id": response_id,
"input": [
{
"type": "function_call_output",
"call_id": event.item.call_id,
"output": _tool_output(event.item.arguments),
}
],
}
)
elif event.type == "response.inject.created":
pending_injections -= 1
elif event.type == "response.inject.failed":
pytest.fail(f"Direct baseline injection failed: {event.error}")
elif event.type == "response.completed":
completed_response = event.response
elif event.type in {"error", "response.failed", "response.incomplete"}:
pytest.fail(f"Direct baseline failed: {event}")
if completed_response is not None and pending_injections == 0:
break
if completed_response is None:
pytest.fail("Direct hosted multi-agent baseline did not complete.")
root_text: list[str] = []
for item in completed_response.output:
if (
item.type == "message"
and getattr(getattr(item, "agent", None), "agent_name", None) == "/root"
and getattr(item, "phase", None) == "final_answer"
):
root_text.extend(part.text for part in item.content if part.type == "output_text")
return "".join(root_text), callers, call_ids
@pytest.mark.asyncio
async def test_live_direct_and_agents_sdk_semantic_parity() -> None:
if os.environ.get("OPENAI_API_KEY") in {None, "", "test_key"}:
pytest.fail("A real OPENAI_API_KEY is required for the live beta test.")
client = AsyncOpenAI()
direct_text, direct_callers, direct_call_ids = await _run_direct_baseline(client)
sdk_callers: set[str] = set()
sdk_call_ids: set[str] = set()
@function_tool
def get_proposal(ctx: ToolContext[Any], proposal: str) -> dict[str, object]:
metadata = get_hosted_agent_metadata(ctx)
sdk_callers.add(metadata.agent_name if metadata else "/root")
sdk_call_ids.add(ctx.tool_call_id)
return _PROPOSALS[proposal]
model = OpenAIHostedMultiAgentModel(
model="gpt-5.6-sol",
openai_client=client,
config=HostedMultiAgentConfig(max_concurrent_subagents=2),
)
agent = Agent(
name="Hosted proposal coordinator",
instructions=(
"Create two subagents. Assign proposal alpha to one and proposal beta to the other. "
"Each subagent must call get_proposal, then the root must synthesize."
),
model=model,
model_settings=ModelSettings(
store=False,
response_include=["reasoning.encrypted_content"],
),
tools=[get_proposal],
)
result = await Runner.run(
agent,
"Compare proposal alpha and proposal beta.",
run_config=RunConfig(tracing_disabled=True),
max_turns=6,
)
assert direct_text
assert result.final_output
assert len(direct_call_ids) == 2
assert len(sdk_call_ids) == 2
assert all(caller != "/root" for caller in direct_callers)
assert all(caller != "/root" for caller in sdk_callers)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,410 @@
"""Tests for AsyncSQLiteSession functionality."""
from __future__ import annotations
import json
import tempfile
from collections.abc import Sequence
from datetime import datetime
from pathlib import Path
from typing import Any, cast
import pytest
pytest.importorskip("aiosqlite") # Skip tests if aiosqlite is not installed
from agents import Agent, Runner, TResponseInputItem
from agents.extensions.memory import AsyncSQLiteSession
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
pytestmark = pytest.mark.asyncio
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
result: list[str] = []
for item in items:
item_dict = cast(dict[str, Any], item)
result.append(cast(str, item_dict["id"]))
return result
async def test_async_sqlite_session_basic_flow():
"""Test AsyncSQLiteSession add/get/clear behavior."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_basic.db"
session = AsyncSQLiteSession("async_basic", db_path)
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
]
await session.add_items(items)
retrieved = await session.get_items()
assert retrieved == items
await session.clear_session()
assert await session.get_items() == []
await session.close()
async def test_async_sqlite_session_pop_item():
"""Test AsyncSQLiteSession pop_item behavior."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_pop.db"
session = AsyncSQLiteSession("async_pop", db_path)
assert await session.pop_item() is None
items: list[TResponseInputItem] = [
{"role": "user", "content": "One"},
{"role": "assistant", "content": "Two"},
]
await session.add_items(items)
popped = await session.pop_item()
assert popped == items[-1]
assert await session.get_items() == items[:-1]
await session.close()
async def test_async_sqlite_session_pop_item_skips_corrupt_most_recent():
"""pop_item skips corrupt newest rows and returns the next valid item."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_pop_corrupt.db"
session = AsyncSQLiteSession("async_pop_corrupt", db_path)
valid_item: TResponseInputItem = {"role": "user", "content": "valid"}
await session.add_items([valid_item])
conn = await session._get_connection()
await conn.execute(
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
(session.session_id, "not valid json {{{"),
)
await conn.commit()
assert await session.pop_item() == valid_item
assert await session.get_items() == []
await session.close()
async def test_async_sqlite_session_pop_item_returns_none_after_dropping_only_corrupt_rows():
"""pop_item removes corrupt rows and returns None when no valid items remain."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_pop_only_corrupt.db"
session = AsyncSQLiteSession("async_pop_only_corrupt", db_path)
conn = await session._get_connection()
await conn.execute(
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
(session.session_id, "not valid json {{{"),
)
await conn.commit()
assert await session.pop_item() is None
assert await session.get_items() == []
await session.close()
async def test_async_sqlite_session_get_items_limit():
"""Test AsyncSQLiteSession get_items limit handling."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_limit.db"
session = AsyncSQLiteSession("async_limit", db_path)
items: list[TResponseInputItem] = [
{"role": "user", "content": "Message 1"},
{"role": "assistant", "content": "Response 1"},
{"role": "user", "content": "Message 2"},
]
await session.add_items(items)
latest = await session.get_items(limit=2)
assert latest == items[-2:]
none = await session.get_items(limit=0)
assert none == []
await session.close()
async def test_async_sqlite_session_session_settings_default():
"""Test that session_settings defaults to empty SessionSettings."""
session = AsyncSQLiteSession("async_default_settings")
assert isinstance(session.session_settings, SessionSettings)
assert session.session_settings.limit is None
await session.close()
async def test_async_sqlite_session_session_settings_constructor():
"""Test passing session_settings via constructor."""
session = AsyncSQLiteSession(
"async_constructor_settings",
session_settings=SessionSettings(limit=5),
)
assert session.session_settings is not None
assert session.session_settings.limit == 5
await session.close()
async def test_async_sqlite_session_get_items_uses_session_settings_limit():
"""Test that get_items uses session_settings.limit as default."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_settings_limit.db"
session = AsyncSQLiteSession(
"async_settings_limit",
db_path,
session_settings=SessionSettings(limit=3),
)
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(5)
]
await session.add_items(items)
retrieved = await session.get_items()
assert retrieved == items[-3:]
await session.close()
async def test_async_sqlite_session_explicit_limit_overrides_session_settings():
"""Test that explicit limit parameter overrides session_settings."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_settings_override.db"
session = AsyncSQLiteSession(
"async_settings_override",
db_path,
session_settings=SessionSettings(limit=5),
)
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(10)
]
await session.add_items(items)
retrieved = await session.get_items(limit=2)
assert retrieved == items[-2:]
no_items = await session.get_items(limit=0)
assert no_items == []
await session.close()
async def test_async_sqlite_session_unicode_content():
"""Test AsyncSQLiteSession stores unicode content."""
session = AsyncSQLiteSession("async_unicode")
items: list[TResponseInputItem] = [
{"role": "user", "content": "こんにちは"},
{"role": "assistant", "content": "Привет"},
]
await session.add_items(items)
retrieved = await session.get_items()
assert retrieved == items
await session.close()
async def test_async_sqlite_session_runner_integration(agent: Agent):
"""Test that AsyncSQLiteSession works correctly with the agent Runner."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_runner_integration.db"
session = AsyncSQLiteSession("runner_integration_test", db_path)
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
session=session,
)
assert result1.final_output == "San Francisco"
agent.model.set_next_output([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
last_input = agent.model.last_turn_args["input"]
assert isinstance(last_input, list)
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
await session.close()
async def test_async_sqlite_session_session_isolation(agent: Agent):
"""Test that different session IDs result in isolated conversation histories."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_isolation.db"
session1 = AsyncSQLiteSession("session_1", db_path)
session2 = AsyncSQLiteSession("session_2", db_path)
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=session1)
agent.model.set_next_output([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=session2)
agent.model.set_next_output([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
await session1.close()
await session2.close()
async def test_async_sqlite_session_add_empty_items_list():
"""Test that adding an empty list of items is a no-op."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_add_empty.db"
session = AsyncSQLiteSession("add_empty_test", db_path)
assert await session.get_items() == []
await session.add_items([])
assert await session.get_items() == []
await session.close()
async def test_async_sqlite_session_pop_from_empty_session():
"""Test that pop_item returns None on an empty session."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_pop_empty.db"
session = AsyncSQLiteSession("empty_session", db_path)
popped = await session.pop_item()
assert popped is None
await session.close()
async def test_async_sqlite_session_get_items_with_limit_more_than_available():
"""Test limit behavior when requesting more items than exist."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_limit_more.db"
session = AsyncSQLiteSession("limit_more_test", db_path)
items: list[TResponseInputItem] = [
{"role": "user", "content": "1"},
{"role": "assistant", "content": "2"},
{"role": "user", "content": "3"},
{"role": "assistant", "content": "4"},
]
await session.add_items(items)
retrieved = await session.get_items(limit=10)
assert retrieved == items
await session.close()
async def test_async_sqlite_session_get_items_same_timestamp_consistent_order():
"""Test that items with identical timestamps keep insertion order."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_same_timestamp.db"
session = AsyncSQLiteSession("same_timestamp_test", db_path)
older_item = cast(
TResponseInputItem, {"id": "older_same_ts", "role": "user", "content": "old"}
)
reasoning_item = cast(TResponseInputItem, {"id": "rs_same_ts", "type": "reasoning"})
message_item = cast(
TResponseInputItem,
{"id": "msg_same_ts", "type": "message", "role": "assistant", "content": []},
)
await session.add_items([older_item])
await session.add_items([reasoning_item, message_item])
conn = await session._get_connection()
cursor = await conn.execute(
f"SELECT id, message_data FROM {session.messages_table} WHERE session_id = ?",
(session.session_id,),
)
rows = await cursor.fetchall()
await cursor.close()
id_map: dict[str, int] = {
cast(str, json.loads(message_json)["id"]): cast(int, row_id)
for row_id, message_json in rows
}
shared = datetime(2025, 10, 15, 17, 26, 39, 132483)
shared_str = shared.strftime("%Y-%m-%d %H:%M:%S.%f")
await conn.execute(
f"""
UPDATE {session.messages_table}
SET created_at = ?
WHERE id IN (?, ?, ?)
""",
(
shared_str,
id_map["older_same_ts"],
id_map["rs_same_ts"],
id_map["msg_same_ts"],
),
)
await conn.commit()
retrieved = await session.get_items()
assert _item_ids(retrieved) == ["older_same_ts", "rs_same_ts", "msg_same_ts"]
latest_two = await session.get_items(limit=2)
assert _item_ids(latest_two) == ["rs_same_ts", "msg_same_ts"]
await session.close()
async def test_async_sqlite_session_pop_item_same_timestamp_returns_latest():
"""Test that pop_item returns the newest item when timestamps tie."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "async_same_timestamp_pop.db"
session = AsyncSQLiteSession("same_timestamp_pop_test", db_path)
reasoning_item = cast(TResponseInputItem, {"id": "rs_pop_same_ts", "type": "reasoning"})
message_item = cast(
TResponseInputItem,
{"id": "msg_pop_same_ts", "type": "message", "role": "assistant", "content": []},
)
await session.add_items([reasoning_item, message_item])
conn = await session._get_connection()
shared = datetime(2025, 10, 15, 17, 26, 39, 132483)
shared_str = shared.strftime("%Y-%m-%d %H:%M:%S.%f")
await conn.execute(
f"UPDATE {session.messages_table} SET created_at = ? WHERE session_id = ?",
(shared_str, session.session_id),
)
await conn.commit()
popped = await session.pop_item()
assert popped is not None
assert cast(dict[str, Any], popped)["id"] == "msg_pop_same_ts"
remaining = await session.get_items()
assert _item_ids(remaining) == ["rs_pop_same_ts"]
await session.close()
@@ -0,0 +1,594 @@
"""
Integration tests for DaprSession with real Dapr sidecar and Redis using testcontainers.
These tests use Docker containers for both Redis and Dapr, with proper networking.
Tests are automatically skipped if dependencies (dapr, testcontainers, docker) are not available.
Run with: pytest tests/extensions/memory/test_dapr_redis_integration.py -v
"""
from __future__ import annotations
import asyncio
import json
import os
import shutil
import sys
import tempfile
import time
import urllib.request
import docker # type: ignore[import-untyped]
import pytest
from docker.errors import DockerException # type: ignore[import-untyped]
# Skip tests if dependencies are not available
pytest.importorskip("dapr") # Skip tests if Dapr is not installed
pytest.importorskip("testcontainers") # Skip if testcontainers is not installed
if sys.platform == "win32":
pytest.skip(
"Dapr Docker integration tests are not supported on Windows",
allow_module_level=True,
)
if shutil.which("docker") is None:
pytest.skip(
"Docker executable is not available; skipping Dapr integration tests",
allow_module_level=True,
)
try:
client = docker.from_env()
client.ping()
except DockerException:
pytest.skip(
"Docker daemon is not available; skipping Dapr integration tests", allow_module_level=True
)
else:
client.close()
from testcontainers.core.container import DockerContainer # type: ignore[import-untyped]
from testcontainers.core.network import Network # type: ignore[import-untyped]
from testcontainers.core.waiting_utils import wait_for_logs # type: ignore[import-untyped]
from agents import Agent, Runner, TResponseInputItem
from agents.extensions.memory import (
DAPR_CONSISTENCY_EVENTUAL,
DAPR_CONSISTENCY_STRONG,
DaprSession,
)
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
# Docker-backed integration tests should stay on the serial test path.
pytestmark = [pytest.mark.asyncio, pytest.mark.serial]
def wait_for_dapr_health(host: str, port: int, timeout: int = 60) -> bool:
"""
Wait for Dapr sidecar to become healthy by checking the HTTP health endpoint.
Args:
host: The host where Dapr is running
port: The HTTP port (typically 3500)
timeout: Maximum time to wait in seconds
Returns:
True if Dapr becomes healthy, False otherwise
"""
health_url = f"http://{host}:{port}/v1.0/healthz/outbound"
start_time = time.time()
while time.time() - start_time < timeout:
try:
with urllib.request.urlopen(health_url, timeout=5) as response:
if 200 <= response.status < 300:
print(f"✓ Dapr health check passed on {health_url}")
return True
except Exception:
pass
time.sleep(1)
print(f"✗ Dapr health check timed out after {timeout}s on {health_url}")
return False
def wait_for_dapr_component(host: str, port: int, component_name: str, timeout: int = 60) -> bool:
"""Wait for a named component to appear in the Dapr metadata endpoint."""
metadata_url = f"http://{host}:{port}/v1.0/metadata"
start_time = time.time()
while time.time() - start_time < timeout:
try:
with urllib.request.urlopen(metadata_url, timeout=5) as response:
if 200 <= response.status < 300:
payload = json.load(response)
components = payload.get("components", [])
if any(component.get("name") == component_name for component in components):
print(f"✓ Dapr component {component_name} loaded via {metadata_url}")
return True
except Exception:
pass
time.sleep(1)
print(f"✗ Dapr component {component_name} did not load after {timeout}s")
return False
@pytest.fixture(scope="module")
def docker_network():
"""Create a Docker network for container-to-container communication."""
with Network() as network:
yield network
@pytest.fixture(scope="module")
def redis_container(docker_network):
"""Start Redis container on the shared network."""
container = (
DockerContainer("redis:7-alpine")
.with_network(docker_network)
.with_network_aliases("redis")
.with_exposed_ports(6379)
)
container.start()
wait_for_logs(container, "Ready to accept connections", timeout=30)
try:
yield container
finally:
container.stop()
@pytest.fixture(scope="module")
def dapr_container(redis_container, docker_network):
"""Start Dapr sidecar container with Redis state store configuration."""
# Create temporary components directory
temp_dir = tempfile.mkdtemp()
os.chmod(temp_dir, 0o755)
components_path = os.path.join(temp_dir, "components")
os.makedirs(components_path, exist_ok=True)
os.chmod(components_path, 0o755)
# Write Redis state store component configuration
# KEY: Use 'redis:6379' (network alias), NOT localhost!
state_store_config = """
apiVersion: dapr.io/v1alpha1
kind: Component
metadata:
name: statestore
spec:
type: state.redis
version: v1
metadata:
- name: redisHost
value: redis:6379
- name: redisPassword
value: ""
- name: actorStateStore
value: "false"
"""
state_store_path = os.path.join(components_path, "statestore.yaml")
with open(state_store_path, "w") as f:
f.write(state_store_config)
os.chmod(state_store_path, 0o644)
# Create Dapr container
container = DockerContainer("daprio/daprd:latest")
container = container.with_network(docker_network) # Join the same network
container = container.with_volume_mapping(components_path, "/components", mode="ro")
container = container.with_command(
[
"./daprd",
"-app-id",
"test-app",
"-dapr-http-port",
"3500", # HTTP API port for health checks
"-dapr-grpc-port",
"50001",
"-resources-path",
"/components",
"-log-level",
"info",
]
)
container = container.with_exposed_ports(3500, 50001) # Expose both ports
container.start()
# Get the exposed HTTP port and host
http_host = container.get_container_host_ip()
http_port = container.get_exposed_port(3500)
# Wait for Dapr to become healthy
if not wait_for_dapr_health(http_host, http_port, timeout=60):
container.stop()
pytest.fail("Dapr container failed to become healthy")
if not wait_for_dapr_component(http_host, http_port, "statestore", timeout=60):
logs = container.get_wrapped_container().logs().decode("utf-8", errors="replace")
container.stop()
pytest.fail(f"Dapr state store component failed to load.\nContainer logs:\n{logs}")
# Set environment variables for Dapr SDK health checks
# The Dapr SDK checks these when creating a client
os.environ["DAPR_HTTP_PORT"] = str(http_port)
os.environ["DAPR_RUNTIME_HOST"] = http_host
yield container
# Cleanup environment variables
os.environ.pop("DAPR_HTTP_PORT", None)
os.environ.pop("DAPR_RUNTIME_HOST", None)
container.stop()
# Cleanup
import shutil
shutil.rmtree(temp_dir, ignore_errors=True)
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
async def test_dapr_redis_integration(dapr_container, monkeypatch):
"""Test DaprSession with real Dapr sidecar and Redis backend."""
# Get Dapr gRPC address (exposed to host)
dapr_host = dapr_container.get_container_host_ip()
dapr_port = dapr_container.get_exposed_port(50001)
dapr_address = f"{dapr_host}:{dapr_port}"
# Monkeypatch the Dapr health check since we already verified it in the fixture
from dapr.clients.health import DaprHealth
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
# Create session using from_address
session = DaprSession.from_address(
session_id="integration_test_session",
state_store_name="statestore",
dapr_address=dapr_address,
)
try:
# Test connectivity
is_connected = await session.ping()
assert is_connected is True
# Clear any existing data
await session.clear_session()
# Test add_items
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello from integration test"},
{"role": "assistant", "content": "Hi there!"},
]
await session.add_items(items)
# Test get_items
retrieved = await session.get_items()
assert len(retrieved) == 2
assert retrieved[0].get("content") == "Hello from integration test"
assert retrieved[1].get("content") == "Hi there!"
# Test get_items with limit
latest_1 = await session.get_items(limit=1)
assert len(latest_1) == 1
assert latest_1[0].get("content") == "Hi there!"
# Test pop_item
popped = await session.pop_item()
assert popped is not None
assert popped.get("content") == "Hi there!"
remaining = await session.get_items()
assert len(remaining) == 1
assert remaining[0].get("content") == "Hello from integration test"
# Test clear_session
await session.clear_session()
cleared = await session.get_items()
assert len(cleared) == 0
finally:
await session.close()
async def test_dapr_runner_integration(agent: Agent, dapr_container, monkeypatch):
"""Test DaprSession with agent Runner using real Dapr sidecar."""
from dapr.clients.health import DaprHealth
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
dapr_host = dapr_container.get_container_host_ip()
dapr_port = dapr_container.get_exposed_port(50001)
dapr_address = f"{dapr_host}:{dapr_port}"
session = DaprSession.from_address(
session_id="runner_integration_test",
state_store_name="statestore",
dapr_address=dapr_address,
)
try:
await session.clear_session()
# First turn
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
session=session,
)
assert result1.final_output == "San Francisco"
# Second turn - should remember context
agent.model.set_next_output([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
# Verify history
last_input = agent.model.last_turn_args["input"]
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
finally:
await session.close()
async def test_dapr_session_isolation(dapr_container, monkeypatch):
"""Test that different session IDs are isolated with real Dapr."""
from dapr.clients.health import DaprHealth
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
dapr_host = dapr_container.get_container_host_ip()
dapr_port = dapr_container.get_exposed_port(50001)
dapr_address = f"{dapr_host}:{dapr_port}"
session1 = DaprSession.from_address(
session_id="isolated_session_1",
state_store_name="statestore",
dapr_address=dapr_address,
)
session2 = DaprSession.from_address(
session_id="isolated_session_2",
state_store_name="statestore",
dapr_address=dapr_address,
)
try:
# Clear both sessions
await session1.clear_session()
await session2.clear_session()
# Add different data to each session
await session1.add_items([{"role": "user", "content": "session 1 data"}])
await session2.add_items([{"role": "user", "content": "session 2 data"}])
# Verify isolation
items1 = await session1.get_items()
items2 = await session2.get_items()
assert len(items1) == 1
assert len(items2) == 1
assert items1[0].get("content") == "session 1 data"
assert items2[0].get("content") == "session 2 data"
finally:
await session1.clear_session()
await session2.clear_session()
await session1.close()
await session2.close()
async def test_dapr_ttl_functionality(dapr_container, monkeypatch):
"""Test TTL functionality with real Dapr and Redis (if supported by state store)."""
from dapr.clients.health import DaprHealth
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
dapr_host = dapr_container.get_container_host_ip()
dapr_port = dapr_container.get_exposed_port(50001)
dapr_address = f"{dapr_host}:{dapr_port}"
# Create session with short TTL
session = DaprSession.from_address(
session_id="ttl_test_session",
state_store_name="statestore",
dapr_address=dapr_address,
ttl=2, # 2 seconds TTL
)
try:
await session.clear_session()
# Add items with TTL
items: list[TResponseInputItem] = [
{"role": "user", "content": "This should expire soon"},
]
await session.add_items(items)
# Verify items exist immediately
retrieved = await session.get_items()
assert len(retrieved) == 1
# Note: Actual expiration testing depends on state store TTL support
# Redis state store supports TTL via ttlInSeconds metadata
finally:
await session.clear_session()
await session.close()
async def test_dapr_consistency_levels(dapr_container, monkeypatch):
"""Test different consistency levels with real Dapr."""
from dapr.clients.health import DaprHealth
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
dapr_host = dapr_container.get_container_host_ip()
dapr_port = dapr_container.get_exposed_port(50001)
dapr_address = f"{dapr_host}:{dapr_port}"
# Test eventual consistency
session_eventual = DaprSession.from_address(
session_id="eventual_consistency_test",
state_store_name="statestore",
dapr_address=dapr_address,
consistency=DAPR_CONSISTENCY_EVENTUAL,
)
# Test strong consistency
session_strong = DaprSession.from_address(
session_id="strong_consistency_test",
state_store_name="statestore",
dapr_address=dapr_address,
consistency=DAPR_CONSISTENCY_STRONG,
)
try:
await session_eventual.clear_session()
await session_strong.clear_session()
# Both should work correctly
items: list[TResponseInputItem] = [{"role": "user", "content": "Consistency test"}]
await session_eventual.add_items(items)
retrieved_eventual = await session_eventual.get_items()
assert len(retrieved_eventual) == 1
await session_strong.add_items(items)
retrieved_strong = await session_strong.get_items()
assert len(retrieved_strong) == 1
finally:
await session_eventual.clear_session()
await session_strong.clear_session()
await session_eventual.close()
await session_strong.close()
async def test_dapr_unicode_and_special_chars(dapr_container, monkeypatch):
"""Test unicode and special characters with real Dapr and Redis."""
from dapr.clients.health import DaprHealth
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
dapr_host = dapr_container.get_container_host_ip()
dapr_port = dapr_container.get_exposed_port(50001)
dapr_address = f"{dapr_host}:{dapr_port}"
session = DaprSession.from_address(
session_id="unicode_test_session",
state_store_name="statestore",
dapr_address=dapr_address,
)
try:
await session.clear_session()
# Test unicode content
items: list[TResponseInputItem] = [
{"role": "user", "content": "こんにちは"},
{"role": "assistant", "content": "😊👍"},
{"role": "user", "content": "Привет"},
{"role": "assistant", "content": '{"nested": "json"}'},
{"role": "user", "content": "Line1\nLine2\tTabbed"},
]
await session.add_items(items)
# Retrieve and verify
retrieved = await session.get_items()
assert len(retrieved) == 5
assert retrieved[0].get("content") == "こんにちは"
assert retrieved[1].get("content") == "😊👍"
assert retrieved[2].get("content") == "Привет"
assert retrieved[3].get("content") == '{"nested": "json"}'
assert retrieved[4].get("content") == "Line1\nLine2\tTabbed"
finally:
await session.clear_session()
await session.close()
async def test_dapr_concurrent_writes_resolution(dapr_container, monkeypatch):
"""
Concurrent writes from multiple session instances should resolve via
optimistic concurrency.
"""
from dapr.clients.health import DaprHealth
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
dapr_host = dapr_container.get_container_host_ip()
dapr_port = dapr_container.get_exposed_port(50001)
dapr_address = f"{dapr_host}:{dapr_port}"
# Use two different session objects pointing to the same logical session_id
# to create real contention.
session_id = "concurrent_integration_session"
s1 = DaprSession.from_address(
session_id=session_id,
state_store_name="statestore",
dapr_address=dapr_address,
)
s2 = DaprSession.from_address(
session_id=session_id,
state_store_name="statestore",
dapr_address=dapr_address,
)
try:
# Clean slate.
await s1.clear_session()
# Fire multiple parallel add_items calls from two different session instances.
tasks: list[asyncio.Task[None]] = []
for i in range(10):
tasks.append(
asyncio.create_task(
s1.add_items(
[
{"role": "user", "content": f"A-{i}"},
]
)
)
)
tasks.append(
asyncio.create_task(
s2.add_items(
[
{"role": "assistant", "content": f"B-{i}"},
]
)
)
)
await asyncio.gather(*tasks)
# Validate all messages were persisted.
# Use a fresh session object for readback to avoid any local caching
# (none expected, but explicit).
s_read = DaprSession.from_address(
session_id=session_id,
state_store_name="statestore",
dapr_address=dapr_address,
)
try:
items = await s_read.get_items()
contents = [item.get("content") for item in items]
# We expect 20 total messages: A-0..9 and B-0..9 (order unspecified).
assert len(contents) == 20
for i in range(10):
assert f"A-{i}" in contents
assert f"B-{i}" in contents
finally:
await s_read.close()
finally:
await s1.close()
await s2.close()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,576 @@
from __future__ import annotations
import tempfile
from pathlib import Path
from typing import cast
import pytest
pytest.importorskip("cryptography") # Skip tests if cryptography is not installed
from cryptography.fernet import Fernet
from agents import Agent, Runner, SessionSettings, SQLiteSession, TResponseInputItem
from agents.extensions.memory.encrypt_session import EncryptedSession
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
# Mark all tests in this file as asyncio
pytestmark = pytest.mark.asyncio
def _invalid_encrypted_envelope() -> TResponseInputItem:
return cast(
TResponseInputItem,
{"__enc__": 1, "v": 1, "kid": "hkdf-v1", "payload": "not-a-valid-token"},
)
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
@pytest.fixture
def encryption_key() -> str:
"""Fixture for a valid Fernet encryption key."""
return str(Fernet.generate_key().decode("utf-8"))
@pytest.fixture
def set_fernet_time(monkeypatch):
"""Freeze Fernet TTL checks so expiration tests avoid real waiting."""
current_time = 1_000
def _set_time(value: int) -> None:
nonlocal current_time
current_time = value
monkeypatch.setattr("cryptography.fernet.time.time", lambda: current_time)
return _set_time
@pytest.fixture
def underlying_session():
"""Fixture for an underlying SQLite session."""
temp_dir = tempfile.mkdtemp()
db_path = Path(temp_dir) / "test_encrypt.db"
return SQLiteSession("test_session", db_path)
async def test_encrypted_session_basic_functionality(
agent: Agent, encryption_key: str, underlying_session: SQLiteSession
):
"""Test basic encryption/decryption functionality."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
ttl=600,
)
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
]
await session.add_items(items)
retrieved = await session.get_items()
assert len(retrieved) == 2
assert retrieved[0].get("content") == "Hello"
assert retrieved[1].get("content") == "Hi there!"
encrypted_items = await underlying_session.get_items()
assert encrypted_items[0].get("__enc__") == 1
assert "payload" in encrypted_items[0]
assert encrypted_items[0].get("content") != "Hello"
underlying_session.close()
async def test_encrypted_session_with_runner(
agent: Agent, encryption_key: str, underlying_session: SQLiteSession
):
"""Test that EncryptedSession works with Runner."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
session=session,
)
assert result1.final_output == "San Francisco"
agent.model.set_next_output([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
last_input = agent.model.last_turn_args["input"]
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
underlying_session.close()
async def test_encrypted_session_pop_item(encryption_key: str, underlying_session: SQLiteSession):
"""Test pop_item functionality."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
items: list[TResponseInputItem] = [
{"role": "user", "content": "First"},
{"role": "assistant", "content": "Second"},
]
await session.add_items(items)
popped = await session.pop_item()
assert popped is not None
assert popped.get("content") == "Second"
remaining = await session.get_items()
assert len(remaining) == 1
assert remaining[0].get("content") == "First"
underlying_session.close()
async def test_encrypted_session_clear(encryption_key: str, underlying_session: SQLiteSession):
"""Test clear_session functionality."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
await session.add_items([{"role": "user", "content": "Test"}])
await session.clear_session()
items = await session.get_items()
assert len(items) == 0
underlying_session.close()
async def test_encrypted_session_ttl_expiration(
encryption_key: str, underlying_session: SQLiteSession, set_fernet_time
):
"""Test TTL expiration - expired items are silently skipped."""
set_fernet_time(1_000)
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
ttl=1, # 1 second TTL
)
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi"},
]
await session.add_items(items)
set_fernet_time(1_002)
retrieved = await session.get_items()
assert len(retrieved) == 0
underlying_items = await underlying_session.get_items()
assert len(underlying_items) == 2
underlying_session.close()
async def test_encrypted_session_pop_expired(
encryption_key: str, underlying_session: SQLiteSession, set_fernet_time
):
"""Test pop_item with expired data."""
set_fernet_time(1_000)
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
ttl=1,
)
await session.add_items([{"role": "user", "content": "Test"}])
set_fernet_time(1_002)
popped = await session.pop_item()
assert popped is None
underlying_session.close()
async def test_encrypted_session_pop_mixed_expired_valid(
encryption_key: str, underlying_session: SQLiteSession, set_fernet_time
):
"""Test pop_item auto-retry with mixed expired and valid items."""
set_fernet_time(1_000)
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
ttl=2, # 2 second TTL
)
await session.add_items(
[
{"role": "user", "content": "Old message 1"},
{"role": "assistant", "content": "Old response 1"},
]
)
set_fernet_time(1_003)
await session.add_items(
[
{"role": "user", "content": "New message"},
{"role": "assistant", "content": "New response"},
]
)
popped = await session.pop_item()
assert popped is not None
assert popped.get("content") == "New response"
popped2 = await session.pop_item()
assert popped2 is not None
assert popped2.get("content") == "New message"
popped3 = await session.pop_item()
assert popped3 is None
underlying_session.close()
async def test_encrypted_session_raw_string_key(underlying_session: SQLiteSession):
"""Test using raw string as encryption key (not base64)."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key="my-secret-password", # Raw string, not Fernet key
)
await session.add_items([{"role": "user", "content": "Test"}])
items = await session.get_items()
assert len(items) == 1
assert items[0].get("content") == "Test"
underlying_session.close()
async def test_encrypted_session_get_items_limit(
encryption_key: str, underlying_session: SQLiteSession
):
"""Test get_items with limit parameter."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(5)
]
await session.add_items(items)
limited = await session.get_items(limit=2)
assert len(limited) == 2
assert limited[0].get("content") == "Message 3" # Latest 2
assert limited[1].get("content") == "Message 4"
underlying_session.close()
async def test_encrypted_session_get_items_limit_skips_invalid_latest_envelope(
encryption_key: str, underlying_session: SQLiteSession
):
"""Test that limit counts valid decrypted items, not encrypted envelopes."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
await session.add_items([{"role": "user", "content": "older valid"}])
await underlying_session.add_items([_invalid_encrypted_envelope()])
all_items = await session.get_items()
assert [item.get("content") for item in all_items] == ["older valid"]
limited = await session.get_items(limit=1)
assert [item.get("content") for item in limited] == ["older valid"]
underlying_session.close()
async def test_encrypted_session_get_items_limit_returns_latest_valid_items_after_invalids(
encryption_key: str, underlying_session: SQLiteSession
):
"""Test that invalid envelopes do not hide earlier valid items from limit."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
await session.add_items(
[
{"role": "user", "content": "valid 0"},
{"role": "assistant", "content": "valid 1"},
]
)
await underlying_session.add_items([_invalid_encrypted_envelope()])
await session.add_items([{"role": "user", "content": "valid 2"}])
limited = await session.get_items(limit=2)
assert [item.get("content") for item in limited] == ["valid 1", "valid 2"]
underlying_session.close()
async def test_encrypted_session_get_items_session_settings_limit_skips_invalid_envelopes(
encryption_key: str, underlying_session: SQLiteSession
):
"""Test that session settings limit counts valid decrypted items."""
underlying_session.session_settings = SessionSettings(limit=3)
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
await session.add_items(
[
{"role": "user", "content": "valid 0"},
{"role": "assistant", "content": "valid 1"},
{"role": "user", "content": "valid 2"},
]
)
await underlying_session.add_items([_invalid_encrypted_envelope()])
items = await session.get_items()
assert [item.get("content") for item in items] == ["valid 0", "valid 1", "valid 2"]
underlying_session.close()
async def test_encrypted_session_unicode_content(
encryption_key: str, underlying_session: SQLiteSession
):
"""Test encryption of international text content."""
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello world"},
{"role": "assistant", "content": "Special chars: áéíóú"},
{"role": "user", "content": "Numbers and symbols: 123!@#"},
]
await session.add_items(items)
retrieved = await session.get_items()
assert retrieved[0].get("content") == "Hello world"
assert retrieved[1].get("content") == "Special chars: áéíóú"
assert retrieved[2].get("content") == "Numbers and symbols: 123!@#"
underlying_session.close()
class CustomSession(SQLiteSession):
"""Mock custom session with additional methods for testing delegation."""
def get_stats(self) -> dict[str, int]:
"""Custom method that should be accessible through delegation."""
return {"custom_method_calls": 42, "test_value": 123}
async def custom_async_method(self) -> str:
"""Custom async method for testing delegation."""
return "custom_async_result"
async def test_encrypted_session_delegation():
"""Test that custom methods on underlying session are accessible through delegation."""
temp_dir = tempfile.mkdtemp()
db_path = Path(temp_dir) / "test_delegation.db"
underlying_session = CustomSession("test_session", db_path)
encryption_key = str(Fernet.generate_key().decode("utf-8"))
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying_session,
encryption_key=encryption_key,
)
stats = session.get_stats()
assert stats == {"custom_method_calls": 42, "test_value": 123}
result = await session.custom_async_method()
assert result == "custom_async_result"
await session.add_items([{"role": "user", "content": "Test delegation"}])
items = await session.get_items()
assert len(items) == 1
assert items[0].get("content") == "Test delegation"
underlying_session.close()
# ============================================================================
# SessionSettings Tests
# ============================================================================
async def test_session_settings_delegated_to_underlying(encryption_key: str):
"""Test that session_settings is correctly delegated to underlying session."""
from agents.memory import SessionSettings
temp_dir = tempfile.mkdtemp()
db_path = Path(temp_dir) / "test_settings.db"
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=5))
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying,
encryption_key=encryption_key,
)
# session_settings should be accessible through EncryptedSession
assert session.session_settings is not None
assert session.session_settings.limit == 5
underlying.close()
async def test_session_settings_get_items_uses_underlying_limit(encryption_key: str):
"""Test that get_items uses underlying session's session_settings.limit."""
from agents.memory import SessionSettings
temp_dir = tempfile.mkdtemp()
db_path = Path(temp_dir) / "test_settings_limit.db"
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=3))
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying,
encryption_key=encryption_key,
)
# Add 5 items
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(5)
]
await session.add_items(items)
# get_items() with no limit should use underlying session_settings.limit=3
retrieved = await session.get_items()
assert len(retrieved) == 3
# Should get the last 3 items
assert retrieved[0].get("content") == "Message 2"
assert retrieved[1].get("content") == "Message 3"
assert retrieved[2].get("content") == "Message 4"
underlying.close()
async def test_session_settings_explicit_limit_overrides_settings(encryption_key: str):
"""Test that explicit limit parameter overrides session_settings."""
from agents.memory import SessionSettings
temp_dir = tempfile.mkdtemp()
db_path = Path(temp_dir) / "test_override.db"
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=5))
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying,
encryption_key=encryption_key,
)
# Add 10 items
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(10)
]
await session.add_items(items)
# Explicit limit=2 should override session_settings.limit=5
retrieved = await session.get_items(limit=2)
assert len(retrieved) == 2
assert retrieved[0].get("content") == "Message 8"
assert retrieved[1].get("content") == "Message 9"
underlying.close()
async def test_session_settings_resolve():
"""Test SessionSettings.resolve() method."""
from agents.memory import SessionSettings
base = SessionSettings(limit=100)
override = SessionSettings(limit=50)
final = base.resolve(override)
assert final.limit == 50 # Override wins
assert base.limit == 100 # Original unchanged
# Resolving with None returns self
final_none = base.resolve(None)
assert final_none.limit == 100
async def test_runner_with_session_settings_override(encryption_key: str):
"""Test that RunConfig can override session's default settings."""
from agents import Agent, RunConfig, Runner
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
temp_dir = tempfile.mkdtemp()
db_path = Path(temp_dir) / "test_runner_override.db"
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=100))
session = EncryptedSession(
session_id="test_session",
underlying_session=underlying,
encryption_key=encryption_key,
)
# Add some history
items: list[TResponseInputItem] = [{"role": "user", "content": f"Turn {i}"} for i in range(10)]
await session.add_items(items)
model = FakeModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("Got it")])
await Runner.run(
agent,
"New question",
session=session,
run_config=RunConfig(
session_settings=SessionSettings(limit=2) # Override to 2
),
)
# Verify the agent received only the last 2 history items + new question
last_input = model.last_turn_args["input"]
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
assert len(history_items) == 2
underlying.close()
@@ -0,0 +1,154 @@
from __future__ import annotations
import importlib.abc
import sys
from types import ModuleType
import pytest
_PACKAGE_EXPORTS: tuple[tuple[str, str, str, str, str], ...] = (
(
"EncryptedSession",
"agents.extensions.memory.encrypt_session",
"agents.extensions.memory.encrypt_session",
"cryptography",
"encrypt",
),
("RedisSession", "agents.extensions.memory.redis_session", "redis.asyncio", "redis", "redis"),
(
"SQLAlchemySession",
"agents.extensions.memory.sqlalchemy_session",
"agents.extensions.memory.sqlalchemy_session",
"sqlalchemy",
"sqlalchemy",
),
("DaprSession", "agents.extensions.memory.dapr_session", "dapr.aio.clients", "dapr", "dapr"),
(
"DAPR_CONSISTENCY_EVENTUAL",
"agents.extensions.memory.dapr_session",
"dapr.aio.clients",
"dapr",
"dapr",
),
(
"DAPR_CONSISTENCY_STRONG",
"agents.extensions.memory.dapr_session",
"dapr.aio.clients",
"dapr",
"dapr",
),
(
"MongoDBSession",
"agents.extensions.memory.mongodb_session",
"pymongo.asynchronous.collection",
"mongodb",
"mongodb",
),
)
_DIRECT_MODULE_IMPORTS: tuple[tuple[str, str, str, str], ...] = (
("agents.extensions.memory.redis_session", "redis.asyncio", "redis", "redis"),
("agents.extensions.memory.dapr_session", "dapr.aio.clients", "dapr", "dapr"),
(
"agents.extensions.memory.mongodb_session",
"pymongo.asynchronous.collection",
"mongodb",
"mongodb",
),
)
class _BrokenImportFinder(importlib.abc.MetaPathFinder):
def __init__(self, broken_module: str, error_cls: type[ImportError]) -> None:
self._broken_module = broken_module
self._error_cls = error_cls
def find_spec(
self,
fullname: str,
path: object | None,
target: ModuleType | None = None,
) -> None:
if fullname == self._broken_module:
raise self._error_cls("simulated dependency import failure")
return None
def _reset_package_imports(
monkeypatch: pytest.MonkeyPatch,
memory_module: ModuleType,
symbol: str,
module_name: str,
broken_module: str,
) -> None:
monkeypatch.delitem(memory_module.__dict__, symbol, raising=False)
_reset_loaded_module(monkeypatch, module_name)
_reset_loaded_module(monkeypatch, broken_module)
def _reset_loaded_module(monkeypatch: pytest.MonkeyPatch, module_name: str) -> None:
monkeypatch.delitem(sys.modules, module_name, raising=False)
parent_name, short_name = module_name.rsplit(".", 1)
parent_module = sys.modules.get(parent_name)
if parent_module is not None:
monkeypatch.delitem(parent_module.__dict__, short_name, raising=False)
def _reset_module_imports(
monkeypatch: pytest.MonkeyPatch,
module_name: str,
broken_module: str,
) -> None:
_reset_loaded_module(monkeypatch, module_name)
_reset_loaded_module(monkeypatch, broken_module)
@pytest.mark.parametrize(
("symbol", "module_name", "broken_module", "dependency_name", "extra_name"),
_PACKAGE_EXPORTS,
)
def test_memory_package_imports_point_to_optional_extra(
monkeypatch: pytest.MonkeyPatch,
symbol: str,
module_name: str,
broken_module: str,
dependency_name: str,
extra_name: str,
) -> None:
import agents.extensions.memory as memory_module
_reset_package_imports(monkeypatch, memory_module, symbol, module_name, broken_module)
finder = _BrokenImportFinder(broken_module, ModuleNotFoundError)
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
with pytest.raises(ImportError) as exc_info:
getattr(memory_module, symbol)
assert f"requires the '{dependency_name}' extra" in str(exc_info.value)
assert f"openai-agents[{extra_name}]" in str(exc_info.value)
assert isinstance(exc_info.value.__cause__, ImportError)
@pytest.mark.parametrize(
("module_name", "broken_module", "dependency_name", "extra_name"),
_DIRECT_MODULE_IMPORTS,
)
@pytest.mark.parametrize("error_cls", [ImportError, ModuleNotFoundError])
def test_memory_direct_module_imports_point_to_optional_extra(
monkeypatch: pytest.MonkeyPatch,
module_name: str,
broken_module: str,
dependency_name: str,
extra_name: str,
error_cls: type[ImportError],
) -> None:
_reset_module_imports(monkeypatch, module_name, broken_module)
finder = _BrokenImportFinder(broken_module, error_cls)
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
with pytest.raises(ImportError) as exc_info:
__import__(module_name)
assert f"requires the '{dependency_name}' extra" in str(exc_info.value)
assert f"openai-agents[{extra_name}]" in str(exc_info.value)
assert isinstance(exc_info.value.__cause__, ImportError)
@@ -0,0 +1,831 @@
"""Tests for MongoDBSession using in-process mock objects.
All tests run without a real MongoDB server — or even the ``pymongo``
package — by injecting lightweight fake classes into ``sys.modules``
before the module under test is imported. This keeps the suite fast and
dependency-free while exercising the full session logic.
"""
from __future__ import annotations
import sys
import types
from collections import defaultdict
from datetime import datetime, timezone
from typing import Any
from unittest.mock import patch
import pytest
from agents import Agent, Runner, TResponseInputItem
from agents.memory.session_settings import SessionSettings
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
pytestmark = pytest.mark.asyncio
# ---------------------------------------------------------------------------
# In-memory fake pymongo async types
# ---------------------------------------------------------------------------
class FakeObjectId:
"""Minimal ObjectId stand-in with a monotonic counter for sort order."""
_counter = 0
def __init__(self) -> None:
FakeObjectId._counter += 1
self._value = FakeObjectId._counter
def __lt__(self, other: FakeObjectId) -> bool:
return self._value < other._value
def __repr__(self) -> str:
return f"FakeObjectId({self._value})"
class FakeCursor:
"""Minimal async cursor returned by ``find()``."""
def __init__(self, docs: list[dict[str, Any]]) -> None:
self._docs = docs
def sort(
self,
key: str | list[tuple[str, int]],
direction: int | None = None,
) -> FakeCursor:
if isinstance(key, list):
pairs = key
else:
direction = direction if direction is not None else 1
pairs = [(key, direction)]
docs = list(self._docs)
for field, dir_ in reversed(pairs):
docs.sort(key=lambda d: d.get(field, 0), reverse=(dir_ == -1))
self._docs = docs
return self
def limit(self, n: int) -> FakeCursor:
self._docs = self._docs[:n]
return self
async def to_list(self) -> list[dict[str, Any]]:
return list(self._docs)
class FakeAsyncCollection:
"""In-memory substitute for pymongo AsyncCollection."""
def __init__(self) -> None:
self._docs: dict[Any, dict[str, Any]] = {}
async def create_index(self, keys: Any, **kwargs: Any) -> str:
return "fake_index"
def find(self, query: dict[str, Any] | None = None) -> FakeCursor:
query = query or {}
results = [doc for doc in self._docs.values() if self._matches(doc, query)]
return FakeCursor(results)
async def find_one_and_delete(
self,
query: dict[str, Any],
sort: list[tuple[str, int]] | None = None,
) -> dict[str, Any] | None:
matches = [doc for doc in self._docs.values() if self._matches(doc, query)]
if not matches:
return None
if sort:
field, dir_ = sort[0]
matches.sort(key=lambda d: d.get(field, 0), reverse=(dir_ == -1))
doc = matches[0]
self._docs.pop(id(doc["_id"]))
return doc
async def insert_many(
self,
documents: list[dict[str, Any]],
ordered: bool = True,
) -> Any:
for doc in documents:
if "_id" not in doc:
doc["_id"] = FakeObjectId()
self._docs[id(doc["_id"])] = dict(doc)
async def find_one_and_update(
self,
query: dict[str, Any],
update: dict[str, Any],
upsert: bool = False,
return_document: bool = False,
) -> dict[str, Any] | None:
for doc in self._docs.values():
if self._matches(doc, query):
# Apply $inc fields.
for field, delta in update.get("$inc", {}).items():
doc[field] = doc.get(field, 0) + delta
for field, value in update.get("$set", {}).items():
doc[field] = value
return dict(doc) if return_document else None
if upsert:
new_doc: dict[str, Any] = {"_id": FakeObjectId()}
new_doc.update(update.get("$setOnInsert", {}))
new_doc.update(update.get("$set", {}))
for field, delta in update.get("$inc", {}).items():
new_doc[field] = new_doc.get(field, 0) + delta
self._docs[id(new_doc["_id"])] = new_doc
return dict(new_doc) if return_document else None
return None
async def update_one(
self,
query: dict[str, Any],
update: dict[str, Any],
upsert: bool = False,
) -> None:
for doc in self._docs.values():
if self._matches(doc, query):
return # Exists — $setOnInsert is a no-op on existing docs.
if upsert:
new_doc2: dict[str, Any] = {"_id": FakeObjectId()}
new_doc2.update(update.get("$setOnInsert", {}))
self._docs[id(new_doc2["_id"])] = new_doc2
async def delete_many(self, query: dict[str, Any]) -> None:
to_remove = [k for k, d in self._docs.items() if self._matches(d, query)]
for key in to_remove:
del self._docs[key]
async def delete_one(self, query: dict[str, Any]) -> None:
for key, doc in list(self._docs.items()):
if self._matches(doc, query):
del self._docs[key]
return
@staticmethod
def _matches(doc: dict[str, Any], query: dict[str, Any]) -> bool:
return all(doc.get(k) == v for k, v in query.items())
class FakeAsyncDatabase:
"""In-memory substitute for a pymongo async Database."""
def __init__(self) -> None:
self._collections: dict[str, FakeAsyncCollection] = defaultdict(FakeAsyncCollection)
def __getitem__(self, name: str) -> FakeAsyncCollection:
return self._collections[name]
class FakeAdminDatabase:
"""Minimal admin database used by ping()."""
def __init__(self) -> None:
self._closed = False
async def command(self, cmd: str) -> dict[str, Any]:
if self._closed:
raise ConnectionError("Client is closed.")
return {"ok": 1}
class FakeDriverInfo:
"""Minimal stand-in for pymongo.driver_info.DriverInfo."""
def __init__(self, name: str, version: str | None = None) -> None:
self.name = name
self.version = version
class FakeAsyncMongoClient:
"""In-memory substitute for pymongo AsyncMongoClient."""
def __init__(self, *args: Any, **kwargs: Any) -> None:
self._databases: dict[str, FakeAsyncDatabase] = defaultdict(FakeAsyncDatabase)
self._closed = False
self.admin = FakeAdminDatabase()
self._metadata_calls: list[FakeDriverInfo] = []
def __getitem__(self, name: str) -> FakeAsyncDatabase:
return self._databases[name]
def append_metadata(self, driver_info: FakeDriverInfo) -> None:
"""Record append_metadata calls for test assertions."""
self._metadata_calls.append(driver_info)
async def close(self) -> None:
"""Async close — matches PyMongo's AsyncMongoClient.close() signature."""
self._closed = True
self.admin._closed = True
# ---------------------------------------------------------------------------
# Inject fake pymongo into sys.modules before importing the module under test
# ---------------------------------------------------------------------------
def _make_fake_pymongo_modules() -> None:
"""Populate sys.modules with stub pymongo async modules."""
pymongo_mod = sys.modules.get("pymongo") or types.ModuleType("pymongo")
async_pkg = types.ModuleType("pymongo.asynchronous")
collection_mod = types.ModuleType("pymongo.asynchronous.collection")
client_mod = types.ModuleType("pymongo.asynchronous.mongo_client")
driver_info_mod = types.ModuleType("pymongo.driver_info")
collection_mod.AsyncCollection = FakeAsyncCollection # type: ignore[attr-defined]
client_mod.AsyncMongoClient = FakeAsyncMongoClient # type: ignore[attr-defined]
driver_info_mod.DriverInfo = FakeDriverInfo # type: ignore[attr-defined]
sys.modules["pymongo"] = pymongo_mod
sys.modules["pymongo.asynchronous"] = async_pkg
sys.modules["pymongo.asynchronous.collection"] = collection_mod
sys.modules["pymongo.asynchronous.mongo_client"] = client_mod
sys.modules["pymongo.driver_info"] = driver_info_mod
_make_fake_pymongo_modules()
# Now it's safe to import the module under test.
from agents.extensions.memory.mongodb_session import MongoDBSession # noqa: E402
# ---------------------------------------------------------------------------
# Helpers / fixtures
# ---------------------------------------------------------------------------
def _make_session(session_id: str = "test-session", **kwargs: Any) -> MongoDBSession:
"""Create a MongoDBSession backed by a FakeAsyncMongoClient."""
client = FakeAsyncMongoClient()
MongoDBSession._init_state.clear()
return MongoDBSession(
session_id,
client=client, # type: ignore[arg-type]
database="agents_test",
**kwargs,
)
@pytest.fixture
def session() -> MongoDBSession:
return _make_session()
@pytest.fixture
def agent() -> Agent:
return Agent(name="test", model=FakeModel())
# ---------------------------------------------------------------------------
# Core CRUD tests
# ---------------------------------------------------------------------------
async def test_add_and_get_items(session: MongoDBSession) -> None:
"""Items added to the session are retrievable in insertion order."""
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
]
await session.add_items(items)
retrieved = await session.get_items()
assert len(retrieved) == 2
assert retrieved[0].get("content") == "Hello"
assert retrieved[1].get("content") == "Hi there!"
async def test_add_empty_list_is_noop(session: MongoDBSession) -> None:
"""Adding an empty list must not create any documents."""
await session.add_items([])
assert await session.get_items() == []
async def test_get_items_empty_session(session: MongoDBSession) -> None:
"""Retrieving items from a brand-new session returns an empty list."""
assert await session.get_items() == []
async def test_pop_item_returns_last(session: MongoDBSession) -> None:
"""pop_item must return and remove the most recently added item."""
items: list[TResponseInputItem] = [
{"role": "user", "content": "first"},
{"role": "assistant", "content": "second"},
]
await session.add_items(items)
popped = await session.pop_item()
assert popped is not None
assert popped.get("content") == "second"
remaining = await session.get_items()
assert len(remaining) == 1
assert remaining[0].get("content") == "first"
async def test_pop_item_empty_session(session: MongoDBSession) -> None:
"""pop_item on an empty session must return None."""
assert await session.pop_item() is None
async def test_clear_session(session: MongoDBSession) -> None:
"""clear_session must remove all items and session metadata."""
await session.add_items([{"role": "user", "content": "x"}])
await session.clear_session()
assert await session.get_items() == []
async def test_multiple_add_calls_accumulate(session: MongoDBSession) -> None:
"""Items from separate add_items calls all appear in get_items."""
await session.add_items([{"role": "user", "content": "a"}])
await session.add_items([{"role": "assistant", "content": "b"}])
await session.add_items([{"role": "user", "content": "c"}])
items = await session.get_items()
assert [i.get("content") for i in items] == ["a", "b", "c"]
async def test_session_metadata_timestamps_are_written(session: MongoDBSession) -> None:
"""Session metadata records creation time and last update time."""
created_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
updated_at = datetime(2026, 1, 2, tzinfo=timezone.utc)
with patch("agents.extensions.memory.mongodb_session.datetime") as mocked_datetime:
mocked_datetime.now.side_effect = [created_at, updated_at]
await session.add_items([{"role": "user", "content": "first"}])
session_doc: dict[str, Any] = next(iter(session._sessions._docs.values()))
assert session_doc["session_id"] == session.session_id
assert session_doc["created_at"] == created_at
assert session_doc["updated_at"] == created_at
await session.add_items([{"role": "assistant", "content": "second"}])
assert session_doc["created_at"] == created_at
assert session_doc["updated_at"] == updated_at
assert session_doc["_seq"] == 2
# ---------------------------------------------------------------------------
# Limit / SessionSettings tests
# ---------------------------------------------------------------------------
async def test_get_items_with_explicit_limit(session: MongoDBSession) -> None:
"""Explicit limit returns the N most recent items in chronological order."""
await session.add_items([{"role": "user", "content": str(i)} for i in range(6)])
result = await session.get_items(limit=3)
assert len(result) == 3
assert [r.get("content") for r in result] == ["3", "4", "5"]
async def test_get_items_limit_zero(session: MongoDBSession) -> None:
"""A limit of 0 must return an empty list immediately."""
await session.add_items([{"role": "user", "content": "x"}])
assert await session.get_items(limit=0) == []
async def test_get_items_limit_exceeds_count(session: MongoDBSession) -> None:
"""Requesting more items than exist returns all items without error."""
await session.add_items([{"role": "user", "content": "only"}])
result = await session.get_items(limit=100)
assert len(result) == 1
async def test_session_settings_limit_used_as_default() -> None:
"""session_settings.limit is applied when no explicit limit is given."""
MongoDBSession._init_state.clear()
s = MongoDBSession(
"ls-test",
client=FakeAsyncMongoClient(), # type: ignore[arg-type]
database="agents_test",
session_settings=SessionSettings(limit=2),
)
await s.add_items([{"role": "user", "content": str(i)} for i in range(5)])
result = await s.get_items()
assert len(result) == 2
assert result[0].get("content") == "3"
assert result[1].get("content") == "4"
async def test_explicit_limit_overrides_session_settings() -> None:
"""An explicit limit passed to get_items must override session_settings.limit."""
MongoDBSession._init_state.clear()
s = MongoDBSession(
"override-test",
client=FakeAsyncMongoClient(), # type: ignore[arg-type]
database="agents_test",
session_settings=SessionSettings(limit=10),
)
await s.add_items([{"role": "user", "content": str(i)} for i in range(8)])
result = await s.get_items(limit=2)
assert len(result) == 2
assert result[0].get("content") == "6"
assert result[1].get("content") == "7"
# ---------------------------------------------------------------------------
# Session isolation
# ---------------------------------------------------------------------------
async def test_sessions_are_isolated() -> None:
"""Two sessions with different IDs must not share data."""
MongoDBSession._init_state.clear()
client = FakeAsyncMongoClient()
s1 = MongoDBSession("alice", client=client, database="agents_test") # type: ignore[arg-type]
s2 = MongoDBSession("bob", client=client, database="agents_test") # type: ignore[arg-type]
await s1.add_items([{"role": "user", "content": "alice msg"}])
await s2.add_items([{"role": "user", "content": "bob msg"}])
assert [i.get("content") for i in await s1.get_items()] == ["alice msg"]
assert [i.get("content") for i in await s2.get_items()] == ["bob msg"]
async def test_clear_does_not_affect_other_sessions() -> None:
"""Clearing one session must leave sibling sessions untouched."""
MongoDBSession._init_state.clear()
client = FakeAsyncMongoClient()
s1 = MongoDBSession("s1", client=client, database="agents_test") # type: ignore[arg-type]
s2 = MongoDBSession("s2", client=client, database="agents_test") # type: ignore[arg-type]
await s1.add_items([{"role": "user", "content": "keep"}])
await s2.add_items([{"role": "user", "content": "delete"}])
await s2.clear_session()
assert len(await s1.get_items()) == 1
assert await s2.get_items() == []
# ---------------------------------------------------------------------------
# Serialisation / unicode safety
# ---------------------------------------------------------------------------
async def test_unicode_content_roundtrip(session: MongoDBSession) -> None:
"""Unicode and emoji content must survive the serialisation round-trip."""
items: list[TResponseInputItem] = [
{"role": "user", "content": "こんにちは"},
{"role": "assistant", "content": "😊👍"},
{"role": "user", "content": "Привет"},
]
await session.add_items(items)
result = await session.get_items()
assert result[0].get("content") == "こんにちは"
assert result[1].get("content") == "😊👍"
assert result[2].get("content") == "Привет"
async def test_json_special_characters(session: MongoDBSession) -> None:
"""Items containing JSON-special strings must be stored without corruption."""
items: list[TResponseInputItem] = [
{"role": "user", "content": '{"nested": "value"}'},
{"role": "assistant", "content": 'Quote: "Hello"'},
{"role": "user", "content": "Line1\nLine2\tTabbed"},
]
await session.add_items(items)
result = await session.get_items()
assert result[0].get("content") == '{"nested": "value"}'
assert result[1].get("content") == 'Quote: "Hello"'
assert result[2].get("content") == "Line1\nLine2\tTabbed"
async def test_corrupted_document_is_skipped(session: MongoDBSession) -> None:
"""Documents with invalid JSON in message_data are silently skipped."""
await session.add_items([{"role": "user", "content": "valid"}])
# Inject a corrupted document directly into the fake collection.
bad_doc = {
"_id": FakeObjectId(),
"session_id": session.session_id,
"message_data": "not valid json {{{",
}
session._messages._docs[id(bad_doc["_id"])] = bad_doc
items = await session.get_items()
assert len(items) == 1
assert items[0].get("content") == "valid"
async def test_missing_message_data_field_is_skipped(session: MongoDBSession) -> None:
"""Documents without a message_data field are silently skipped."""
await session.add_items([{"role": "user", "content": "valid"}])
bad_doc = {"_id": FakeObjectId(), "session_id": session.session_id}
session._messages._docs[id(bad_doc["_id"])] = bad_doc
items = await session.get_items()
assert len(items) == 1
async def test_non_string_message_data_is_skipped(session: MongoDBSession) -> None:
"""Documents whose message_data is a non-string BSON type are silently skipped."""
await session.add_items([{"role": "user", "content": "valid"}])
# Inject a document where message_data is an integer — json.loads raises TypeError.
bad_doc = {"_id": FakeObjectId(), "session_id": session.session_id, "message_data": 42}
session._messages._docs[id(bad_doc["_id"])] = bad_doc
items = await session.get_items()
assert len(items) == 1
assert items[0].get("content") == "valid"
async def test_pop_item_skips_corrupt_most_recent(session: MongoDBSession) -> None:
"""pop_item must skip a corrupt most-recent document and return the next valid one."""
await session.add_items([{"role": "user", "content": "valid"}])
# Inject a corrupt document with a higher seq so it sorts as "most recent".
bad_doc = {
"_id": FakeObjectId(),
"session_id": session.session_id,
"seq": 999,
"message_data": "not valid json {{{",
}
session._messages._docs[id(bad_doc["_id"])] = bad_doc
popped = await session.pop_item()
assert popped is not None
assert popped.get("content") == "valid"
# Both the corrupt doc and the valid one are now gone.
assert await session.get_items() == []
async def test_pop_item_returns_none_when_only_corrupt_docs_remain(
session: MongoDBSession,
) -> None:
"""pop_item must drop every corrupt doc and return None when nothing valid remains."""
bad1 = {
"_id": FakeObjectId(),
"session_id": session.session_id,
"seq": 1,
"message_data": "garbage",
}
bad2 = {
"_id": FakeObjectId(),
"session_id": session.session_id,
"seq": 2,
"message_data": 42, # non-string — TypeError
}
session._messages._docs[id(bad1["_id"])] = bad1
session._messages._docs[id(bad2["_id"])] = bad2
assert await session.pop_item() is None
# Both corrupt docs must have been removed in the process.
assert session._messages._docs == {}
# ---------------------------------------------------------------------------
# Index initialisation (idempotency)
# ---------------------------------------------------------------------------
async def test_index_creation_runs_only_once(session: MongoDBSession) -> None:
"""_ensure_indexes must call create_index only on the very first call."""
call_count = 0
original_messages = session._messages.create_index
original_sessions = session._sessions.create_index
async def counting(*args: Any, **kwargs: Any) -> str:
nonlocal call_count
call_count += 1
return "fake_index"
session._messages.create_index = counting # type: ignore[method-assign]
session._sessions.create_index = counting # type: ignore[method-assign]
await session._ensure_indexes()
await session._ensure_indexes() # Second call must be a no-op.
# Exactly one call per collection (sessions + messages).
assert call_count == 2
session._messages.create_index = original_messages # type: ignore[method-assign]
session._sessions.create_index = original_sessions # type: ignore[method-assign]
async def test_different_clients_each_run_index_init() -> None:
"""Each distinct AsyncMongoClient gets its own index-creation pass."""
MongoDBSession._init_state.clear()
client_a = FakeAsyncMongoClient()
client_b = FakeAsyncMongoClient()
call_counts: dict[str, int] = {"a": 0, "b": 0}
async def counting_a(*args: Any, **kwargs: Any) -> str:
call_counts["a"] += 1
return "fake_index"
async def counting_b(*args: Any, **kwargs: Any) -> str:
call_counts["b"] += 1
return "fake_index"
s_a = MongoDBSession("x", client=client_a, database="agents_test") # type: ignore[arg-type]
s_b = MongoDBSession("x", client=client_b, database="agents_test") # type: ignore[arg-type]
s_a._messages.create_index = counting_a # type: ignore[method-assign]
s_a._sessions.create_index = counting_a # type: ignore[method-assign]
s_b._messages.create_index = counting_b # type: ignore[method-assign]
s_b._sessions.create_index = counting_b # type: ignore[method-assign]
await s_a._ensure_indexes()
await s_b._ensure_indexes()
# Each client must trigger its own index creation (2 calls = sessions + messages).
assert call_counts["a"] == 2
assert call_counts["b"] == 2
# ---------------------------------------------------------------------------
# Connectivity and lifecycle
# ---------------------------------------------------------------------------
async def test_ping_success(session: MongoDBSession) -> None:
"""ping() must return True when the client responds normally."""
assert await session.ping() is True
async def test_ping_failure(session: MongoDBSession) -> None:
"""ping() must return False when the server raises an exception."""
original = session._client.admin.command
async def _fail(*args: Any, **kwargs: Any) -> dict[str, Any]:
raise ConnectionError("unreachable")
session._client.admin.command = _fail # type: ignore[method-assign, assignment]
assert await session.ping() is False
session._client.admin.command = original # type: ignore[method-assign]
async def test_close_external_client_not_closed() -> None:
"""close() must NOT close a client that was injected externally."""
MongoDBSession._init_state.clear()
client = FakeAsyncMongoClient()
s = MongoDBSession("x", client=client, database="agents_test") # type: ignore[arg-type]
assert s._owns_client is False
await s.close()
assert not client._closed
async def test_close_owned_client_is_closed() -> None:
"""close() must close a client created by from_uri."""
MongoDBSession._init_state.clear()
fake_client = FakeAsyncMongoClient()
with patch(
"agents.extensions.memory.mongodb_session.AsyncMongoClient",
return_value=fake_client,
):
s = MongoDBSession.from_uri("owned", uri="mongodb://localhost:27017", database="t")
assert s._owns_client is True
await s.close()
assert fake_client._closed
# ---------------------------------------------------------------------------
# Runner integration
# ---------------------------------------------------------------------------
async def test_runner_integration(agent: Agent) -> None:
"""MongoDBSession must supply conversation history to the Runner."""
session = _make_session("runner-test")
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
result1 = await Runner.run(agent, "Where is the Golden Gate Bridge?", session=session)
assert result1.final_output == "San Francisco"
agent.model.set_next_output([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
last_input = agent.model.last_turn_args["input"]
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
async def test_runner_session_isolation(agent: Agent) -> None:
"""Two independent sessions must not bleed history into each other."""
MongoDBSession._init_state.clear()
client = FakeAsyncMongoClient()
s1 = MongoDBSession("user-a", client=client, database="agents_test") # type: ignore[arg-type]
s2 = MongoDBSession("user-b", client=client, database="agents_test") # type: ignore[arg-type]
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=s1)
agent.model.set_next_output([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=s2)
agent.model.set_next_output([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I mention?", session=s1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
async def test_runner_with_session_settings_limit(agent: Agent) -> None:
"""RunConfig.session_settings.limit must cap the history sent to the model."""
from agents import RunConfig
MongoDBSession._init_state.clear()
session = MongoDBSession(
"limit-test",
client=FakeAsyncMongoClient(), # type: ignore[arg-type]
database="agents_test",
session_settings=SessionSettings(limit=100),
)
history: list[TResponseInputItem] = [
{"role": "user", "content": f"Turn {i}"} for i in range(10)
]
await session.add_items(history)
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("Got it")])
await Runner.run(
agent,
"New question",
session=session,
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
)
last_input = agent.model.last_turn_args["input"]
history_items = [i for i in last_input if i.get("content") != "New question"]
assert len(history_items) == 2
# ---------------------------------------------------------------------------
# Client metadata (driver handshake)
# ---------------------------------------------------------------------------
async def test_injected_client_receives_append_metadata() -> None:
"""Append_metadata is called on a caller-supplied client."""
MongoDBSession._init_state.clear()
client = FakeAsyncMongoClient()
MongoDBSession("meta-test", client=client, database="agents_test") # type: ignore[arg-type]
assert len(client._metadata_calls) == 1
info = client._metadata_calls[0]
assert info.name == "openai-agents"
async def test_from_uri_passes_driver_info_to_constructor() -> None:
"""driver=_DRIVER_INFO is forwarded to AsyncMongoClient via from_uri."""
MongoDBSession._init_state.clear()
captured_kwargs: dict[str, Any] = {}
def _fake_client(uri: str, **kwargs: Any) -> FakeAsyncMongoClient:
captured_kwargs.update(kwargs)
return FakeAsyncMongoClient()
with patch(
"agents.extensions.memory.mongodb_session.AsyncMongoClient",
side_effect=_fake_client,
):
MongoDBSession.from_uri("uri-test", uri="mongodb://localhost:27017", database="t")
assert "driver" in captured_kwargs
assert captured_kwargs["driver"].name == "openai-agents"
async def test_caller_supplied_driver_info_is_not_overwritten() -> None:
"""A caller-supplied driver kwarg must not be silently replaced."""
MongoDBSession._init_state.clear()
captured_kwargs: dict[str, Any] = {}
custom_info = FakeDriverInfo(name="MyApp")
def _fake_client(uri: str, **kwargs: Any) -> FakeAsyncMongoClient:
captured_kwargs.update(kwargs)
return FakeAsyncMongoClient()
with patch(
"agents.extensions.memory.mongodb_session.AsyncMongoClient",
side_effect=_fake_client,
):
MongoDBSession.from_uri(
"uri-test",
uri="mongodb://localhost:27017",
database="t",
client_kwargs={"driver": custom_info},
)
# The caller's value must be preserved — setdefault must not overwrite it.
assert captured_kwargs["driver"] is custom_info
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,956 @@
from __future__ import annotations
import asyncio
import json
import threading
from collections.abc import Iterable, Sequence
from contextlib import asynccontextmanager
from datetime import datetime, timedelta
from typing import Any, cast
import pytest
from openai.types.responses.response_output_message_param import ResponseOutputMessageParam
from openai.types.responses.response_output_text_param import ResponseOutputTextParam
from openai.types.responses.response_reasoning_item_param import (
ResponseReasoningItemParam,
Summary,
)
from sqlalchemy import insert, select, text, update
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlalchemy.sql import Select
pytest.importorskip("sqlalchemy") # Skip tests if SQLAlchemy is not installed
from agents import Agent, Runner, TResponseInputItem
from agents.extensions.memory.sqlalchemy_session import SQLAlchemySession
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
# Mark all tests in this file as asyncio
pytestmark = pytest.mark.asyncio
# Use in-memory SQLite for tests
DB_URL = "sqlite+aiosqlite:///:memory:"
def _make_message_item(item_id: str, text_value: str) -> TResponseInputItem:
content: ResponseOutputTextParam = {
"type": "output_text",
"text": text_value,
"annotations": [],
"logprobs": [],
}
message: ResponseOutputMessageParam = {
"id": item_id,
"type": "message",
"role": "assistant",
"status": "completed",
"content": [content],
}
return cast(TResponseInputItem, message)
def _make_reasoning_item(item_id: str, summary_text: str) -> TResponseInputItem:
summary: Summary = {"type": "summary_text", "text": summary_text}
reasoning: ResponseReasoningItemParam = {
"id": item_id,
"type": "reasoning",
"summary": [summary],
}
return cast(TResponseInputItem, reasoning)
def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
result: list[str] = []
for item in items:
item_dict = cast(dict[str, Any], item)
result.append(cast(str, item_dict["id"]))
return result
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
async def test_sqlalchemy_session_direct_ops(agent: Agent):
"""Test direct database operations of SQLAlchemySession."""
session_id = "direct_ops_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
# 1. Add items
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
]
await session.add_items(items)
# 2. Get items and verify
retrieved = await session.get_items()
assert len(retrieved) == 2
assert retrieved[0].get("content") == "Hello"
assert retrieved[1].get("content") == "Hi there!"
# 3. Pop item
popped = await session.pop_item()
assert popped is not None
assert popped.get("content") == "Hi there!"
retrieved_after_pop = await session.get_items()
assert len(retrieved_after_pop) == 1
assert retrieved_after_pop[0].get("content") == "Hello"
# 4. Clear session
await session.clear_session()
retrieved_after_clear = await session.get_items()
assert len(retrieved_after_clear) == 0
async def test_sqlalchemy_session_defaults_to_escaped_non_ascii_storage():
"""Default storage keeps the historical escaped non-ASCII JSON representation."""
session = SQLAlchemySession.from_url("default_ascii_storage", url=DB_URL, create_tables=True)
item: TResponseInputItem = {"role": "user", "content": "café"}
await session.add_items([item])
async with session._session_factory() as sess:
rows = await sess.execute(
select(session._messages.c.message_data).where(
session._messages.c.session_id == session.session_id
)
)
stored = rows.scalar_one()
assert "\\u00e9" in stored
assert "café" not in stored
assert await session.get_items() == [item]
async def test_sqlalchemy_session_can_store_non_ascii_without_escaping():
"""ensure_ascii=False stores multilingual content readably while preserving round-trip data."""
session = SQLAlchemySession.from_url(
"non_ascii_storage",
url=DB_URL,
create_tables=True,
ensure_ascii=False,
)
item: TResponseInputItem = {"role": "user", "content": "café"}
await session.add_items([item])
async with session._session_factory() as sess:
rows = await sess.execute(
select(session._messages.c.message_data).where(
session._messages.c.session_id == session.session_id
)
)
stored = rows.scalar_one()
assert "café" in stored
assert "\\u00e9" not in stored
assert await session.get_items() == [item]
async def test_runner_integration(agent: Agent):
"""Test that SQLAlchemySession works correctly with the agent Runner."""
session_id = "runner_integration_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
# First turn
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
result1 = await Runner.run(
agent,
"What city is the Golden Gate Bridge in?",
session=session,
)
assert result1.final_output == "San Francisco"
# Second turn
agent.model.set_next_output([get_text_message("California")])
result2 = await Runner.run(agent, "What state is it in?", session=session)
assert result2.final_output == "California"
# Verify history was passed to the model on the second turn
last_input = agent.model.last_turn_args["input"]
assert len(last_input) > 1
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
async def test_session_isolation(agent: Agent):
"""Test that different session IDs result in isolated conversation histories."""
session_id_1 = "session_1"
session1 = SQLAlchemySession.from_url(session_id_1, url=DB_URL, create_tables=True)
session_id_2 = "session_2"
session2 = SQLAlchemySession.from_url(session_id_2, url=DB_URL, create_tables=True)
# Interact with session 1
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("I like cats.")])
await Runner.run(agent, "I like cats.", session=session1)
# Interact with session 2
agent.model.set_next_output([get_text_message("I like dogs.")])
await Runner.run(agent, "I like dogs.", session=session2)
# Go back to session 1 and check its memory
agent.model.set_next_output([get_text_message("You said you like cats.")])
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
assert "cats" in result.final_output.lower()
assert "dogs" not in result.final_output.lower()
async def test_get_items_with_limit(agent: Agent):
"""Test the limit parameter in get_items."""
session_id = "limit_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
items: list[TResponseInputItem] = [
{"role": "user", "content": "1"},
{"role": "assistant", "content": "2"},
{"role": "user", "content": "3"},
{"role": "assistant", "content": "4"},
]
await session.add_items(items)
# Get last 2 items
latest_2 = await session.get_items(limit=2)
assert len(latest_2) == 2
assert latest_2[0].get("content") == "3"
assert latest_2[1].get("content") == "4"
# Get all items
all_items = await session.get_items()
assert len(all_items) == 4
# Get more than available
more_than_all = await session.get_items(limit=10)
assert len(more_than_all) == 4
async def test_pop_from_empty_session():
"""Test that pop_item returns None on an empty session."""
session = SQLAlchemySession.from_url("empty_session", url=DB_URL, create_tables=True)
popped = await session.pop_item()
assert popped is None
async def test_pop_item_skips_corrupt_most_recent():
"""pop_item skips corrupt newest rows and returns the next valid item."""
session = SQLAlchemySession.from_url("pop_corrupt", url=DB_URL, create_tables=True)
valid_item: TResponseInputItem = {"role": "user", "content": "valid"}
await session.add_items([valid_item])
await session._ensure_tables()
async with session._session_factory() as sess:
async with sess.begin():
await sess.execute(
insert(session._messages).values(
{"session_id": session.session_id, "message_data": "not valid json {{{"}
)
)
assert await session.pop_item() == valid_item
assert await session.get_items() == []
async def test_pop_item_returns_none_after_dropping_only_corrupt_rows():
"""pop_item removes corrupt rows and returns None when no valid items remain."""
session = SQLAlchemySession.from_url("pop_only_corrupt", url=DB_URL, create_tables=True)
await session._ensure_tables()
async with session._session_factory() as sess:
async with sess.begin():
await sess.execute(
insert(session._messages).values(
{"session_id": session.session_id, "message_data": "not valid json {{{"}
)
)
assert await session.pop_item() is None
assert await session.get_items() == []
async def test_add_empty_items_list():
"""Test that adding an empty list of items is a no-op."""
session_id = "add_empty_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
initial_items = await session.get_items()
assert len(initial_items) == 0
await session.add_items([])
items_after_add = await session.get_items()
assert len(items_after_add) == 0
async def test_add_items_concurrent_first_access_with_create_tables(tmp_path):
"""Concurrent first writes should not race table creation or drop items."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_first_access.db'}"
session = SQLAlchemySession.from_url(
"concurrent_first_access",
url=db_url,
create_tables=True,
)
submitted = [f"msg-{i}" for i in range(25)]
async def worker(content: str) -> None:
await session.add_items([{"role": "user", "content": content}])
results = await asyncio.gather(
*(worker(content) for content in submitted),
return_exceptions=True,
)
assert [result for result in results if isinstance(result, Exception)] == []
stored = await session.get_items()
assert len(stored) == len(submitted)
stored_contents: list[str] = []
for item in stored:
content = item.get("content")
assert isinstance(content, str)
stored_contents.append(content)
assert sorted(stored_contents) == sorted(submitted)
async def test_add_items_concurrent_first_write_after_tables_exist(tmp_path):
"""Concurrent first writes should not race parent session creation."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_first_write.db'}"
setup_session = SQLAlchemySession.from_url(
"concurrent_first_write",
url=db_url,
create_tables=True,
)
await setup_session.get_items()
session = SQLAlchemySession.from_url(
"concurrent_first_write",
url=db_url,
create_tables=False,
)
submitted = [f"msg-{i}" for i in range(25)]
async def worker(content: str) -> None:
await session.add_items([{"role": "user", "content": content}])
results = await asyncio.gather(
*(worker(content) for content in submitted),
return_exceptions=True,
)
assert [result for result in results if isinstance(result, Exception)] == []
stored = await session.get_items()
assert len(stored) == len(submitted)
stored_contents: list[str] = []
for item in stored:
content = item.get("content")
assert isinstance(content, str)
stored_contents.append(content)
assert sorted(stored_contents) == sorted(submitted)
async def test_add_items_waits_for_transient_sqlite_write_lock(tmp_path):
"""SQLite writes should wait briefly for a transient lock instead of failing."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'sqlite_write_lock_retry.db'}"
session = SQLAlchemySession.from_url(
"sqlite_write_lock_retry",
url=db_url,
create_tables=True,
)
await session.get_items()
async with session.engine.connect() as conn:
await conn.execute(text("BEGIN IMMEDIATE"))
blocked_write = asyncio.create_task(
session.add_items([{"role": "user", "content": "after-lock"}])
)
await asyncio.sleep(0.1)
await conn.rollback()
await asyncio.wait_for(blocked_write, timeout=5)
stored = await session.get_items()
assert len(stored) == 1
assert stored[0].get("content") == "after-lock"
async def test_add_items_concurrent_first_access_across_sessions_with_shared_engine(tmp_path):
"""Concurrent first writes should not race table creation across session instances."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_shared_engine.db'}"
engine = create_async_engine(db_url)
try:
session_a = SQLAlchemySession("shared_engine_a", engine=engine, create_tables=True)
session_b = SQLAlchemySession("shared_engine_b", engine=engine, create_tables=True)
results = await asyncio.gather(
session_a.add_items([{"role": "user", "content": "one"}]),
session_b.add_items([{"role": "user", "content": "two"}]),
return_exceptions=True,
)
assert [result for result in results if isinstance(result, Exception)] == []
stored_a = await session_a.get_items()
assert len(stored_a) == 1
assert stored_a[0].get("content") == "one"
stored_b = await session_b.get_items()
assert len(stored_b) == 1
assert stored_b[0].get("content") == "two"
finally:
await engine.dispose()
async def test_add_items_concurrent_first_access_across_from_url_sessions(tmp_path):
"""Concurrent first writes should not race table creation across from_url sessions."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_from_url.db'}"
session_a = SQLAlchemySession.from_url("from_url_a", url=db_url, create_tables=True)
session_b = SQLAlchemySession.from_url("from_url_b", url=db_url, create_tables=True)
try:
results = await asyncio.gather(
session_a.add_items([{"role": "user", "content": "one"}]),
session_b.add_items([{"role": "user", "content": "two"}]),
return_exceptions=True,
)
assert [result for result in results if isinstance(result, Exception)] == []
stored_a = await session_a.get_items()
assert len(stored_a) == 1
assert stored_a[0].get("content") == "one"
stored_b = await session_b.get_items()
assert len(stored_b) == 1
assert stored_b[0].get("content") == "two"
finally:
await session_a.engine.dispose()
await session_b.engine.dispose()
async def test_add_items_concurrent_first_access_across_from_url_sessions_cross_loop(tmp_path):
"""Concurrent first writes should not race or hang across event loops."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_from_url_cross_loop.db'}"
barrier = threading.Barrier(2)
results: list[tuple[str, str, Any]] = []
results_lock = threading.Lock()
def worker(session_id: str, content: str) -> None:
async def run() -> tuple[str, Any]:
session = SQLAlchemySession.from_url(session_id, url=db_url, create_tables=True)
barrier.wait()
try:
await asyncio.wait_for(
session.add_items([{"role": "user", "content": content}]),
timeout=5,
)
stored = await session.get_items()
return ("ok", stored)
finally:
await session.engine.dispose()
try:
status, payload = asyncio.run(run())
except Exception as exc:
status, payload = type(exc).__name__, str(exc)
with results_lock:
results.append((session_id, status, payload))
threads = [
threading.Thread(target=worker, args=("from_url_cross_loop_a", "one")),
threading.Thread(target=worker, args=("from_url_cross_loop_b", "two")),
]
for thread in threads:
thread.start()
for thread in threads:
await asyncio.to_thread(thread.join)
assert len(results) == 2
assert [status for _, status, _ in results] == ["ok", "ok"]
stored_by_session = {
session_id: cast(list[TResponseInputItem], payload) for session_id, _, payload in results
}
assert stored_by_session["from_url_cross_loop_a"][0].get("content") == "one"
assert stored_by_session["from_url_cross_loop_b"][0].get("content") == "two"
async def test_add_items_concurrent_first_access_with_shared_session_cross_loop(tmp_path):
"""A shared session instance should not hang when used from two event loops."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'shared_session_cross_loop.db'}"
session = SQLAlchemySession.from_url(
"shared_session_cross_loop",
url=db_url,
create_tables=True,
)
barrier = threading.Barrier(2)
results: list[tuple[str, str]] = []
results_lock = threading.Lock()
def worker(content: str) -> None:
async def run() -> None:
barrier.wait()
await asyncio.wait_for(
session.add_items([{"role": "user", "content": content}]),
timeout=5,
)
try:
asyncio.run(run())
status = "ok"
except Exception as exc:
status = type(exc).__name__
with results_lock:
results.append((content, status))
threads = [
threading.Thread(target=worker, args=("one",)),
threading.Thread(target=worker, args=("two",)),
]
try:
for thread in threads:
thread.start()
for thread in threads:
await asyncio.to_thread(thread.join)
assert sorted(results) == [("one", "ok"), ("two", "ok")]
stored = await session.get_items()
stored_contents: list[str] = []
for item in stored:
content = item.get("content")
assert isinstance(content, str)
stored_contents.append(content)
assert sorted(stored_contents) == ["one", "two"]
finally:
await session.engine.dispose()
async def test_add_items_cancelled_waiter_does_not_strand_table_init_lock(tmp_path):
"""Cancelling a waiting initializer must not leave the shared init lock acquired."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'cancelled_table_init_waiter.db'}"
holder = SQLAlchemySession.from_url("holder", url=db_url, create_tables=True)
waiter = SQLAlchemySession.from_url("waiter", url=db_url, create_tables=True)
follower = SQLAlchemySession.from_url("follower", url=db_url, create_tables=True)
assert holder._init_lock is waiter._init_lock
assert waiter._init_lock is follower._init_lock
assert holder._init_lock is not None
acquired = holder._init_lock.acquire(blocking=False)
assert acquired
try:
blocked = asyncio.create_task(waiter.add_items([{"role": "user", "content": "waiter"}]))
await asyncio.sleep(0.05)
blocked.cancel()
with pytest.raises(asyncio.CancelledError):
await blocked
finally:
holder._init_lock.release()
try:
await asyncio.wait_for(
follower.add_items([{"role": "user", "content": "follower"}]),
timeout=2,
)
stored = await follower.get_items()
assert len(stored) == 1
assert stored[0].get("content") == "follower"
finally:
await holder.engine.dispose()
await waiter.engine.dispose()
await follower.engine.dispose()
async def test_create_tables_false_does_not_allocate_shared_init_lock(tmp_path):
"""Sessions that skip auto-create should not populate the shared lock map."""
db_url = f"sqlite+aiosqlite:///{tmp_path / 'no_create_tables_lock.db'}"
before = len(SQLAlchemySession._table_init_locks)
session = SQLAlchemySession.from_url("no_create_tables_lock", url=db_url, create_tables=False)
try:
assert session._init_lock is None
assert len(SQLAlchemySession._table_init_locks) == before
finally:
await session.engine.dispose()
async def test_get_items_same_timestamp_consistent_order():
"""Test that items with identical timestamps keep insertion order."""
session_id = "same_timestamp_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
older_item = _make_message_item("older_same_ts", "old")
reasoning_item = _make_reasoning_item("rs_same_ts", "...")
message_item = _make_message_item("msg_same_ts", "...")
await session.add_items([older_item])
await session.add_items([reasoning_item, message_item])
async with session._session_factory() as sess:
rows = await sess.execute(
select(session._messages.c.id, session._messages.c.message_data).where(
session._messages.c.session_id == session.session_id
)
)
id_map = {
json.loads(message_json)["id"]: row_id for row_id, message_json in rows.fetchall()
}
shared = datetime(2025, 10, 15, 17, 26, 39, 132483)
older = shared - timedelta(milliseconds=1)
await sess.execute(
update(session._messages)
.where(
session._messages.c.id.in_(
[
id_map["rs_same_ts"],
id_map["msg_same_ts"],
]
)
)
.values(created_at=shared)
)
await sess.execute(
update(session._messages)
.where(session._messages.c.id == id_map["older_same_ts"])
.values(created_at=older)
)
await sess.commit()
real_factory = session._session_factory
class FakeResult:
def __init__(self, rows: Iterable[Any]):
self._rows = list(rows)
def all(self) -> list[Any]:
return list(self._rows)
def needs_shuffle(statement: Any) -> bool:
if not isinstance(statement, Select):
return False
orderings = list(statement._order_by_clause)
if not orderings:
return False
id_asc = session._messages.c.id.asc()
id_desc = session._messages.c.id.desc()
def references_id(clause) -> bool:
try:
return bool(clause.compare(id_asc) or clause.compare(id_desc))
except AttributeError:
return False
if any(references_id(clause) for clause in orderings):
return False
# Only shuffle queries that target the messages table.
target_tables: set[str] = set()
for from_clause in statement.get_final_froms():
name_attr = getattr(from_clause, "name", None)
if isinstance(name_attr, str):
target_tables.add(name_attr)
table_name_obj = getattr(session._messages, "name", "")
table_name = table_name_obj if isinstance(table_name_obj, str) else ""
return bool(table_name in target_tables)
@asynccontextmanager
async def shuffled_session():
async with real_factory() as inner:
original_execute = inner.execute
async def execute_with_shuffle(statement: Any, *args: Any, **kwargs: Any) -> Any:
result = await original_execute(statement, *args, **kwargs)
if needs_shuffle(statement):
rows = result.all()
shuffled = list(rows)
shuffled.reverse()
return FakeResult(shuffled)
return result
cast(Any, inner).execute = execute_with_shuffle
try:
yield inner
finally:
cast(Any, inner).execute = original_execute
session._session_factory = cast(Any, shuffled_session)
try:
retrieved = await session.get_items()
assert _item_ids(retrieved) == ["older_same_ts", "rs_same_ts", "msg_same_ts"]
latest_two = await session.get_items(limit=2)
assert _item_ids(latest_two) == ["rs_same_ts", "msg_same_ts"]
finally:
session._session_factory = real_factory
async def test_pop_item_same_timestamp_returns_latest():
"""Test that pop_item returns the newest item when timestamps tie."""
session_id = "same_timestamp_pop_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
reasoning_item = _make_reasoning_item("rs_pop_same_ts", "...")
message_item = _make_message_item("msg_pop_same_ts", "...")
await session.add_items([reasoning_item, message_item])
async with session._session_factory() as sess:
await sess.execute(
text(
"UPDATE agent_messages SET created_at = :created_at WHERE session_id = :session_id"
),
{
"created_at": "2025-10-15 17:26:39.132483",
"session_id": session.session_id,
},
)
await sess.commit()
popped = await session.pop_item()
assert popped is not None
assert cast(dict[str, Any], popped)["id"] == "msg_pop_same_ts"
remaining = await session.get_items()
assert _item_ids(remaining) == ["rs_pop_same_ts"]
async def test_get_items_orders_by_id_for_ties():
"""Test that get_items adds id ordering to break timestamp ties."""
session_id = "order_by_id_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
await session.add_items(
[
_make_reasoning_item("rs_first", "..."),
_make_message_item("msg_second", "..."),
]
)
real_factory = session._session_factory
recorded: list[Any] = []
@asynccontextmanager
async def wrapped_session():
async with real_factory() as inner:
original_execute = inner.execute
async def recording_execute(statement: Any, *args: Any, **kwargs: Any) -> Any:
recorded.append(statement)
return await original_execute(statement, *args, **kwargs)
cast(Any, inner).execute = recording_execute
try:
yield inner
finally:
cast(Any, inner).execute = original_execute
session._session_factory = cast(Any, wrapped_session)
try:
retrieved_full = await session.get_items()
retrieved_limited = await session.get_items(limit=2)
finally:
session._session_factory = real_factory
assert len(recorded) >= 2
orderings_full = [str(clause) for clause in recorded[0]._order_by_clause]
assert orderings_full == [
"agent_messages.created_at ASC",
"agent_messages.id ASC",
]
orderings_limited = [str(clause) for clause in recorded[1]._order_by_clause]
assert orderings_limited == [
"agent_messages.created_at DESC",
"agent_messages.id DESC",
]
assert _item_ids(retrieved_full) == ["rs_first", "msg_second"]
assert _item_ids(retrieved_limited) == ["rs_first", "msg_second"]
async def test_engine_property_from_url():
"""Test that the engine property returns the AsyncEngine from from_url."""
session_id = "engine_property_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
# Verify engine property returns an AsyncEngine instance
assert isinstance(session.engine, AsyncEngine)
# Verify we can use the engine for advanced operations
# For example, check pool status
assert session.engine.pool is not None
# Verify we can manually dispose the engine
await session.engine.dispose()
async def test_engine_property_from_external_engine():
"""Test that the engine property returns the external engine."""
session_id = "external_engine_test"
# Create engine externally
external_engine = create_async_engine(DB_URL)
# Create session with external engine
session = SQLAlchemySession(session_id, engine=external_engine, create_tables=True)
# Verify engine property returns the same engine instance
assert session.engine is external_engine
# Verify we can use the engine
assert isinstance(session.engine, AsyncEngine)
# Clean up - user is responsible for disposing external engine
await external_engine.dispose()
async def test_engine_property_is_read_only():
"""Test that the engine property cannot be modified."""
session_id = "readonly_engine_test"
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
# Verify engine property exists
assert hasattr(session, "engine")
# Verify it's a property (read-only, cannot be set)
# Type ignore needed because mypy correctly detects this is read-only
with pytest.raises(AttributeError):
session.engine = create_async_engine(DB_URL) # type: ignore[misc]
# Clean up
await session.engine.dispose()
async def test_session_settings_default():
"""Test that session_settings defaults to empty SessionSettings."""
from agents.memory import SessionSettings
session = SQLAlchemySession.from_url("default_settings_test", url=DB_URL, create_tables=True)
# Should have default SessionSettings
assert isinstance(session.session_settings, SessionSettings)
assert session.session_settings.limit is None
async def test_session_settings_from_url():
"""Test passing session_settings via from_url."""
from agents.memory import SessionSettings
session = SQLAlchemySession.from_url(
"from_url_settings_test",
url=DB_URL,
create_tables=True,
session_settings=SessionSettings(limit=5),
)
assert session.session_settings is not None
assert session.session_settings.limit == 5
async def test_get_items_uses_session_settings_limit():
"""Test that get_items uses session_settings.limit as default."""
from agents.memory import SessionSettings
session = SQLAlchemySession.from_url(
"uses_settings_limit_test",
url=DB_URL,
create_tables=True,
session_settings=SessionSettings(limit=3),
)
# Add 5 items
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(5)
]
await session.add_items(items)
# get_items() with no limit should use session_settings.limit=3
retrieved = await session.get_items()
assert len(retrieved) == 3
# Should get the last 3 items
assert retrieved[0].get("content") == "Message 2"
assert retrieved[1].get("content") == "Message 3"
assert retrieved[2].get("content") == "Message 4"
async def test_get_items_explicit_limit_overrides_session_settings():
"""Test that explicit limit parameter overrides session_settings."""
from agents.memory import SessionSettings
session = SQLAlchemySession.from_url(
"explicit_override_test",
url=DB_URL,
create_tables=True,
session_settings=SessionSettings(limit=5),
)
# Add 10 items
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(10)
]
await session.add_items(items)
# Explicit limit=2 should override session_settings.limit=5
retrieved = await session.get_items(limit=2)
assert len(retrieved) == 2
assert retrieved[0].get("content") == "Message 8"
assert retrieved[1].get("content") == "Message 9"
async def test_session_settings_resolve():
"""Test SessionSettings.resolve() method."""
from agents.memory import SessionSettings
base = SessionSettings(limit=100)
override = SessionSettings(limit=50)
final = base.resolve(override)
assert final.limit == 50 # Override wins
assert base.limit == 100 # Original unchanged
# Resolving with None returns self
final_none = base.resolve(None)
assert final_none.limit == 100
async def test_runner_with_session_settings_override(agent: Agent):
"""Test that RunConfig can override session's default settings."""
from agents import RunConfig
from agents.memory import SessionSettings
# Session with default limit=100
session = SQLAlchemySession.from_url(
"runner_override_test",
url=DB_URL,
create_tables=True,
session_settings=SessionSettings(limit=100),
)
# Add some history
items: list[TResponseInputItem] = [{"role": "user", "content": f"Turn {i}"} for i in range(10)]
await session.add_items(items)
# Use RunConfig to override limit to 2
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("Got it")])
await Runner.run(
agent,
"New question",
session=session,
run_config=RunConfig(
session_settings=SessionSettings(limit=2) # Override to 2
),
)
# Verify the agent received only the last 2 history items + new question
last_input = agent.model.last_turn_args["input"]
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
assert len(history_items) == 2
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,345 @@
from __future__ import annotations
import importlib.util
import sys
import types
from pathlib import Path
from typing import Any, cast
import pytest
def _load_example_module() -> Any:
path = (
Path(__file__).resolve().parents[3]
/ "examples"
/ "sandbox"
/ "extensions"
/ "runloop"
/ "capabilities.py"
)
module_name = "tests.extensions.sandbox.runloop_capabilities_example"
spec = importlib.util.spec_from_file_location(module_name, path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
class _FakeNotFoundError(Exception):
def __init__(self) -> None:
self.status_code = 404
self.response = types.SimpleNamespace(status_code=404)
class _FakeConflictError(Exception):
def __init__(self, message: str) -> None:
self.status_code = 400
self.response = types.SimpleNamespace(status_code=400)
self.body = {"message": message}
class _FakeSecret:
def __init__(self, name: str, secret_id: str) -> None:
self.id = secret_id
self.name = name
class _FakeSecretsClient:
def __init__(self) -> None:
self.secrets: dict[str, _FakeSecret] = {}
self.create_calls: list[tuple[str, str]] = []
self.delete_calls: list[str] = []
self._counter = 0
def add(self, name: str) -> _FakeSecret:
self._counter += 1
secret = _FakeSecret(name=name, secret_id=f"secret-{self._counter}")
self.secrets[name] = secret
return secret
async def get(self, name: str) -> _FakeSecret:
if name not in self.secrets:
raise _FakeNotFoundError()
return self.secrets[name]
async def create(self, *, name: str, value: str) -> _FakeSecret:
self.create_calls.append((name, value))
return self.add(name)
class _FakePolicy:
def __init__(self, policy_id: str, name: str, description: str | None = None) -> None:
self.id = policy_id
self.name = name
self.description = description
class _FakePolicyRef:
def __init__(self, policy: _FakePolicy) -> None:
self._policy = policy
async def get_info(self) -> object:
return types.SimpleNamespace(
id=self._policy.id,
name=self._policy.name,
description=self._policy.description,
)
class _FakeNetworkPoliciesClient:
def __init__(self) -> None:
self.policies: dict[str, _FakePolicy] = {}
self.create_calls: list[dict[str, object]] = []
self.delete_calls: list[str] = []
self._counter = 0
def add(self, name: str, description: str | None = None) -> _FakePolicy:
self._counter += 1
policy = _FakePolicy(
policy_id=f"np-{self._counter}",
name=name,
description=description,
)
self.policies[policy.id] = policy
return policy
async def list(self, **params: object) -> list[_FakePolicy]:
name = params.get("name")
policies = list(self.policies.values())
if isinstance(name, str):
return [policy for policy in policies if policy.name == name]
return policies
async def create(self, **params: object) -> _FakePolicy:
self.create_calls.append(dict(params))
name = str(params["name"])
if any(policy.name == name for policy in self.policies.values()):
raise _FakeConflictError(f"NetworkPolicy with name '{name}' already exists")
description = cast(
str | None,
params.get("description") if isinstance(params.get("description"), str) else None,
)
return self.add(
name=name,
description=description,
)
def get(self, policy_id: str) -> _FakePolicyRef:
return _FakePolicyRef(self.policies[policy_id])
class _FakePlatformClient:
def __init__(self) -> None:
self.secrets = _FakeSecretsClient()
self.network_policies = _FakeNetworkPoliciesClient()
class _FakeRunloopClient:
def __init__(self) -> None:
self.platform = _FakePlatformClient()
@pytest.mark.asyncio
async def test_query_runloop_secret_returns_non_sensitive_metadata() -> None:
module = _load_example_module()
client = _FakeRunloopClient()
secret = client.platform.secrets.add("RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN")
result = await module._query_runloop_secret( # noqa: SLF001
client,
name=secret.name,
)
assert result.found is True
assert result.id == secret.id
assert "value" not in result.model_dump(mode="json")
@pytest.mark.asyncio
async def test_query_runloop_secret_reports_missing_before_create() -> None:
module = _load_example_module()
client = _FakeRunloopClient()
result = await module._query_runloop_secret( # noqa: SLF001
client,
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
)
assert result.found is False
assert result.id is None
@pytest.mark.asyncio
async def test_query_runloop_network_policy_reports_existing_resource() -> None:
module = _load_example_module()
client = _FakeRunloopClient()
policy = client.platform.network_policies.add(
"runloop-capabilities-example-policy",
description="Persistent example policy.",
)
result = await module._query_runloop_network_policy( # noqa: SLF001
client,
name=policy.name,
)
assert result.found is True
assert result.id == policy.id
assert result.description == "Persistent example policy."
@pytest.mark.asyncio
async def test_bootstrap_persistent_resources_reuses_existing_resources_without_cleanup() -> None:
module = _load_example_module()
client = _FakeRunloopClient()
secret = client.platform.secrets.add("RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN")
policy = client.platform.network_policies.add("runloop-capabilities-example-policy")
query_results = {
"secret": module.RunloopResourceQueryResult(
resource_type="secret",
name=secret.name,
found=True,
id=secret.id,
),
"network_policy": module.RunloopResourceQueryResult(
resource_type="network_policy",
name=policy.name,
found=True,
id=policy.id,
),
}
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
client,
managed_secret_name=secret.name,
managed_secret_value="runloop-capabilities-example-token",
network_policy_name=policy.name,
network_policy_id_override=None,
query_results=query_results,
axon_name=None,
)
secret_bootstrap = bootstrap["secret"]
network_policy_bootstrap = bootstrap["network_policy"]
assert secret_bootstrap.action == "reused"
assert network_policy_bootstrap.action == "reused"
assert client.platform.secrets.create_calls == []
assert client.platform.network_policies.create_calls == []
assert client.platform.secrets.delete_calls == []
assert client.platform.network_policies.delete_calls == []
@pytest.mark.asyncio
async def test_bootstrap_persistent_resources_creates_missing_resources() -> None:
module = _load_example_module()
client = _FakeRunloopClient()
query_results = {
"secret": module.RunloopResourceQueryResult(
resource_type="secret",
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
found=False,
),
"network_policy": module.RunloopResourceQueryResult(
resource_type="network_policy",
name="runloop-capabilities-example-policy",
found=False,
),
}
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
client,
managed_secret_name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
managed_secret_value="runloop-capabilities-example-token",
network_policy_name="runloop-capabilities-example-policy",
network_policy_id_override=None,
query_results=query_results,
axon_name=None,
)
secret_bootstrap = bootstrap["secret"]
network_policy_bootstrap = bootstrap["network_policy"]
assert secret_bootstrap.action == "created"
assert network_policy_bootstrap.action == "created"
assert client.platform.secrets.create_calls == [
("RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN", "runloop-capabilities-example-token")
]
assert client.platform.network_policies.create_calls == [
{
"name": "runloop-capabilities-example-policy",
"allow_all": True,
"description": "Persistent network policy for the Runloop capabilities example.",
}
]
@pytest.mark.asyncio
async def test_bootstrap_persistent_resources_respects_policy_override() -> None:
module = _load_example_module()
client = _FakeRunloopClient()
query_results = {
"secret": module.RunloopResourceQueryResult(
resource_type="secret",
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
found=False,
),
"network_policy": module.RunloopResourceQueryResult(
resource_type="network_policy",
name="runloop-capabilities-example-policy",
found=False,
),
}
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
client,
managed_secret_name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
managed_secret_value="runloop-capabilities-example-token",
network_policy_name="runloop-capabilities-example-policy",
network_policy_id_override="np-override",
query_results=query_results,
axon_name=None,
)
network_policy_bootstrap = bootstrap["network_policy"]
assert network_policy_bootstrap.action == "override"
assert network_policy_bootstrap.id == "np-override"
assert client.platform.network_policies.create_calls == []
@pytest.mark.asyncio
async def test_bootstrap_persistent_resources_recovers_from_existing_policy_conflict() -> None:
module = _load_example_module()
client = _FakeRunloopClient()
policy = client.platform.network_policies.add(
"runloop-capabilities-example-policy",
description="Persistent example policy.",
)
query_results = {
"secret": module.RunloopResourceQueryResult(
resource_type="secret",
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
found=False,
),
"network_policy": module.RunloopResourceQueryResult(
resource_type="network_policy",
name=policy.name,
found=False,
),
}
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
client,
managed_secret_name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
managed_secret_value="runloop-capabilities-example-token",
network_policy_name=policy.name,
network_policy_id_override=None,
query_results=query_results,
axon_name=None,
)
network_policy_bootstrap = bootstrap["network_policy"]
assert network_policy_bootstrap.action == "reused"
assert network_policy_bootstrap.found_before_bootstrap is True
assert network_policy_bootstrap.id == policy.id
@@ -0,0 +1,224 @@
from __future__ import annotations
import io
import types
import uuid
from pathlib import Path
from typing import Any, cast
import pytest
from agents.sandbox import Manifest
from agents.sandbox.entries import RcloneMountPattern, S3Mount
from agents.sandbox.errors import MountConfigError
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
from agents.sandbox.types import ExecResult
class _FakeRunloopMountSession(BaseSandboxSession):
def __init__(self, results: list[ExecResult] | None = None) -> None:
self.state = cast(
Any,
types.SimpleNamespace(
session_id=uuid.uuid4(),
manifest=Manifest(root="/workspace"),
),
)
self._results = list(results or [])
self.exec_calls: list[str] = []
async def _exec_internal(
self,
*command: str | Path,
timeout: float | None = None,
) -> ExecResult:
_ = timeout
cmd_str = " ".join(str(c) for c in command)
self.exec_calls.append(cmd_str)
if self._results:
return self._results.pop(0)
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
async def read(self, path: Path, *, user: object = None) -> io.IOBase:
_ = (path, user)
return io.BytesIO(b"")
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
_ = (path, data, user)
async def persist_workspace(self) -> io.IOBase:
raise AssertionError("not expected")
async def hydrate_workspace(self, data: io.IOBase) -> None:
_ = data
raise AssertionError("not expected")
async def running(self) -> bool:
return True
_FakeRunloopMountSession.__name__ = "RunloopSandboxSession"
def _exec_ok(stdout: bytes = b"") -> ExecResult:
return ExecResult(stdout=stdout, stderr=b"", exit_code=0)
def _exec_fail() -> ExecResult:
return ExecResult(stdout=b"", stderr=b"", exit_code=1)
def test_runloop_package_re_exports_cloud_bucket_strategy() -> None:
package_module = __import__(
"agents.extensions.sandbox.runloop",
fromlist=["RunloopCloudBucketMountStrategy"],
)
assert hasattr(package_module, "RunloopCloudBucketMountStrategy")
def test_runloop_extension_re_exports_cloud_bucket_strategy() -> None:
package_module = __import__(
"agents.extensions.sandbox",
fromlist=["RunloopCloudBucketMountStrategy"],
)
assert hasattr(package_module, "RunloopCloudBucketMountStrategy")
def test_runloop_mount_strategy_type_and_default_pattern() -> None:
from agents.extensions.sandbox.runloop.mounts import RunloopCloudBucketMountStrategy
strategy = RunloopCloudBucketMountStrategy()
assert strategy.type == "runloop_cloud_bucket"
assert isinstance(strategy.pattern, RcloneMountPattern)
assert strategy.pattern.mode == "fuse"
def test_runloop_mount_strategy_round_trips_through_manifest() -> None:
from agents.extensions.sandbox.runloop.mounts import RunloopCloudBucketMountStrategy
manifest = Manifest.model_validate(
{
"root": "/workspace",
"entries": {
"bucket": {
"type": "s3_mount",
"bucket": "my-bucket",
"mount_strategy": {"type": "runloop_cloud_bucket"},
}
},
}
)
mount = manifest.entries["bucket"]
assert isinstance(mount, S3Mount)
assert isinstance(mount.mount_strategy, RunloopCloudBucketMountStrategy)
def test_runloop_session_guard_rejects_wrong_type() -> None:
from agents.extensions.sandbox.runloop.mounts import _assert_runloop_session
class _WrongSession:
pass
with pytest.raises(MountConfigError, match="RunloopSandboxSession"):
_assert_runloop_session(_WrongSession()) # type: ignore[arg-type]
def test_runloop_session_guard_accepts_correct_type() -> None:
from agents.extensions.sandbox.runloop.mounts import _assert_runloop_session
_assert_runloop_session(_FakeRunloopMountSession())
@pytest.mark.asyncio
async def test_runloop_ensure_rclone_installs_with_root_apt() -> None:
from agents.extensions.sandbox._rclone import ensure_rclone
session = _FakeRunloopMountSession(
[
_exec_fail(),
_exec_ok(),
_exec_ok(),
_exec_ok(),
_exec_ok(),
]
)
await ensure_rclone(session)
assert session.exec_calls[:2] == [
"sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone",
"sh -lc command -v apt-get >/dev/null 2>&1",
]
assert session.exec_calls[2] == (
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 update -qq"
)
assert session.exec_calls[3] == (
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 install -y -qq "
"curl unzip ca-certificates"
)
assert (
session.exec_calls[4]
== "sudo -u root -- sh -lc curl -fsSL https://rclone.org/install.sh | bash"
)
assert session.exec_calls[5] == (
"sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone"
)
@pytest.mark.asyncio
async def test_runloop_ensure_fuse_installs_missing_fusermount() -> None:
from agents.extensions.sandbox.runloop.mounts import _ensure_fuse_support
session = _FakeRunloopMountSession(
[
_exec_ok(),
_exec_ok(),
_exec_fail(),
_exec_ok(),
_exec_ok(),
_exec_ok(),
_exec_ok(),
_exec_ok(),
]
)
await _ensure_fuse_support(session)
assert session.exec_calls == [
"sh -lc test -c /dev/fuse",
"sh -lc grep -qw fuse /proc/filesystems",
"sh -lc command -v fusermount3 >/dev/null 2>&1 || command -v fusermount >/dev/null 2>&1",
"sh -lc command -v apt-get >/dev/null 2>&1",
(
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 update -qq"
),
(
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 install -y -qq fuse3"
),
"sh -lc command -v fusermount3 >/dev/null 2>&1 || command -v fusermount >/dev/null 2>&1",
(
"sudo -u root -- sh -lc chmod a+rw /dev/fuse && "
"touch /etc/fuse.conf && "
"(grep -qxF user_allow_other /etc/fuse.conf || "
"printf '\\nuser_allow_other\\n' >> /etc/fuse.conf)"
),
]
@pytest.mark.asyncio
async def test_runloop_rclone_pattern_adds_fuse_access_args() -> None:
from agents.extensions.sandbox._rclone import rclone_pattern_for_session
session = _FakeRunloopMountSession([_exec_ok(stdout=b"1000\n1000\n")])
pattern = await rclone_pattern_for_session(session, RcloneMountPattern(mode="fuse"))
assert pattern.extra_args == ["--allow-other", "--uid", "1000", "--gid", "1000"]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,635 @@
"""Tests for ToolOutputTrimmer — the built-in call_model_input_filter for trimming
large tool outputs from older conversation turns.
"""
from __future__ import annotations
import copy
import json
from typing import Any, cast
from unittest.mock import MagicMock
import pytest
from agents.extensions.tool_output_trimmer import ToolOutputTrimmer
from agents.run_config import CallModelData, ModelInputData
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _user(text: str = "hello") -> dict[str, Any]:
return {"role": "user", "content": text}
def _assistant(text: str = "response") -> dict[str, Any]:
return {"role": "assistant", "content": text}
def _func_call(call_id: str, name: str, *, namespace: str | None = None) -> dict[str, Any]:
item = {"type": "function_call", "call_id": call_id, "name": name, "arguments": "{}"}
if namespace is not None:
item["namespace"] = namespace
return item
def _func_output(call_id: str, output: str) -> dict[str, Any]:
return {"type": "function_call_output", "call_id": call_id, "output": output}
def _make_data(items: list[Any]) -> CallModelData[Any]:
model_data = ModelInputData(input=items, instructions="You are helpful.")
return CallModelData(model_data=model_data, agent=MagicMock(), context=None)
def _output(result: ModelInputData, idx: int) -> Any:
"""Extract the ``output`` field from a result item (untyped for test convenience)."""
item: Any = result.input[idx]
return item["output"]
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
class TestDefaults:
def test_default_values(self) -> None:
trimmer = ToolOutputTrimmer()
assert trimmer.recent_turns == 2
assert trimmer.max_output_chars == 500
assert trimmer.preview_chars == 200
assert trimmer.trimmable_tools is None
def test_trimmable_tools_coerced_to_frozenset(self) -> None:
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"a", "b"}))
assert isinstance(trimmer.trimmable_tools, frozenset)
assert trimmer.trimmable_tools == frozenset({"a", "b"})
def test_trimmable_tools_from_list(self) -> None:
trimmer = ToolOutputTrimmer(trimmable_tools=["search", "run_code"])
assert isinstance(trimmer.trimmable_tools, frozenset)
assert "search" in trimmer.trimmable_tools
assert "run_code" in trimmer.trimmable_tools
def test_trimmable_tools_from_string(self) -> None:
trimmer = ToolOutputTrimmer(trimmable_tools="search")
assert isinstance(trimmer.trimmable_tools, frozenset)
assert trimmer.trimmable_tools == frozenset({"search"})
# ---------------------------------------------------------------------------
# Input validation
# ---------------------------------------------------------------------------
class TestValidation:
def test_recent_turns_zero_raises(self) -> None:
with pytest.raises(ValueError, match="recent_turns must be >= 1"):
ToolOutputTrimmer(recent_turns=0)
def test_recent_turns_negative_raises(self) -> None:
with pytest.raises(ValueError, match="recent_turns must be >= 1"):
ToolOutputTrimmer(recent_turns=-1)
def test_max_output_chars_zero_raises(self) -> None:
with pytest.raises(ValueError, match="max_output_chars must be >= 1"):
ToolOutputTrimmer(max_output_chars=0)
def test_preview_chars_negative_raises(self) -> None:
with pytest.raises(ValueError, match="preview_chars must be >= 0"):
ToolOutputTrimmer(preview_chars=-1)
def test_preview_chars_zero_allowed(self) -> None:
trimmer = ToolOutputTrimmer(preview_chars=0)
assert trimmer.preview_chars == 0
def test_trimmable_tools_bytes_raises(self) -> None:
with pytest.raises(ValueError, match="trimmable_tools must be a string or iterable"):
ToolOutputTrimmer(trimmable_tools=b"search") # type: ignore[arg-type]
# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------
class TestRecentBoundary:
def test_empty_items(self) -> None:
trimmer = ToolOutputTrimmer()
assert trimmer._find_recent_boundary([]) == 0
def test_single_user_message(self) -> None:
trimmer = ToolOutputTrimmer()
assert trimmer._find_recent_boundary([_user()]) == 0
def test_two_user_messages_boundary_at_first(self) -> None:
items = [_user("q1"), _assistant("a1"), _user("q2"), _assistant("a2")]
trimmer = ToolOutputTrimmer(recent_turns=2)
assert trimmer._find_recent_boundary(items) == 0
def test_three_user_messages(self) -> None:
items = [
_user("q1"),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(recent_turns=2)
assert trimmer._find_recent_boundary(items) == 2
def test_custom_recent_turns(self) -> None:
items = [
_user("q1"),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
_user("q4"),
_assistant("a4"),
]
trimmer = ToolOutputTrimmer(recent_turns=3)
# q4 at 6 (count=1), q3 at 4 (count=2), q2 at 2 (count=3) -> boundary=2
assert trimmer._find_recent_boundary(items) == 2
# ---------------------------------------------------------------------------
# Trimming behavior
# ---------------------------------------------------------------------------
class TestTrimming:
def test_empty_input(self) -> None:
trimmer = ToolOutputTrimmer()
data = _make_data([])
result = trimmer(data)
assert result.input == []
def test_no_trimming_when_all_recent(self) -> None:
"""With only 1 user message, everything is recent."""
large = "x" * 1000
items = [
_user("q"),
_func_call("c1", "search"),
_func_output("c1", large),
_assistant("a"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
assert _output(result, 2) == large
def test_trims_large_old_output(self) -> None:
"""Large output in an old turn should be trimmed."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
trimmed = _output(result, 2)
assert "[Trimmed:" in trimmed
assert "search" in trimmed
assert "1000 chars" in trimmed
assert len(trimmed) < len(large)
def test_preserves_small_old_output(self) -> None:
"""Small outputs should never be trimmed."""
small = "x" * 100
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", small),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(max_output_chars=500)
result = trimmer(_make_data(items))
assert _output(result, 2) == small
def test_respects_trimmable_tools_allowlist(self) -> None:
"""Only outputs from tools in trimmable_tools should be trimmed."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", large),
_func_call("c2", "resolve_entity"),
_func_output("c2", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"search"}))
result = trimmer(_make_data(items))
# search output trimmed
assert "[Trimmed:" in _output(result, 2)
# resolve_entity output preserved
assert _output(result, 4) == large
def test_string_trimmable_tools_allowlist_matches_single_tool_name(self) -> None:
"""A string trimmable_tools value should match one tool name, not characters."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", large),
_func_call("c2", "s"),
_func_output("c2", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(trimmable_tools="search")
result = trimmer(_make_data(items))
assert "[Trimmed:" in _output(result, 2)
assert _output(result, 4) == large
def test_respects_qualified_tool_names_allowlist(self) -> None:
"""Qualified allowlist entries should match namespaced function tools."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "lookup_account", namespace="billing"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"billing.lookup_account"}))
result = trimmer(_make_data(items))
assert "[Trimmed:" in _output(result, 2)
assert "billing.lookup_account" in _output(result, 2)
def test_namespaced_tools_still_match_bare_allowlist_entries(self) -> None:
"""Bare allowlist entries remain valid for namespaced tools."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "lookup_account", namespace="billing"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"lookup_account"}))
result = trimmer(_make_data(items))
assert "[Trimmed:" in _output(result, 2)
assert "billing.lookup_account" in _output(result, 2)
def test_synthetic_same_name_namespace_uses_bare_display_name(self) -> None:
"""Deferred synthetic namespaces should not display as `name.name`."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "get_weather", namespace="get_weather"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"get_weather"}))
result = trimmer(_make_data(items))
assert "[Trimmed:" in _output(result, 2)
assert "get_weather.get_weather" not in _output(result, 2)
assert "get_weather" in _output(result, 2)
def test_trims_tool_search_output_tool_definitions(self) -> None:
"""Large tool_search_output tool definitions should be structurally trimmed."""
verbose_schema = {
"type": "object",
"description": "schema " * 200,
"properties": {
"customer_id": {
"type": "string",
"description": "customer id " * 200,
"default": "cust_123",
}
},
"required": ["customer_id"],
}
items = [
_user("q1"),
{"type": "tool_search_call", "call_id": "ts1", "arguments": {"query": "profile"}},
{
"type": "tool_search_output",
"call_id": "ts1",
"tools": [
{
"type": "function",
"name": "lookup_account",
"description": "tool description " * 200,
"parameters": verbose_schema,
}
],
},
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
original_len = len(json.dumps(items[2]["tools"], sort_keys=True))
trimmer = ToolOutputTrimmer(max_output_chars=400, preview_chars=60)
result = trimmer(_make_data(items))
trimmed_item_dict = cast(dict[str, Any], result.input[2])
assert trimmed_item_dict["type"] == "tool_search_output"
trimmed_tools = list(trimmed_item_dict["tools"])
assert trimmed_tools[0]["name"] == "lookup_account"
assert "description" not in trimmed_tools[0]["parameters"]
assert trimmed_tools[0]["parameters"]["properties"]["customer_id"]["default"] == "cust_123"
assert len(json.dumps(trimmed_tools, sort_keys=True)) < original_len
def test_trims_legacy_tool_search_output_results(self) -> None:
"""Legacy tool_search_output snapshots with free-text results should still trim."""
large = "x" * 2000
items = [
_user("q1"),
{"type": "tool_search_call", "call_id": "ts1", "arguments": {"query": "profile"}},
{
"type": "tool_search_output",
"call_id": "ts1",
"results": [{"text": large}],
},
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(max_output_chars=400, preview_chars=80)
result = trimmer(_make_data(items))
trimmed_item = cast(dict[str, Any], result.input[2])
assert trimmed_item["type"] == "tool_search_output"
assert "[Trimmed: tool_search output" in trimmed_item["results"][0]["text"]
def test_trims_all_tools_when_allowlist_is_none(self) -> None:
"""When trimmable_tools is None, all tools are eligible."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "any_tool"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(trimmable_tools=None)
result = trimmer(_make_data(items))
assert "[Trimmed:" in _output(result, 2)
def test_preserves_recent_large_output(self) -> None:
"""Large outputs in recent turns should never be trimmed."""
large = "x" * 1000
items = [
_user("q1"),
_assistant("a1"),
_user("q2"),
_func_call("c1", "search"),
_func_output("c1", large),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
assert _output(result, 4) == large
def test_does_not_mutate_original_items(self) -> None:
"""The filter must not mutate the original input items."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
original = copy.deepcopy(items)
trimmer = ToolOutputTrimmer()
trimmer(_make_data(items))
assert items == original
def test_preserves_instructions(self) -> None:
"""The instructions field should pass through unchanged."""
items: list[Any] = [_user("hi")]
model_data = ModelInputData(input=items, instructions="Custom prompt")
data: CallModelData[Any] = CallModelData(
model_data=model_data, agent=MagicMock(), context=None
)
trimmer = ToolOutputTrimmer()
result = trimmer(data)
assert result.instructions == "Custom prompt"
def test_multiple_old_outputs_trimmed(self) -> None:
"""Multiple large outputs in old turns should all be trimmed."""
large1 = "a" * 1000
large2 = "b" * 2000
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", large1),
_func_call("c2", "execute"),
_func_output("c2", large2),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
assert "[Trimmed:" in _output(result, 2)
assert "[Trimmed:" in _output(result, 4)
assert "search" in _output(result, 2)
assert "execute" in _output(result, 4)
def test_custom_preview_chars(self) -> None:
"""Preview length should respect the preview_chars setting."""
large = "abcdefghij" * 100 # 1000 chars
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(preview_chars=50)
result = trimmer(_make_data(items))
trimmed = _output(result, 2)
# The preview portion should be exactly 50 chars of the original
assert "abcdefghij" * 5 in trimmed
def test_preserves_user_and_assistant_messages(self) -> None:
"""User and assistant messages are never modified."""
items = [
_user("important"),
_assistant("detailed " * 100),
_user("follow up"),
_assistant("another"),
_user("final"),
_assistant("done"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
assert result.input == items
# ---------------------------------------------------------------------------
# Sliding window behavior
# ---------------------------------------------------------------------------
class TestSlidingWindow:
"""Verify the trimmer acts as a sliding window across turns."""
def test_turn3_trims_turn1(self) -> None:
"""On turn 3, turn 1 outputs should be trimmed."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_func_call("c2", "search"),
_func_output("c2", large),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
# Turn 1 (old) trimmed
assert "[Trimmed:" in _output(result, 2)
# Turn 2 (recent) preserved
assert _output(result, 6) == large
def test_turn4_trims_turns_1_and_2(self) -> None:
"""On turn 4, turns 1 and 2 outputs should both be trimmed."""
large = "x" * 1000
items = [
_user("q1"),
_func_call("c1", "s"),
_func_output("c1", large),
_assistant("a1"),
_user("q2"),
_func_call("c2", "s"),
_func_output("c2", large),
_assistant("a2"),
_user("q3"),
_func_call("c3", "s"),
_func_output("c3", large),
_assistant("a3"),
_user("q4"),
_assistant("a4"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
# Turns 1 and 2 trimmed
assert "[Trimmed:" in _output(result, 2)
assert "[Trimmed:" in _output(result, 6)
# Turn 3 (recent) preserved
assert _output(result, 10) == large
# ---------------------------------------------------------------------------
# Edge cases
# ---------------------------------------------------------------------------
class TestEdgeCases:
def test_skips_trim_when_summary_would_exceed_original(self) -> None:
"""When preview_chars is large relative to the output, the summary can be
longer than the original. In that case the output should be left untouched."""
# Output is 501 chars (just above default max_output_chars=500).
# With preview_chars=490, the summary header + 490-char preview + "..." will
# easily exceed 501 chars, so trimming should be skipped.
borderline = "x" * 501
items = [
_user("q1"),
_func_call("c1", "search"),
_func_output("c1", borderline),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(max_output_chars=500, preview_chars=490)
result = trimmer(_make_data(items))
# Output left untouched because summary would be longer
assert _output(result, 2) == borderline
def test_unknown_tool_name_fallback(self) -> None:
"""When a function_call_output has no matching function_call, the summary
should show 'unknown_tool' instead of a blank name."""
large = "x" * 1000
# Deliberately omit the _func_call so the call_id has no name mapping
items = [
_user("q1"),
_func_output("orphan_id", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer()
result = trimmer(_make_data(items))
trimmed = _output(result, 1)
assert "unknown_tool" in trimmed
assert "[Trimmed:" in trimmed
def test_unresolved_tool_skipped_with_allowlist(self) -> None:
"""When trimmable_tools is set and the tool name can't be resolved,
the output should NOT be trimmed (empty string won't match the allowlist)."""
large = "x" * 1000
items = [
_user("q1"),
_func_output("orphan_id", large),
_assistant("a1"),
_user("q2"),
_assistant("a2"),
_user("q3"),
_assistant("a3"),
]
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"search"}))
result = trimmer(_make_data(items))
# Unresolved tool name is "" which is not in the allowlist — left untouched
assert _output(result, 1) == large
+367
View File
@@ -0,0 +1,367 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any
from openai.types.responses import (
Response,
ResponseApplyPatchToolCall,
ResponseCompletedEvent,
ResponseContentPartAddedEvent,
ResponseContentPartDoneEvent,
ResponseCreatedEvent,
ResponseFunctionCallArgumentsDeltaEvent,
ResponseFunctionCallArgumentsDoneEvent,
ResponseFunctionToolCall,
ResponseInProgressEvent,
ResponseOutputItemAddedEvent,
ResponseOutputItemDoneEvent,
ResponseOutputMessage,
ResponseOutputText,
ResponseReasoningSummaryPartAddedEvent,
ResponseReasoningSummaryPartDoneEvent,
ResponseReasoningSummaryTextDeltaEvent,
ResponseReasoningSummaryTextDoneEvent,
ResponseTextDeltaEvent,
ResponseTextDoneEvent,
ResponseUsage,
)
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
from openai.types.responses.response_reasoning_summary_part_added_event import (
Part as AddedEventPart,
)
from openai.types.responses.response_reasoning_summary_part_done_event import Part as DoneEventPart
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
from agents.agent_output import AgentOutputSchemaBase
from agents.handoffs import Handoff
from agents.items import (
ModelResponse,
TResponseInputItem,
TResponseOutputItem,
TResponseStreamEvent,
)
from agents.model_settings import ModelSettings
from agents.models.interface import Model, ModelTracing
from agents.tool import Tool
from agents.tracing import SpanError, generation_span
from agents.usage import Usage
class FakeModel(Model):
def __init__(
self,
tracing_enabled: bool = False,
initial_output: list[TResponseOutputItem] | Exception | None = None,
):
if initial_output is None:
initial_output = []
self.turn_outputs: list[list[TResponseOutputItem] | Exception] = (
[initial_output] if initial_output else []
)
self.tracing_enabled = tracing_enabled
self.last_turn_args: dict[str, Any] = {}
self.first_turn_args: dict[str, Any] | None = None
self.hardcoded_usage: Usage | None = None
def set_hardcoded_usage(self, usage: Usage):
self.hardcoded_usage = usage
def set_next_output(self, output: list[TResponseOutputItem] | Exception):
self.turn_outputs.append(output)
def add_multiple_turn_outputs(self, outputs: list[list[TResponseOutputItem] | Exception]):
self.turn_outputs.extend(outputs)
def get_next_output(self) -> list[TResponseOutputItem] | Exception:
if not self.turn_outputs:
return []
return self.turn_outputs.pop(0)
async def get_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None,
conversation_id: str | None,
prompt: Any | None,
) -> ModelResponse:
turn_args = {
"system_instructions": system_instructions,
"input": input,
"model_settings": model_settings,
"tools": tools,
"output_schema": output_schema,
"previous_response_id": previous_response_id,
"conversation_id": conversation_id,
}
if self.first_turn_args is None:
self.first_turn_args = turn_args.copy()
self.last_turn_args = turn_args
with generation_span(disabled=not self.tracing_enabled) as span:
output = self.get_next_output()
if isinstance(output, Exception):
span.set_error(
SpanError(
message="Error",
data={
"name": output.__class__.__name__,
"message": str(output),
},
)
)
raise output
converted_output = []
for item in output:
if isinstance(item, dict) and item.get("type") == "apply_patch_call":
call_id = str(item.get("call_id") or item.get("id") or "")
converted_output.append(
ResponseApplyPatchToolCall(
type="apply_patch_call",
id=str(item.get("id") or call_id),
call_id=call_id,
status=item.get("status") or "completed",
operation=item.get("operation"),
)
)
else:
converted_output.append(item)
return ModelResponse(
output=converted_output,
usage=self.hardcoded_usage or Usage(),
response_id="resp-789",
)
async def stream_response(
self,
system_instructions: str | None,
input: str | list[TResponseInputItem],
model_settings: ModelSettings,
tools: list[Tool],
output_schema: AgentOutputSchemaBase | None,
handoffs: list[Handoff],
tracing: ModelTracing,
*,
previous_response_id: str | None = None,
conversation_id: str | None = None,
prompt: Any | None = None,
) -> AsyncIterator[TResponseStreamEvent]:
turn_args = {
"system_instructions": system_instructions,
"input": input,
"model_settings": model_settings,
"tools": tools,
"output_schema": output_schema,
"previous_response_id": previous_response_id,
"conversation_id": conversation_id,
}
if self.first_turn_args is None:
self.first_turn_args = turn_args.copy()
self.last_turn_args = turn_args
with generation_span(disabled=not self.tracing_enabled) as span:
output = self.get_next_output()
if isinstance(output, Exception):
span.set_error(
SpanError(
message="Error",
data={
"name": output.__class__.__name__,
"message": str(output),
},
)
)
raise output
response = get_response_obj(output, usage=self.hardcoded_usage)
sequence_number = 0
yield ResponseCreatedEvent(
type="response.created",
response=response,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseInProgressEvent(
type="response.in_progress",
response=response,
sequence_number=sequence_number,
)
sequence_number += 1
for output_index, output_item in enumerate(output):
yield ResponseOutputItemAddedEvent(
type="response.output_item.added",
item=output_item,
output_index=output_index,
sequence_number=sequence_number,
)
sequence_number += 1
if isinstance(output_item, ResponseReasoningItem):
if output_item.summary:
for summary_index, summary in enumerate(output_item.summary):
yield ResponseReasoningSummaryPartAddedEvent(
type="response.reasoning_summary_part.added",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
part=AddedEventPart(text=summary.text, type=summary.type),
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseReasoningSummaryTextDeltaEvent(
type="response.reasoning_summary_text.delta",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
delta=summary.text,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseReasoningSummaryTextDoneEvent(
type="response.reasoning_summary_text.done",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
text=summary.text,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseReasoningSummaryPartDoneEvent(
type="response.reasoning_summary_part.done",
item_id=output_item.id,
output_index=output_index,
summary_index=summary_index,
part=DoneEventPart(text=summary.text, type=summary.type),
sequence_number=sequence_number,
)
sequence_number += 1
elif isinstance(output_item, ResponseFunctionToolCall):
yield ResponseFunctionCallArgumentsDeltaEvent(
type="response.function_call_arguments.delta",
item_id=output_item.call_id,
output_index=output_index,
delta=output_item.arguments,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseFunctionCallArgumentsDoneEvent(
type="response.function_call_arguments.done",
item_id=output_item.call_id,
output_index=output_index,
arguments=output_item.arguments,
name=output_item.name,
sequence_number=sequence_number,
)
sequence_number += 1
elif isinstance(output_item, ResponseOutputMessage):
for content_index, content_part in enumerate(output_item.content or []):
if isinstance(content_part, ResponseOutputText):
yield ResponseContentPartAddedEvent(
type="response.content_part.added",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
part=content_part,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseTextDeltaEvent(
type="response.output_text.delta",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
delta=content_part.text,
logprobs=[],
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseTextDoneEvent(
type="response.output_text.done",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
text=content_part.text,
logprobs=[],
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseContentPartDoneEvent(
type="response.content_part.done",
item_id=output_item.id,
output_index=output_index,
content_index=content_index,
part=content_part,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseOutputItemDoneEvent(
type="response.output_item.done",
item=output_item,
output_index=output_index,
sequence_number=sequence_number,
)
sequence_number += 1
yield ResponseCompletedEvent(
type="response.completed",
response=response,
sequence_number=sequence_number,
)
class PromptCacheFakeModel(FakeModel):
def _supports_default_prompt_cache_key(self) -> bool:
return True
def get_response_obj(
output: list[TResponseOutputItem],
response_id: str | None = None,
usage: Usage | None = None,
) -> Response:
return Response(
id=response_id or "resp-789",
created_at=123,
model="test_model",
object="response",
output=output,
tool_choice="none",
tools=[],
top_p=None,
parallel_tool_calls=False,
usage=ResponseUsage(
input_tokens=usage.input_tokens if usage else 0,
output_tokens=usage.output_tokens if usage else 0,
total_tokens=usage.total_tokens if usage else 0,
input_tokens_details=InputTokensDetails.model_validate(
{"cache_write_tokens": 0, "cached_tokens": 0}
),
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
),
)
View File
+30
View File
@@ -0,0 +1,30 @@
from collections.abc import AsyncIterator
from fastapi import FastAPI
from starlette.responses import StreamingResponse
from agents import Agent, Runner, RunResultStreaming
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant.",
)
app = FastAPI()
@app.post("/stream")
async def stream():
result = Runner.run_streamed(agent, input="Tell me a joke")
stream_handler = StreamHandler(result)
return StreamingResponse(stream_handler.stream_events(), media_type="application/x-ndjson")
class StreamHandler:
def __init__(self, result: RunResultStreaming):
self.result = result
async def stream_events(self) -> AsyncIterator[str]:
async for event in self.result.stream_events():
yield f"{event.type}\n\n"
+41
View File
@@ -0,0 +1,41 @@
import pytest
from httpx import ASGITransport, AsyncClient
from inline_snapshot import snapshot
from ..fake_model import FakeModel
from ..test_responses import get_text_message
from .streaming_app import agent, app
@pytest.mark.asyncio
async def test_streaming_context():
"""This ensures that FastAPI streaming works. The context for this test is that the Runner
method was called in one async context, and the streaming was ended in another context,
leading to a tracing error because the context was closed in the wrong context. This test
ensures that this actually works.
"""
model = FakeModel()
agent.model = model
model.set_next_output([get_text_message("done")])
transport = ASGITransport(app)
async with AsyncClient(transport=transport, base_url="http://test") as ac:
async with ac.stream("POST", "/stream") as r:
assert r.status_code == 200
body = (await r.aread()).decode("utf-8")
lines = [line for line in body.splitlines() if line]
assert lines == snapshot(
[
"agent_updated_stream_event",
"raw_response_event", # ResponseCreatedEvent
"raw_response_event", # ResponseInProgressEvent
"raw_response_event", # ResponseOutputItemAddedEvent
"raw_response_event", # ResponseContentPartAddedEvent
"raw_response_event", # ResponseTextDeltaEvent
"raw_response_event", # ResponseTextDoneEvent
"raw_response_event", # ResponseContentPartDoneEvent
"raw_response_event", # ResponseOutputItemDoneEvent
"raw_response_event", # ResponseCompletedEvent
"run_item_stream_event", # MessageOutputItem
]
)
View File
+164
View File
@@ -0,0 +1,164 @@
from __future__ import annotations
import asyncio
import json
import shutil
from typing import Any
from mcp import Tool as MCPTool
from mcp.types import (
CallToolResult,
Content,
GetPromptResult,
ListPromptsResult,
ListResourcesResult,
ListResourceTemplatesResult,
PromptMessage,
ReadResourceResult,
TextContent,
)
from agents.mcp import MCPServer
from agents.mcp.server import _UNSET, _MCPServerWithClientSession, _UnsetType
from agents.mcp.util import MCPToolCustomDataExtractor, MCPToolMetaResolver, ToolFilter
from agents.tool import ToolErrorFunction
tee = shutil.which("tee") or ""
assert tee, "tee not found"
# Added dummy stream classes for patching stdio_client to avoid real I/O during tests
class DummyStream:
async def send(self, msg):
pass
async def receive(self):
raise Exception("Dummy receive not implemented")
class DummyStreamsContextManager:
async def __aenter__(self):
return (DummyStream(), DummyStream())
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
class _TestFilterServer(_MCPServerWithClientSession):
"""Minimal implementation of _MCPServerWithClientSession for testing tool filtering"""
def __init__(self, tool_filter: ToolFilter, server_name: str):
# Initialize parent class properly to avoid type errors
super().__init__(
cache_tools_list=False,
client_session_timeout_seconds=None,
tool_filter=tool_filter,
)
self._server_name: str = server_name
# Override some attributes for test isolation
self.session = None
self._cleanup_lock = asyncio.Lock()
def create_streams(self):
raise NotImplementedError("Not needed for filtering tests")
@property
def name(self) -> str:
return self._server_name
class FakeMCPServer(MCPServer):
def __init__(
self,
tools: list[MCPTool] | None = None,
tool_filter: ToolFilter = None,
server_name: str = "fake_mcp_server",
require_approval: object | None = None,
failure_error_function: ToolErrorFunction | None | _UnsetType = _UNSET,
tool_meta_resolver: MCPToolMetaResolver | None = None,
custom_data_extractor: MCPToolCustomDataExtractor | None = None,
):
super().__init__(
use_structured_content=False,
require_approval=require_approval, # type: ignore[arg-type]
failure_error_function=failure_error_function,
tool_meta_resolver=tool_meta_resolver,
custom_data_extractor=custom_data_extractor,
)
self.tools: list[MCPTool] = tools or []
self.tool_calls: list[str] = []
self.tool_results: list[str] = []
self.tool_metas: list[dict[str, Any] | None] = []
self.tool_filter = tool_filter
self._server_name = server_name
self._custom_content: list[Content] | None = None
self._response_meta: dict[str, Any] | None = None
def add_tool(self, name: str, input_schema: dict[str, Any]):
self.tools.append(MCPTool(name=name, inputSchema=input_schema))
async def connect(self):
pass
async def cleanup(self):
pass
async def list_tools(self, run_context=None, agent=None):
tools = self.tools
# Apply tool filtering using the REAL implementation
if self.tool_filter is not None:
# Use the real _MCPServerWithClientSession filtering logic
filter_server = _TestFilterServer(self.tool_filter, self.name)
tools = await filter_server._apply_tool_filter(tools, run_context, agent)
return tools
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None,
meta: dict[str, Any] | None = None,
) -> CallToolResult:
self.tool_calls.append(tool_name)
self.tool_results.append(f"result_{tool_name}_{json.dumps(arguments)}")
self.tool_metas.append(meta)
# Allow testing custom content scenarios
if self._custom_content is not None:
return CallToolResult(content=self._custom_content)
return CallToolResult(
content=[TextContent(text=self.tool_results[-1], type="text")],
_meta=self._response_meta,
)
async def list_prompts(self, run_context=None, agent=None) -> ListPromptsResult:
"""Return empty list of prompts for fake server"""
return ListPromptsResult(prompts=[])
async def get_prompt(
self, name: str, arguments: dict[str, Any] | None = None
) -> GetPromptResult:
"""Return a simple prompt result for fake server"""
content = f"Fake prompt content for {name}"
message = PromptMessage(role="user", content=TextContent(type="text", text=content))
return GetPromptResult(description=f"Fake prompt: {name}", messages=[message])
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
"""Return empty list of resources for fake server."""
return ListResourcesResult(resources=[])
async def list_resource_templates(
self, cursor: str | None = None
) -> ListResourceTemplatesResult:
"""Return empty list of resource templates for fake server."""
return ListResourceTemplatesResult(resourceTemplates=[])
async def read_resource(self, uri: str) -> ReadResourceResult:
"""Return empty resource contents for fake server."""
return ReadResourceResult(contents=[])
@property
def name(self) -> str:
return self._server_name
+63
View File
@@ -0,0 +1,63 @@
from unittest.mock import AsyncMock, patch
import pytest
from mcp.types import ListToolsResult, Tool as MCPTool
from agents import Agent
from agents.mcp import MCPServerStdio
from agents.run_context import RunContextWrapper
from .helpers import DummyStreamsContextManager, tee
@pytest.mark.asyncio
@patch("mcp.client.stdio.stdio_client", return_value=DummyStreamsContextManager())
@patch("mcp.client.session.ClientSession.initialize", new_callable=AsyncMock, return_value=None)
@patch("mcp.client.session.ClientSession.list_tools")
async def test_server_caching_works(
mock_list_tools: AsyncMock, mock_initialize: AsyncMock, mock_stdio_client
):
"""Test that if we turn caching on, the list of tools is cached and not fetched from the server
on each call to `list_tools()`.
"""
server = MCPServerStdio(
params={
"command": tee,
},
cache_tools_list=True,
)
tools = [
MCPTool(name="tool1", inputSchema={}),
MCPTool(name="tool2", inputSchema={}),
]
mock_list_tools.return_value = ListToolsResult(tools=tools)
async with server:
# Create test context and agent
run_context = RunContextWrapper(context=None)
agent = Agent(name="test_agent", instructions="Test agent")
# Call list_tools() multiple times
result_tools = await server.list_tools(run_context, agent)
assert result_tools == tools
assert mock_list_tools.call_count == 1, "list_tools() should have been called once"
# Call list_tools() again, should return the cached value
result_tools = await server.list_tools(run_context, agent)
assert result_tools == tools
assert mock_list_tools.call_count == 1, "list_tools() should not have been called again"
# Invalidate the cache and call list_tools() again
server.invalidate_tools_cache()
result_tools = await server.list_tools(run_context, agent)
assert result_tools == tools
assert mock_list_tools.call_count == 2, "list_tools() should be called again"
# Without invalidating the cache, calling list_tools() again should return the cached value
result_tools = await server.list_tools(run_context, agent)
assert result_tools == tools
+574
View File
@@ -0,0 +1,574 @@
import asyncio
import sys
from contextlib import asynccontextmanager
from typing import cast
import httpx
import pytest
from anyio import ClosedResourceError
from mcp import ClientSession, Tool as MCPTool
from mcp.shared.exceptions import McpError
from mcp.types import CallToolResult, ErrorData, GetPromptResult, ListPromptsResult, ListToolsResult
from agents.exceptions import UserError
from agents.mcp.server import MCPServerStreamableHttp, _MCPServerWithClientSession
if sys.version_info < (3, 11):
from exceptiongroup import BaseExceptionGroup # pyright: ignore[reportMissingImports]
class DummySession:
def __init__(self, fail_call_tool: int = 0, fail_list_tools: int = 0):
self.fail_call_tool = fail_call_tool
self.fail_list_tools = fail_list_tools
self.call_tool_attempts = 0
self.list_tools_attempts = 0
async def call_tool(self, tool_name, arguments, meta=None):
self.call_tool_attempts += 1
if self.call_tool_attempts <= self.fail_call_tool:
raise RuntimeError("call_tool failure")
return CallToolResult(content=[])
async def list_tools(self):
self.list_tools_attempts += 1
if self.list_tools_attempts <= self.fail_list_tools:
raise RuntimeError("list_tools failure")
return ListToolsResult(tools=[MCPTool(name="tool", inputSchema={})])
class DummyServer(_MCPServerWithClientSession):
def __init__(self, session: DummySession, retries: int, *, serialize_requests: bool = False):
super().__init__(
cache_tools_list=False,
client_session_timeout_seconds=None,
max_retry_attempts=retries,
retry_backoff_seconds_base=0,
)
self.session = cast(ClientSession, session)
self._serialize_session_requests = serialize_requests
def create_streams(self):
raise NotImplementedError
@property
def name(self) -> str:
return "dummy"
@pytest.mark.asyncio
async def test_call_tool_retries_until_success():
session = DummySession(fail_call_tool=2)
server = DummyServer(session=session, retries=2)
result = await server.call_tool("tool", None)
assert isinstance(result, CallToolResult)
assert session.call_tool_attempts == 3
@pytest.mark.asyncio
async def test_list_tools_unlimited_retries():
session = DummySession(fail_list_tools=3)
server = DummyServer(session=session, retries=-1)
tools = await server.list_tools()
assert len(tools) == 1
assert tools[0].name == "tool"
assert session.list_tools_attempts == 4
@pytest.mark.asyncio
async def test_call_tool_validates_required_parameters_before_remote_call():
session = DummySession()
server = DummyServer(session=session, retries=0)
server._tools_list = [ # noqa: SLF001
MCPTool(
name="tool",
inputSchema={
"type": "object",
"properties": {"param_a": {"type": "string"}},
"required": ["param_a"],
},
)
]
with pytest.raises(UserError, match="missing required parameters: param_a"):
await server.call_tool("tool", {})
assert session.call_tool_attempts == 0
@pytest.mark.asyncio
async def test_call_tool_with_required_parameters_still_calls_remote_tool():
session = DummySession()
server = DummyServer(session=session, retries=0)
server._tools_list = [ # noqa: SLF001
MCPTool(
name="tool",
inputSchema={
"type": "object",
"properties": {"param_a": {"type": "string"}},
"required": ["param_a"],
},
)
]
result = await server.call_tool("tool", {"param_a": "value"})
assert isinstance(result, CallToolResult)
assert session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_call_tool_skips_validation_when_tool_is_missing_from_cache():
session = DummySession()
server = DummyServer(session=session, retries=0)
server._tools_list = [MCPTool(name="different_tool", inputSchema={"required": ["param_a"]})] # noqa: SLF001
await server.call_tool("tool", {})
assert session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_call_tool_skips_validation_when_required_list_is_absent():
session = DummySession()
server = DummyServer(session=session, retries=0)
server._tools_list = [MCPTool(name="tool", inputSchema={"type": "object"})] # noqa: SLF001
await server.call_tool("tool", None)
assert session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_call_tool_validates_required_parameters_when_arguments_is_none():
session = DummySession()
server = DummyServer(session=session, retries=0)
server._tools_list = [MCPTool(name="tool", inputSchema={"required": ["param_a"]})] # noqa: SLF001
with pytest.raises(UserError, match="missing required parameters: param_a"):
await server.call_tool("tool", None)
assert session.call_tool_attempts == 0
@pytest.mark.asyncio
async def test_call_tool_rejects_non_object_arguments_before_remote_call():
session = DummySession()
server = DummyServer(session=session, retries=0)
server._tools_list = [MCPTool(name="tool", inputSchema={"required": ["param_a"]})] # noqa: SLF001
with pytest.raises(UserError, match="arguments must be an object"):
await server.call_tool("tool", cast(dict[str, object] | None, ["bad"]))
assert session.call_tool_attempts == 0
class ConcurrentCancellationSession:
def __init__(self):
self._slow_task: asyncio.Task[CallToolResult] | None = None
self._slow_started = asyncio.Event()
async def call_tool(self, tool_name, arguments, meta=None):
if tool_name == "slow":
self._slow_task = cast(asyncio.Task[CallToolResult], asyncio.current_task())
self._slow_started.set()
await asyncio.sleep(0.1)
return CallToolResult(content=[])
await self._slow_started.wait()
assert self._slow_task is not None
self._slow_task.cancel()
raise RuntimeError("synthetic request failure")
class CancelledToolSession:
async def call_tool(self, tool_name, arguments, meta=None):
raise asyncio.CancelledError("synthetic call cancellation")
class MixedExceptionGroupSession:
async def call_tool(self, tool_name, arguments, meta=None):
req = httpx.Request("POST", "https://example.test/mcp")
resp = httpx.Response(401, request=req)
raise BaseExceptionGroup(
"mixed request failure",
[
asyncio.CancelledError("synthetic call cancellation"),
httpx.HTTPStatusError("HTTP error 401", request=req, response=resp),
],
)
class SharedHttpStatusSession:
def __init__(self, status_code: int):
self.status_code = status_code
async def call_tool(self, tool_name, arguments, meta=None):
req = httpx.Request("POST", "https://example.test/mcp")
resp = httpx.Response(self.status_code, request=req)
raise httpx.HTTPStatusError(
f"HTTP error {self.status_code}",
request=req,
response=resp,
)
class TimeoutSession:
def __init__(self, message: str = "timed out"):
self.call_tool_attempts = 0
self.message = message
async def call_tool(self, tool_name, arguments, meta=None):
self.call_tool_attempts += 1
raise httpx.TimeoutException(self.message)
class ClosedResourceSession:
def __init__(self):
self.call_tool_attempts = 0
async def call_tool(self, tool_name, arguments, meta=None):
self.call_tool_attempts += 1
raise ClosedResourceError()
class McpRequestTimeoutSession:
def __init__(self, message: str = "timed out"):
self.call_tool_attempts = 0
self.message = message
async def call_tool(self, tool_name, arguments, meta=None):
self.call_tool_attempts += 1
raise McpError(
ErrorData(code=httpx.codes.REQUEST_TIMEOUT, message=self.message),
)
class IsolatedRetrySession:
def __init__(self):
self.call_tool_attempts = 0
async def call_tool(self, tool_name, arguments, meta=None):
self.call_tool_attempts += 1
return CallToolResult(content=[])
class HangingSession:
async def call_tool(self, tool_name, arguments, meta=None):
await asyncio.sleep(10)
class DummyStreamableHttpServer(MCPServerStreamableHttp):
def __init__(self, shared_session: object, isolated_session: object):
super().__init__(
params={"url": "https://example.test/mcp"},
client_session_timeout_seconds=None,
max_retry_attempts=0,
)
self.session = cast(ClientSession, shared_session)
self._isolated_session = cast(ClientSession, isolated_session)
@asynccontextmanager
async def _isolated_client_session(self):
yield self._isolated_session
async def list_tools(self, run_context=None, agent=None):
return [MCPTool(name="tool", inputSchema={})]
async def list_prompts(self):
return ListPromptsResult(prompts=[])
async def get_prompt(self, name, arguments=None):
raise NotImplementedError
class IsolatedSessionEnterFailure:
def __init__(self, server: "EnterFailingStreamableHttpServer", message: str):
self.server = server
self.message = message
async def __aenter__(self):
self.server.isolated_enter_attempts += 1
raise httpx.TimeoutException(self.message)
async def __aexit__(self, exc_type, exc, tb):
return False
class EnterFailingStreamableHttpServer(DummyStreamableHttpServer):
def __init__(self, shared_session: object, *, isolated_message: str):
super().__init__(shared_session, IsolatedRetrySession())
self.isolated_enter_attempts = 0
self._isolated_message = isolated_message
def _isolated_client_session(self):
return IsolatedSessionEnterFailure(self, self._isolated_message)
@pytest.mark.asyncio
async def test_streamable_http_retries_cancelled_request_on_isolated_session():
shared_session = CancelledToolSession()
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(shared_session, isolated_session)
server.max_retry_attempts = 1
result = await server.call_tool("tool", None)
assert isinstance(result, CallToolResult)
assert isolated_session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_streamable_http_retries_5xx_on_isolated_session():
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(SharedHttpStatusSession(504), isolated_session)
server.max_retry_attempts = 1
result = await server.call_tool("tool", None)
assert isinstance(result, CallToolResult)
assert isolated_session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_streamable_http_retries_closed_resource_on_isolated_session():
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(ClosedResourceSession(), isolated_session)
server.max_retry_attempts = 1
result = await server.call_tool("tool", None)
assert isinstance(result, CallToolResult)
assert isolated_session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_streamable_http_retries_mcp_408_on_isolated_session():
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(
McpRequestTimeoutSession("Timed out while waiting for response to ClientRequest."),
isolated_session,
)
server.max_retry_attempts = 1
result = await server.call_tool("tool", None)
assert isinstance(result, CallToolResult)
assert isolated_session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_streamable_http_does_not_retry_4xx_on_isolated_session():
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(SharedHttpStatusSession(401), isolated_session)
with pytest.raises(UserError, match="HTTP error 401"):
await server.call_tool("tool", None)
assert isolated_session.call_tool_attempts == 0
@pytest.mark.asyncio
async def test_streamable_http_does_not_isolated_retry_without_retry_budget():
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(CancelledToolSession(), isolated_session)
server.max_retry_attempts = 0
with pytest.raises(asyncio.CancelledError):
await server.call_tool("tool", None)
assert isolated_session.call_tool_attempts == 0
@pytest.mark.asyncio
async def test_streamable_http_counts_isolated_retry_against_retry_budget():
shared_session = TimeoutSession("shared timed out")
isolated_session = TimeoutSession("isolated timed out")
server = DummyStreamableHttpServer(shared_session, isolated_session)
server.max_retry_attempts = 2
with pytest.raises(httpx.TimeoutException, match="shared timed out"):
await server.call_tool("tool", None)
assert shared_session.call_tool_attempts == 2
assert isolated_session.call_tool_attempts == 1
@pytest.mark.asyncio
async def test_streamable_http_counts_isolated_session_setup_failure_against_retry_budget():
shared_session = TimeoutSession("shared timed out")
server = EnterFailingStreamableHttpServer(
shared_session,
isolated_message="isolated setup timed out",
)
server.max_retry_attempts = 2
with pytest.raises(httpx.TimeoutException, match="shared timed out"):
await server.call_tool("tool", None)
assert shared_session.call_tool_attempts == 2
assert server.isolated_enter_attempts == 1
@pytest.mark.asyncio
async def test_streamable_http_does_not_retry_mixed_exception_groups():
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(MixedExceptionGroupSession(), isolated_session)
server.max_retry_attempts = 1
with pytest.raises(UserError, match="HTTP error 401"):
await server.call_tool("tool", None)
assert isolated_session.call_tool_attempts == 0
@pytest.mark.asyncio
async def test_streamable_http_preserves_outer_cancellation():
isolated_session = IsolatedRetrySession()
server = DummyStreamableHttpServer(HangingSession(), isolated_session)
task = asyncio.create_task(server.call_tool("slow", None))
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert isolated_session.call_tool_attempts == 0
@pytest.mark.asyncio
async def test_streamable_http_preserves_outer_cancellation_during_isolated_retry():
server = DummyStreamableHttpServer(CancelledToolSession(), HangingSession())
server.max_retry_attempts = 1
task = asyncio.create_task(server.call_tool("tool", None))
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
class ConcurrentPromptCancellationSession(ConcurrentCancellationSession):
async def list_tools(self):
return ListToolsResult(tools=[MCPTool(name="tool", inputSchema={})])
async def list_prompts(self):
await self._slow_started.wait()
assert self._slow_task is not None
self._slow_task.cancel()
raise RuntimeError("synthetic request failure")
async def get_prompt(self, name, arguments=None):
await self._slow_started.wait()
assert self._slow_task is not None
self._slow_task.cancel()
raise RuntimeError("synthetic request failure")
class OverlapTrackingSession:
def __init__(self):
self.in_flight = 0
self.max_in_flight = 0
@asynccontextmanager
async def _enter_request(self):
self.in_flight += 1
self.max_in_flight = max(self.max_in_flight, self.in_flight)
try:
await asyncio.sleep(0.02)
yield
finally:
self.in_flight -= 1
async def call_tool(self, tool_name, arguments, meta=None):
async with self._enter_request():
return CallToolResult(content=[])
async def list_prompts(self):
async with self._enter_request():
return ListPromptsResult(prompts=[])
async def get_prompt(self, name, arguments=None):
async with self._enter_request():
return GetPromptResult(
description=None,
messages=[],
)
class DummyPromptStreamableHttpServer(DummyStreamableHttpServer):
def __init__(
self,
shared_session: OverlapTrackingSession,
isolated_session: IsolatedRetrySession,
):
super().__init__(shared_session, isolated_session)
self.session = cast(ClientSession, shared_session)
async def list_prompts(self):
session = self.session
assert session is not None
return await self._maybe_serialize_request(lambda: session.list_prompts())
async def get_prompt(self, name, arguments=None):
session = self.session
assert session is not None
return await self._maybe_serialize_request(lambda: session.get_prompt(name, arguments))
@pytest.mark.asyncio
async def test_serialized_session_requests_prevent_sibling_cancellation():
session = ConcurrentPromptCancellationSession()
server = DummyServer(session=cast(DummySession, session), retries=0, serialize_requests=True)
results = await asyncio.gather(
server.call_tool("slow", None),
server.call_tool("fail", None),
return_exceptions=True,
)
assert isinstance(results[0], CallToolResult)
assert isinstance(results[1], RuntimeError)
@pytest.mark.asyncio
@pytest.mark.parametrize("prompt_method", ["list_prompts", "get_prompt"])
async def test_serialized_prompt_requests_prevent_tool_cancellation(prompt_method: str):
session = ConcurrentPromptCancellationSession()
server = DummyServer(session=cast(DummySession, session), retries=0, serialize_requests=True)
prompt_request = (
server.list_prompts() if prompt_method == "list_prompts" else server.get_prompt("prompt")
)
results = await asyncio.gather(
server.call_tool("slow", None),
prompt_request,
return_exceptions=True,
)
assert isinstance(results[0], CallToolResult)
assert isinstance(results[1], RuntimeError)
@pytest.mark.asyncio
@pytest.mark.parametrize("prompt_method", ["list_prompts", "get_prompt"])
async def test_streamable_http_serializes_call_tool_with_prompt_requests(prompt_method: str):
shared_session = OverlapTrackingSession()
isolated_session = IsolatedRetrySession()
server = DummyPromptStreamableHttpServer(shared_session, isolated_session)
prompt_request = (
server.list_prompts() if prompt_method == "list_prompts" else server.get_prompt("prompt")
)
results = await asyncio.gather(
server.call_tool("slow", None),
prompt_request,
return_exceptions=True,
)
assert isinstance(results[0], CallToolResult)
if prompt_method == "list_prompts":
assert isinstance(results[1], ListPromptsResult)
else:
assert isinstance(results[1], GetPromptResult)
assert shared_session.max_in_flight == 1
assert isolated_session.call_tool_attempts == 0
+69
View File
@@ -0,0 +1,69 @@
from unittest.mock import AsyncMock, patch
import pytest
from mcp.types import ListToolsResult, Tool as MCPTool
from agents.mcp import MCPServerStdio
from .helpers import DummyStreamsContextManager, tee
@pytest.mark.asyncio
@patch("mcp.client.stdio.stdio_client", return_value=DummyStreamsContextManager())
@patch("mcp.client.session.ClientSession.initialize", new_callable=AsyncMock, return_value=None)
@patch("mcp.client.session.ClientSession.list_tools")
async def test_async_ctx_manager_works(
mock_list_tools: AsyncMock, mock_initialize: AsyncMock, mock_stdio_client
):
"""Test that the async context manager works."""
server = MCPServerStdio(
params={
"command": tee,
},
cache_tools_list=True,
)
tools = [
MCPTool(name="tool1", inputSchema={}),
MCPTool(name="tool2", inputSchema={}),
]
mock_list_tools.return_value = ListToolsResult(tools=tools)
assert server.session is None, "Server should not be connected"
async with server:
assert server.session is not None, "Server should be connected"
assert server.session is None, "Server should be disconnected"
@pytest.mark.asyncio
@patch("mcp.client.stdio.stdio_client", return_value=DummyStreamsContextManager())
@patch("mcp.client.session.ClientSession.initialize", new_callable=AsyncMock, return_value=None)
@patch("mcp.client.session.ClientSession.list_tools")
async def test_manual_connect_disconnect_works(
mock_list_tools: AsyncMock, mock_initialize: AsyncMock, mock_stdio_client
):
"""Test that the async context manager works."""
server = MCPServerStdio(
params={
"command": tee,
},
cache_tools_list=True,
)
tools = [
MCPTool(name="tool1", inputSchema={}),
MCPTool(name="tool2", inputSchema={}),
]
mock_list_tools.return_value = ListToolsResult(tools=tools)
assert server.session is None, "Server should not be connected"
await server.connect()
assert server.session is not None, "Server should be connected"
await server.cleanup()
assert server.session is None, "Server should be disconnected"
+242
View File
@@ -0,0 +1,242 @@
import asyncio
import pytest
from mcp.types import Tool as MCPTool
from agents import Agent, RunContextWrapper, Runner
from agents.exceptions import UserError
from ..fake_model import FakeModel
from ..test_responses import get_function_tool_call, get_text_message
from ..utils.hitl import queue_function_call_and_text, resume_after_first_approval
from .helpers import FakeMCPServer
@pytest.mark.asyncio
async def test_mcp_require_approval_pauses_and_resumes():
"""MCP servers should honor require_approval for non-hosted tools."""
server = FakeMCPServer(require_approval="always")
server.add_tool("add", {"type": "object", "properties": {}})
model = FakeModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
model,
get_function_tool_call("add", "{}"),
followup=[get_text_message("done")],
)
first = await Runner.run(agent, "call add")
assert first.interruptions, "MCP tool should request approval"
assert first.interruptions[0].tool_name == "add"
resumed = await resume_after_first_approval(agent, first, always_approve=True)
assert not resumed.interruptions
assert server.tool_calls == ["add"]
assert resumed.final_output == "done"
@pytest.mark.asyncio
async def test_mcp_require_approval_tool_lists():
"""TS-style requireApproval toolNames should map to needs_approval."""
require_approval: dict[str, object] = {
"always": {"tool_names": ["add"]},
"never": {"tool_names": ["noop"]},
}
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("add", {"type": "object", "properties": {}})
model = FakeModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
model,
get_function_tool_call("add", "{}"),
followup=[get_text_message("done")],
)
first = await Runner.run(agent, "call add")
assert first.interruptions, "add should require approval via require_approval toolNames"
resumed = await resume_after_first_approval(agent, first, always_approve=True)
assert resumed.final_output == "done"
assert server.tool_calls == ["add"]
@pytest.mark.asyncio
async def test_mcp_require_approval_tool_mapping():
"""Tool-name require_approval mappings should map to needs_approval."""
require_approval = {"add": "always", "noop": "never"}
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("add", {"type": "object", "properties": {}})
model = FakeModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
model,
get_function_tool_call("add", "{}"),
followup=[get_text_message("done")],
)
first = await Runner.run(agent, "call add")
assert first.interruptions, "add should require approval via require_approval mapping"
resumed = await resume_after_first_approval(agent, first, always_approve=True)
assert resumed.final_output == "done"
assert server.tool_calls == ["add"]
@pytest.mark.asyncio
async def test_mcp_require_approval_mapping_allows_policy_keyword_tool_names():
"""Tool-name mappings should treat literal 'always'/'never' as tool names."""
require_approval = {"always": "always", "never": "never"}
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("always", {"type": "object", "properties": {}})
server.add_tool("never", {"type": "object", "properties": {}})
model = FakeModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
model,
get_function_tool_call("always", "{}"),
followup=[get_text_message("done")],
)
first = await Runner.run(agent, "call always")
assert first.interruptions, "tool named 'always' should require approval"
assert first.interruptions[0].tool_name == "always"
resumed = await resume_after_first_approval(agent, first, always_approve=True)
assert resumed.final_output == "done"
queue_function_call_and_text(
model,
get_function_tool_call("never", "{}"),
followup=[get_text_message("done")],
)
second = await Runner.run(agent, "call never")
assert not second.interruptions, "tool named 'never' should not require approval"
@pytest.mark.parametrize(
("require_approval", "message"),
[
("alwyas", "expected 'always' or 'never'"),
({"delete": "alwyas"}, "delete"),
(
{
"always": {"tool_names": ["delete"]},
"never": {"tool_names": ["delete"]},
},
"both always and never",
),
],
)
def test_mcp_require_approval_rejects_invalid_fail_open_policies(require_approval, message):
"""Invalid MCP approval policies should not silently disable approvals."""
with pytest.raises(UserError, match=message):
FakeMCPServer(require_approval=require_approval)
@pytest.mark.asyncio
async def test_mcp_require_approval_callable_can_allow_and_block_by_tool_name():
"""Callable policies should decide approval dynamically for each MCP tool."""
seen: list[str] = []
def require_approval(
_run_context: RunContextWrapper[object | None],
_agent: Agent,
tool: MCPTool,
) -> bool:
seen.append(tool.name)
return tool.name == "guarded"
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("guarded", {"type": "object", "properties": {}})
server.add_tool("safe", {"type": "object", "properties": {}})
model = FakeModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
model,
get_function_tool_call("guarded", "{}"),
followup=[get_text_message("guarded done")],
)
first = await Runner.run(agent, "call guarded")
assert first.interruptions, "guarded should require approval via callable policy"
assert first.interruptions[0].tool_name == "guarded"
resumed = await resume_after_first_approval(agent, first, always_approve=True)
assert resumed.final_output == "guarded done"
queue_function_call_and_text(
model,
get_function_tool_call("safe", "{}"),
followup=[get_text_message("safe done")],
)
second = await Runner.run(agent, "call safe")
assert not second.interruptions, "safe should bypass approval via callable policy"
assert second.final_output == "safe done"
assert seen == ["guarded", "guarded", "safe"]
@pytest.mark.asyncio
async def test_mcp_require_approval_async_callable_uses_run_context():
"""Async callable policies should receive the run context and be awaited."""
seen_contexts: list[object | None] = []
async def require_approval(
run_context: RunContextWrapper[dict[str, bool] | None],
_agent: Agent,
_tool,
) -> bool:
seen_contexts.append(run_context.context)
await asyncio.sleep(0)
return bool(run_context.context and run_context.context.get("needs_approval"))
server = FakeMCPServer(require_approval=require_approval)
server.add_tool("conditional", {"type": "object", "properties": {}})
model = FakeModel()
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
queue_function_call_and_text(
model,
get_function_tool_call("conditional", "{}"),
followup=[get_text_message("approved path")],
)
first = await Runner.run(agent, "call conditional", context={"needs_approval": True})
assert first.interruptions, "run context should be able to trigger approval"
resumed = await resume_after_first_approval(agent, first, always_approve=True)
assert resumed.final_output == "approved path"
queue_function_call_and_text(
model,
get_function_tool_call("conditional", "{}"),
followup=[get_text_message("no approval path")],
)
second = await Runner.run(agent, "call conditional", context={"needs_approval": False})
assert not second.interruptions, "run context should be able to skip approval"
assert second.final_output == "no approval path"
assert seen_contexts == [
{"needs_approval": True},
{"needs_approval": True},
{"needs_approval": False},
]
+179
View File
@@ -0,0 +1,179 @@
"""Tests for auth and httpx_client_factory params on MCPServerSse and MCPServerStreamableHttp."""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import httpx
import pytest
from agents.mcp import MCPServerSse, MCPServerStreamableHttp
from agents.mcp.server import _create_default_streamable_http_client
class TestMCPServerSseAuthAndFactory:
"""Tests for auth and httpx_client_factory added to MCPServerSseParams."""
@pytest.mark.asyncio
async def test_sse_default_no_auth_no_factory(self):
"""SSE create_streams falls back to the hardened default httpx_client_factory."""
with patch("agents.mcp.server.sse_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerSse(params={"url": "http://localhost:8000/sse"})
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/sse",
headers=None,
timeout=5,
sse_read_timeout=300,
httpx_client_factory=_create_default_streamable_http_client,
)
@pytest.mark.asyncio
async def test_sse_with_auth(self):
"""SSE create_streams forwards auth and still applies the hardened default factory."""
auth = httpx.BasicAuth(username="user", password="pass")
with patch("agents.mcp.server.sse_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerSse(params={"url": "http://localhost:8000/sse", "auth": auth})
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/sse",
headers=None,
timeout=5,
sse_read_timeout=300,
auth=auth,
httpx_client_factory=_create_default_streamable_http_client,
)
@pytest.mark.asyncio
async def test_sse_with_httpx_client_factory(self):
"""SSE create_streams forwards a custom httpx_client_factory when provided."""
def custom_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(verify=False) # pragma: no cover
with patch("agents.mcp.server.sse_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerSse(
params={
"url": "http://localhost:8000/sse",
"httpx_client_factory": custom_factory,
}
)
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/sse",
headers=None,
timeout=5,
sse_read_timeout=300,
httpx_client_factory=custom_factory,
)
@pytest.mark.asyncio
async def test_sse_with_auth_and_factory(self):
"""SSE create_streams forwards both auth and httpx_client_factory together."""
auth = httpx.BasicAuth(username="user", password="pass")
def custom_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(verify=False) # pragma: no cover
with patch("agents.mcp.server.sse_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerSse(
params={
"url": "http://localhost:8000/sse",
"headers": {"X-Token": "abc"},
"auth": auth,
"httpx_client_factory": custom_factory,
}
)
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/sse",
headers={"X-Token": "abc"},
timeout=5,
sse_read_timeout=300,
auth=auth,
httpx_client_factory=custom_factory,
)
class TestMCPServerStreamableHttpAuth:
"""Tests for the auth parameter added to MCPServerStreamableHttpParams."""
@pytest.mark.asyncio
async def test_streamable_http_default_no_auth(self):
"""StreamableHttp create_streams omits auth when not provided."""
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(params={"url": "http://localhost:8000/mcp"})
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/mcp",
headers=None,
timeout=5,
sse_read_timeout=300,
terminate_on_close=True,
httpx_client_factory=_create_default_streamable_http_client,
)
@pytest.mark.asyncio
async def test_streamable_http_with_auth(self):
"""StreamableHttp create_streams forwards the auth parameter when provided."""
auth = httpx.BasicAuth(username="user", password="pass")
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={"url": "http://localhost:8000/mcp", "auth": auth}
)
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/mcp",
headers=None,
timeout=5,
sse_read_timeout=300,
terminate_on_close=True,
httpx_client_factory=_create_default_streamable_http_client,
auth=auth,
)
@pytest.mark.asyncio
async def test_streamable_http_with_auth_and_factory(self):
"""StreamableHttp create_streams forwards both auth and httpx_client_factory."""
auth = httpx.BasicAuth(username="user", password="pass")
def custom_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(verify=False) # pragma: no cover
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "http://localhost:8000/mcp",
"auth": auth,
"httpx_client_factory": custom_factory,
}
)
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/mcp",
headers=None,
timeout=5,
sse_read_timeout=300,
terminate_on_close=True,
auth=auth,
httpx_client_factory=custom_factory,
)
+73
View File
@@ -0,0 +1,73 @@
from __future__ import annotations
import importlib
import importlib.abc
import sys
from types import ModuleType
import pytest
_SERVER_EXPORTS = (
"LocalMCPApprovalCallable",
"MCPServer",
"MCPServerSse",
"MCPServerSseParams",
"MCPServerStdio",
"MCPServerStdioParams",
"MCPServerStreamableHttp",
"MCPServerStreamableHttpParams",
)
class _BrokenMCPServerImportFinder(importlib.abc.MetaPathFinder):
def find_spec(
self,
fullname: str,
path: object | None,
target: ModuleType | None = None,
) -> None:
if fullname == "agents.mcp.server":
raise ImportError("simulated dependency import failure")
return None
def _clear_mcp_server_imports(
monkeypatch: pytest.MonkeyPatch,
mcp_module: ModuleType,
) -> None:
monkeypatch.delitem(sys.modules, "agents.mcp.server", raising=False)
monkeypatch.delitem(mcp_module.__dict__, "server", raising=False)
for name in _SERVER_EXPORTS:
monkeypatch.delitem(mcp_module.__dict__, name, raising=False)
def test_mcp_package_import_does_not_eagerly_import_server(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import agents.mcp as mcp_module
_clear_mcp_server_imports(monkeypatch, mcp_module)
finder = _BrokenMCPServerImportFinder()
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
reloaded_mcp = importlib.reload(mcp_module)
assert reloaded_mcp.MCPUtil is not None
def test_mcp_server_reexport_preserves_underlying_import_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import agents.mcp as mcp_module
_clear_mcp_server_imports(monkeypatch, mcp_module)
finder = _BrokenMCPServerImportFinder()
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
namespace: dict[str, object] = {}
with pytest.raises(ImportError) as exc_info:
exec("from agents.mcp import MCPServerStreamableHttp", namespace)
assert "Failed to import MCPServerStreamableHttp from agents.mcp" in str(exc_info.value)
assert isinstance(exc_info.value.__cause__, ImportError)
assert "simulated dependency import failure" in str(exc_info.value.__cause__)
+175
View File
@@ -0,0 +1,175 @@
"""Tests for MCP server list_resources, list_resource_templates, and read_resource."""
from unittest.mock import AsyncMock, MagicMock
import pytest
from mcp.types import (
ListResourcesResult,
ListResourceTemplatesResult,
ReadResourceResult,
Resource,
ResourceTemplate,
TextResourceContents,
)
from pydantic import AnyUrl
from agents.mcp import MCPServerStreamableHttp
@pytest.fixture
def server():
return MCPServerStreamableHttp(params={"url": "http://localhost:8000/mcp"})
@pytest.mark.asyncio
async def test_list_resources_raises_when_not_connected(server: MCPServerStreamableHttp):
"""list_resources raises UserError when server has not been connected."""
from agents.exceptions import UserError
with pytest.raises(UserError, match="Server not initialized"):
await server.list_resources()
@pytest.mark.asyncio
async def test_list_resource_templates_raises_when_not_connected(server: MCPServerStreamableHttp):
"""list_resource_templates raises UserError when server has not been connected."""
from agents.exceptions import UserError
with pytest.raises(UserError, match="Server not initialized"):
await server.list_resource_templates()
@pytest.mark.asyncio
async def test_read_resource_raises_when_not_connected(server: MCPServerStreamableHttp):
"""read_resource raises UserError when server has not been connected."""
from agents.exceptions import UserError
with pytest.raises(UserError, match="Server not initialized"):
await server.read_resource("file:///etc/hosts")
@pytest.mark.asyncio
async def test_list_resources_returns_result(server: MCPServerStreamableHttp):
"""list_resources delegates to the underlying MCP session."""
mock_session = MagicMock()
expected = ListResourcesResult(
resources=[
Resource(uri=AnyUrl("file:///readme.md"), name="readme.md", mimeType="text/markdown"),
]
)
mock_session.list_resources = AsyncMock(return_value=expected)
server.session = mock_session
result = await server.list_resources()
assert result is expected
mock_session.list_resources.assert_awaited_once_with(None)
@pytest.mark.asyncio
async def test_list_resources_forwards_cursor(server: MCPServerStreamableHttp):
"""list_resources forwards the cursor argument for pagination."""
mock_session = MagicMock()
page2 = ListResourcesResult(resources=[])
mock_session.list_resources = AsyncMock(return_value=page2)
server.session = mock_session
result = await server.list_resources(cursor="tok_abc")
assert result is page2
mock_session.list_resources.assert_awaited_once_with("tok_abc")
@pytest.mark.asyncio
async def test_list_resource_templates_returns_result(server: MCPServerStreamableHttp):
"""list_resource_templates delegates to the underlying MCP session."""
mock_session = MagicMock()
expected = ListResourceTemplatesResult(
resourceTemplates=[
ResourceTemplate(uriTemplate="file:///{path}", name="file"),
]
)
mock_session.list_resource_templates = AsyncMock(return_value=expected)
server.session = mock_session
result = await server.list_resource_templates()
assert result is expected
mock_session.list_resource_templates.assert_awaited_once_with(None)
@pytest.mark.asyncio
async def test_list_resource_templates_forwards_cursor(server: MCPServerStreamableHttp):
"""list_resource_templates forwards the cursor argument for pagination."""
mock_session = MagicMock()
page2 = ListResourceTemplatesResult(resourceTemplates=[])
mock_session.list_resource_templates = AsyncMock(return_value=page2)
server.session = mock_session
result = await server.list_resource_templates(cursor="tok_xyz")
assert result is page2
mock_session.list_resource_templates.assert_awaited_once_with("tok_xyz")
@pytest.mark.asyncio
async def test_read_resource_returns_result(server: MCPServerStreamableHttp):
"""read_resource delegates to the underlying MCP session with the given URI."""
mock_session = MagicMock()
uri = "file:///readme.md"
expected = ReadResourceResult(
contents=[
TextResourceContents(uri=AnyUrl(uri), text="# Hello", mimeType="text/markdown"),
]
)
mock_session.read_resource = AsyncMock(return_value=expected)
server.session = mock_session
result = await server.read_resource(uri)
assert result is expected
mock_session.read_resource.assert_awaited_once_with(AnyUrl(uri))
@pytest.mark.asyncio
async def test_base_methods_raise_not_implemented():
"""Bare MCPServer subclasses that don't override resource methods get NotImplementedError."""
from mcp.types import CallToolResult, GetPromptResult, ListPromptsResult
from agents.mcp import MCPServer
class MinimalServer(MCPServer):
"""Minimal subclass implementing only the truly abstract methods."""
@property
def name(self) -> str:
return "minimal"
async def connect(self) -> None:
pass
async def cleanup(self) -> None:
pass
async def list_tools(self, run_context=None, agent=None):
return []
async def call_tool(self, tool_name, tool_arguments, run_context=None, agent=None):
return CallToolResult(content=[])
async def list_prompts(self):
return ListPromptsResult(prompts=[])
async def get_prompt(self, name, arguments=None):
return GetPromptResult(messages=[])
s = MinimalServer()
with pytest.raises(NotImplementedError, match="list_resources"):
await s.list_resources()
with pytest.raises(NotImplementedError, match="list_resource_templates"):
await s.list_resource_templates()
with pytest.raises(NotImplementedError, match="read_resource"):
await s.read_resource("file:///test.txt")
+552
View File
@@ -0,0 +1,552 @@
import asyncio
from typing import Any, cast
import pytest
from mcp.types import (
CallToolResult,
GetPromptResult,
ListPromptsResult,
ListResourcesResult,
ListResourceTemplatesResult,
ReadResourceResult,
Tool as MCPTool,
)
from agents.mcp import MCPServer, MCPServerManager
from agents.run_context import RunContextWrapper
class TaskBoundServer(MCPServer):
def __init__(self) -> None:
super().__init__()
self._connect_task: asyncio.Task[object] | None = None
self.cleaned = False
@property
def name(self) -> str:
return "task-bound"
async def connect(self) -> None:
self._connect_task = asyncio.current_task()
async def cleanup(self) -> None:
if self._connect_task is None:
raise RuntimeError("Server was not connected")
if asyncio.current_task() is not self._connect_task:
raise RuntimeError("Attempted to exit cancel scope in a different task")
self.cleaned = True
async def list_tools(
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
) -> list[MCPTool]:
raise NotImplementedError
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None,
meta: dict[str, Any] | None = None,
) -> CallToolResult:
raise NotImplementedError
async def list_prompts(self) -> ListPromptsResult:
raise NotImplementedError
async def get_prompt(
self, name: str, arguments: dict[str, Any] | None = None
) -> GetPromptResult:
raise NotImplementedError
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
return ListResourcesResult(resources=[])
async def list_resource_templates(
self, cursor: str | None = None
) -> ListResourceTemplatesResult:
return ListResourceTemplatesResult(resourceTemplates=[])
async def read_resource(self, uri: str) -> ReadResourceResult:
return ReadResourceResult(contents=[])
class FlakyServer(MCPServer):
def __init__(self, failures: int) -> None:
super().__init__()
self.failures_remaining = failures
self.connect_calls = 0
@property
def name(self) -> str:
return "flaky"
async def connect(self) -> None:
self.connect_calls += 1
if self.failures_remaining > 0:
self.failures_remaining -= 1
raise RuntimeError("connect failed")
async def cleanup(self) -> None:
return None
async def list_tools(
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
) -> list[MCPTool]:
raise NotImplementedError
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None,
meta: dict[str, Any] | None = None,
) -> CallToolResult:
raise NotImplementedError
async def list_prompts(self) -> ListPromptsResult:
raise NotImplementedError
async def get_prompt(
self, name: str, arguments: dict[str, Any] | None = None
) -> GetPromptResult:
raise NotImplementedError
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
return ListResourcesResult(resources=[])
async def list_resource_templates(
self, cursor: str | None = None
) -> ListResourceTemplatesResult:
return ListResourceTemplatesResult(resourceTemplates=[])
async def read_resource(self, uri: str) -> ReadResourceResult:
return ReadResourceResult(contents=[])
class CleanupAwareServer(MCPServer):
def __init__(self) -> None:
super().__init__()
self.connect_calls = 0
self.cleanup_calls = 0
@property
def name(self) -> str:
return "cleanup-aware"
async def connect(self) -> None:
if self.connect_calls > self.cleanup_calls:
raise RuntimeError("connect called without cleanup")
self.connect_calls += 1
async def cleanup(self) -> None:
self.cleanup_calls += 1
async def list_tools(
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
) -> list[MCPTool]:
raise NotImplementedError
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None,
meta: dict[str, Any] | None = None,
) -> CallToolResult:
raise NotImplementedError
async def list_prompts(self) -> ListPromptsResult:
raise NotImplementedError
async def get_prompt(
self, name: str, arguments: dict[str, Any] | None = None
) -> GetPromptResult:
raise NotImplementedError
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
return ListResourcesResult(resources=[])
async def list_resource_templates(
self, cursor: str | None = None
) -> ListResourceTemplatesResult:
return ListResourceTemplatesResult(resourceTemplates=[])
async def read_resource(self, uri: str) -> ReadResourceResult:
return ReadResourceResult(contents=[])
class CancelledServer(MCPServer):
@property
def name(self) -> str:
return "cancelled"
async def connect(self) -> None:
raise asyncio.CancelledError()
async def cleanup(self) -> None:
return None
async def list_tools(
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
) -> list[MCPTool]:
raise NotImplementedError
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None,
meta: dict[str, Any] | None = None,
) -> CallToolResult:
raise NotImplementedError
async def list_prompts(self) -> ListPromptsResult:
raise NotImplementedError
async def get_prompt(
self, name: str, arguments: dict[str, Any] | None = None
) -> GetPromptResult:
raise NotImplementedError
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
return ListResourcesResult(resources=[])
async def list_resource_templates(
self, cursor: str | None = None
) -> ListResourceTemplatesResult:
return ListResourceTemplatesResult(resourceTemplates=[])
async def read_resource(self, uri: str) -> ReadResourceResult:
return ReadResourceResult(contents=[])
class FailingTaskBoundServer(TaskBoundServer):
@property
def name(self) -> str:
return "failing-task-bound"
async def connect(self) -> None:
await super().connect()
raise RuntimeError("connect failed")
class FatalError(BaseException):
pass
class FatalTaskBoundServer(TaskBoundServer):
@property
def name(self) -> str:
return "fatal-task-bound"
async def connect(self) -> None:
await super().connect()
raise FatalError("fatal connect failed")
class CleanupFailingServer(TaskBoundServer):
@property
def name(self) -> str:
return "cleanup-failing"
async def cleanup(self) -> None:
await super().cleanup()
raise RuntimeError("cleanup failed")
@pytest.mark.asyncio
async def test_manager_keeps_connect_and_cleanup_in_same_task() -> None:
server = TaskBoundServer()
async with MCPServerManager([server]) as manager:
assert manager.active_servers == [server]
assert server.cleaned is True
@pytest.mark.asyncio
async def test_manager_connects_in_worker_tasks_when_parallel() -> None:
server = TaskBoundServer()
async with MCPServerManager([server], connect_in_parallel=True) as manager:
assert manager.active_servers == [server]
assert server._connect_task is not None
assert server._connect_task is not asyncio.current_task()
assert server.cleaned is True
@pytest.mark.asyncio
async def test_cross_task_cleanup_raises_without_manager() -> None:
server = TaskBoundServer()
connect_task = asyncio.create_task(server.connect())
await connect_task
with pytest.raises(RuntimeError, match="cancel scope"):
await server.cleanup()
@pytest.mark.asyncio
async def test_manager_reconnect_failed_only() -> None:
server = FlakyServer(failures=1)
async with MCPServerManager([server]) as manager:
assert manager.active_servers == []
assert manager.failed_servers == [server]
await manager.reconnect()
assert manager.active_servers == [server]
assert manager.failed_servers == []
@pytest.mark.asyncio
async def test_manager_reconnect_deduplicates_failures() -> None:
server = FlakyServer(failures=2)
async with MCPServerManager([server], connect_in_parallel=True) as manager:
assert manager.active_servers == []
assert manager.failed_servers == [server]
assert server.connect_calls == 1
await manager.reconnect()
assert manager.active_servers == []
assert manager.failed_servers == [server]
assert server.connect_calls == 2
await manager.reconnect()
assert manager.active_servers == [server]
assert manager.failed_servers == []
assert server.connect_calls == 3
@pytest.mark.asyncio
async def test_manager_connect_all_retries_all_servers() -> None:
server = FlakyServer(failures=1)
manager = MCPServerManager([server])
try:
await manager.connect_all()
assert manager.active_servers == []
assert manager.failed_servers == [server]
assert server.connect_calls == 1
await manager.connect_all()
assert manager.active_servers == [server]
assert manager.failed_servers == []
assert server.connect_calls == 2
finally:
await manager.cleanup_all()
@pytest.mark.asyncio
async def test_manager_connect_all_is_idempotent() -> None:
server = CleanupAwareServer()
async with MCPServerManager([server]) as manager:
assert server.connect_calls == 1
await manager.connect_all()
@pytest.mark.asyncio
async def test_manager_reconnect_all_avoids_duplicate_connections() -> None:
server = CleanupAwareServer()
async with MCPServerManager([server]) as manager:
assert server.connect_calls == 1
await manager.reconnect(failed_only=False)
@pytest.mark.asyncio
async def test_manager_strict_reconnect_refreshes_active_servers() -> None:
server_a = FlakyServer(failures=1)
server_b = FlakyServer(failures=2)
async with MCPServerManager([server_a, server_b]) as manager:
assert manager.active_servers == []
manager.strict = True
with pytest.raises(RuntimeError, match="connect failed"):
await manager.reconnect()
assert manager.active_servers == [server_a]
assert manager.failed_servers == [server_b]
@pytest.mark.asyncio
async def test_manager_strict_connect_preserves_existing_active_servers() -> None:
connected_server = TaskBoundServer()
failing_server = FlakyServer(failures=2)
manager = MCPServerManager([connected_server, failing_server])
try:
await manager.connect_all()
assert manager.active_servers == [connected_server]
assert manager.failed_servers == [failing_server]
manager.strict = True
with pytest.raises(RuntimeError, match="connect failed"):
await manager.connect_all()
assert manager.active_servers == [connected_server]
assert manager.failed_servers == [failing_server]
finally:
await manager.cleanup_all()
@pytest.mark.asyncio
async def test_manager_strict_connect_cleans_up_connected_servers() -> None:
connected_server = TaskBoundServer()
failing_server = FlakyServer(failures=1)
manager = MCPServerManager([connected_server, failing_server], strict=True)
with pytest.raises(RuntimeError, match="connect failed"):
await manager.connect_all()
assert connected_server.cleaned is True
assert manager.active_servers == []
@pytest.mark.asyncio
async def test_manager_strict_connect_cleans_up_failed_server() -> None:
failing_server = FailingTaskBoundServer()
manager = MCPServerManager([failing_server], strict=True)
with pytest.raises(RuntimeError, match="connect failed"):
await manager.connect_all()
assert failing_server.cleaned is True
@pytest.mark.asyncio
async def test_manager_strict_connect_parallel_cleans_up_failed_server() -> None:
failing_server = FailingTaskBoundServer()
manager = MCPServerManager([failing_server], strict=True, connect_in_parallel=True)
with pytest.raises(RuntimeError, match="connect failed"):
await manager.connect_all()
assert failing_server.cleaned is True
@pytest.mark.asyncio
async def test_manager_strict_connect_parallel_cleans_up_workers() -> None:
connected_server = TaskBoundServer()
failing_server = FailingTaskBoundServer()
manager = MCPServerManager(
[connected_server, failing_server], strict=True, connect_in_parallel=True
)
with pytest.raises(RuntimeError, match="connect failed"):
await manager.connect_all()
assert connected_server.cleaned is True
assert failing_server.cleaned is True
assert manager._workers == {}
@pytest.mark.asyncio
async def test_manager_parallel_cleanup_clears_worker_on_failure() -> None:
server = CleanupFailingServer()
manager = MCPServerManager([server], connect_in_parallel=True)
await manager.connect_all()
await manager.cleanup_all()
assert server not in manager._workers
assert server not in manager._connected_servers
@pytest.mark.asyncio
async def test_manager_parallel_cleanup_drops_worker_after_error() -> None:
class HangingCleanupWorker:
def __init__(self) -> None:
self.cleanup_calls = 0
@property
def is_done(self) -> bool:
return False
async def cleanup(self) -> None:
self.cleanup_calls += 1
raise RuntimeError("cleanup failed")
server = FlakyServer(failures=0)
manager = MCPServerManager([server], connect_in_parallel=True)
manager._workers[server] = cast(Any, HangingCleanupWorker())
await manager.cleanup_all()
assert manager._workers == {}
@pytest.mark.asyncio
async def test_manager_parallel_suppresses_cancelled_error_in_strict_mode() -> None:
server = CancelledServer()
manager = MCPServerManager([server], connect_in_parallel=True, strict=True)
try:
await manager.connect_all()
assert manager.active_servers == []
assert manager.failed_servers == [server]
finally:
await manager.cleanup_all()
@pytest.mark.asyncio
async def test_manager_parallel_propagates_cancelled_error_when_unsuppressed() -> None:
server = CancelledServer()
manager = MCPServerManager([server], connect_in_parallel=True, suppress_cancelled_error=False)
try:
with pytest.raises(asyncio.CancelledError):
await manager.connect_all()
finally:
await manager.cleanup_all()
@pytest.mark.asyncio
async def test_manager_sequential_propagates_base_exception() -> None:
server = FatalTaskBoundServer()
manager = MCPServerManager([server])
with pytest.raises(FatalError, match="fatal connect failed"):
await manager.connect_all()
assert server.cleaned is True
assert manager.failed_servers == [server]
@pytest.mark.asyncio
async def test_manager_parallel_propagates_base_exception() -> None:
server = FatalTaskBoundServer()
manager = MCPServerManager([server], connect_in_parallel=True)
with pytest.raises(FatalError, match="fatal connect failed"):
await manager.connect_all()
assert server.cleaned is True
assert manager._workers == {}
@pytest.mark.asyncio
async def test_manager_parallel_prefers_cancelled_error_when_unsuppressed() -> None:
cancelled_server = CancelledServer()
fatal_server = FatalTaskBoundServer()
manager = MCPServerManager(
[fatal_server, cancelled_server],
connect_in_parallel=True,
suppress_cancelled_error=False,
)
try:
with pytest.raises(asyncio.CancelledError):
await manager.connect_all()
finally:
await manager.cleanup_all()
@pytest.mark.asyncio
async def test_manager_cleanup_runs_on_cancelled_error_during_connect() -> None:
server = CleanupAwareServer()
cancelled_server = CancelledServer()
manager = MCPServerManager(
[server, cancelled_server],
suppress_cancelled_error=False,
)
try:
with pytest.raises(asyncio.CancelledError):
await manager.connect_all()
assert server.cleanup_calls == 1
finally:
await manager.cleanup_all()
+274
View File
@@ -0,0 +1,274 @@
import pytest
from inline_snapshot import snapshot
from agents import Agent, RunConfig, Runner
from ..fake_model import FakeModel
from ..test_responses import get_function_tool, get_function_tool_call, get_text_message
from ..testing_processor import SPAN_PROCESSOR_TESTING, fetch_normalized_spans
from .helpers import FakeMCPServer
@pytest.mark.asyncio
async def test_mcp_tracing():
model = FakeModel()
server = FakeMCPServer()
server.add_tool("test_tool_1", {})
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
tools=[get_function_tool("non_mcp_tool", "tool_result")],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_1", "")],
# Second turn: text message
[get_text_message("done")],
]
)
# First run: should list MCP tools before first and second steps
x = Runner.run_streamed(agent, input="first_test")
async for _ in x.stream_events():
pass
assert x.final_output == "done"
spans = fetch_normalized_spans()
# Should have a single tool listing, and the function span should have MCP data
assert spans == snapshot(
[
{
"workflow_name": "Agent workflow",
"children": [
{
"type": "mcp_tools",
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
},
{
"type": "agent",
"data": {
"name": "test",
"handoffs": [],
"tools": ["test_tool_1", "non_mcp_tool"],
"output_type": "str",
},
"children": [
{
"type": "function",
"data": {
"name": "test_tool_1",
"input": "",
"output": "{'type': 'text', 'text': 'result_test_tool_1_{}'}", # noqa: E501
"mcp_data": {"server": "fake_mcp_server"},
},
},
{
"type": "mcp_tools",
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
},
],
},
],
}
]
)
server.add_tool("test_tool_2", {})
SPAN_PROCESSOR_TESTING.clear()
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[
get_text_message("a_message"),
get_function_tool_call("non_mcp_tool", ""),
get_function_tool_call("test_tool_2", ""),
],
# Second turn: text message
[get_text_message("done")],
]
)
await Runner.run(agent, input="second_test")
spans = fetch_normalized_spans()
# Should have a single tool listing, and the function span should have MCP data, and the non-mcp
# tool function span should not have MCP data
assert spans == snapshot(
[
{
"workflow_name": "Agent workflow",
"children": [
{
"type": "mcp_tools",
"data": {
"server": "fake_mcp_server",
"result": ["test_tool_1", "test_tool_2"],
},
},
{
"type": "agent",
"data": {
"name": "test",
"handoffs": [],
"tools": ["test_tool_1", "test_tool_2", "non_mcp_tool"],
"output_type": "str",
},
"children": [
{
"type": "function",
"data": {
"name": "non_mcp_tool",
"input": "",
"output": "tool_result",
},
},
{
"type": "function",
"data": {
"name": "test_tool_2",
"input": "",
"output": "{'type': 'text', 'text': 'result_test_tool_2_{}'}", # noqa: E501
"mcp_data": {"server": "fake_mcp_server"},
},
},
{
"type": "mcp_tools",
"data": {
"server": "fake_mcp_server",
"result": ["test_tool_1", "test_tool_2"],
},
},
],
},
],
}
]
)
SPAN_PROCESSOR_TESTING.clear()
# Add more tools to the server
server.add_tool("test_tool_3", {})
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
# Second turn: text message
[get_text_message("done")],
]
)
await Runner.run(agent, input="third_test")
spans = fetch_normalized_spans()
# Should have a single tool listing, and the function span should have MCP data, and the non-mcp
# tool function span should not have MCP data
assert spans == snapshot(
[
{
"workflow_name": "Agent workflow",
"children": [
{
"type": "mcp_tools",
"data": {
"server": "fake_mcp_server",
"result": ["test_tool_1", "test_tool_2", "test_tool_3"],
},
},
{
"type": "agent",
"data": {
"name": "test",
"handoffs": [],
"tools": ["test_tool_1", "test_tool_2", "test_tool_3", "non_mcp_tool"],
"output_type": "str",
},
"children": [
{
"type": "function",
"data": {
"name": "test_tool_3",
"input": "",
"output": "{'type': 'text', 'text': 'result_test_tool_3_{}'}", # noqa: E501
"mcp_data": {"server": "fake_mcp_server"},
},
},
{
"type": "mcp_tools",
"data": {
"server": "fake_mcp_server",
"result": ["test_tool_1", "test_tool_2", "test_tool_3"],
},
},
],
},
],
}
]
)
@pytest.mark.asyncio
async def test_mcp_tracing_redacts_output_when_sensitive_data_disabled():
model = FakeModel()
server = FakeMCPServer()
server.add_tool("test_tool_1", {})
agent = Agent(name="test", model=model, mcp_servers=[server])
model.add_multiple_turn_outputs(
[
[get_function_tool_call("test_tool_1", "")],
[get_text_message("done")],
]
)
await Runner.run(
agent,
input="redaction_test",
run_config=RunConfig(trace_include_sensitive_data=False),
)
spans = fetch_normalized_spans()
assert spans == snapshot(
[
{
"workflow_name": "Agent workflow",
"children": [
{
"type": "mcp_tools",
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
},
{
"type": "agent",
"data": {
"name": "test",
"handoffs": [],
"tools": ["test_tool_1"],
"output_type": "str",
},
"children": [
{
"type": "function",
"data": {
"name": "test_tool_1",
"mcp_data": {"server": "fake_mcp_server"},
},
},
{
"type": "mcp_tools",
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
},
],
},
],
}
]
)
File diff suppressed because it is too large Load Diff
+133
View File
@@ -0,0 +1,133 @@
from __future__ import annotations
import contextlib
from typing import Union
import anyio
import pytest
from mcp.client.session import MessageHandlerFnT
from mcp.shared.message import SessionMessage
from mcp.shared.session import RequestResponder
from mcp.types import (
ClientResult,
Implementation,
InitializeResult,
ServerCapabilities,
ServerNotification,
ServerRequest,
)
from agents.mcp.server import (
MCPServerSse,
MCPServerStdio,
MCPServerStreamableHttp,
_MCPServerWithClientSession,
)
HandlerMessage = Union[ # noqa: UP007
RequestResponder[ServerRequest, ClientResult], ServerNotification, Exception
]
class _StubClientSession:
"""Stub ClientSession that records the configured message handler."""
def __init__(
self,
read_stream,
write_stream,
read_timeout_seconds,
*,
message_handler=None,
**_: object,
) -> None:
self.message_handler = message_handler
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
async def initialize(self) -> InitializeResult:
capabilities = ServerCapabilities.model_construct()
server_info = Implementation.model_construct(name="stub", version="1.0")
return InitializeResult(
protocolVersion="2024-11-05",
capabilities=capabilities,
serverInfo=server_info,
)
class _MessageHandlerTestServer(_MCPServerWithClientSession):
def __init__(self, handler: MessageHandlerFnT | None):
super().__init__(
cache_tools_list=False,
client_session_timeout_seconds=None,
message_handler=handler,
)
def create_streams(self):
@contextlib.asynccontextmanager
async def _streams():
send_stream, recv_stream = anyio.create_memory_object_stream[
SessionMessage | Exception
](1)
try:
yield recv_stream, send_stream, None
finally:
await recv_stream.aclose()
await send_stream.aclose()
return _streams()
@property
def name(self) -> str:
return "test-server"
@pytest.mark.asyncio
async def test_client_session_receives_message_handler(monkeypatch):
captured: dict[str, object] = {}
def _recording_client_session(*args, **kwargs):
session = _StubClientSession(*args, **kwargs)
captured["message_handler"] = session.message_handler
return session
monkeypatch.setattr("agents.mcp.server.ClientSession", _recording_client_session)
class _AsyncHandler:
async def __call__(self, message: HandlerMessage) -> None:
del message
handler: MessageHandlerFnT = _AsyncHandler()
server = _MessageHandlerTestServer(handler)
try:
await server.connect()
finally:
await server.cleanup()
assert captured["message_handler"] is handler
@pytest.mark.parametrize(
"server_cls, params",
[
(MCPServerSse, {"url": "https://example.com"}),
(MCPServerStreamableHttp, {"url": "https://example.com"}),
(MCPServerStdio, {"command": "python"}),
],
)
def test_message_handler_propagates_to_server_base(server_cls, params):
class _AsyncHandler:
async def __call__(self, message: HandlerMessage) -> None:
del message
handler: MessageHandlerFnT = _AsyncHandler()
server = server_cls(params, message_handler=handler)
assert server.message_handler is handler
+324
View File
@@ -0,0 +1,324 @@
from typing import Any
import pytest
from mcp.types import ListResourcesResult, ListResourceTemplatesResult, ReadResourceResult
from agents import Agent, Runner
from agents.mcp import MCPServer, MCPToolMetaResolver
from ..fake_model import FakeModel
from ..test_responses import get_text_message
class FakeMCPPromptServer(MCPServer):
"""Fake MCP server for testing prompt functionality"""
def __init__(
self,
server_name: str = "fake_prompt_server",
tool_meta_resolver: MCPToolMetaResolver | None = None,
):
super().__init__(tool_meta_resolver=tool_meta_resolver)
self.prompts: list[Any] = []
self.prompt_results: dict[str, str] = {}
self._server_name = server_name
def add_prompt(self, name: str, description: str, arguments: dict[str, Any] | None = None):
"""Add a prompt to the fake server"""
from mcp.types import Prompt
prompt = Prompt(name=name, description=description, arguments=[])
self.prompts.append(prompt)
def set_prompt_result(self, name: str, result: str):
"""Set the result that should be returned for a prompt"""
self.prompt_results[name] = result
async def connect(self):
pass
async def cleanup(self):
pass
async def list_prompts(self, run_context=None, agent=None):
"""List available prompts"""
from mcp.types import ListPromptsResult
return ListPromptsResult(prompts=self.prompts)
async def get_prompt(self, name: str, arguments: dict[str, Any] | None = None):
"""Get a prompt with arguments"""
from mcp.types import GetPromptResult, PromptMessage, TextContent
if name not in self.prompt_results:
raise ValueError(f"Prompt '{name}' not found")
content = self.prompt_results[name]
# If it's a format string, try to format it with arguments
if arguments and "{" in content:
try:
content = content.format(**arguments)
except KeyError:
pass # Use original content if formatting fails
message = PromptMessage(role="user", content=TextContent(type="text", text=content))
return GetPromptResult(description=f"Generated prompt for {name}", messages=[message])
async def list_tools(self, run_context=None, agent=None):
return []
async def call_tool(
self,
tool_name: str,
arguments: dict[str, Any] | None = None,
meta: dict[str, Any] | None = None,
):
raise NotImplementedError("This fake server doesn't support tools")
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
return ListResourcesResult(resources=[])
async def list_resource_templates(
self, cursor: str | None = None
) -> ListResourceTemplatesResult:
return ListResourceTemplatesResult(resourceTemplates=[])
async def read_resource(self, uri: str) -> ReadResourceResult:
return ReadResourceResult(contents=[])
@property
def name(self) -> str:
return self._server_name
@pytest.mark.asyncio
async def test_list_prompts():
"""Test listing available prompts"""
server = FakeMCPPromptServer()
server.add_prompt(
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
)
result = await server.list_prompts()
assert len(result.prompts) == 1
assert result.prompts[0].name == "generate_code_review_instructions"
assert result.prompts[0].description is not None
assert "code review" in result.prompts[0].description
@pytest.mark.asyncio
async def test_get_prompt_without_arguments():
"""Test getting a prompt without arguments"""
server = FakeMCPPromptServer()
server.add_prompt("simple_prompt", "A simple prompt")
server.set_prompt_result("simple_prompt", "You are a helpful assistant.")
result = await server.get_prompt("simple_prompt")
assert len(result.messages) == 1
assert result.messages[0].content.text == "You are a helpful assistant."
@pytest.mark.asyncio
async def test_get_prompt_with_arguments():
"""Test getting a prompt with arguments"""
server = FakeMCPPromptServer()
server.add_prompt(
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
)
server.set_prompt_result(
"generate_code_review_instructions",
"You are a senior {language} code review specialist. Focus on {focus}.",
)
result = await server.get_prompt(
"generate_code_review_instructions",
{"focus": "security vulnerabilities", "language": "python"},
)
assert len(result.messages) == 1
expected_text = (
"You are a senior python code review specialist. Focus on security vulnerabilities."
)
assert result.messages[0].content.text == expected_text
@pytest.mark.asyncio
async def test_get_prompt_not_found():
"""Test getting a prompt that doesn't exist"""
server = FakeMCPPromptServer()
with pytest.raises(ValueError, match="Prompt 'nonexistent' not found"):
await server.get_prompt("nonexistent")
@pytest.mark.asyncio
async def test_agent_with_prompt_instructions():
"""Test using prompt-generated instructions with an agent"""
server = FakeMCPPromptServer()
server.add_prompt(
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
)
server.set_prompt_result(
"generate_code_review_instructions",
"You are a code reviewer. Analyze the provided code for security issues.",
)
# Get instructions from prompt
prompt_result = await server.get_prompt("generate_code_review_instructions")
instructions = prompt_result.messages[0].content.text
# Create agent with prompt-generated instructions
model = FakeModel()
agent = Agent(name="prompt_agent", instructions=instructions, model=model, mcp_servers=[server])
# Mock model response
model.add_multiple_turn_outputs(
[[get_text_message("Code analysis complete. Found security vulnerability.")]]
)
# Run the agent
result = await Runner.run(agent, input="Review this code: def unsafe_exec(cmd): os.system(cmd)")
assert "Code analysis complete" in result.final_output
assert (
agent.instructions
== "You are a code reviewer. Analyze the provided code for security issues."
)
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_agent_with_prompt_instructions_streaming(streaming: bool):
"""Test using prompt-generated instructions with streaming and non-streaming"""
server = FakeMCPPromptServer()
server.add_prompt(
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
)
server.set_prompt_result(
"generate_code_review_instructions",
"You are a {language} code reviewer focusing on {focus}.",
)
# Get instructions from prompt with arguments
prompt_result = await server.get_prompt(
"generate_code_review_instructions", {"language": "Python", "focus": "security"}
)
instructions = prompt_result.messages[0].content.text
# Create agent
model = FakeModel()
agent = Agent(
name="streaming_prompt_agent", instructions=instructions, model=model, mcp_servers=[server]
)
model.add_multiple_turn_outputs([[get_text_message("Security analysis complete.")]])
if streaming:
streaming_result = Runner.run_streamed(agent, input="Review code")
async for _ in streaming_result.stream_events():
pass
final_result = streaming_result.final_output
else:
result = await Runner.run(agent, input="Review code")
final_result = result.final_output
assert "Security analysis complete" in final_result
assert agent.instructions == "You are a Python code reviewer focusing on security."
@pytest.mark.asyncio
async def test_multiple_prompts():
"""Test server with multiple prompts"""
server = FakeMCPPromptServer()
# Add multiple prompts
server.add_prompt(
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
)
server.add_prompt(
"generate_testing_instructions", "Generate agent instructions for testing tasks"
)
server.set_prompt_result("generate_code_review_instructions", "You are a code reviewer.")
server.set_prompt_result("generate_testing_instructions", "You are a test engineer.")
# Test listing prompts
prompts_result = await server.list_prompts()
assert len(prompts_result.prompts) == 2
prompt_names = [p.name for p in prompts_result.prompts]
assert "generate_code_review_instructions" in prompt_names
assert "generate_testing_instructions" in prompt_names
# Test getting each prompt
review_result = await server.get_prompt("generate_code_review_instructions")
assert review_result.messages[0].content.text == "You are a code reviewer."
testing_result = await server.get_prompt("generate_testing_instructions")
assert testing_result.messages[0].content.text == "You are a test engineer."
@pytest.mark.asyncio
async def test_prompt_with_complex_arguments():
"""Test prompt with complex argument formatting"""
server = FakeMCPPromptServer()
server.add_prompt(
"generate_detailed_instructions", "Generate detailed instructions with multiple parameters"
)
server.set_prompt_result(
"generate_detailed_instructions",
"You are a {role} specialist. Your focus is on {focus}. "
+ "You work with {language} code. Your experience level is {level}.",
)
arguments = {
"role": "security",
"focus": "vulnerability detection",
"language": "Python",
"level": "senior",
}
result = await server.get_prompt("generate_detailed_instructions", arguments)
expected = (
"You are a security specialist. Your focus is on vulnerability detection. "
"You work with Python code. Your experience level is senior."
)
assert result.messages[0].content.text == expected
@pytest.mark.asyncio
async def test_prompt_with_missing_arguments():
"""Test prompt with missing arguments in format string"""
server = FakeMCPPromptServer()
server.add_prompt("incomplete_prompt", "Prompt with missing arguments")
server.set_prompt_result("incomplete_prompt", "You are a {role} working on {task}.")
# Only provide one of the required arguments
result = await server.get_prompt("incomplete_prompt", {"role": "developer"})
# Should return the original string since formatting fails
assert result.messages[0].content.text == "You are a {role} working on {task}."
@pytest.mark.asyncio
async def test_prompt_server_cleanup():
"""Test that prompt server cleanup works correctly"""
server = FakeMCPPromptServer()
server.add_prompt("test_prompt", "Test prompt")
server.set_prompt_result("test_prompt", "Test result")
# Test that server works before cleanup
result = await server.get_prompt("test_prompt")
assert result.messages[0].content.text == "Test result"
# Cleanup should not raise any errors
await server.cleanup()
# Server should still work after cleanup (in this fake implementation)
result = await server.get_prompt("test_prompt")
assert result.messages[0].content.text == "Test result"
+402
View File
@@ -0,0 +1,402 @@
import json
from typing import Any
import pytest
from pydantic import BaseModel
from agents import (
Agent,
FunctionTool,
ModelBehaviorError,
RunContextWrapper,
Runner,
UserError,
default_tool_error_function,
handoff,
)
from agents.exceptions import AgentsException
from ..fake_model import FakeModel
from ..test_responses import get_function_tool_call, get_text_message
from .helpers import FakeMCPServer
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_calls_mcp_tool(streaming: bool):
"""Test that the runner calls an MCP tool when the model produces a tool call."""
server = FakeMCPServer()
server.add_tool("test_tool_1", {})
server.add_tool("test_tool_2", {})
server.add_tool("test_tool_3", {})
model = FakeModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
# Second turn: text message
[get_text_message("done")],
]
)
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
assert server.tool_calls == ["test_tool_2"]
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_asserts_when_mcp_tool_not_found(streaming: bool):
"""Test that the runner asserts when an MCP tool is not found."""
server = FakeMCPServer()
server.add_tool("test_tool_1", {})
server.add_tool("test_tool_2", {})
server.add_tool("test_tool_3", {})
model = FakeModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_doesnt_exist", "")],
# Second turn: text message
[get_text_message("done")],
]
)
with pytest.raises(ModelBehaviorError):
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_works_with_multiple_mcp_servers(streaming: bool):
"""Test that the runner works with multiple MCP servers."""
server1 = FakeMCPServer()
server1.add_tool("test_tool_1", {})
server2 = FakeMCPServer()
server2.add_tool("test_tool_2", {})
server2.add_tool("test_tool_3", {})
model = FakeModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server1, server2],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
# Second turn: text message
[get_text_message("done")],
]
)
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
assert server1.tool_calls == []
assert server2.tool_calls == ["test_tool_2"]
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_errors_when_mcp_tools_clash(streaming: bool):
"""Test that the runner errors when multiple servers have the same tool name."""
server1 = FakeMCPServer()
server1.add_tool("test_tool_1", {})
server1.add_tool("test_tool_2", {})
server2 = FakeMCPServer()
server2.add_tool("test_tool_2", {})
server2.add_tool("test_tool_3", {})
model = FakeModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server1, server2],
)
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
# Second turn: text message
[get_text_message("done")],
]
)
with pytest.raises(UserError):
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_can_call_server_prefixed_mcp_tool_names(streaming: bool):
server1 = FakeMCPServer(server_name="docs")
server1.add_tool("search", {})
server2 = FakeMCPServer(server_name="calendar")
server2.add_tool("search", {})
model = FakeModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server1, server2],
mcp_config={"include_server_in_tool_names": True},
)
model.add_multiple_turn_outputs(
[
[get_text_message("a_message"), get_function_tool_call("mcp_calendar__search", "")],
[get_text_message("done")],
]
)
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
assert server1.tool_calls == []
assert server2.tool_calls == ["search"]
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_agent_tools(streaming: bool):
server1 = FakeMCPServer(server_name="docs")
server1.add_tool("search", {})
server2 = FakeMCPServer(server_name="calendar")
server2.add_tool("search", {})
local_tool_calls: list[str] = []
async def invoke_local_tool(context: Any, input_json: str) -> str:
local_tool_calls.append(input_json)
return "local"
local_tool = FunctionTool(
name="mcp_calendar__search",
description="Local tool that intentionally collides with the natural MCP prefix.",
params_json_schema={"type": "object", "properties": {}, "additionalProperties": False},
on_invoke_tool=invoke_local_tool,
)
model = FakeModel()
agent = Agent(
name="test",
model=model,
tools=[local_tool],
mcp_servers=[server1, server2],
mcp_config={"include_server_in_tool_names": True},
)
mcp_tools = await agent.get_mcp_tools(RunContextWrapper(context=None))
calendar_search_tool_name = next(
tool.name
for tool in mcp_tools
if getattr(getattr(tool, "_tool_origin", None), "mcp_server_name", None) == "calendar"
)
assert calendar_search_tool_name != "mcp_calendar__search"
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
model.add_multiple_turn_outputs(
[
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
[get_text_message("done")],
]
)
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
assert local_tool_calls == []
assert server1.tool_calls == []
assert server2.tool_calls == ["search"]
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(streaming: bool):
server = FakeMCPServer(server_name="calendar")
server.add_tool("search", {})
target_model = FakeModel()
target_agent = Agent(name="calendar_agent", model=target_model)
target_model.add_multiple_turn_outputs([[get_text_message("handoff target")]])
model = FakeModel()
agent = Agent(
name="test",
model=model,
handoffs=[handoff(target_agent, tool_name_override="mcp_calendar__search")],
mcp_servers=[server],
mcp_config={"include_server_in_tool_names": True},
)
mcp_tools = await agent.get_mcp_tools(RunContextWrapper(context=None))
assert len(mcp_tools) == 1
calendar_search_tool_name = mcp_tools[0].name
assert calendar_search_tool_name != "mcp_calendar__search"
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
model.add_multiple_turn_outputs(
[
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
[get_text_message("done")],
]
)
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
assert server.tool_calls == ["search"]
assert target_model.first_turn_args is None
class Foo(BaseModel):
bar: str
baz: int
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_calls_mcp_tool_with_args(streaming: bool):
"""Test that the runner calls an MCP tool when the model produces a tool call."""
server = FakeMCPServer()
await server.connect()
server.add_tool("test_tool_1", {})
server.add_tool("test_tool_2", Foo.model_json_schema())
server.add_tool("test_tool_3", {})
model = FakeModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
)
json_args = json.dumps(Foo(bar="baz", baz=1).model_dump())
model.add_multiple_turn_outputs(
[
# First turn: a message and tool call
[get_text_message("a_message"), get_function_tool_call("test_tool_2", json_args)],
# Second turn: text message
[get_text_message("done")],
]
)
if streaming:
result = Runner.run_streamed(agent, input="user_message")
async for _ in result.stream_events():
pass
else:
await Runner.run(agent, input="user_message")
assert server.tool_calls == ["test_tool_2"]
assert server.tool_results == [f"result_test_tool_2_{json_args}"]
await server.cleanup()
class CrashingFakeMCPServer(FakeMCPServer):
async def call_tool(
self,
tool_name: str,
arguments: dict[str, object] | None,
meta: dict[str, object] | None = None,
):
raise Exception("Crash!")
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_emits_mcp_error_tool_call_output_item(streaming: bool):
"""Runner should emit tool_call_output_item with failure output when MCP tool raises."""
server = CrashingFakeMCPServer()
server.add_tool("crashing_tool", {})
model = FakeModel()
agent = Agent(
name="test",
model=model,
mcp_servers=[server],
)
model.add_multiple_turn_outputs(
[
[get_text_message("a_message"), get_function_tool_call("crashing_tool", "{}")],
[get_text_message("done")],
]
)
if streaming:
streamed_result = Runner.run_streamed(agent, input="user_message")
async for _ in streamed_result.stream_events():
pass
tool_output_items = [
item for item in streamed_result.new_items if item.type == "tool_call_output_item"
]
assert streamed_result.final_output == "done"
else:
non_streamed_result = await Runner.run(agent, input="user_message")
tool_output_items = [
item for item in non_streamed_result.new_items if item.type == "tool_call_output_item"
]
assert non_streamed_result.final_output == "done"
assert tool_output_items, "Expected tool_call_output_item for MCP failure"
wrapped_error = AgentsException(
"Error invoking MCP tool crashing_tool on server 'fake_mcp_server': Crash!"
)
expected_error_message = default_tool_error_function(
RunContextWrapper(context=None),
wrapped_error,
)
assert tool_output_items[0].output == expected_error_message
+85
View File
@@ -0,0 +1,85 @@
import builtins
import sys
from unittest.mock import MagicMock, patch
import httpx
import pytest
from agents import Agent
from agents.exceptions import UserError
from agents.mcp.server import MCPServerStreamableHttp, _MCPServerWithClientSession
from agents.run_context import RunContextWrapper
# Handle Python version compatibility for ExceptionGroups
if sys.version_info < (3, 11):
from exceptiongroup import BaseExceptionGroup
else:
BaseExceptionGroup = builtins.BaseExceptionGroup
class CrashingClientSessionServer(_MCPServerWithClientSession):
def __init__(self):
super().__init__(cache_tools_list=False, client_session_timeout_seconds=5)
self.cleanup_called = False
def create_streams(self):
raise ValueError("Crash!")
async def cleanup(self):
self.cleanup_called = True
await super().cleanup()
@property
def name(self) -> str:
return "crashing_client_session_server"
@pytest.mark.asyncio
async def test_server_errors_cause_error_and_cleanup_called():
server = CrashingClientSessionServer()
with pytest.raises(ValueError):
await server.connect()
assert server.cleanup_called
@pytest.mark.asyncio
async def test_not_calling_connect_causes_error():
server = CrashingClientSessionServer()
run_context = RunContextWrapper(context=None)
agent = Agent(name="test_agent", instructions="Test agent")
with pytest.raises(UserError):
await server.list_tools(run_context, agent)
with pytest.raises(UserError):
await server.call_tool("foo", {})
@pytest.mark.asyncio
async def test_call_tool_nested_exception_group_mapping():
"""
Regression test ensuring that nested ExceptionGroups containing HTTP errors
are recursively extracted and mapped to a UserError in call_tool().
"""
# 1. Initialize the server with mock streamable parameters
server = MCPServerStreamableHttp(params={"url": "http://fake-mcp-server"})
# 2. Simulate an active connection by mocking the session object
server.session = MagicMock()
# 3. Construct a nested ExceptionGroup hierarchy containing a connection error
http_error = httpx.ConnectError("Network unreachable")
inner_group = BaseExceptionGroup("inner_failures", [http_error])
outer_group = BaseExceptionGroup("outer_failures", [inner_group])
# 4 & 5. Mock the internal retry handler to raise the nested group, and assert UserError
with patch.object(server, "_call_tool_with_isolated_retry", side_effect=outer_group):
with pytest.raises(UserError) as exc_info:
await server.call_tool(tool_name="test_tool", arguments={})
# 6. Verify that the user-facing message is mapped correctly based on the root cause
assert "Connection lost" in str(exc_info.value)
assert exc_info.value.__cause__ is http_error
@@ -0,0 +1,442 @@
"""Tests for MCPServerStreamableHttp httpx_client_factory functionality."""
from __future__ import annotations
import base64
from unittest.mock import MagicMock, patch
import httpx
import pytest
from anyio import create_memory_object_stream
from mcp.shared.message import SessionMessage
from mcp.types import JSONRPCMessage, JSONRPCNotification, JSONRPCRequest
from agents.mcp import MCPServerStreamableHttp
from agents.mcp.server import (
_create_default_streamable_http_client,
_InitializedNotificationTolerantStreamableHTTPTransport,
_streamablehttp_client_with_transport,
)
class TestMCPServerStreamableHttpClientFactory:
"""Test cases for custom httpx_client_factory parameter."""
@pytest.mark.asyncio
async def test_default_httpx_client_factory(self):
"""Test that default behavior works when no custom factory is provided."""
# Mock the streamablehttp_client to avoid actual network calls
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "http://localhost:8000/mcp",
"headers": {"Authorization": "Bearer token"},
"timeout": 10,
}
)
server.create_streams()
# Verify streamablehttp_client was called with the hardened default factory.
mock_client.assert_called_once_with(
url="http://localhost:8000/mcp",
headers={"Authorization": "Bearer token"},
timeout=10,
sse_read_timeout=300, # Default value
terminate_on_close=True, # Default value
httpx_client_factory=_create_default_streamable_http_client,
)
@pytest.mark.asyncio
async def test_custom_httpx_client_factory(self):
"""Test that custom httpx_client_factory is passed correctly."""
# Create a custom factory function
def custom_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(
verify=False, # Disable SSL verification for testing
timeout=httpx.Timeout(60.0),
headers={"X-Custom-Header": "test"},
)
# Mock the streamablehttp_client to avoid actual network calls
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "http://localhost:8000/mcp",
"headers": {"Authorization": "Bearer token"},
"timeout": 10,
"httpx_client_factory": custom_factory,
}
)
# Create streams should pass the custom factory
server.create_streams()
# Verify streamablehttp_client was called with the custom factory
mock_client.assert_called_once_with(
url="http://localhost:8000/mcp",
headers={"Authorization": "Bearer token"},
timeout=10,
sse_read_timeout=300, # Default value
terminate_on_close=True, # Default value
httpx_client_factory=custom_factory,
)
@pytest.mark.asyncio
async def test_custom_httpx_client_factory_with_ssl_cert(self):
"""Test custom factory with SSL certificate configuration."""
def ssl_cert_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(
verify="/path/to/cert.pem", # Custom SSL certificate
timeout=httpx.Timeout(120.0),
)
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "https://secure-server.com/mcp",
"timeout": 30,
"httpx_client_factory": ssl_cert_factory,
}
)
server.create_streams()
mock_client.assert_called_once_with(
url="https://secure-server.com/mcp",
headers=None,
timeout=30,
sse_read_timeout=300,
terminate_on_close=True,
httpx_client_factory=ssl_cert_factory,
)
@pytest.mark.asyncio
async def test_custom_httpx_client_factory_with_proxy(self):
"""Test custom factory with proxy configuration."""
def proxy_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(
proxy="http://proxy.example.com:8080",
timeout=httpx.Timeout(60.0),
)
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "http://localhost:8000/mcp",
"httpx_client_factory": proxy_factory,
}
)
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/mcp",
headers=None,
timeout=5, # Default value
sse_read_timeout=300,
terminate_on_close=True,
httpx_client_factory=proxy_factory,
)
@pytest.mark.asyncio
async def test_custom_httpx_client_factory_with_retry_logic(self):
"""Test custom factory with retry logic configuration."""
def retry_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(
timeout=httpx.Timeout(30.0),
# Note: httpx doesn't have built-in retry, but this shows how
# a custom factory could be used to configure retry behavior
# through middleware or other mechanisms
)
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "http://localhost:8000/mcp",
"httpx_client_factory": retry_factory,
}
)
server.create_streams()
mock_client.assert_called_once_with(
url="http://localhost:8000/mcp",
headers=None,
timeout=5,
sse_read_timeout=300,
terminate_on_close=True,
httpx_client_factory=retry_factory,
)
def test_httpx_client_factory_type_annotation(self):
"""Test that the type annotation is correct for httpx_client_factory."""
from agents.mcp.server import MCPServerStreamableHttpParams
# This test ensures the type annotation is properly set
# We can't easily test the TypedDict at runtime, but we can verify
# that the import works and the type is available
assert hasattr(MCPServerStreamableHttpParams, "__annotations__")
# Verify that the httpx_client_factory parameter is in the annotations
annotations = MCPServerStreamableHttpParams.__annotations__
assert "httpx_client_factory" in annotations
# The annotation should contain the string representation of the type
annotation_str = str(annotations["httpx_client_factory"])
assert "HttpClientFactory" in annotation_str
@pytest.mark.asyncio
async def test_all_parameters_with_custom_factory(self):
"""Test that all parameters work together with custom factory."""
def comprehensive_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
return httpx.AsyncClient(
verify=False,
timeout=httpx.Timeout(90.0),
headers={"X-Test": "value"},
)
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "https://api.example.com/mcp",
"headers": {"Authorization": "Bearer token"},
"timeout": 45,
"sse_read_timeout": 600,
"terminate_on_close": False,
"httpx_client_factory": comprehensive_factory,
}
)
server.create_streams()
mock_client.assert_called_once_with(
url="https://api.example.com/mcp",
headers={"Authorization": "Bearer token"},
timeout=45,
sse_read_timeout=600,
terminate_on_close=False,
httpx_client_factory=comprehensive_factory,
)
@pytest.mark.asyncio
async def test_initialized_notification_failure_returns_synthetic_success():
async def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(503, request=request)
transport = _InitializedNotificationTolerantStreamableHTTPTransport("https://example.test/mcp")
read_stream_writer, _ = create_memory_object_stream[SessionMessage | Exception](0)
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
try:
ctx = MagicMock()
ctx.client = client
ctx.read_stream_writer = read_stream_writer
ctx.session_message = SessionMessage(
JSONRPCMessage(
JSONRPCNotification(
jsonrpc="2.0",
method="notifications/initialized",
params={},
)
)
)
await transport._handle_post_request(ctx)
finally:
await client.aclose()
await read_stream_writer.aclose()
@pytest.mark.asyncio
async def test_initialized_notification_transport_exception_returns_synthetic_success():
async def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("boom", request=request)
transport = _InitializedNotificationTolerantStreamableHTTPTransport("https://example.test/mcp")
read_stream_writer, _ = create_memory_object_stream[SessionMessage | Exception](0)
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
try:
ctx = MagicMock()
ctx.client = client
ctx.read_stream_writer = read_stream_writer
ctx.session_message = SessionMessage(
JSONRPCMessage(
JSONRPCNotification(
jsonrpc="2.0",
method="notifications/initialized",
params={},
)
)
)
await transport._handle_post_request(ctx)
finally:
await client.aclose()
await read_stream_writer.aclose()
@pytest.mark.asyncio
async def test_streamable_http_server_passes_ignore_initialized_notification_failure():
with patch("agents.mcp.server._streamablehttp_client_with_transport") as mock_client:
mock_client.return_value = MagicMock()
server = MCPServerStreamableHttp(
params={
"url": "http://localhost:8000/mcp",
"ignore_initialized_notification_failure": True,
}
)
server.create_streams()
kwargs = mock_client.call_args.kwargs
assert kwargs["url"] == "http://localhost:8000/mcp"
assert kwargs["headers"] is None
assert kwargs["timeout"] == 5
assert kwargs["sse_read_timeout"] == 300
assert kwargs["terminate_on_close"] is True
assert kwargs["httpx_client_factory"] is _create_default_streamable_http_client
assert (
kwargs["transport_factory"] is _InitializedNotificationTolerantStreamableHTTPTransport
)
@pytest.mark.asyncio
async def test_transport_preserves_non_initialized_failures():
async def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("boom", request=request)
transport = _InitializedNotificationTolerantStreamableHTTPTransport("https://example.test/mcp")
read_stream_writer, _ = create_memory_object_stream[SessionMessage | Exception](0)
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
try:
ctx = MagicMock()
ctx.client = client
ctx.read_stream_writer = read_stream_writer
ctx.session_message = SessionMessage(
JSONRPCMessage(
JSONRPCRequest(
jsonrpc="2.0",
id=1,
method="tools/list",
params={},
)
)
)
with pytest.raises(httpx.ConnectError):
await transport._handle_post_request(ctx)
finally:
await client.aclose()
await read_stream_writer.aclose()
@pytest.mark.asyncio
async def test_stream_client_preserves_custom_factory_headers_timeout_and_auth():
seen: dict[str, object] = {}
class RecordingAuth(httpx.Auth):
def auth_flow(self, request: httpx.Request):
request.headers["Authorization"] = f"Basic {base64.b64encode(b'user:pass').decode()}"
yield request
async def handler(request: httpx.Request) -> httpx.Response:
seen["request_headers"] = dict(request.headers)
return httpx.Response(200, request=request)
def base_factory(
headers: dict[str, str] | None = None,
timeout: httpx.Timeout | None = None,
auth: httpx.Auth | None = None,
) -> httpx.AsyncClient:
seen["factory_headers"] = headers
seen["factory_timeout"] = timeout
seen["factory_auth"] = auth
return httpx.AsyncClient(
headers=headers,
timeout=timeout,
auth=auth,
transport=httpx.MockTransport(handler),
)
timeout = httpx.Timeout(12.0)
auth = RecordingAuth()
async with _streamablehttp_client_with_transport(
"https://example.test/mcp",
headers={"X-Test": "value"},
timeout=12.0,
sse_read_timeout=30.0,
httpx_client_factory=base_factory,
auth=auth,
transport_factory=_InitializedNotificationTolerantStreamableHTTPTransport,
):
pass
assert seen["factory_headers"] == {"X-Test": "value"}
seen_timeout = seen["factory_timeout"]
assert isinstance(seen_timeout, httpx.Timeout)
assert seen_timeout.connect == timeout.connect
assert seen_timeout.read == 30.0
assert seen_timeout.write == timeout.write
assert seen_timeout.pool == timeout.pool
assert seen["factory_auth"] is auth
@pytest.mark.asyncio
async def test_default_streamable_http_client_matches_expected_defaults():
timeout = httpx.Timeout(12.0)
auth = httpx.BasicAuth("user", "pass")
client = _create_default_streamable_http_client(
headers={"X-Test": "value"},
timeout=timeout,
auth=auth,
)
try:
assert client.headers["X-Test"] == "value"
assert client.timeout.connect == timeout.connect
assert client.timeout.read == timeout.read
assert client.timeout.write == timeout.write
assert client.timeout.pool == timeout.pool
assert client.auth is auth
assert client.follow_redirects is False
finally:
await client.aclose()
@@ -0,0 +1,115 @@
"""Tests for MCPServerStreamableHttp.session_id property (issue #924)."""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from agents.mcp import MCPServerStreamableHttp
class TestStreamableHttpSessionId:
"""Tests that the session_id property is correctly exposed."""
def test_session_id_is_none_before_connect(self):
"""session_id should be None when the server has not been connected yet."""
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
assert server.session_id is None
def test_session_id_returns_none_when_callback_is_none(self):
"""session_id should be None when _get_session_id callback is None."""
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
server._get_session_id = None
assert server.session_id is None
def test_session_id_returns_callback_value(self):
"""session_id should return the value from the get_session_id callback."""
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
mock_get_session_id = MagicMock(return_value="test-session-abc123")
server._get_session_id = mock_get_session_id
assert server.session_id == "test-session-abc123"
mock_get_session_id.assert_called_once()
def test_session_id_returns_none_when_callback_returns_none(self):
"""session_id should return None when the callback itself returns None."""
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
mock_get_session_id = MagicMock(return_value=None)
server._get_session_id = mock_get_session_id
assert server.session_id is None
def test_session_id_reflects_updated_callback_value(self):
"""session_id should reflect the latest value from the callback each time."""
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
call_count = 0
def changing_callback() -> str | None:
nonlocal call_count
call_count += 1
return f"session-{call_count}"
server._get_session_id = changing_callback
assert server.session_id == "session-1"
assert server.session_id == "session-2"
@pytest.mark.asyncio
async def test_connect_captures_get_session_id_callback(self):
"""connect() should capture the third element of the transport tuple as _get_session_id."""
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
mock_read = AsyncMock()
mock_write = AsyncMock()
mock_get_session_id = MagicMock(return_value="captured-session-xyz")
mock_initialize_result = MagicMock()
mock_session = AsyncMock()
mock_session.initialize = AsyncMock(return_value=mock_initialize_result)
# Simulate the full 3-tuple that streamablehttp_client returns
transport_tuple = (mock_read, mock_write, mock_get_session_id)
with patch("agents.mcp.server.ClientSession") as mock_client_session_cls:
mock_client_session_cls.return_value.__aenter__ = AsyncMock(return_value=mock_session)
mock_client_session_cls.return_value.__aexit__ = AsyncMock(return_value=None)
with patch.object(
server,
"create_streams",
) as mock_create_streams:
mock_cm = MagicMock()
mock_cm.__aenter__ = AsyncMock(return_value=transport_tuple)
mock_cm.__aexit__ = AsyncMock(return_value=None)
mock_create_streams.return_value = mock_cm
with patch.object(server.exit_stack, "enter_async_context") as mock_enter:
# First call returns transport, second call returns session
mock_enter.side_effect = [transport_tuple, mock_session]
mock_session.initialize.return_value = mock_initialize_result
await server.connect()
# After connect, _get_session_id should be the callable from the transport
assert server._get_session_id is mock_get_session_id
assert server.session_id == "captured-session-xyz"
@pytest.mark.asyncio
async def test_session_id_is_none_after_cleanup():
"""session_id must return None after disconnect (cleanup clears _get_session_id)."""
server = MCPServerStreamableHttp(params={"url": "http://localhost:8000/mcp"})
mock_get_session_id = MagicMock(return_value="session-to-clear")
# Manually inject a session-id callback to simulate a connected state
server._get_session_id = mock_get_session_id
server.session = MagicMock() # pretend connected
assert server.session_id == "session-to-clear"
# Now simulate cleanup completing (exit_stack.aclose is a no-op here)
with patch.object(server.exit_stack, "aclose", new_callable=AsyncMock):
await server.cleanup()
# After cleanup both session and _get_session_id must be None
assert server.session is None
assert server._get_session_id is None
assert server.session_id is None
+246
View File
@@ -0,0 +1,246 @@
"""
Tool filtering tests use FakeMCPServer instead of real MCPServer implementations to avoid
external dependencies (processes, network connections) and ensure fast, reliable unit tests.
FakeMCPServer delegates filtering logic to the real _MCPServerWithClientSession implementation.
"""
import asyncio
import pytest
from mcp import Tool as MCPTool
from agents import Agent
from agents.mcp import ToolFilterContext, create_static_tool_filter
from agents.run_context import RunContextWrapper
from .helpers import FakeMCPServer
def create_test_agent(name: str = "test_agent") -> Agent:
"""Create a test agent for filtering tests."""
return Agent(name=name, instructions="Test agent")
def create_test_context() -> RunContextWrapper:
"""Create a test run context for filtering tests."""
return RunContextWrapper(context=None)
# === Static Tool Filtering Tests ===
@pytest.mark.asyncio
async def test_static_tool_filtering():
"""Test all static tool filtering scenarios: allowed, blocked, both, none, etc."""
server = FakeMCPServer(server_name="test_server")
server.add_tool("tool1", {})
server.add_tool("tool2", {})
server.add_tool("tool3", {})
server.add_tool("tool4", {})
# Create test context and agent for all calls
run_context = create_test_context()
agent = create_test_agent()
# Test allowed_tool_names only
server.tool_filter = {"allowed_tool_names": ["tool1", "tool2"]}
tools = await server.list_tools(run_context, agent)
assert len(tools) == 2
assert {t.name for t in tools} == {"tool1", "tool2"}
# Test blocked_tool_names only
server.tool_filter = {"blocked_tool_names": ["tool3", "tool4"]}
tools = await server.list_tools(run_context, agent)
assert len(tools) == 2
assert {t.name for t in tools} == {"tool1", "tool2"}
# Test both filters together (allowed first, then blocked)
server.tool_filter = {
"allowed_tool_names": ["tool1", "tool2", "tool3"],
"blocked_tool_names": ["tool3"],
}
tools = await server.list_tools(run_context, agent)
assert len(tools) == 2
assert {t.name for t in tools} == {"tool1", "tool2"}
# Test no filter
server.tool_filter = None
tools = await server.list_tools(run_context, agent)
assert len(tools) == 4
# Test helper function
server.tool_filter = create_static_tool_filter(
allowed_tool_names=["tool1", "tool2"], blocked_tool_names=["tool2"]
)
tools = await server.list_tools(run_context, agent)
assert len(tools) == 1
assert tools[0].name == "tool1"
# === Dynamic Tool Filtering Core Tests ===
@pytest.mark.asyncio
async def test_dynamic_filter_sync_and_async():
"""Test both synchronous and asynchronous dynamic filters"""
server = FakeMCPServer(server_name="test_server")
server.add_tool("allowed_tool", {})
server.add_tool("blocked_tool", {})
server.add_tool("restricted_tool", {})
# Create test context and agent
run_context = create_test_context()
agent = create_test_agent()
# Test sync filter
def sync_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
return tool.name.startswith("allowed")
server.tool_filter = sync_filter
tools = await server.list_tools(run_context, agent)
assert len(tools) == 1
assert tools[0].name == "allowed_tool"
# Test async filter
async def async_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
await asyncio.sleep(0.001) # Simulate async operation
return "restricted" not in tool.name
server.tool_filter = async_filter
tools = await server.list_tools(run_context, agent)
assert len(tools) == 2
assert {t.name for t in tools} == {"allowed_tool", "blocked_tool"}
@pytest.mark.asyncio
async def test_dynamic_filter_context_handling():
"""Test dynamic filters with context access"""
server = FakeMCPServer(server_name="test_server")
server.add_tool("admin_tool", {})
server.add_tool("user_tool", {})
server.add_tool("guest_tool", {})
# Test context-independent filter
def context_independent_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
return not tool.name.startswith("admin")
server.tool_filter = context_independent_filter
run_context = create_test_context()
agent = create_test_agent()
tools = await server.list_tools(run_context, agent)
assert len(tools) == 2
assert {t.name for t in tools} == {"user_tool", "guest_tool"}
# Test context-dependent filter (needs context)
def context_dependent_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
assert context is not None
assert context.run_context is not None
assert context.agent is not None
assert context.server_name == "test_server"
# Only admin tools for agents with "admin" in name
if "admin" in context.agent.name.lower():
return True
else:
return not tool.name.startswith("admin")
server.tool_filter = context_dependent_filter
# Should work with context
run_context = RunContextWrapper(context=None)
regular_agent = create_test_agent("regular_user")
tools = await server.list_tools(run_context, regular_agent)
assert len(tools) == 2
assert {t.name for t in tools} == {"user_tool", "guest_tool"}
admin_agent = create_test_agent("admin_user")
tools = await server.list_tools(run_context, admin_agent)
assert len(tools) == 3
@pytest.mark.asyncio
async def test_dynamic_filter_error_handling():
"""Test error handling in dynamic filters"""
server = FakeMCPServer(server_name="test_server")
server.add_tool("good_tool", {})
server.add_tool("error_tool", {})
server.add_tool("another_good_tool", {})
def error_prone_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
if tool.name == "error_tool":
raise ValueError("Simulated filter error")
return True
server.tool_filter = error_prone_filter
# Test with server call
run_context = create_test_context()
agent = create_test_agent()
tools = await server.list_tools(run_context, agent)
assert len(tools) == 2
assert {t.name for t in tools} == {"good_tool", "another_good_tool"}
# === Integration Tests ===
@pytest.mark.asyncio
async def test_agent_dynamic_filtering_integration():
"""Test dynamic filtering integration with Agent methods"""
server = FakeMCPServer()
server.add_tool("file_read", {"type": "object", "properties": {"path": {"type": "string"}}})
server.add_tool(
"file_write",
{
"type": "object",
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
},
)
server.add_tool(
"database_query", {"type": "object", "properties": {"query": {"type": "string"}}}
)
server.add_tool(
"network_request", {"type": "object", "properties": {"url": {"type": "string"}}}
)
# Role-based filter for comprehensive testing
async def role_based_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
# Simulate async permission check
await asyncio.sleep(0.001)
agent_name = context.agent.name.lower()
if "admin" in agent_name:
return True
elif "readonly" in agent_name:
return "read" in tool.name or "query" in tool.name
else:
return tool.name.startswith("file_")
server.tool_filter = role_based_filter
# Test admin agent
admin_agent = Agent(name="admin_user", instructions="Admin", mcp_servers=[server])
run_context = RunContextWrapper(context=None)
admin_tools = await admin_agent.get_mcp_tools(run_context)
assert len(admin_tools) == 4
# Test readonly agent
readonly_agent = Agent(name="readonly_viewer", instructions="Read-only", mcp_servers=[server])
readonly_tools = await readonly_agent.get_mcp_tools(run_context)
assert len(readonly_tools) == 2
assert {t.name for t in readonly_tools} == {"file_read", "database_query"}
# Test regular agent
regular_agent = Agent(name="regular_user", instructions="Regular", mcp_servers=[server])
regular_tools = await regular_agent.get_mcp_tools(run_context)
assert len(regular_tools) == 2
assert {t.name for t in regular_tools} == {"file_read", "file_write"}
# Test get_all_tools method
all_tools = await regular_agent.get_all_tools(run_context)
mcp_tool_names = {
t.name
for t in all_tools
if t.name in {"file_read", "file_write", "database_query", "network_request"}
}
assert mcp_tool_names == {"file_read", "file_write"}
@@ -0,0 +1,475 @@
"""Tests for OpenAI Conversations Session functionality."""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from agents import Agent, Runner, TResponseInputItem
from agents.memory.openai_conversations_session import (
OpenAIConversationsSession,
start_openai_conversations_session,
)
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
@pytest.fixture
def mock_openai_client():
"""Create a mock OpenAI client for testing."""
client = AsyncMock()
# Mock conversations.create
client.conversations.create.return_value = MagicMock(id="test_conversation_id")
# Mock conversations.delete
client.conversations.delete.return_value = None
# Mock conversations.items.create
client.conversations.items.create.return_value = None
# Mock conversations.items.delete
client.conversations.items.delete.return_value = None
return client
@pytest.fixture
def agent() -> Agent:
"""Fixture for a basic agent with a fake model."""
return Agent(name="test", model=FakeModel())
class TestStartOpenAIConversationsSession:
"""Test the standalone start_openai_conversations_session function."""
@pytest.mark.asyncio
async def test_start_with_provided_client(self, mock_openai_client):
"""Test starting a conversation session with a provided client."""
conversation_id = await start_openai_conversations_session(mock_openai_client)
assert conversation_id == "test_conversation_id"
mock_openai_client.conversations.create.assert_called_once_with(items=[])
@pytest.mark.asyncio
async def test_start_with_none_client(self):
"""Test starting a conversation session with None client (uses default)."""
with patch(
"agents.memory.openai_conversations_session.get_default_openai_client"
) as mock_get_default:
with patch("agents.memory.openai_conversations_session.AsyncOpenAI"):
# Test case 1: get_default_openai_client returns a client
mock_default_client = AsyncMock()
mock_default_client.conversations.create.return_value = MagicMock(
id="default_client_id"
)
mock_get_default.return_value = mock_default_client
conversation_id = await start_openai_conversations_session(None)
assert conversation_id == "default_client_id"
mock_get_default.assert_called_once()
mock_default_client.conversations.create.assert_called_once_with(items=[])
@pytest.mark.asyncio
async def test_start_with_none_client_fallback(self):
"""Test starting a conversation session when get_default_openai_client returns None."""
with patch(
"agents.memory.openai_conversations_session.get_default_openai_client"
) as mock_get_default:
with patch(
"agents.memory.openai_conversations_session.AsyncOpenAI"
) as mock_async_openai:
# Test case 2: get_default_openai_client returns None, fallback to AsyncOpenAI()
mock_get_default.return_value = None
mock_fallback_client = AsyncMock()
mock_fallback_client.conversations.create.return_value = MagicMock(
id="fallback_client_id"
)
mock_async_openai.return_value = mock_fallback_client
conversation_id = await start_openai_conversations_session(None)
assert conversation_id == "fallback_client_id"
mock_get_default.assert_called_once()
mock_async_openai.assert_called_once()
mock_fallback_client.conversations.create.assert_called_once_with(items=[])
class TestOpenAIConversationsSessionConstructor:
"""Test OpenAIConversationsSession constructor and client handling."""
def test_init_with_conversation_id_and_client(self, mock_openai_client):
"""Test constructor with both conversation_id and openai_client provided."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
assert session._session_id == "test_id"
assert session._openai_client is mock_openai_client
def test_init_with_conversation_id_only(self):
"""Test constructor with only conversation_id, client should be created."""
with patch(
"agents.memory.openai_conversations_session.get_default_openai_client"
) as mock_get_default:
with patch("agents.memory.openai_conversations_session.AsyncOpenAI"):
mock_default_client = AsyncMock()
mock_get_default.return_value = mock_default_client
session = OpenAIConversationsSession(conversation_id="test_id")
assert session._session_id == "test_id"
assert session._openai_client is mock_default_client
mock_get_default.assert_called_once()
def test_init_with_client_only(self, mock_openai_client):
"""Test constructor with only openai_client, no conversation_id."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
assert session._session_id is None
assert session._openai_client is mock_openai_client
def test_init_with_no_args_fallback(self):
"""Test constructor with no args, should create default client."""
with patch(
"agents.memory.openai_conversations_session.get_default_openai_client"
) as mock_get_default:
with patch(
"agents.memory.openai_conversations_session.AsyncOpenAI"
) as mock_async_openai:
# Test fallback when get_default_openai_client returns None
mock_get_default.return_value = None
mock_fallback_client = AsyncMock()
mock_async_openai.return_value = mock_fallback_client
session = OpenAIConversationsSession()
assert session._session_id is None
assert session._openai_client is mock_fallback_client
mock_get_default.assert_called_once()
mock_async_openai.assert_called_once()
class TestOpenAIConversationsSessionLifecycle:
"""Test session ID lifecycle management."""
@pytest.mark.asyncio
async def test_get_session_id_with_existing_id(self, mock_openai_client):
"""Test _get_session_id when session_id already exists."""
session = OpenAIConversationsSession(
conversation_id="existing_id", openai_client=mock_openai_client
)
session_id = await session._get_session_id()
assert session_id == "existing_id"
# Should not call conversations.create since ID already exists
mock_openai_client.conversations.create.assert_not_called()
@pytest.mark.asyncio
async def test_get_session_id_creates_new_conversation(self, mock_openai_client):
"""Test _get_session_id when session_id is None, should create new conversation."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
session_id = await session._get_session_id()
assert session_id == "test_conversation_id"
assert session._session_id == "test_conversation_id"
mock_openai_client.conversations.create.assert_called_once_with(items=[])
@pytest.mark.asyncio
async def test_clear_session_id(self, mock_openai_client):
"""Test _clear_session_id sets session_id to None."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
await session._clear_session_id()
assert session._session_id is None
class TestOpenAIConversationsSessionBasicOperations:
"""Test basic CRUD operations with simple mocking."""
@pytest.mark.asyncio
async def test_add_items_simple(self, mock_openai_client):
"""Test adding items to the conversation."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
]
await session.add_items(items)
mock_openai_client.conversations.items.create.assert_called_once_with(
conversation_id="test_id", items=items
)
@pytest.mark.asyncio
async def test_add_items_creates_session_id(self, mock_openai_client):
"""Test that add_items creates session_id if it doesn't exist."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
items: list[TResponseInputItem] = [{"role": "user", "content": "Hello"}]
await session.add_items(items)
# Should create conversation first
mock_openai_client.conversations.create.assert_called_once_with(items=[])
# Then add items
mock_openai_client.conversations.items.create.assert_called_once_with(
conversation_id="test_conversation_id", items=items
)
@pytest.mark.asyncio
async def test_pop_item_with_items(self, mock_openai_client):
"""Test popping item when items exist using method patching."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
# Mock get_items to return one item
latest_item = {"id": "item_123", "role": "assistant", "content": "Latest message"}
with patch.object(session, "get_items", return_value=[latest_item]):
popped_item = await session.pop_item()
assert popped_item == latest_item
mock_openai_client.conversations.items.delete.assert_called_once_with(
conversation_id="test_id", item_id="item_123"
)
@pytest.mark.asyncio
async def test_pop_item_empty_session(self, mock_openai_client):
"""Test popping item from empty session."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
# Mock get_items to return empty list
with patch.object(session, "get_items", return_value=[]):
popped_item = await session.pop_item()
assert popped_item is None
mock_openai_client.conversations.items.delete.assert_not_called()
@pytest.mark.asyncio
async def test_clear_session(self, mock_openai_client):
"""Test clearing the entire session."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
await session.clear_session()
# Should delete the conversation and clear session ID
mock_openai_client.conversations.delete.assert_called_once_with(conversation_id="test_id")
assert session._session_id is None
@pytest.mark.asyncio
async def test_clear_session_creates_session_id_first(self, mock_openai_client):
"""Test that clear_session creates session_id if it doesn't exist."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
await session.clear_session()
# Should create conversation first, then delete it
mock_openai_client.conversations.create.assert_called_once_with(items=[])
mock_openai_client.conversations.delete.assert_called_once_with(
conversation_id="test_conversation_id"
)
assert session._session_id is None
class TestOpenAIConversationsSessionRunnerIntegration:
"""Test integration with Agent Runner using simple mocking."""
@pytest.mark.asyncio
async def test_runner_integration_basic(self, agent: Agent, mock_openai_client):
"""Test that OpenAIConversationsSession works with Agent Runner."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
# Mock the session methods to avoid complex async iterator setup
with patch.object(session, "get_items", return_value=[]):
with patch.object(session, "add_items") as mock_add_items:
# Run the agent
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("San Francisco")])
result = await Runner.run(
agent, "What city is the Golden Gate Bridge in?", session=session
)
assert result.final_output == "San Francisco"
# Verify session interactions occurred
mock_add_items.assert_called()
@pytest.mark.asyncio
async def test_runner_with_conversation_history(self, agent: Agent, mock_openai_client):
"""Test that conversation history is preserved across Runner calls."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
# Mock conversation history
conversation_history = [
{"role": "user", "content": "What city is the Golden Gate Bridge in?"},
{"role": "assistant", "content": "San Francisco"},
]
with patch.object(session, "get_items", return_value=conversation_history):
with patch.object(session, "add_items"):
# Second turn - should have access to previous conversation
assert isinstance(agent.model, FakeModel)
agent.model.set_next_output([get_text_message("California")])
result = await Runner.run(agent, "What state is it in?", session=session)
assert result.final_output == "California"
# Verify that the model received the conversation history
last_input = agent.model.last_turn_args["input"]
assert len(last_input) > 1 # Should include previous messages
# Check that previous conversation is included
input_contents = [str(item.get("content", "")) for item in last_input]
assert any("Golden Gate Bridge" in content for content in input_contents)
class TestOpenAIConversationsSessionErrorHandling:
"""Test error handling for various failure scenarios."""
@pytest.mark.asyncio
async def test_api_failure_during_conversation_creation(self, mock_openai_client):
"""Test handling of API failures during conversation creation."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
# Mock API failure
mock_openai_client.conversations.create.side_effect = Exception("API Error")
with pytest.raises(Exception, match="API Error"):
await session._get_session_id()
@pytest.mark.asyncio
async def test_api_failure_during_add_items(self, mock_openai_client):
"""Test handling of API failures during add_items."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
mock_openai_client.conversations.items.create.side_effect = Exception("Add items failed")
items: list[TResponseInputItem] = [{"role": "user", "content": "Hello"}]
with pytest.raises(Exception, match="Add items failed"):
await session.add_items(items)
@pytest.mark.asyncio
async def test_api_failure_during_clear_session(self, mock_openai_client):
"""Test handling of API failures during clear_session."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
mock_openai_client.conversations.delete.side_effect = Exception("Clear session failed")
with pytest.raises(Exception, match="Clear session failed"):
await session.clear_session()
@pytest.mark.asyncio
async def test_invalid_item_id_in_pop_item(self, mock_openai_client):
"""Test handling of invalid item ID during pop_item."""
session = OpenAIConversationsSession(
conversation_id="test_id", openai_client=mock_openai_client
)
# Mock item without ID
invalid_item = {"role": "assistant", "content": "No ID"}
with patch.object(session, "get_items", return_value=[invalid_item]):
# This should raise a KeyError because 'id' field is missing
with pytest.raises(KeyError, match="'id'"):
await session.pop_item()
class TestOpenAIConversationsSessionConcurrentAccess:
"""Test concurrent access patterns with simple scenarios."""
@pytest.mark.asyncio
async def test_multiple_sessions_different_conversation_ids(self, mock_openai_client):
"""Test that multiple sessions with different conversation IDs are isolated."""
session1 = OpenAIConversationsSession(
conversation_id="conversation_1", openai_client=mock_openai_client
)
session2 = OpenAIConversationsSession(
conversation_id="conversation_2", openai_client=mock_openai_client
)
items1: list[TResponseInputItem] = [{"role": "user", "content": "Session 1 message"}]
items2: list[TResponseInputItem] = [{"role": "user", "content": "Session 2 message"}]
# Add items to both sessions
await session1.add_items(items1)
await session2.add_items(items2)
# Verify calls were made with correct conversation IDs
assert mock_openai_client.conversations.items.create.call_count == 2
# Check the calls
calls = mock_openai_client.conversations.items.create.call_args_list
assert calls[0][1]["conversation_id"] == "conversation_1"
assert calls[0][1]["items"] == items1
assert calls[1][1]["conversation_id"] == "conversation_2"
assert calls[1][1]["items"] == items2
@pytest.mark.asyncio
async def test_session_id_lazy_creation_consistency(self, mock_openai_client):
"""Test that session ID creation is consistent across multiple calls."""
session = OpenAIConversationsSession(openai_client=mock_openai_client)
# Call _get_session_id multiple times
id1 = await session._get_session_id()
id2 = await session._get_session_id()
id3 = await session._get_session_id()
# All should return the same session ID
assert id1 == id2 == id3 == "test_conversation_id"
# Conversation should only be created once
mock_openai_client.conversations.create.assert_called_once()
# ============================================================================
# SessionSettings Tests
# ============================================================================
class TestOpenAIConversationsSessionSettings:
"""Test SessionSettings integration with OpenAIConversationsSession."""
def test_session_settings_default(self, mock_openai_client):
"""Test that session_settings defaults to empty SessionSettings."""
from agents.memory import SessionSettings
session = OpenAIConversationsSession(openai_client=mock_openai_client)
# Should have default SessionSettings
assert isinstance(session.session_settings, SessionSettings)
assert session.session_settings.limit is None
def test_session_settings_constructor(self, mock_openai_client):
"""Test passing session_settings via constructor."""
from agents.memory import SessionSettings
session = OpenAIConversationsSession(
openai_client=mock_openai_client, session_settings=SessionSettings(limit=5)
)
assert session.session_settings is not None
assert session.session_settings.limit == 5
File diff suppressed because it is too large Load Diff
+808
View File
@@ -0,0 +1,808 @@
"""Tests for session memory functionality."""
import asyncio
import sqlite3
import tempfile
from pathlib import Path
import pytest
from agents import Agent, RunConfig, Runner, SQLiteSession, TResponseInputItem
from tests.fake_model import FakeModel
from tests.test_responses import get_text_message
# Helper functions for parametrized testing of different Runner methods
def _run_sync_wrapper(agent, input_data, **kwargs):
"""Wrapper for run_sync that properly sets up an event loop."""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
return Runner.run_sync(agent, input_data, **kwargs)
finally:
loop.close()
async def run_agent_async(runner_method: str, agent, input_data, **kwargs):
"""Helper function to run agent with different methods."""
if runner_method == "run":
return await Runner.run(agent, input_data, **kwargs)
elif runner_method == "run_sync":
# For run_sync, we need to run it in a thread with its own event loop
return await asyncio.to_thread(_run_sync_wrapper, agent, input_data, **kwargs)
elif runner_method == "run_streamed":
result = Runner.run_streamed(agent, input_data, **kwargs)
# For streaming, we first try to get at least one event to trigger any early exceptions
# If there's an exception in setup (like memory validation), it will be raised here
try:
first_event = None
async for event in result.stream_events():
if first_event is None:
first_event = event
# Continue consuming all events
pass
except Exception:
# If an exception occurs during streaming, we let it propagate up
raise
return result
else:
raise ValueError(f"Unknown runner method: {runner_method}")
# Parametrized tests for different runner methods
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_memory_basic_functionality_parametrized(runner_method):
"""Test basic session memory functionality with SQLite backend across all runner methods."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_memory.db"
session_id = "test_session_123"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
agent = Agent(name="test", model=model)
# First turn
model.set_next_output([get_text_message("San Francisco")])
result1 = await run_agent_async(
runner_method,
agent,
"What city is the Golden Gate Bridge in?",
session=session,
)
assert result1.final_output == "San Francisco"
# Second turn - should have conversation history
model.set_next_output([get_text_message("California")])
result2 = await run_agent_async(
runner_method,
agent,
"What state is it in?",
session=session,
)
assert result2.final_output == "California"
# Verify that the input to the second turn includes the previous conversation
# The model should have received the full conversation history
last_input = model.last_turn_args["input"]
assert len(last_input) > 1 # Should have more than just the current message
session.close()
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_memory_with_explicit_instance_parametrized(runner_method):
"""Test session memory with an explicit SQLiteSession instance across all runner methods."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_memory.db"
session_id = "test_session_456"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
agent = Agent(name="test", model=model)
# First turn
model.set_next_output([get_text_message("Hello")])
result1 = await run_agent_async(runner_method, agent, "Hi there", session=session)
assert result1.final_output == "Hello"
# Second turn
model.set_next_output([get_text_message("I remember you said hi")])
result2 = await run_agent_async(
runner_method,
agent,
"Do you remember what I said?",
session=session,
)
assert result2.final_output == "I remember you said hi"
session.close()
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_memory_disabled_parametrized(runner_method):
"""Test that session memory is disabled when session=None across all runner methods."""
model = FakeModel()
agent = Agent(name="test", model=model)
# First turn (no session parameters = disabled)
model.set_next_output([get_text_message("Hello")])
result1 = await run_agent_async(runner_method, agent, "Hi there")
assert result1.final_output == "Hello"
# Second turn - should NOT have conversation history
model.set_next_output([get_text_message("I don't remember")])
result2 = await run_agent_async(runner_method, agent, "Do you remember what I said?")
assert result2.final_output == "I don't remember"
# Verify that the input to the second turn is just the current message
last_input = model.last_turn_args["input"]
assert len(last_input) == 1 # Should only have the current message
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_memory_different_sessions_parametrized(runner_method):
"""Test that different session IDs maintain separate conversation histories across all runner
methods."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_memory.db"
model = FakeModel()
agent = Agent(name="test", model=model)
# Session 1
session_id_1 = "session_1"
session_1 = SQLiteSession(session_id_1, db_path)
model.set_next_output([get_text_message("I like cats")])
result1 = await run_agent_async(runner_method, agent, "I like cats", session=session_1)
assert result1.final_output == "I like cats"
# Session 2 - different session
session_id_2 = "session_2"
session_2 = SQLiteSession(session_id_2, db_path)
model.set_next_output([get_text_message("I like dogs")])
result2 = await run_agent_async(runner_method, agent, "I like dogs", session=session_2)
assert result2.final_output == "I like dogs"
# Back to Session 1 - should remember cats, not dogs
model.set_next_output([get_text_message("Yes, you mentioned cats")])
result3 = await run_agent_async(
runner_method,
agent,
"What did I say I like?",
session=session_1,
)
assert result3.final_output == "Yes, you mentioned cats"
session_1.close()
session_2.close()
@pytest.mark.asyncio
async def test_sqlite_session_memory_direct():
"""Test SQLiteSession class directly."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_direct.db"
session_id = "direct_test"
session = SQLiteSession(session_id, db_path)
# Test adding and retrieving items
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
]
await session.add_items(items)
retrieved = await session.get_items()
assert len(retrieved) == 2
assert retrieved[0].get("role") == "user"
assert retrieved[0].get("content") == "Hello"
assert retrieved[1].get("role") == "assistant"
assert retrieved[1].get("content") == "Hi there!"
# Test clearing session
await session.clear_session()
retrieved_after_clear = await session.get_items()
assert len(retrieved_after_clear) == 0
session.close()
@pytest.mark.asyncio
async def test_sqlite_session_close_closes_worker_thread_connections():
"""Test that close cleans up connections opened by async worker threads."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_worker_thread_close.db"
session = SQLiteSession("worker_thread_close", db_path)
await session.add_items([{"role": "user", "content": "Hello"}])
connections = list(session._connections)
assert connections
session.close()
assert session._connections == set()
with pytest.raises(sqlite3.ProgrammingError):
connections[0].execute("SELECT 1")
@pytest.mark.asyncio
async def test_sqlite_session_memory_pop_item():
"""Test SQLiteSession pop_item functionality."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_pop.db"
session_id = "pop_test"
session = SQLiteSession(session_id, db_path)
# Test popping from empty session
popped = await session.pop_item()
assert popped is None
# Add items
items: list[TResponseInputItem] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there!"},
{"role": "user", "content": "How are you?"},
]
await session.add_items(items)
# Verify all items are there
retrieved = await session.get_items()
assert len(retrieved) == 3
# Pop the most recent item
popped = await session.pop_item()
assert popped is not None
assert popped.get("role") == "user"
assert popped.get("content") == "How are you?"
# Verify item was removed
retrieved_after_pop = await session.get_items()
assert len(retrieved_after_pop) == 2
assert retrieved_after_pop[-1].get("content") == "Hi there!"
# Pop another item
popped2 = await session.pop_item()
assert popped2 is not None
assert popped2.get("role") == "assistant"
assert popped2.get("content") == "Hi there!"
# Pop the last item
popped3 = await session.pop_item()
assert popped3 is not None
assert popped3.get("role") == "user"
assert popped3.get("content") == "Hello"
# Try to pop from empty session again
popped4 = await session.pop_item()
assert popped4 is None
# Verify session is empty
final_items = await session.get_items()
assert len(final_items) == 0
session.close()
@pytest.mark.asyncio
async def test_session_memory_pop_different_sessions():
"""Test that pop_item only affects the specified session."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_pop_sessions.db"
session_1_id = "session_1"
session_2_id = "session_2"
session_1 = SQLiteSession(session_1_id, db_path)
session_2 = SQLiteSession(session_2_id, db_path)
# Add items to both sessions
items_1: list[TResponseInputItem] = [
{"role": "user", "content": "Session 1 message"},
]
items_2: list[TResponseInputItem] = [
{"role": "user", "content": "Session 2 message 1"},
{"role": "user", "content": "Session 2 message 2"},
]
await session_1.add_items(items_1)
await session_2.add_items(items_2)
# Pop from session 2
popped = await session_2.pop_item()
assert popped is not None
assert popped.get("content") == "Session 2 message 2"
# Verify session 1 is unaffected
session_1_items = await session_1.get_items()
assert len(session_1_items) == 1
assert session_1_items[0].get("content") == "Session 1 message"
# Verify session 2 has one item left
session_2_items = await session_2.get_items()
assert len(session_2_items) == 1
assert session_2_items[0].get("content") == "Session 2 message 1"
session_1.close()
session_2.close()
@pytest.mark.asyncio
async def test_sqlite_session_pop_item_skips_corrupt_most_recent():
"""pop_item skips corrupt newest rows and returns the next valid item."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_pop_corrupt.db"
session = SQLiteSession("pop_corrupt", db_path)
valid_item: TResponseInputItem = {"role": "user", "content": "valid"}
await session.add_items([valid_item])
with session._locked_connection() as conn:
conn.execute(
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
(session.session_id, "not valid json {{{"),
)
conn.commit()
assert await session.pop_item() == valid_item
assert await session.get_items() == []
session.close()
@pytest.mark.asyncio
async def test_sqlite_session_pop_item_returns_none_after_dropping_only_corrupt_rows():
"""pop_item removes corrupt rows and returns None when no valid items remain."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_pop_only_corrupt.db"
session = SQLiteSession("pop_only_corrupt", db_path)
with session._locked_connection() as conn:
conn.execute(
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
(session.session_id, "not valid json {{{"),
)
conn.commit()
assert await session.pop_item() is None
assert await session.get_items() == []
session.close()
@pytest.mark.asyncio
async def test_sqlite_session_get_items_with_limit():
"""Test SQLiteSession get_items with limit parameter."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_count.db"
session_id = "count_test"
session = SQLiteSession(session_id, db_path)
# Add multiple items
items: list[TResponseInputItem] = [
{"role": "user", "content": "Message 1"},
{"role": "assistant", "content": "Response 1"},
{"role": "user", "content": "Message 2"},
{"role": "assistant", "content": "Response 2"},
{"role": "user", "content": "Message 3"},
{"role": "assistant", "content": "Response 3"},
]
await session.add_items(items)
# Test getting all items (default behavior)
all_items = await session.get_items()
assert len(all_items) == 6
assert all_items[0].get("content") == "Message 1"
assert all_items[-1].get("content") == "Response 3"
# Test getting latest 2 items
latest_2 = await session.get_items(limit=2)
assert len(latest_2) == 2
assert latest_2[0].get("content") == "Message 3"
assert latest_2[1].get("content") == "Response 3"
# Test getting latest 4 items
latest_4 = await session.get_items(limit=4)
assert len(latest_4) == 4
assert latest_4[0].get("content") == "Message 2"
assert latest_4[1].get("content") == "Response 2"
assert latest_4[2].get("content") == "Message 3"
assert latest_4[3].get("content") == "Response 3"
# Test getting more items than available
latest_10 = await session.get_items(limit=10)
assert len(latest_10) == 6 # Should return all available items
assert latest_10[0].get("content") == "Message 1"
assert latest_10[-1].get("content") == "Response 3"
# Test getting 0 items
latest_0 = await session.get_items(limit=0)
assert len(latest_0) == 0
session.close()
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_memory_appends_list_input_by_default(runner_method):
"""Test that list inputs are appended to session history when no callback is provided."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_validation.db"
session_id = "test_validation_parametrized"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
agent = Agent(name="test", model=model)
initial_history: list[TResponseInputItem] = [
{"role": "user", "content": "Earlier message"},
{"role": "assistant", "content": "Saved reply"},
]
await session.add_items(initial_history)
list_input = [{"role": "user", "content": "Test message"}]
model.set_next_output([get_text_message("This should run")])
await run_agent_async(runner_method, agent, list_input, session=session)
assert model.last_turn_args["input"] == initial_history + list_input
session.close()
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_callback_prepared_input(runner_method):
"""Test if the user passes a list of items and want to append them."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_memory.db"
model = FakeModel()
agent = Agent(name="test", model=model)
# Session
session_id = "session_1"
session = SQLiteSession(session_id, db_path)
# Add first messages manually
initial_history: list[TResponseInputItem] = [
{"role": "user", "content": "Hello there."},
{"role": "assistant", "content": "Hi, I'm here to assist you."},
]
try:
await session.add_items(initial_history)
def filter_assistant_messages(history, new_input):
# Only include user messages from history
return [item for item in history if item["role"] == "user"] + new_input
new_turn_input = [{"role": "user", "content": "What your name?"}]
model.set_next_output([get_text_message("I'm gpt-4o")])
# Run the agent with the callable
await run_agent_async(
runner_method,
agent,
new_turn_input,
session=session,
run_config=RunConfig(session_input_callback=filter_assistant_messages),
)
expected_model_input = [
initial_history[0], # From history
new_turn_input[0], # New input
]
assert len(model.last_turn_args["input"]) == 2
assert model.last_turn_args["input"] == expected_model_input
finally:
session.close()
@pytest.mark.asyncio
async def test_sqlite_session_unicode_content():
"""Test that session correctly stores and retrieves unicode/non-ASCII content."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_unicode.db"
session_id = "unicode_test"
session = SQLiteSession(session_id, db_path)
# Add unicode content to the session
items: list[TResponseInputItem] = [
{"role": "user", "content": "こんにちは"},
{"role": "assistant", "content": "😊👍"},
{"role": "user", "content": "Привет"},
]
await session.add_items(items)
# Retrieve items and verify unicode content
retrieved = await session.get_items()
assert retrieved[0].get("content") == "こんにちは"
assert retrieved[1].get("content") == "😊👍"
assert retrieved[2].get("content") == "Привет"
session.close()
@pytest.mark.asyncio
async def test_sqlite_session_special_characters_and_sql_injection():
"""
Test that session safely stores and retrieves items with special characters and SQL keywords.
"""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_special_chars.db"
session_id = "special_chars_test"
session = SQLiteSession(session_id, db_path)
# Add items with special characters and SQL keywords
items: list[TResponseInputItem] = [
{"role": "user", "content": "O'Reilly"},
{"role": "assistant", "content": "DROP TABLE sessions;"},
{"role": "user", "content": ('"SELECT * FROM users WHERE name = "admin";"')},
{"role": "assistant", "content": "Robert'); DROP TABLE students;--"},
{"role": "user", "content": "Normal message"},
]
await session.add_items(items)
# Retrieve all items and verify they are stored correctly
retrieved = await session.get_items()
assert len(retrieved) == len(items)
assert retrieved[0].get("content") == "O'Reilly"
assert retrieved[1].get("content") == "DROP TABLE sessions;"
assert retrieved[2].get("content") == '"SELECT * FROM users WHERE name = "admin";"'
assert retrieved[3].get("content") == "Robert'); DROP TABLE students;--"
assert retrieved[4].get("content") == "Normal message"
session.close()
@pytest.mark.asyncio
async def test_sqlite_session_concurrent_access():
"""
Test concurrent access to the same session to verify data integrity.
"""
import concurrent.futures
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_concurrent.db"
session_id = "concurrent_test"
session = SQLiteSession(session_id, db_path)
# Add initial item
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(10)
]
# Use ThreadPoolExecutor to simulate concurrent writes
def add_item(item):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(session.add_items([item]))
loop.close()
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
executor.map(add_item, items)
# Retrieve all items and verify all are present
retrieved = await session.get_items()
contents = {
content
for item in retrieved
for content in [item.get("content")]
if isinstance(content, str)
}
expected = {f"Message {i}" for i in range(10)}
assert contents == expected
session.close()
@pytest.mark.asyncio
async def test_sqlite_session_file_lock_is_shared_across_instances():
"""File-backed sessions pointing at the same DB path should reuse one process-local lock."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_shared_lock.db"
lock_path = db_path.resolve()
session_1 = SQLiteSession("session_1", db_path)
session_2 = SQLiteSession("session_2", db_path)
assert session_1._lock is session_2._lock
assert SQLiteSession._file_lock_counts[lock_path] == 2
await asyncio.gather(
session_1.add_items([{"role": "user", "content": "session_1"}]),
session_2.add_items([{"role": "user", "content": "session_2"}]),
)
assert [item.get("content") for item in await session_1.get_items()] == ["session_1"]
assert [item.get("content") for item in await session_2.get_items()] == ["session_2"]
session_1.close()
assert SQLiteSession._file_lock_counts[lock_path] == 1
assert lock_path in SQLiteSession._file_locks
session_2.close()
assert lock_path not in SQLiteSession._file_lock_counts
assert lock_path not in SQLiteSession._file_locks
@pytest.mark.asyncio
async def test_session_add_items_exception_propagates_in_streamed():
"""Test that exceptions from session.add_items are properly propagated
in run_streamed instead of causing the stream to hang forever.
Regression test for https://github.com/openai/openai-agents-python/issues/2130
"""
session = SQLiteSession("test_exception_session")
async def _failing_add_items(_items):
raise RuntimeError("Simulated session.add_items failure")
session.add_items = _failing_add_items # type: ignore[method-assign]
model = FakeModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("This should not be reached")])
result = Runner.run_streamed(agent, "Hello", session=session)
async def consume_stream():
async for _event in result.stream_events():
pass
with pytest.raises(RuntimeError, match="Simulated session.add_items failure"):
# Timeout ensures test fails fast instead of hanging forever if bug regresses
await asyncio.wait_for(consume_stream(), timeout=5.0)
session.close()
# ============================================================================
# SessionSettings Tests
# ============================================================================
@pytest.mark.asyncio
async def test_session_settings_default():
"""Test that session_settings defaults to empty SessionSettings."""
from agents.memory import SessionSettings
session = SQLiteSession("default_settings_test")
# Should have default SessionSettings
assert isinstance(session.session_settings, SessionSettings)
assert session.session_settings.limit is None
session.close()
@pytest.mark.asyncio
async def test_session_settings_constructor():
"""Test passing session_settings via constructor."""
from agents.memory import SessionSettings
session = SQLiteSession("constructor_settings_test", session_settings=SessionSettings(limit=5))
assert session.session_settings is not None
assert session.session_settings.limit == 5
session.close()
@pytest.mark.asyncio
async def test_get_items_uses_session_settings_limit():
"""Test that get_items uses session_settings.limit as default."""
from agents.memory import SessionSettings
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_settings_limit.db"
session = SQLiteSession(
"uses_settings_limit_test", db_path, session_settings=SessionSettings(limit=3)
)
# Add 5 items
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(5)
]
await session.add_items(items)
# get_items() with no limit should use session_settings.limit=3
retrieved = await session.get_items()
assert len(retrieved) == 3
# Should get the last 3 items
assert retrieved[0].get("content") == "Message 2"
assert retrieved[1].get("content") == "Message 3"
assert retrieved[2].get("content") == "Message 4"
session.close()
@pytest.mark.asyncio
async def test_get_items_explicit_limit_overrides_session_settings():
"""Test that explicit limit parameter overrides session_settings."""
from agents.memory import SessionSettings
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_override.db"
session = SQLiteSession(
"explicit_override_test", db_path, session_settings=SessionSettings(limit=5)
)
# Add 10 items
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Message {i}"} for i in range(10)
]
await session.add_items(items)
# Explicit limit=2 should override session_settings.limit=5
retrieved = await session.get_items(limit=2)
assert len(retrieved) == 2
assert retrieved[0].get("content") == "Message 8"
assert retrieved[1].get("content") == "Message 9"
session.close()
@pytest.mark.asyncio
async def test_session_settings_resolve():
"""Test SessionSettings.resolve() method."""
from agents.memory import SessionSettings
base = SessionSettings(limit=100)
override = SessionSettings(limit=50)
final = base.resolve(override)
assert final.limit == 50 # Override wins
assert base.limit == 100 # Original unchanged
# Resolving with None returns self
final_none = base.resolve(None)
assert final_none.limit == 100
@pytest.mark.asyncio
async def test_runner_with_session_settings_override():
"""Test that RunConfig can override session's default settings."""
from agents.memory import SessionSettings
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_runner_override.db"
# Session with default limit=100
session = SQLiteSession(
"runner_override_test", db_path, session_settings=SessionSettings(limit=100)
)
# Add some history
items: list[TResponseInputItem] = [
{"role": "user", "content": f"Turn {i}"} for i in range(10)
]
await session.add_items(items)
model = FakeModel()
agent = Agent(name="test", model=model)
model.set_next_output([get_text_message("Got it")])
await Runner.run(
agent,
"New question",
session=session,
run_config=RunConfig(
session_settings=SessionSettings(limit=2) # Override to 2
),
)
# Verify the agent received only the last 2 history items + new question
last_input = model.last_turn_args["input"]
# Filter out the new "New question" input
history_items = [item for item in last_input if item.get("content") != "New question"]
# Should have 2 history items (last two from the 10 we added)
assert len(history_items) == 2
session.close()
+176
View File
@@ -0,0 +1,176 @@
"""Test session_limit parameter functionality via SessionSettings."""
import tempfile
from pathlib import Path
import pytest
from agents import Agent, RunConfig, SQLiteSession
from agents.memory import SessionSettings
from tests.fake_model import FakeModel
from tests.memory.test_session import run_agent_async
from tests.test_responses import get_text_message
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_limit_parameter(runner_method):
"""Test that session_limit parameter correctly limits conversation history
retrieved from session across all Runner methods."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_limit.db"
session_id = "limit_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
agent = Agent(name="test", model=model)
# Build up a longer conversation history
model.set_next_output([get_text_message("Reply 1")])
await run_agent_async(runner_method, agent, "Message 1", session=session)
model.set_next_output([get_text_message("Reply 2")])
await run_agent_async(runner_method, agent, "Message 2", session=session)
model.set_next_output([get_text_message("Reply 3")])
await run_agent_async(runner_method, agent, "Message 3", session=session)
# Verify we have 6 items in total (3 user + 3 assistant)
all_items = await session.get_items()
assert len(all_items) == 6
# Test session_limit via RunConfig - should only get last 2 history items + new input
model.set_next_output([get_text_message("Reply 4")])
await run_agent_async(
runner_method,
agent,
"Message 4",
session=session,
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
)
# Verify model received limited history
last_input = model.last_turn_args["input"]
# Should have: 2 history items + 1 new message = 3 total
assert len(last_input) == 3
# First item should be "Message 3" (not Message 1 or 2)
assert last_input[0].get("content") == "Message 3"
# Assistant message has content as a list
assert last_input[1].get("content")[0]["text"] == "Reply 3"
assert last_input[2].get("content") == "Message 4"
session.close()
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_limit_zero(runner_method):
"""Test that session_limit=0 provides no history, only new message."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_limit_zero.db"
session_id = "limit_zero_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
agent = Agent(name="test", model=model)
# Build conversation history
model.set_next_output([get_text_message("Reply 1")])
await run_agent_async(runner_method, agent, "Message 1", session=session)
model.set_next_output([get_text_message("Reply 2")])
await run_agent_async(runner_method, agent, "Message 2", session=session)
# Test with limit=0 - should get NO history, just new message
model.set_next_output([get_text_message("Reply 3")])
await run_agent_async(
runner_method,
agent,
"Message 3",
session=session,
run_config=RunConfig(session_settings=SessionSettings(limit=0)),
)
# Verify model received only the new message
last_input = model.last_turn_args["input"]
assert len(last_input) == 1
assert last_input[0].get("content") == "Message 3"
session.close()
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_limit_none_gets_all_history(runner_method):
"""Test that session_limit=None retrieves entire history (default behavior)."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_limit_none.db"
session_id = "limit_none_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
agent = Agent(name="test", model=model)
# Build longer conversation
for i in range(1, 6):
model.set_next_output([get_text_message(f"Reply {i}")])
await run_agent_async(runner_method, agent, f"Message {i}", session=session)
# Verify 10 items in session (5 user + 5 assistant)
all_items = await session.get_items()
assert len(all_items) == 10
# Test with session_limit=None (default) - should get all history
model.set_next_output([get_text_message("Reply 6")])
await run_agent_async(
runner_method,
agent,
"Message 6",
session=session,
run_config=RunConfig(session_settings=SessionSettings(limit=None)),
)
# Verify model received all history + new message
last_input = model.last_turn_args["input"]
assert len(last_input) == 11 # 10 history + 1 new
assert last_input[0].get("content") == "Message 1"
assert last_input[-1].get("content") == "Message 6"
session.close()
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
@pytest.mark.asyncio
async def test_session_limit_larger_than_history(runner_method):
"""Test that session_limit larger than history size returns all items."""
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "test_limit_large.db"
session_id = "limit_large_test"
session = SQLiteSession(session_id, db_path)
model = FakeModel()
agent = Agent(name="test", model=model)
# Build small conversation
model.set_next_output([get_text_message("Reply 1")])
await run_agent_async(runner_method, agent, "Message 1", session=session)
# Test with limit=100 (much larger than actual history)
model.set_next_output([get_text_message("Reply 2")])
await run_agent_async(
runner_method,
agent,
"Message 2",
session=session,
run_config=RunConfig(session_settings=SessionSettings(limit=100)),
)
# Verify model received all available history + new message
last_input = model.last_turn_args["input"]
assert len(last_input) == 3 # 2 history + 1 new
assert last_input[0].get("content") == "Message 1"
# Assistant message has content as a list
assert last_input[1].get("content")[0]["text"] == "Reply 1"
assert last_input[2].get("content") == "Message 2"
session.close()
@@ -0,0 +1,135 @@
from __future__ import annotations
from typing import Any, cast
import pytest
from agents.items import TResponseInputItem
from agents.run_internal.session_persistence import _sanitize_openai_conversation_item
def _sanitize(item: dict[str, Any]) -> dict[str, Any]:
return cast(dict[str, Any], _sanitize_openai_conversation_item(cast(TResponseInputItem, item)))
@pytest.mark.parametrize(
"item_type",
[
"file_search_call",
"web_search_call",
"computer_call",
"code_interpreter_call",
"image_generation_call",
"local_shell_call",
"local_shell_call_output",
"mcp_list_tools",
"mcp_approval_request",
"mcp_call",
"item_reference",
],
)
def test_sanitize_preserves_ids_required_by_openai_conversation_items(item_type: str) -> None:
item = {"type": item_type, "id": f"{item_type}_abc123", "status": "completed"}
sanitized = _sanitize(item)
assert sanitized["id"] == f"{item_type}_abc123"
assert sanitized["type"] == item_type
def test_sanitize_preserves_file_search_call_payload_id() -> None:
item = {
"type": "file_search_call",
"id": "fs_call_abc",
"queries": ["latest q3 revenue"],
"status": "completed",
"results": [{"file_id": "file_1", "filename": "q3.pdf", "score": 0.9, "text": "..."}],
}
sanitized = _sanitize(item)
assert sanitized["id"] == "fs_call_abc"
assert sanitized["queries"] == ["latest q3 revenue"]
assert sanitized["status"] == "completed"
@pytest.mark.parametrize(
"item",
[
{
"type": "message",
"id": "msg_abc",
"role": "assistant",
"content": [{"type": "output_text", "text": "hi"}],
},
{
"type": "function_call",
"id": "fc_abc",
"call_id": "call_abc",
"name": "get_weather",
"arguments": "{}",
},
{"type": "function_call_output", "id": "out_abc", "call_id": "call_abc", "output": "{}"},
{"type": "computer_call_output", "id": "ccout_abc", "call_id": "call_abc", "output": {}},
{"type": "tool_search_call", "id": "ts_abc", "status": "completed"},
{"type": "shell_call", "id": "sh_abc", "call_id": "call_abc", "action": {}},
],
)
def test_sanitize_strips_optional_or_policy_controlled_ids(item: dict[str, Any]) -> None:
sanitized = _sanitize(item)
assert "id" not in sanitized
assert sanitized["type"] == item["type"]
def test_sanitize_preserves_reasoning_id_for_openai_conversations() -> None:
item = {
"type": "reasoning",
"id": "rs_abc",
"summary": [],
"content": [],
"provider_data": {"server": "metadata"},
}
sanitized = _sanitize(item)
assert sanitized["id"] == "rs_abc"
assert "provider_data" not in sanitized
def test_sanitize_preserves_reasoning_encrypted_content() -> None:
item = {
"type": "reasoning",
"summary": [],
"content": [],
"encrypted_content": "encrypted",
}
sanitized = _sanitize(item)
assert sanitized["encrypted_content"] == "encrypted"
def test_sanitize_always_strips_provider_data() -> None:
item = {
"type": "file_search_call",
"id": "fs_keep",
"status": "completed",
"provider_data": {"model": "gpt-4.1-mini"},
}
sanitized = _sanitize(item)
assert sanitized["id"] == "fs_keep"
assert "provider_data" not in sanitized
def test_sanitize_passes_through_non_dict_items() -> None:
class DummyItem:
pass
item = DummyItem()
sanitized: Any = _sanitize_openai_conversation_item(cast(TResponseInputItem, item))
assert sanitized is item
+359
View File
@@ -0,0 +1,359 @@
import json
from dataclasses import fields
from openai.types.shared import Reasoning
from pydantic import TypeAdapter
from pydantic_core import to_json
from agents.model_settings import MCPToolChoice, ModelSettings
from agents.retry import ModelRetryBackoffSettings, ModelRetrySettings, retry_policies
def verify_serialization(model_settings: ModelSettings) -> None:
"""Verify that ModelSettings can be serialized to a JSON string."""
json_dict = model_settings.to_json_dict()
json_string = json.dumps(json_dict)
assert json_string is not None
def test_basic_serialization() -> None:
"""Tests whether ModelSettings can be serialized to a JSON string."""
# First, lets create a ModelSettings instance
model_settings = ModelSettings(
temperature=0.5,
top_p=0.9,
max_tokens=100,
)
# Now, lets serialize the ModelSettings instance to a JSON string
verify_serialization(model_settings)
def test_mcp_tool_choice_serialization() -> None:
"""Tests whether ModelSettings with MCPToolChoice can be serialized to a JSON string."""
# First, lets create a ModelSettings instance
model_settings = ModelSettings(
temperature=0.5,
tool_choice=MCPToolChoice(server_label="mcp", name="mcp_tool"),
)
# Now, lets serialize the ModelSettings instance to a JSON string
verify_serialization(model_settings)
def test_all_fields_serialization() -> None:
"""Tests whether ModelSettings can be serialized to a JSON string."""
# First, lets create a ModelSettings instance
model_settings = ModelSettings(
temperature=0.5,
top_p=0.9,
frequency_penalty=0.0,
presence_penalty=0.0,
tool_choice="auto",
parallel_tool_calls=True,
truncation="auto",
max_tokens=100,
reasoning=Reasoning(),
metadata={"foo": "bar"},
store=False,
prompt_cache_retention="24h",
include_usage=False,
response_include=["reasoning.encrypted_content"],
top_logprobs=1,
verbosity="low",
extra_query={"foo": "bar"},
extra_body={"foo": "bar"},
extra_headers={"foo": "bar"},
extra_args={"custom_param": "value", "another_param": 42},
retry=ModelRetrySettings(
max_retries=2,
backoff=ModelRetryBackoffSettings(
initial_delay=0.1,
max_delay=1.0,
multiplier=2.0,
jitter=False,
),
),
context_management=[{"type": "compaction", "compact_threshold": 200000}],
prompt_cache_options={"mode": "explicit", "ttl": "30m"},
)
# Verify that every single field is set to a non-None value
for field in fields(model_settings):
assert getattr(model_settings, field.name) is not None, (
f"You must set the {field.name} field"
)
# Now, lets serialize the ModelSettings instance to a JSON string
verify_serialization(model_settings)
def test_gpt_5_6_reasoning_and_prompt_cache_serialization() -> None:
model_settings = ModelSettings(
reasoning=Reasoning(mode="pro", effort="max", context="all_turns"),
prompt_cache_options={"mode": "explicit", "ttl": "30m"},
)
serialized_reasoning = model_settings.to_json_dict()["reasoning"]
assert serialized_reasoning["context"] == "all_turns"
assert serialized_reasoning["effort"] == "max"
assert serialized_reasoning["mode"] == "pro"
assert model_settings.to_traceable_dict()["prompt_cache_options"] == {
"mode": "explicit",
"ttl": "30m",
}
def test_prompt_cache_options_is_appended_to_public_field_order() -> None:
field_names = [field.name for field in fields(ModelSettings)]
assert field_names[-2:] == ["context_management", "prompt_cache_options"]
def test_extra_args_serialization() -> None:
"""Test that extra_args are properly serialized."""
model_settings = ModelSettings(
temperature=0.5,
extra_args={"custom_param": "value", "another_param": 42, "nested": {"key": "value"}},
)
json_dict = model_settings.to_json_dict()
assert json_dict["extra_args"] == {
"custom_param": "value",
"another_param": 42,
"nested": {"key": "value"},
}
# Verify serialization works
verify_serialization(model_settings)
def test_traceable_serialization_omits_request_extras() -> None:
model_settings = ModelSettings(
temperature=0.5,
extra_headers={"Authorization": "Bearer provider-token"},
extra_query={"api-key": "query-token"},
extra_body={"secret": "body-token"},
extra_args={"api_key": "arg-token"},
)
json_dict = model_settings.to_json_dict()
assert json_dict["extra_headers"] == {"Authorization": "Bearer provider-token"}
assert json_dict["extra_query"] == {"api-key": "query-token"}
assert json_dict["extra_body"] == {"secret": "body-token"}
assert json_dict["extra_args"] == {"api_key": "arg-token"}
traceable = model_settings.to_traceable_dict()
assert traceable["temperature"] == 0.5
assert "extra_headers" not in traceable
assert "extra_query" not in traceable
assert "extra_body" not in traceable
assert "extra_args" not in traceable
def test_extra_args_resolve() -> None:
"""Test that extra_args are properly merged in the resolve method."""
base_settings = ModelSettings(
temperature=0.5, extra_args={"param1": "base_value", "param2": "base_only"}
)
override_settings = ModelSettings(
top_p=0.9, extra_args={"param1": "override_value", "param3": "override_only"}
)
resolved = base_settings.resolve(override_settings)
# Check that regular fields are properly resolved
assert resolved.temperature == 0.5 # from base
assert resolved.top_p == 0.9 # from override
# Check that extra_args are properly merged
expected_extra_args = {
"param1": "override_value", # override wins
"param2": "base_only", # from base
"param3": "override_only", # from override
}
assert resolved.extra_args == expected_extra_args
def test_extra_args_resolve_with_none() -> None:
"""Test that resolve works properly when one side has None extra_args."""
# Base with extra_args, override with None
base_settings = ModelSettings(extra_args={"param1": "value1"})
override_settings = ModelSettings(temperature=0.8)
resolved = base_settings.resolve(override_settings)
assert resolved.extra_args == {"param1": "value1"}
assert resolved.temperature == 0.8
# Base with None, override with extra_args
base_settings = ModelSettings(temperature=0.5)
override_settings = ModelSettings(extra_args={"param2": "value2"})
resolved = base_settings.resolve(override_settings)
assert resolved.extra_args == {"param2": "value2"}
assert resolved.temperature == 0.5
def test_extra_args_resolve_both_none() -> None:
"""Test that resolve works when both sides have None extra_args."""
base_settings = ModelSettings(temperature=0.5)
override_settings = ModelSettings(top_p=0.9)
resolved = base_settings.resolve(override_settings)
assert resolved.extra_args is None
assert resolved.temperature == 0.5
assert resolved.top_p == 0.9
def test_pydantic_serialization() -> None:
"""Tests whether ModelSettings can be serialized with Pydantic."""
# First, lets create a ModelSettings instance
model_settings = ModelSettings(
temperature=0.5,
top_p=0.9,
frequency_penalty=0.0,
presence_penalty=0.0,
tool_choice="auto",
parallel_tool_calls=True,
truncation="auto",
max_tokens=100,
reasoning=Reasoning(),
metadata={"foo": "bar"},
store=False,
include_usage=False,
top_logprobs=1,
extra_query={"foo": "bar"},
extra_body={"foo": "bar"},
extra_headers={"foo": "bar"},
extra_args={"custom_param": "value", "another_param": 42},
)
json = to_json(model_settings)
deserialized = TypeAdapter(ModelSettings).validate_json(json)
assert model_settings == deserialized
def test_retry_policy_is_excluded_from_json_dict() -> None:
"""Tests whether runtime-only retry policies are omitted from JSON serialization."""
model_settings = ModelSettings(
retry=ModelRetrySettings(
max_retries=1,
backoff=ModelRetryBackoffSettings(initial_delay=0.1),
policy=retry_policies.http_status([429]),
)
)
json_dict = model_settings.to_json_dict()
assert json_dict["retry"] == {
"max_retries": 1,
"backoff": {
"initial_delay": 0.1,
"max_delay": None,
"multiplier": None,
"jitter": None,
},
}
verify_serialization(model_settings)
def test_retry_resolve_deep_merges_backoff() -> None:
"""Tests whether retry settings are deep-merged in resolve()."""
base_settings = ModelSettings(
retry=ModelRetrySettings(
max_retries=1,
backoff=ModelRetryBackoffSettings(initial_delay=0.1, max_delay=1.0),
)
)
override_settings = ModelSettings(
retry=ModelRetrySettings(
backoff=ModelRetryBackoffSettings(multiplier=3.0, jitter=False),
policy=retry_policies.never(),
)
)
resolved = base_settings.resolve(override_settings)
assert resolved.retry is not None
assert resolved.retry.max_retries == 1
assert resolved.retry.policy is not None
assert resolved.retry.backoff == ModelRetryBackoffSettings(
initial_delay=0.1,
max_delay=1.0,
multiplier=3.0,
jitter=False,
)
def test_retry_policy_is_omitted_from_pydantic_round_trip() -> None:
"""Tests whether runtime-only retry policies are omitted from Pydantic serialization."""
model_settings = ModelSettings(
retry=ModelRetrySettings(
max_retries=2,
backoff=ModelRetryBackoffSettings(initial_delay=0.5),
policy=retry_policies.http_status([429]),
)
)
serialized = to_json(model_settings)
deserialized = TypeAdapter(ModelSettings).validate_json(serialized)
assert deserialized.retry is not None
assert deserialized.retry.max_retries == 2
assert deserialized.retry.backoff == ModelRetryBackoffSettings(initial_delay=0.5)
assert deserialized.retry.policy is None
def test_retry_backoff_validate_python_accepts_nested_dict_input() -> None:
"""Tests whether nested retry/backoff dict input is coerced to dataclasses."""
deserialized = TypeAdapter(ModelSettings).validate_python(
{
"retry": {
"max_retries": 3,
"backoff": {
"initial_delay": 0.25,
"max_delay": 2.0,
"multiplier": 3.0,
"jitter": False,
},
}
}
)
assert deserialized.retry is not None
assert deserialized.retry.max_retries == 3
assert deserialized.retry.backoff == ModelRetryBackoffSettings(
initial_delay=0.25,
max_delay=2.0,
multiplier=3.0,
jitter=False,
)
def test_retry_backoff_validate_python_preserves_falsey_values() -> None:
"""Tests whether falsey-only retry backoff input survives validation and serialization."""
deserialized = TypeAdapter(ModelRetrySettings).validate_python(
{
"max_retries": 1,
"backoff": {
"jitter": False,
},
}
)
assert deserialized.backoff == ModelRetryBackoffSettings(jitter=False)
assert deserialized.to_json_dict()["backoff"] == {
"initial_delay": None,
"max_delay": None,
"multiplier": None,
"jitter": False,
}
View File
+168
View File
@@ -0,0 +1,168 @@
from __future__ import annotations
import pytest
from agents import (
OpenAIAgentRegistrationConfig,
RunConfig,
set_default_openai_agent_registration,
set_default_openai_harness,
)
from agents.models.multi_provider import MultiProvider
from agents.models.openai_agent_registration import (
OPENAI_HARNESS_ID_TRACE_METADATA_KEY,
resolve_openai_agent_registration_config,
resolve_openai_harness_id_for_model_provider,
)
from agents.models.openai_provider import OpenAIProvider
from agents.run_internal.agent_runner_helpers import resolve_trace_settings
from agents.tracing import agent_span, trace
def test_agent_registration_config_precedence(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
set_default_openai_agent_registration(
OpenAIAgentRegistrationConfig(harness_id="default-harness")
)
try:
resolved = resolve_openai_agent_registration_config(
OpenAIAgentRegistrationConfig(harness_id="explicit-harness")
)
finally:
set_default_openai_agent_registration(None)
assert resolved is not None
assert resolved.harness_id == "explicit-harness"
def test_agent_registration_uses_default_before_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
set_default_openai_agent_registration(
OpenAIAgentRegistrationConfig(harness_id="default-harness")
)
try:
resolved = resolve_openai_agent_registration_config(None)
finally:
set_default_openai_agent_registration(None)
assert resolved is not None
assert resolved.harness_id == "default-harness"
def test_agent_registration_uses_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
resolved = resolve_openai_agent_registration_config(None)
assert resolved is not None
assert resolved.harness_id == "env-harness"
def test_set_default_openai_harness(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
set_default_openai_harness("helper-harness")
try:
resolved = resolve_openai_agent_registration_config(None)
finally:
set_default_openai_harness(None)
assert resolved is not None
assert resolved.harness_id == "helper-harness"
def test_agent_registration_disabled_without_config(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("OPENAI_AGENT_HARNESS_ID", raising=False)
assert resolve_openai_agent_registration_config(None) is None
def test_agent_registration_provider_constructor_config() -> None:
config = OpenAIAgentRegistrationConfig(harness_id="provider-harness")
openai_provider = OpenAIProvider(agent_registration=config)
multi_provider = MultiProvider(openai_agent_registration=config)
assert openai_provider.agent_registration is not None
assert openai_provider.agent_registration.harness_id == "provider-harness"
assert multi_provider.openai_provider.agent_registration is not None
assert multi_provider.openai_provider.agent_registration.harness_id == "provider-harness"
def test_harness_id_resolves_private_agent_registration() -> None:
class Provider:
_agent_registration = OpenAIAgentRegistrationConfig(harness_id="private-harness")
assert resolve_openai_harness_id_for_model_provider(Provider()) == "private-harness"
def test_harness_id_is_added_to_trace_metadata() -> None:
provider = OpenAIProvider(
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
)
_, _, _, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(model_provider=provider),
)
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
def test_harness_id_preserves_explicit_trace_metadata() -> None:
provider = OpenAIProvider(
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
)
_, _, _, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(
model_provider=provider,
trace_metadata={
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
"source": "test",
},
),
)
assert metadata == {
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
"source": "test",
}
def test_env_harness_id_is_added_to_trace_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
_, _, _, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(),
)
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "env-harness"}
def test_harness_id_trace_metadata_propagates_to_spans() -> None:
provider = OpenAIProvider(
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
)
workflow_name, trace_id, group_id, metadata, _ = resolve_trace_settings(
run_state=None,
run_config=RunConfig(model_provider=provider),
)
with trace(
workflow_name=workflow_name,
trace_id=trace_id,
group_id=group_id,
metadata=metadata,
):
with agent_span(name="agent") as span:
assert span.trace_metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
span_export = span.export()
assert span_export is not None
assert span_export["metadata"] == {
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"
}
@@ -0,0 +1,418 @@
"""
Test for Anthropic thinking blocks in conversation history.
This test validates the fix for issue #1704:
- Thinking blocks are properly preserved from Anthropic responses
- Reasoning items are stored in session but not sent back in conversation history
- Non-reasoning models are unaffected
- Token usage is not increased for non-reasoning scenarios
"""
from __future__ import annotations
from typing import Any, cast
from openai.types.chat import ChatCompletionMessageToolCall
from openai.types.chat.chat_completion_message_tool_call import Function
from agents.extensions.models.litellm_model import InternalChatCompletionMessage
from agents.models.chatcmpl_converter import Converter
def create_mock_anthropic_response_with_thinking() -> InternalChatCompletionMessage:
"""Create a mock Anthropic response with thinking blocks (like real response)."""
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather in Paris for you.",
reasoning_content="I need to call the weather function for Paris",
thinking_blocks=[
{
"type": "thinking",
"thinking": "I need to call the weather function for Paris",
"signature": "EqMDCkYIBxgCKkBAFZO8EyZwN1hiLctq0YjZnP0KeKgprr+C0PzgDv4GSggnFwrPQHIZ9A5s+paH+DrQBI1+Vnfq3mLAU5lJnoetEgzUEWx/Cv1022ieAvcaDCXdmg1XkMK0tZ8uCCIwURYAAX0uf2wFdnWt9n8whkhmy8ARQD5G2za4R8X5vTqBq8jpJ15T3c1Jcf3noKMZKooCWFVf0/W5VQqpZTgwDkqyTau7XraS+u48YlmJGSfyWMPO8snFLMZLGaGmVJgHfEI5PILhOEuX/R2cEeLuC715f51LMVuxTNzlOUV/037JV6P2ten7D66FnWU9JJMMJJov+DjMb728yQFHwHz4roBJ5ePHaaFP6mDwpqYuG/hai6pVv2TAK1IdKUui/oXrYtU+0gxb6UF2kS1bspqDuN++R8JdL7CMSU5l28pQ8TsH1TpVF4jZpsFbp1Du4rQIULFsCFFg+Edf9tPgyKZOq6xcskIjT7oylAPO37/jhdNknDq2S82PaSKtke3ViOigtM5uJfG521ZscBJQ1K3kwoI/repIdV9PatjOYdsYAQ==", # noqa: E501
}
],
)
return message
def test_converter_skips_reasoning_items():
"""
Unit test to verify that reasoning items are skipped when converting items to messages.
"""
# Create test items including a reasoning item
test_items: list[dict[str, Any]] = [
{"role": "user", "content": "Hello"},
{
"id": "reasoning_123",
"type": "reasoning",
"summary": [{"text": "User said hello", "type": "summary_text"}],
},
{
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Hi there!"}],
"status": "completed",
},
]
# Convert to messages
messages = Converter.items_to_messages(test_items) # type: ignore[arg-type]
# Should have user message and assistant message, but no reasoning content
assert len(messages) == 2
assert messages[0]["role"] == "user"
assert messages[1]["role"] == "assistant"
# Verify no thinking blocks in assistant message
assistant_msg = messages[1]
content = assistant_msg.get("content")
if isinstance(content, list):
for part in content:
assert part.get("type") != "thinking"
def test_reasoning_items_preserved_in_message_conversion():
"""
Test that reasoning content and thinking blocks are properly extracted
from Anthropic responses and stored in reasoning items.
"""
# Create mock message with thinking blocks
mock_message = create_mock_anthropic_response_with_thinking()
# Convert to output items
output_items = Converter.message_to_output_items(mock_message)
# Should have reasoning item, message item, and tool call items
reasoning_items = [
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
]
assert len(reasoning_items) == 1
reasoning_item = reasoning_items[0]
assert reasoning_item.summary[0].text == "I need to call the weather function for Paris"
# Verify thinking blocks are stored if we preserve them
if (
hasattr(reasoning_item, "content")
and reasoning_item.content
and len(reasoning_item.content) > 0
):
thinking_block = reasoning_item.content[0]
assert thinking_block.type == "reasoning_text"
assert thinking_block.text == "I need to call the weather function for Paris"
def test_anthropic_thinking_blocks_with_tool_calls():
"""
Test for models with extended thinking and interleaved thinking with tool calls.
This test verifies the Anthropic's API's requirements for thinking blocks
to be the first content in assistant messages when reasoning is enabled and tool
calls are present.
"""
# Create a message with reasoning, thinking blocks and tool calls
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather for you.",
reasoning_content="The user wants weather information, I need to call the weather function",
thinking_blocks=[
{
"type": "thinking",
"thinking": (
"The user is asking about weather. "
"Let me use the weather tool to get this information."
),
"signature": "TestSignature123",
},
{
"type": "thinking",
"thinking": ("We should use the city Tokyo as the city."),
"signature": "TestSignature456",
},
],
tool_calls=[
ChatCompletionMessageToolCall(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
],
)
# Step 1: Convert message to output items
output_items = Converter.message_to_output_items(message)
# Verify reasoning item exists and contains thinking blocks
reasoning_items = [
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
]
assert len(reasoning_items) == 1, "Should have exactly two reasoning items"
reasoning_item = reasoning_items[0]
# Verify thinking text is stored in content
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
"Reasoning item should have content"
)
assert reasoning_item.content[0].type == "reasoning_text", (
"Content should be reasoning_text type"
)
# Verify signature is stored in encrypted_content
assert hasattr(reasoning_item, "encrypted_content"), (
"Reasoning item should have encrypted_content"
)
assert reasoning_item.encrypted_content == "TestSignature123\nTestSignature456", (
"Signature should be preserved"
)
# Verify tool calls are present
tool_call_items = [
item for item in output_items if hasattr(item, "type") and item.type == "function_call"
]
assert len(tool_call_items) == 1, "Should have exactly one tool call"
# Step 2: Convert output items back to messages
# Convert items to dicts for the converter (simulating serialization/deserialization)
items_as_dicts: list[dict[str, Any]] = []
for item in output_items:
if hasattr(item, "model_dump"):
items_as_dicts.append(item.model_dump())
else:
items_as_dicts.append(cast(dict[str, Any], item))
messages = Converter.items_to_messages(
items_as_dicts, # type: ignore[arg-type]
model="anthropic/claude-4-opus",
preserve_thinking_blocks=True,
)
# Find the assistant message with tool calls
assistant_messages = [
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
]
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
assistant_msg = assistant_messages[0]
# Content must start with thinking blocks, not text
content = assistant_msg.get("content")
assert content is not None, "Assistant message should have content"
assert isinstance(content, list) and len(content) > 0, (
"Assistant message content should be a non-empty list"
)
first_content = content[0]
assert first_content.get("type") == "thinking", (
f"First content must be 'thinking' type for Anthropic compatibility, "
f"but got '{first_content.get('type')}'"
)
expected_thinking = (
"The user is asking about weather. Let me use the weather tool to get this information."
)
assert first_content.get("thinking") == expected_thinking, (
"Thinking content should be preserved"
)
# Signature should also be preserved
assert first_content.get("signature") == "TestSignature123", (
"Signature should be preserved in thinking block"
)
second_content = content[1]
assert second_content.get("type") == "thinking", (
f"Second content must be 'thinking' type for Anthropic compatibility, "
f"but got '{second_content.get('type')}'"
)
expected_thinking = "We should use the city Tokyo as the city."
assert second_content.get("thinking") == expected_thinking, (
"Thinking content should be preserved"
)
# Signature should also be preserved
assert second_content.get("signature") == "TestSignature456", (
"Signature should be preserved in thinking block"
)
last_content = content[2]
assert last_content.get("type") == "text", (
f"First content must be 'text' type but got '{last_content.get('type')}'"
)
expected_text = "I'll check the weather for you."
assert last_content.get("text") == expected_text, "Content text should be preserved"
# Verify tool calls are preserved
tool_calls = assistant_msg.get("tool_calls", [])
assert len(cast(list[Any], tool_calls)) == 1, "Tool calls should be preserved"
assert cast(list[Any], tool_calls)[0]["function"]["name"] == "get_weather"
def test_items_to_messages_preserves_positional_bool_arguments():
"""
Preserve positional compatibility for the released items_to_messages signature.
"""
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather for you.",
reasoning_content="The user wants weather information, I need to call the weather function",
thinking_blocks=[
{
"type": "thinking",
"thinking": (
"The user is asking about weather. "
"Let me use the weather tool to get this information."
),
"signature": "TestSignature123",
}
],
tool_calls=[
ChatCompletionMessageToolCall(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
],
)
output_items = Converter.message_to_output_items(message)
items_as_dicts: list[dict[str, Any]] = []
for item in output_items:
if hasattr(item, "model_dump"):
items_as_dicts.append(item.model_dump())
else:
items_as_dicts.append(cast(dict[str, Any], item))
messages = Converter.items_to_messages(
items_as_dicts, # type: ignore[arg-type]
"anthropic/claude-4-opus",
True,
True,
)
assistant_messages = [
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
]
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
assistant_msg = assistant_messages[0]
content = assistant_msg.get("content")
assert isinstance(content, list) and len(content) > 0, (
"Positional bool arguments should still preserve thinking blocks"
)
assert content[0].get("type") == "thinking", (
"The third positional argument must continue to map to preserve_thinking_blocks"
)
def test_anthropic_thinking_blocks_without_tool_calls():
"""
Test for models with extended thinking WITHOUT tool calls.
This test verifies that thinking blocks are properly attached to assistant
messages even when there are no tool calls (fixes issue #2195).
"""
# Create a message with reasoning and thinking blocks but NO tool calls
message = InternalChatCompletionMessage(
role="assistant",
content="The weather in Paris is sunny with a temperature of 22°C.",
reasoning_content="The user wants to know about the weather in Paris.",
thinking_blocks=[
{
"type": "thinking",
"thinking": "Let me think about the weather in Paris.",
"signature": "TestSignatureNoTools123",
}
],
tool_calls=None, # No tool calls
)
# Step 1: Convert message to output items
output_items = Converter.message_to_output_items(message)
# Verify reasoning item exists and contains thinking blocks
reasoning_items = [
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
]
assert len(reasoning_items) == 1, "Should have exactly one reasoning item"
reasoning_item = reasoning_items[0]
# Verify thinking text is stored in content
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
"Reasoning item should have content"
)
assert reasoning_item.content[0].type == "reasoning_text", (
"Content should be reasoning_text type"
)
assert reasoning_item.content[0].text == "Let me think about the weather in Paris.", (
"Thinking text should be preserved"
)
# Verify signature is stored in encrypted_content
assert hasattr(reasoning_item, "encrypted_content"), (
"Reasoning item should have encrypted_content"
)
assert reasoning_item.encrypted_content == "TestSignatureNoTools123", (
"Signature should be preserved"
)
# Verify message item exists
message_items = [
item for item in output_items if hasattr(item, "type") and item.type == "message"
]
assert len(message_items) == 1, "Should have exactly one message item"
# Step 2: Convert output items back to messages with preserve_thinking_blocks=True
items_as_dicts: list[dict[str, Any]] = []
for item in output_items:
if hasattr(item, "model_dump"):
items_as_dicts.append(item.model_dump())
else:
items_as_dicts.append(cast(dict[str, Any], item))
messages = Converter.items_to_messages(
items_as_dicts, # type: ignore[arg-type]
model="anthropic/claude-4-opus",
preserve_thinking_blocks=True,
)
# Should have one assistant message
assistant_messages = [msg for msg in messages if msg.get("role") == "assistant"]
assert len(assistant_messages) == 1, "Should have exactly one assistant message"
assistant_msg = assistant_messages[0]
# Content must start with thinking blocks even WITHOUT tool calls
content = assistant_msg.get("content")
assert content is not None, "Assistant message should have content"
assert isinstance(content, list), (
f"Assistant message content should be a list when thinking blocks are present, "
f"but got {type(content)}"
)
assert len(content) >= 2, (
f"Assistant message should have at least 2 content items "
f"(thinking + text), got {len(content)}"
)
# First content should be thinking block
first_content = content[0]
assert first_content.get("type") == "thinking", (
f"First content must be 'thinking' type for Anthropic compatibility, "
f"but got '{first_content.get('type')}'"
)
assert first_content.get("thinking") == "Let me think about the weather in Paris.", (
"Thinking content should be preserved"
)
assert first_content.get("signature") == "TestSignatureNoTools123", (
"Signature should be preserved in thinking block"
)
# Second content should be text
second_content = content[1]
assert second_content.get("type") == "text", (
f"Second content must be 'text' type, but got '{second_content.get('type')}'"
)
assert (
second_content.get("text") == "The weather in Paris is sunny with a temperature of 22°C."
), "Text content should be preserved"
+904
View File
@@ -0,0 +1,904 @@
from __future__ import annotations
import importlib
import sys
import types as pytypes
from collections.abc import AsyncIterator
from typing import Any, Literal, cast
import pytest
from openai.types.chat import (
ChatCompletion,
ChatCompletionChunk,
ChatCompletionMessage,
ChatCompletionMessageFunctionToolCall,
)
from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from openai.types.completion_usage import CompletionUsage, PromptTokensDetails
from openai.types.responses import Response, ResponseCompletedEvent, ResponseOutputMessage
from openai.types.responses.response_error_event import ResponseErrorEvent
from openai.types.responses.response_failed_event import ResponseFailedEvent
from openai.types.responses.response_incomplete_event import ResponseIncompleteEvent
from openai.types.responses.response_output_text import ResponseOutputText
from openai.types.responses.response_usage import (
InputTokensDetails,
OutputTokensDetails,
ResponseUsage,
)
from pydantic import BaseModel
from agents import (
Agent,
Handoff,
ModelBehaviorError,
ModelSettings,
ModelTracing,
Tool,
TResponseInputItem,
__version__,
)
from agents.exceptions import UserError
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
from agents.models.fake_id import FAKE_RESPONSES_ID
class FakeAnyLLMProvider:
def __init__(
self,
*,
supports_responses: bool,
chat_response: Any | None = None,
responses_response: Any | None = None,
) -> None:
self.SUPPORTS_RESPONSES = supports_responses
self.chat_response = chat_response
self.responses_response = responses_response
self.chat_calls: list[dict[str, Any]] = []
self.responses_calls: list[dict[str, Any]] = []
self.private_responses_calls: list[dict[str, Any]] = []
async def acompletion(self, **kwargs: Any) -> Any:
self.chat_calls.append(kwargs)
return self.chat_response
async def aresponses(self, **kwargs: Any) -> Any:
self.responses_calls.append(kwargs)
return self.responses_response
async def _aresponses(self, params: Any, **kwargs: Any) -> Any:
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
return self.responses_response
def _import_any_llm_module(
monkeypatch: pytest.MonkeyPatch,
provider: FakeAnyLLMProvider,
) -> tuple[Any, list[dict[str, Any]]]:
create_calls: list[dict[str, Any]] = []
class FakeAnyLLMFactory:
@staticmethod
def create(provider_name: str, api_key: str | None = None, api_base: str | None = None):
create_calls.append(
{
"provider_name": provider_name,
"api_key": api_key,
"api_base": api_base,
}
)
return provider
fake_any_llm: Any = pytypes.ModuleType("any_llm")
fake_any_llm.AnyLLM = FakeAnyLLMFactory
sys.modules.pop("agents.extensions.models.any_llm_model", None)
monkeypatch.setitem(sys.modules, "any_llm", fake_any_llm)
module = importlib.import_module("agents.extensions.models.any_llm_model")
monkeypatch.setattr(module, "AnyLLM", FakeAnyLLMFactory, raising=True)
return module, create_calls
def _chat_completion(text: str) -> ChatCompletion:
return ChatCompletion(
id="chatcmpl_123",
created=0,
model="fake-model",
object="chat.completion",
choices=[
Choice(
index=0,
finish_reason="stop",
message=ChatCompletionMessage(role="assistant", content=text),
)
],
usage=CompletionUsage(
completion_tokens=5,
prompt_tokens=7,
total_tokens=12,
prompt_tokens_details=PromptTokensDetails.model_validate(
{"cached_tokens": 2, "cache_write_tokens": 4}
),
),
)
def _responses_output(text: str) -> list[Any]:
return [
ResponseOutputMessage(
id="msg_123",
role="assistant",
status="completed",
type="message",
content=[
ResponseOutputText(
text=text,
type="output_text",
annotations=[],
logprobs=[],
)
],
)
]
def _response(text: str, response_id: str = "resp_123") -> Response:
return Response(
id=response_id,
created_at=123,
model="fake-model",
object="response",
output=_responses_output(text),
tool_choice="none",
tools=[],
parallel_tool_calls=False,
usage=ResponseUsage(
input_tokens=11,
output_tokens=13,
total_tokens=24,
input_tokens_details=InputTokensDetails.model_validate(
{"cache_write_tokens": 0, "cached_tokens": 0}
),
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
),
)
def _chat_completion_with_tool_call(*, thought_signature: str) -> ChatCompletion:
return ChatCompletion(
id="chatcmpl_tool_123",
created=0,
model="fake-model",
object="chat.completion",
choices=[
Choice(
index=0,
finish_reason="tool_calls",
message=ChatCompletionMessage(
role="assistant",
content="Calling a tool.",
tool_calls=[
ChatCompletionMessageFunctionToolCall.model_validate(
{
"id": "call_123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city":"Paris"}',
},
"extra_content": {
"google": {"thought_signature": thought_signature}
},
}
)
],
),
)
],
usage=CompletionUsage(
completion_tokens=5,
prompt_tokens=7,
total_tokens=12,
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
),
)
class GenericChatCompletionPayload(BaseModel):
id: str
created: int
model: str
object: str
choices: list[Any]
usage: Any
async def _empty_chat_stream() -> AsyncIterator[ChatCompletionChunk]:
if False:
yield ChatCompletionChunk(
id="chunk_123",
created=0,
model="fake-model",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason=None)],
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
async def test_user_agent_header_any_llm_chat(override_ua: str | None, monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
expected_ua = override_ua or f"Agents/Python {__version__}"
if override_ua is not None:
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
else:
token = None
try:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
finally:
if token is not None:
HEADERS_OVERRIDE.reset(token)
assert provider.chat_calls[0]["extra_headers"]["User-Agent"] == expected_ua
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_chat_path_is_used_when_responses_are_unsupported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
module, create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="router-key")
response = await model.get_response(
system_instructions="You are terse.",
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
assert create_calls == [
{
"provider_name": "openrouter",
"api_key": "router-key",
"api_base": None,
}
]
assert len(provider.chat_calls) == 1
assert provider.responses_calls == []
assert provider.chat_calls[0]["model"] == "openai/gpt-5.4-mini"
assert response.response_id is None
assert response.output[0].content[0].text == "Hello"
assert response.usage.input_tokens_details.cached_tokens == 2
assert getattr(response.usage.input_tokens_details, "cache_write_tokens", None) == 4
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
"chat_response",
[
pytest.param(_chat_completion("Hello").model_dump(), id="dict"),
pytest.param(
GenericChatCompletionPayload.model_validate(_chat_completion("Hello").model_dump()),
id="basemodel",
),
],
)
async def test_any_llm_chat_path_normalizes_non_stream_payloads(
monkeypatch,
chat_response: Any,
) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=chat_response)
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
assert response.response_id is None
assert response.output[0].content[0].text == "Hello"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_chat_path_preserves_gemini_tool_call_metadata(monkeypatch) -> None:
provider = FakeAnyLLMProvider(
supports_responses=False,
chat_response=_chat_completion_with_tool_call(thought_signature="sig_123"),
)
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="gemini/gemini-2.0-flash")
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
function_calls = [
item for item in response.output if getattr(item, "type", None) == "function_call"
]
assert len(function_calls) == 1
provider_data = function_calls[0].model_dump()["provider_data"]
assert provider_data["model"] == "gemini/gemini-2.0-flash"
assert provider_data["response_id"] == "chatcmpl_tool_123"
assert provider_data["thought_signature"] == "sig_123"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_path_is_used_when_supported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
module, create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="gpt-5.4-mini", api_key="openai-key")
response = await model.get_response(
system_instructions="You are terse.",
input="hi",
model_settings=ModelSettings(store=True),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
assert create_calls == [
{
"provider_name": "openai",
"api_key": "openai-key",
"api_base": None,
}
]
assert provider.chat_calls == []
assert provider.responses_calls == []
assert len(provider.private_responses_calls) == 1
params = provider.private_responses_calls[0]["params"]
kwargs = provider.private_responses_calls[0]["kwargs"]
assert params.model == "gpt-5.4-mini"
assert params.previous_response_id == "resp_prev"
assert params.conversation == "conv_123"
assert kwargs["extra_headers"]["User-Agent"] == f"Agents/Python {__version__}"
assert response.response_id == "resp_123"
assert response.output[0].content[0].text == "Hello"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_can_force_chat_completions_when_responses_are_supported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(
supports_responses=True,
chat_response=_chat_completion("Hello from chat"),
responses_response=_response("Hello from responses"),
)
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-4.1-mini", api="chat_completions")
response = await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
assert len(provider.chat_calls) == 1
assert provider.responses_calls == []
assert response.response_id is None
assert response.output[0].content[0].text == "Hello from chat"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_forced_responses_errors_when_provider_does_not_support_it(
monkeypatch,
) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openrouter/openai/gpt-4.1-mini", api="responses")
with pytest.raises(UserError, match="does not support the Responses API"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_stream_uses_chat_handler_when_responses_are_unsupported(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_empty_chat_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
completed = ResponseCompletedEvent(
type="response.completed",
response=_response("Hello from stream"),
sequence_number=1,
)
async def fake_handle_stream(response, stream, model=None):
assert model == "openrouter/openai/gpt-5.4-mini"
async for _chunk in stream:
pass
yield completed
monkeypatch.setattr(module.ChatCmplStreamHandler, "handle_stream", fake_handle_stream)
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
events = [
event
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
]
assert [event.type for event in events] == ["response.completed"]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_stream_passthrough_uses_responses_when_supported(monkeypatch) -> None:
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
yield ResponseCompletedEvent(
type="response.completed",
response=_response("Hello from responses stream"),
sequence_number=1,
)
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
events = [
event
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id="resp_prev",
conversation_id="conv_123",
prompt=None,
)
]
assert [event.type for event in events] == ["response.completed"]
assert provider.responses_calls == []
assert provider.private_responses_calls[0]["params"].previous_response_id == "resp_prev"
assert provider.private_responses_calls[0]["params"].conversation == "conv_123"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
("terminal_event_type", "terminal_event_cls"),
[
("response.incomplete", ResponseIncompleteEvent),
("response.failed", ResponseFailedEvent),
],
)
async def test_any_llm_responses_stream_rejects_failed_terminal_events(
monkeypatch,
terminal_event_type: str,
terminal_event_cls: type[Any],
) -> None:
async def response_stream() -> AsyncIterator[Any]:
yield terminal_event_cls(
type=terminal_event_type,
response=_response("partial", response_id="resp-terminal"),
sequence_number=1,
)
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
events = []
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
events.append(event)
assert len(events) == 1
assert events[0].type == terminal_event_type
assert events[0].response.id == "resp-terminal"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_stream_rejects_error_event(monkeypatch) -> None:
async def response_stream() -> AsyncIterator[ResponseErrorEvent]:
yield ResponseErrorEvent(
type="error",
code="invalid_request_error",
message="bad request",
param=None,
sequence_number=1,
)
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
events = []
with pytest.raises(ModelBehaviorError, match="invalid_request_error"):
async for event in model.stream_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
events.append(event)
assert len(events) == 1
assert events[0].type == "error"
assert events[0].code == "invalid_request_error"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_path_passes_transport_kwargs_via_private_provider_api(
monkeypatch,
) -> None:
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(
extra_headers={"X-Test-Header": "test"},
extra_query={"trace": "1"},
extra_body={"foo": "bar"},
),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
assert provider.responses_calls == []
assert len(provider.private_responses_calls) == 1
call = provider.private_responses_calls[0]
assert call["kwargs"]["extra_headers"]["X-Test-Header"] == "test"
assert call["kwargs"]["extra_query"] == {"trace": "1"}
assert call["kwargs"]["extra_body"] == {"foo": "bar"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_prompt_requests_fail_fast(monkeypatch) -> None:
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
with pytest.raises(Exception, match="prompt-managed requests"):
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt={"id": "pmpt_123"},
)
def test_any_llm_responses_input_sanitizer_strips_none_fields_from_reasoning_items() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from agents.extensions.models.any_llm_model import AnyLLMModel
model = AnyLLMModel(model="openai/gpt-5.4-mini")
raw_input = [
{
"id": "rid1",
"summary": [{"text": "why", "type": "summary_text"}],
"type": "reasoning",
"content": [{"type": "reasoning_text", "text": "thinking"}],
"status": None,
"encrypted_content": None,
}
]
cleaned = model._sanitize_any_llm_responses_input(raw_input)
assert cleaned == [
{
"id": "rid1",
"summary": [{"text": "why", "type": "summary_text"}],
"type": "reasoning",
"content": [{"type": "reasoning_text", "text": "thinking"}],
}
]
ResponsesParams = importlib.import_module("any_llm.types.responses").ResponsesParams
params = ResponsesParams(model="dummy", input=cleaned)
assert isinstance(params.input, list)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_any_llm_responses_path_sanitizes_replayed_items_before_validation() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from agents.extensions.models.any_llm_model import AnyLLMModel
class ValidatingProvider:
SUPPORTS_RESPONSES = True
def __init__(self) -> None:
self.private_responses_calls: list[dict[str, Any]] = []
async def aresponses(self, **kwargs: Any) -> Any:
raise AssertionError("public aresponses path should not be used in this test")
async def _aresponses(self, params: Any, **kwargs: Any) -> Response:
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
return _response("Hello from sanitized replay")
class TestAnyLLMModel(AnyLLMModel):
def __init__(self, provider: ValidatingProvider) -> None:
super().__init__(model="openai/gpt-5.4-mini", api="responses")
self._provider = provider
def _get_provider(self) -> Any:
return self._provider
provider = ValidatingProvider()
model = TestAnyLLMModel(provider)
tools: list[Tool] = []
handoffs: list[Handoff[Any, Agent[Any]]] = []
stream_flag: Literal[False] = False
replay_input = cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": FAKE_RESPONSES_ID,
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": [{"type": "reasoning_text", "text": "thinking"}],
"status": None,
"provider_data": {"model": "anthropic/fake-responses-model"},
},
{
"id": FAKE_RESPONSES_ID,
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"status": None,
"provider_data": {"model": "anthropic/fake-responses-model"},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
response = await model._fetch_responses_response(
system_instructions=None,
input=replay_input,
model_settings=ModelSettings(),
tools=tools,
output_schema=None,
handoffs=handoffs,
previous_response_id=None,
conversation_id=None,
stream=stream_flag,
prompt=None,
)
assert response.id == "resp_123"
assert len(provider.private_responses_calls) == 1
params = provider.private_responses_calls[0]["params"]
assert params.input == [
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
]
def test_any_llm_provider_passes_api_override() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from agents.extensions.models.any_llm_model import AnyLLMModel
from agents.extensions.models.any_llm_provider import AnyLLMProvider
provider = AnyLLMProvider(api="chat_completions")
model = provider.get_model("openai/gpt-4.1-mini")
assert isinstance(model, AnyLLMModel)
assert model.api == "chat_completions"
def test_any_llm_reasoning_objects_prefer_content_attributes_over_iterable_pairs() -> None:
pytest.importorskip(
"any_llm",
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
)
from any_llm.types.completion import Reasoning
from agents.extensions.models.any_llm_model import _extract_any_llm_reasoning_text
delta = pytypes.SimpleNamespace(reasoning=Reasoning(content="用户"))
assert _extract_any_llm_reasoning_text(delta) == "用户"
def test_any_llm_split_does_not_duplicate_content_or_thinking(monkeypatch) -> None:
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
Anthropic's extended thinking API rejects requests that include the same signed
thinking block more than once, and duplicated assistant text corrupts conversation
history. Only the first split should retain content, thinking_blocks, and
reasoning_content; subsequent splits should carry the tool_call alone.
"""
provider = FakeAnyLLMProvider(supports_responses=False)
module, _ = _import_any_llm_module(monkeypatch, provider)
AnyLLMModel = module.AnyLLMModel
model = AnyLLMModel(model="anthropic/claude-3-5-sonnet")
messages: list[Any] = [
{"role": "user", "content": "Search both"},
{
"role": "assistant",
"content": "Looking up both queries.",
"thinking_blocks": [{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}],
"reasoning_content": "internal plan",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
]
result = model._fix_tool_message_ordering(messages)
assistants = [m for m in result if m.get("role") == "assistant"]
assert len(assistants) == 2
# First split keeps the shared fields.
assert assistants[0].get("content") == "Looking up both queries."
assert "thinking_blocks" in assistants[0]
assert "reasoning_content" in assistants[0]
# Second split must NOT duplicate them.
assert "content" not in assistants[1]
assert "thinking_blocks" not in assistants[1]
assert "reasoning_content" not in assistants[1]
# Tool calls are still split one-per-message.
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
@@ -0,0 +1,361 @@
from typing import Any
import litellm
import pytest
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
Function,
Message,
ModelResponse,
Usage,
)
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models.chatcmpl_converter import Converter
from agents.models.interface import ModelTracing
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_deepseek_reasoning_content_preserved_in_tool_calls(monkeypatch):
"""
Ensure DeepSeek reasoning_content is preserved when converting items to messages.
DeepSeek requires reasoning_content field in assistant messages with tool_calls.
This test verifies that reasoning content from reasoning items is correctly
extracted and added to assistant messages during conversion.
"""
# Capture the messages sent to the model
captured_calls: list[dict[str, Any]] = []
async def fake_acompletion(model, messages=None, **kwargs):
captured_calls.append({"model": model, "messages": messages, **kwargs})
# First call: model returns reasoning_content + tool_call
if len(captured_calls) == 1:
tool_call = ChatCompletionMessageToolCall(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
msg = Message(
role="assistant",
content=None,
tool_calls=[tool_call],
)
# DeepSeek adds reasoning_content to the message
msg.reasoning_content = "Let me think about getting the weather for Tokyo..."
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
# Second call: model returns final response
msg = Message(role="assistant", content="The weather in Tokyo is sunny.")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(model="deepseek/deepseek-reasoner")
# First call: get the tool call response
first_response = await model.get_response(
system_instructions="You are a helpful assistant.",
input="What's the weather in Tokyo?",
model_settings=ModelSettings(),
tools=[], # We'll simulate the tool response manually
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert len(first_response.output) >= 1
input_items: list[Any] = []
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
for item in first_response.output:
if hasattr(item, "model_dump"):
input_items.append(item.model_dump())
else:
input_items.append(item)
input_items.append(
{
"type": "function_call_output",
"call_id": "call_123",
"output": "The weather in Tokyo is sunny.",
}
)
messages = Converter.items_to_messages(
input_items,
model="deepseek/deepseek-reasoner",
)
assistant_messages_with_tool_calls = [
m
for m in messages
if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls")
]
assert len(assistant_messages_with_tool_calls) > 0
assistant_msg = assistant_messages_with_tool_calls[0]
assert "reasoning_content" in assistant_msg
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_deepseek_reasoning_content_in_multi_turn_conversation(monkeypatch):
"""
Verify reasoning_content is included in assistant messages during multi-turn conversations.
When DeepSeek returns reasoning_content with tool_calls, subsequent API calls must
include the reasoning_content field in the assistant message to avoid 400 errors.
"""
captured_calls: list[dict[str, Any]] = []
async def fake_acompletion(model, messages=None, **kwargs):
captured_calls.append({"model": model, "messages": messages, **kwargs})
# First call: model returns reasoning_content + tool_call
if len(captured_calls) == 1:
tool_call = ChatCompletionMessageToolCall(
id="call_weather_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
)
msg = Message(
role="assistant",
content=None,
tool_calls=[tool_call],
)
# DeepSeek adds reasoning_content
msg.reasoning_content = "I need to get the weather for Tokyo first."
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
# Second call: check if reasoning_content was in the request
# In real DeepSeek API, this would fail with 400 if reasoning_content is missing
msg = Message(
role="assistant", content="Based on my findings, the weather in Tokyo is sunny."
)
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(model="deepseek/deepseek-reasoner")
# First call
first_response = await model.get_response(
system_instructions="You are a helpful assistant.",
input="What's the weather in Tokyo?",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
input_items: list[Any] = []
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
for item in first_response.output:
if hasattr(item, "model_dump"):
input_items.append(item.model_dump())
else:
input_items.append(item)
input_items.append(
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
}
)
await model.get_response(
system_instructions="You are a helpful assistant.",
input=input_items,
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
assert len(captured_calls) == 2
second_call_messages = captured_calls[1]["messages"]
assistant_with_tools = None
for msg in second_call_messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
assert "reasoning_content" in assistant_with_tools
def test_deepseek_reasoning_content_with_openai_chatcompletions_path():
"""
Verify reasoning_content works when using OpenAIChatCompletionsModel.
This ensures the fix works for both LiteLLM and OpenAI ChatCompletions code paths.
"""
from agents.models.chatcmpl_converter import Converter
input_items: list[Any] = [
{"role": "user", "content": "What's the weather in Paris?"},
{
"id": "__fake_id__",
"summary": [{"text": "I need to check the weather in Paris.", "type": "summary_text"}],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": "deepseek-reasoner", "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Paris"}',
"call_id": "call_weather_456",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": "deepseek-reasoner"},
},
{
"type": "function_call_output",
"call_id": "call_weather_456",
"output": "The weather in Paris is cloudy and 15°C.",
},
]
messages = Converter.items_to_messages(
input_items,
model="deepseek-reasoner",
)
assistant_with_tools = None
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
assert "reasoning_content" in assistant_with_tools
# Use type: ignore since reasoning_content is a dynamic field not in OpenAI's TypedDict
assert assistant_with_tools["reasoning_content"] == "I need to check the weather in Paris." # type: ignore[typeddict-item]
def test_reasoning_content_from_other_provider_not_attached_to_deepseek():
"""
Verify reasoning_content from non-DeepSeek providers is NOT attached to DeepSeek messages.
When switching models mid-conversation (e.g., from Claude to DeepSeek), reasoning items
that originated from Claude should not have their summaries attached as reasoning_content
to DeepSeek assistant messages, as this would leak unrelated reasoning and may trigger
DeepSeek 400 errors.
"""
from agents.models.chatcmpl_converter import Converter
input_items: list[Any] = [
{"role": "user", "content": "What's the weather in Paris?"},
{
"id": "__fake_id__",
"summary": [{"text": "Claude's reasoning about the weather.", "type": "summary_text"}],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
# this one came from Claude, not DeepSeek
"provider_data": {"model": "claude-sonnet-4-20250514", "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Paris"}',
"call_id": "call_weather_789",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": "claude-sonnet-4-20250514"},
},
{
"type": "function_call_output",
"call_id": "call_weather_789",
"output": "The weather in Paris is cloudy.",
},
]
messages = Converter.items_to_messages(
input_items,
model="deepseek-reasoner",
)
assistant_with_tools = None
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
# reasoning_content should NOT be present since the reasoning came from Claude, not DeepSeek
assert "reasoning_content" not in assistant_with_tools
def test_reasoning_content_without_provider_data_attached_for_backward_compat():
"""
Verify reasoning_content from items without provider_data is attached for backward compat.
For older items that don't have provider_data (before provider tracking was added),
we should still attach reasoning_content to maintain backward compatibility.
"""
from agents.models.chatcmpl_converter import Converter
# Reasoning item without provider_data (older format)
input_items: list[Any] = [
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [{"text": "Reasoning without provider info.", "type": "summary_text"}],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
# No provider_data
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_101",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
},
{
"type": "function_call_output",
"call_id": "call_weather_101",
"output": "The weather in Tokyo is sunny.",
},
]
messages = Converter.items_to_messages(
input_items,
model="deepseek-reasoner",
)
assistant_with_tools = None
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
assistant_with_tools = msg
break
assert assistant_with_tools is not None
# reasoning_content SHOULD be present for backward compatibility
assert "reasoning_content" in assistant_with_tools
assert assistant_with_tools["reasoning_content"] == "Reasoning without provider info." # type: ignore[typeddict-item]
+201
View File
@@ -0,0 +1,201 @@
import os
from typing import Literal
from unittest.mock import patch
import pytest
from openai.types.shared.reasoning import Reasoning
from agents import Agent
from agents.model_settings import ModelSettings
from agents.models import (
get_default_model,
get_default_model_settings,
gpt_5_reasoning_settings_required,
is_gpt_5_default,
)
def _gpt_5_default_settings(
reasoning_effort: Literal["none", "low", "medium"] | None,
) -> ModelSettings:
if reasoning_effort is None:
return ModelSettings(verbosity="low")
return ModelSettings(reasoning=Reasoning(effort=reasoning_effort), verbosity="low")
def test_default_model_is_gpt_5_4_mini():
assert get_default_model() == "gpt-5.4-mini"
assert is_gpt_5_default() is True
assert gpt_5_reasoning_settings_required(get_default_model()) is True
assert get_default_model_settings() == _gpt_5_default_settings("none")
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.4"})
def test_is_gpt_5_default_with_real_model_name():
assert get_default_model() == "gpt-5.4"
assert is_gpt_5_default() is True
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-4.1"})
def test_is_gpt_5_default_returns_false_for_non_gpt_5_default_model():
assert get_default_model() == "gpt-4.1"
assert is_gpt_5_default() is False
def test_gpt_5_reasoning_settings_required_detects_gpt_5_models_while_ignoring_chat_latest():
assert gpt_5_reasoning_settings_required("gpt-5") is True
assert gpt_5_reasoning_settings_required("gpt-5.1") is True
assert gpt_5_reasoning_settings_required("gpt-5.2") is True
assert gpt_5_reasoning_settings_required("gpt-5.2-codex") is True
assert gpt_5_reasoning_settings_required("gpt-5.2-pro") is True
assert gpt_5_reasoning_settings_required("gpt-5.4-pro") is True
assert gpt_5_reasoning_settings_required("gpt-5.5") is True
assert gpt_5_reasoning_settings_required("gpt-5-mini") is True
assert gpt_5_reasoning_settings_required("gpt-5-nano") is True
assert gpt_5_reasoning_settings_required("gpt-5-chat-latest") is False
assert gpt_5_reasoning_settings_required("gpt-5.1-chat-latest") is False
assert gpt_5_reasoning_settings_required("gpt-5.2-chat-latest") is False
assert gpt_5_reasoning_settings_required("gpt-5.3-chat-latest") is False
def test_gpt_5_reasoning_settings_required_returns_false_for_non_gpt_5_models():
assert gpt_5_reasoning_settings_required("gpt-4.1") is False
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_1_models():
assert get_default_model_settings("gpt-5.1") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.1-2025-11-13") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_2_models():
assert get_default_model_settings("gpt-5.2") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.2-2025-12-11") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_3_codex_models():
assert get_default_model_settings("gpt-5.3-codex") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_models():
assert get_default_model_settings("gpt-5.4") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_snapshot_families():
assert get_default_model_settings("gpt-5.4-2026-03-05") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.4-mini-2026-03-17") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.4-nano-2026-03-17") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_mini_and_nano():
assert get_default_model_settings("gpt-5.4-mini") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.4-nano") == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_5_models():
assert get_default_model_settings("gpt-5.5") == _gpt_5_default_settings("none")
assert get_default_model_settings("gpt-5.5-2026-04-23") == _gpt_5_default_settings("none")
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_6_models(
model: str,
):
assert get_default_model_settings(model) == _gpt_5_default_settings("none")
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
def test_agent_uses_gpt_5_6_model_settings_from_default_model_env(
model: str, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", model.upper())
agent = Agent(name="test")
assert get_default_model() == model
assert agent.model is None
assert agent.model_settings == _gpt_5_default_settings("none")
def test_get_default_model_settings_returns_low_reasoning_defaults_for_base_gpt_5():
assert get_default_model_settings("gpt-5") == _gpt_5_default_settings("low")
assert get_default_model_settings("gpt-5-2025-08-07") == _gpt_5_default_settings("low")
def test_get_default_model_settings_returns_low_reasoning_defaults_for_gpt_5_2_codex():
assert get_default_model_settings("gpt-5.2-codex") == _gpt_5_default_settings("low")
def test_get_default_model_settings_returns_medium_reasoning_defaults_for_gpt_5_pro_models():
assert get_default_model_settings("gpt-5.2-pro") == _gpt_5_default_settings("medium")
assert get_default_model_settings("gpt-5.2-pro-2025-12-11") == _gpt_5_default_settings("medium")
assert get_default_model_settings("gpt-5.4-pro") == _gpt_5_default_settings("medium")
assert get_default_model_settings("gpt-5.4-pro-2026-03-05") == _gpt_5_default_settings("medium")
def test_get_default_model_settings_omits_reasoning_for_unconfirmed_gpt_5_variants():
assert get_default_model_settings("gpt-5-mini") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5-mini-2025-08-07") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5-nano") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5-nano-2025-08-07") == _gpt_5_default_settings(None)
assert get_default_model_settings("gpt-5.1-codex") == _gpt_5_default_settings(None)
def test_get_default_model_settings_returns_empty_settings_for_gpt_5_chat_latest_aliases():
assert get_default_model_settings("gpt-5-chat-latest") == ModelSettings()
assert get_default_model_settings("gpt-5.1-chat-latest") == ModelSettings()
assert get_default_model_settings("gpt-5.2-chat-latest") == ModelSettings()
assert get_default_model_settings("gpt-5.3-chat-latest") == ModelSettings()
def test_get_default_model_settings_returns_empty_settings_for_non_gpt_5_models():
assert get_default_model_settings("gpt-4.1") == ModelSettings()
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
def test_agent_uses_gpt_5_default_model_settings():
"""Agent should inherit GPT-5 default model settings."""
agent = Agent(name="test")
assert agent.model is None
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
assert agent.model_settings.verbosity == "low"
def test_agent_uses_model_specific_settings_for_explicit_gpt_5_models():
"""Agent should not apply the fallback model's GPT-5 settings to explicit GPT-5 models."""
agent = Agent(name="test", model="gpt-5")
assert agent.model == "gpt-5"
assert agent.model_settings == get_default_model_settings("gpt-5")
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
def test_agent_uses_empty_settings_for_explicit_non_gpt_5_models():
"""Agent should not apply GPT-5 defaults to explicit non-GPT-5 models."""
agent = Agent(name="test", model="gpt-4.1")
assert agent.model == "gpt-4.1"
assert agent.model_settings == ModelSettings()
def test_agent_clone_recomputes_implicit_settings_when_model_changes():
"""Agent.clone should keep implicit model settings aligned with the cloned model."""
agent = Agent(name="test", model="gpt-5")
cloned = agent.clone(model="gpt-5.4-mini")
assert cloned.model == "gpt-5.4-mini"
assert cloned.model_settings == get_default_model_settings("gpt-5.4-mini")
assert cloned.model_settings.reasoning.effort == "none" # type: ignore[union-attr]
def test_agent_clone_preserves_explicit_settings_when_model_changes():
"""Agent.clone should not recompute model settings that were explicitly customized."""
model_settings = ModelSettings(temperature=0.3)
agent = Agent(name="test", model="gpt-5", model_settings=model_settings)
cloned = agent.clone(model="gpt-5.4-mini")
assert cloned.model == "gpt-5.4-mini"
assert cloned.model_settings == model_settings
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
def test_agent_resets_model_settings_for_non_gpt_5_models():
"""Agent should reset default GPT-5 settings when using a non-GPT-5 model."""
agent = Agent(name="test", model="gpt-4.1")
assert agent.model == "gpt-4.1"
assert agent.model_settings == ModelSettings()
@@ -0,0 +1,354 @@
"""Tests for the extended thinking message order bug fix in LitellmModel."""
from __future__ import annotations
from typing import Any, cast
from openai.types.chat import ChatCompletionMessageParam
from agents.extensions.models.litellm_model import LitellmModel
class TestExtendedThinkingMessageOrder:
"""Test the _fix_tool_message_ordering method."""
def test_basic_reordering_tool_result_before_call(self):
"""Test that a tool result appearing before its tool call gets reordered correctly."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{"role": "tool", "tool_call_id": "call_123", "content": "Result for call_123"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_123",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "user", "content": "Thanks"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should reorder to: user, assistant+tool_call, tool_result, user
assert len(result) == 4
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_123" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_123"
assert result[3]["role"] == "user"
def test_consecutive_tool_calls_get_separated(self):
"""Test that consecutive assistant messages with tool calls get properly paired with results.""" # noqa: E501
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test1", "arguments": "{}"},
}
],
},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_2",
"type": "function",
"function": {"name": "test2", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should pair each tool call with its result immediately
assert len(result) == 5
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "assistant"
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
assert result[4]["role"] == "tool"
assert result[4]["tool_call_id"] == "call_2"
def test_unmatched_tool_results_preserved(self):
"""Test that tool results without matching tool calls are preserved."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Matched result"},
{"role": "tool", "tool_call_id": "call_orphan", "content": "Orphaned result"},
{"role": "user", "content": "End"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should preserve the orphaned tool result
assert len(result) == 5
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "tool" # Orphaned result preserved
assert result[3]["tool_call_id"] == "call_orphan"
assert result[4]["role"] == "user"
def test_tool_calls_without_results_preserved(self):
"""Test that tool calls without results are still included."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "user", "content": "End"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should preserve the tool call even without a result
assert len(result) == 3
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "user"
def test_correctly_ordered_messages_unchanged(self):
"""Test that correctly ordered messages remain in the same order."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Result"},
{"role": "assistant", "content": "Done"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should remain exactly the same
assert len(result) == 4
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "assistant"
def test_multiple_tool_calls_single_message(self):
"""Test assistant message with multiple tool calls gets split properly."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "test1", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "test2", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should split the multi-tool message and pair each properly
assert len(result) == 5
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert len(result[1]["tool_calls"]) == 1 # type: ignore
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
assert result[2]["role"] == "tool"
assert result[2]["tool_call_id"] == "call_1"
assert result[3]["role"] == "assistant"
assert len(result[3]["tool_calls"]) == 1 # type: ignore
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
assert result[4]["role"] == "tool"
assert result[4]["tool_call_id"] == "call_2"
def test_split_does_not_duplicate_content_or_thinking(self):
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
Anthropic's extended thinking API rejects requests that include the same signed
thinking block more than once, and duplicated assistant text corrupts conversation
history. Only the first split should retain content, thinking_blocks, and
reasoning_content; subsequent splits should carry the tool_call alone.
"""
# Build the assistant message via cast so mypy doesn't reject the
# extra keys (`thinking_blocks`, `reasoning_content`) which are not
# part of the upstream ChatCompletionAssistantMessageParam TypedDict
# but are surfaced by litellm for Anthropic extended thinking.
assistant_msg = cast(
ChatCompletionMessageParam,
{
"role": "assistant",
"content": "Looking up both queries.",
"thinking_blocks": [
{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}
],
"reasoning_content": "internal plan",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "s", "arguments": "{}"},
},
],
},
)
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Search both"},
assistant_msg,
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
]
model = LitellmModel("claude-3-5-sonnet")
result = model._fix_tool_message_ordering(messages)
assistants = [cast(dict[str, Any], m) for m in result if m.get("role") == "assistant"]
assert len(assistants) == 2
# First split keeps the shared fields.
assert assistants[0].get("content") == "Looking up both queries."
assert "thinking_blocks" in assistants[0]
assert "reasoning_content" in assistants[0]
# Second split must NOT duplicate them.
assert "content" not in assistants[1]
assert "thinking_blocks" not in assistants[1]
assert "reasoning_content" not in assistants[1]
# Tool calls are still split one-per-message.
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
def test_empty_messages_list(self):
"""Test that empty message list is handled correctly."""
messages: list[ChatCompletionMessageParam] = []
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
assert result == []
def test_no_tool_messages(self):
"""Test that messages without tool calls are left unchanged."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi there"},
{"role": "user", "content": "How are you?"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
assert result == messages
def test_complex_mixed_scenario(self):
"""Test a complex scenario with various message types and orderings."""
messages: list[ChatCompletionMessageParam] = [
{"role": "user", "content": "Start"},
{
"role": "tool",
"tool_call_id": "call_out_of_order",
"content": "Out of order result",
}, # This comes before its call
{"role": "assistant", "content": "Regular response"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_out_of_order",
"type": "function",
"function": {"name": "test", "arguments": "{}"},
}
],
},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_normal",
"type": "function",
"function": {"name": "test2", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_normal", "content": "Normal result"},
{
"role": "tool",
"tool_call_id": "call_orphan",
"content": "Orphaned result",
}, # No matching call
{"role": "user", "content": "End"},
]
model = LitellmModel("test-model")
result = model._fix_tool_message_ordering(messages)
# Should reorder properly while preserving all messages
assert len(result) == 8
assert result[0]["role"] == "user" # Start
assert result[1]["role"] == "assistant" # Regular response
assert result[2]["role"] == "assistant" # call_out_of_order
assert result[2]["tool_calls"][0]["id"] == "call_out_of_order" # type: ignore
assert result[3]["role"] == "tool" # Out of order result (now properly paired)
assert result[3]["tool_call_id"] == "call_out_of_order"
assert result[4]["role"] == "assistant" # call_normal
assert result[4]["tool_calls"][0]["id"] == "call_normal" # type: ignore
assert result[5]["role"] == "tool" # Normal result
assert result[5]["tool_call_id"] == "call_normal"
assert result[6]["role"] == "tool" # Orphaned result (preserved)
assert result[6]["tool_call_id"] == "call_orphan"
assert result[7]["role"] == "user" # End
@@ -0,0 +1,126 @@
"""
Test for Gemini thought signatures in function calling.
Validates that thought signatures are preserved through the bidirectional roundtrip:
- Gemini chatcmpl message response item back to message
"""
from __future__ import annotations
from typing import Any
from openai.types.chat.chat_completion_message_tool_call import Function
from agents.extensions.models.litellm_model import InternalChatCompletionMessage, InternalToolCall
from agents.models.chatcmpl_converter import Converter
def test_gemini_thought_signature_roundtrip():
"""Test that thought signatures are preserved from Gemini responses to messages."""
# Create mock Gemini response with thought signature in new extra_content structure
class MockToolCall(InternalToolCall):
def __init__(self):
super().__init__(
id="call_123",
type="function",
function=Function(name="get_weather", arguments='{"city": "Paris"}'),
extra_content={"google": {"thought_signature": "test_signature_abc"}},
)
message = InternalChatCompletionMessage(
role="assistant",
content="I'll check the weather.",
reasoning_content="",
tool_calls=[MockToolCall()],
)
# Step 1: Convert to items
provider_data = {"model": "gemini/gemini-3-pro", "response_id": "gemini-response-id-123"}
items = Converter.message_to_output_items(message, provider_data=provider_data)
func_calls = [item for item in items if hasattr(item, "type") and item.type == "function_call"]
assert len(func_calls) == 1
# Verify thought_signature is stored in items with our provider_data structure
func_call_dict = func_calls[0].model_dump()
assert func_call_dict["provider_data"]["model"] == "gemini/gemini-3-pro"
assert func_call_dict["provider_data"]["response_id"] == "gemini-response-id-123"
assert func_call_dict["provider_data"]["thought_signature"] == "test_signature_abc"
# Step 2: Convert back to messages
items_as_dicts = [item.model_dump() for item in items]
messages = Converter.items_to_messages(
[{"role": "user", "content": "test"}] + items_as_dicts,
model="gemini/gemini-3-pro",
)
# Verify thought_signature is restored in extra_content format
assistant_msg = [msg for msg in messages if msg.get("role") == "assistant"][0]
tool_call = assistant_msg["tool_calls"][0] # type: ignore[index, typeddict-item]
assert tool_call["extra_content"]["google"]["thought_signature"] == "test_signature_abc"
def test_gemini_multiple_tool_calls_with_thought_signatures():
"""Test multiple tool calls each preserve their own thought signatures."""
tool_call_1 = InternalToolCall(
id="call_1",
type="function",
function=Function(name="func_a", arguments='{"x": 1}'),
extra_content={"google": {"thought_signature": "sig_aaa"}},
)
tool_call_2 = InternalToolCall(
id="call_2",
type="function",
function=Function(name="func_b", arguments='{"y": 2}'),
extra_content={"google": {"thought_signature": "sig_bbb"}},
)
message = InternalChatCompletionMessage(
role="assistant",
content="Calling two functions.",
reasoning_content="",
tool_calls=[tool_call_1, tool_call_2],
)
provider_data = {"model": "gemini/gemini-3-pro"}
items = Converter.message_to_output_items(message, provider_data=provider_data)
func_calls = [i for i in items if hasattr(i, "type") and i.type == "function_call"]
assert len(func_calls) == 2
assert func_calls[0].model_dump()["provider_data"]["thought_signature"] == "sig_aaa"
assert func_calls[1].model_dump()["provider_data"]["thought_signature"] == "sig_bbb"
def test_gemini_thought_signature_items_to_messages():
"""Test that items_to_messages restores extra_content from provider_data for Gemini."""
# Create a function call item with provider_data containing thought_signature
func_call_item = {
"id": "fake-id",
"call_id": "call_restore",
"name": "restore_func",
"arguments": '{"test": true}',
"type": "function_call",
"provider_data": {
"model": "gemini/gemini-3-pro",
"response_id": "gemini-response-id-123",
"thought_signature": "restored_sig_xyz",
},
}
items = [{"role": "user", "content": "test"}, func_call_item]
messages = Converter.items_to_messages(items, model="gemini/gemini-3-pro") # type: ignore[arg-type]
# Find the assistant message with tool_calls
assistant_msgs = [m for m in messages if m.get("role") == "assistant"]
assert len(assistant_msgs) == 1
tool_calls: list[dict[str, Any]] = assistant_msgs[0].get("tool_calls", []) # type: ignore[assignment]
assert len(tool_calls) == 1
# Verify extra_content is restored in Google format
assert tool_calls[0]["extra_content"]["google"]["thought_signature"] == "restored_sig_xyz"
@@ -0,0 +1,210 @@
"""
Test for Gemini thought signatures in streaming function calls.
Validates that thought signatures are captured from streaming chunks
and included in the final function call events.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any, cast
import pytest
from openai.types.chat import ChatCompletionChunk
from openai.types.chat.chat_completion_chunk import (
Choice,
ChoiceDelta,
ChoiceDeltaToolCall,
ChoiceDeltaToolCallFunction,
)
from openai.types.responses import Response
from agents.models.chatcmpl_stream_handler import ChatCmplStreamHandler
# ========== Helper Functions ==========
def create_tool_call_delta(
index: int,
tool_call_id: str | None = None,
function_name: str | None = None,
arguments: str | None = None,
provider_specific_fields: dict[str, Any] | None = None,
extra_content: dict[str, Any] | None = None,
) -> ChoiceDeltaToolCall:
"""Create a tool call delta for streaming."""
function = ChoiceDeltaToolCallFunction(
name=function_name,
arguments=arguments,
)
delta = ChoiceDeltaToolCall(
index=index,
id=tool_call_id,
type="function" if tool_call_id else None,
function=function,
)
# Add provider_specific_fields (litellm format)
if provider_specific_fields:
delta_any = cast(Any, delta)
delta_any.provider_specific_fields = provider_specific_fields
# Add extra_content (Google chatcmpl format)
if extra_content:
delta_any = cast(Any, delta)
delta_any.extra_content = extra_content
return delta
def create_chunk(
tool_calls: list[ChoiceDeltaToolCall] | None = None,
content: str | None = None,
include_usage: bool = False,
) -> ChatCompletionChunk:
"""Create a ChatCompletionChunk for testing."""
delta = ChoiceDelta(
content=content,
role="assistant" if content or tool_calls else None,
tool_calls=tool_calls,
)
chunk = ChatCompletionChunk(
id="chunk-id-123",
created=1,
model="gemini/gemini-3-pro",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=delta, finish_reason=None)],
)
if include_usage:
from openai.types.completion_usage import CompletionUsage
chunk.usage = CompletionUsage(
completion_tokens=10,
prompt_tokens=5,
total_tokens=15,
)
return chunk
def create_final_chunk() -> ChatCompletionChunk:
"""Create a final chunk with finish_reason='tool_calls'."""
return ChatCompletionChunk(
id="chunk-id-456",
created=1,
model="gemini/gemini-3-pro",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="tool_calls")],
)
async def create_fake_stream(
chunks: list[ChatCompletionChunk],
) -> AsyncIterator[ChatCompletionChunk]:
"""Create an async iterator from chunks."""
for chunk in chunks:
yield chunk
def create_mock_response() -> Response:
"""Create a mock Response object."""
return Response(
id="resp-id",
created_at=0,
model="gemini/gemini-3-pro",
object="response",
output=[],
tool_choice="auto",
tools=[],
parallel_tool_calls=False,
)
# ========== Tests ==========
@pytest.mark.asyncio
async def test_stream_captures_litellmprovider_specific_fields_thought_signature():
"""Test streaming captures thought_signature from litellm's provider_specific_fields."""
chunks = [
create_chunk(
tool_calls=[
create_tool_call_delta(
index=0,
tool_call_id="call_stream_1",
function_name="get_weather",
provider_specific_fields={"thought_signature": "litellm_sig_123"},
)
]
),
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"city": "Tokyo"}')]),
create_final_chunk(),
]
response = create_mock_response()
stream = create_fake_stream(chunks)
events = []
async for event in ChatCmplStreamHandler.handle_stream(
response,
stream, # type: ignore[arg-type]
model="gemini/gemini-3-pro",
):
events.append(event)
# Find function call done event
done_events = [e for e in events if e.type == "response.output_item.done"]
func_done = [
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
]
assert len(func_done) == 1
provider_data = func_done[0].item.model_dump().get("provider_data", {})
assert provider_data.get("thought_signature") == "litellm_sig_123"
assert provider_data["model"] == "gemini/gemini-3-pro"
assert provider_data["response_id"] == "chunk-id-123"
@pytest.mark.asyncio
async def test_stream_captures_google_extra_content_thought_signature():
"""Test streaming captures thought_signature from Google's extra_content format."""
chunks = [
create_chunk(
tool_calls=[
create_tool_call_delta(
index=0,
tool_call_id="call_stream_2",
function_name="search",
extra_content={"google": {"thought_signature": "google_sig_456"}},
)
]
),
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"query": "test"}')]),
create_final_chunk(),
]
response = create_mock_response()
stream = create_fake_stream(chunks)
events = []
async for event in ChatCmplStreamHandler.handle_stream(
response,
stream, # type: ignore[arg-type]
model="gemini/gemini-3-pro",
):
events.append(event)
done_events = [e for e in events if e.type == "response.output_item.done"]
func_done = [
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
]
assert len(func_done) == 1
provider_data = func_done[0].item.model_dump().get("provider_data", {})
assert provider_data.get("thought_signature") == "google_sig_456"
assert provider_data["model"] == "gemini/gemini-3-pro"
assert provider_data["response_id"] == "chunk-id-123"
+317
View File
@@ -0,0 +1,317 @@
import httpx
import litellm
import pytest
from httpx import Headers, Response
from litellm.exceptions import RateLimitError
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from openai import APIConnectionError
from openai.types.chat.chat_completion import ChatCompletion, Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.completion_usage import CompletionUsage
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models._retry_runtime import provider_managed_retries_disabled
from agents.models.interface import ModelTracing
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.retry import ModelRetryAdviceRequest, ModelRetrySettings
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_litellm_kwargs_forwarded(monkeypatch):
"""
Test that kwargs from ModelSettings are forwarded to litellm.acompletion.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
temperature=0.5,
extra_args={
"custom_param": "custom_value",
"seed": 42,
"stop": ["END"],
"logit_bias": {123: -100},
},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
)
# Verify that all kwargs were passed through
assert captured["custom_param"] == "custom_value"
assert captured["seed"] == 42
assert captured["stop"] == ["END"]
assert captured["logit_bias"] == {123: -100}
# Verify regular parameters are still passed
assert captured["temperature"] == 0.5
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_openai_chatcompletions_kwargs_forwarded(monkeypatch):
"""
Test that kwargs from ModelSettings are forwarded to OpenAI chat completions API.
"""
captured: dict[str, object] = {}
class MockChatCompletions:
async def create(self, **kwargs):
captured.update(kwargs)
msg = ChatCompletionMessage(role="assistant", content="test response")
choice = Choice(index=0, message=msg, finish_reason="stop")
return ChatCompletion(
id="test-id",
created=0,
model="gpt-4",
object="chat.completion",
choices=[choice],
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
)
class MockChat:
def __init__(self):
self.completions = MockChatCompletions()
class MockClient:
def __init__(self):
self.chat = MockChat()
self.base_url = "https://api.openai.com/v1"
settings = ModelSettings(
temperature=0.7,
extra_args={
"seed": 123,
"logit_bias": {456: 10},
"stop": ["STOP", "END"],
"user": "test-user",
},
)
mock_client = MockClient()
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=mock_client) # type: ignore
await model.get_response(
system_instructions="Test system",
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# Verify that all kwargs were passed through
assert captured["seed"] == 123
assert captured["logit_bias"] == {456: 10}
assert captured["stop"] == ["STOP", "END"]
assert captured["user"] == "test-user"
# Verify regular parameters are still passed
assert captured["temperature"] == 0.7
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_empty_kwargs_handling(monkeypatch):
"""
Test that empty or None kwargs are handled gracefully.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# Test with None kwargs
settings_none = ModelSettings(temperature=0.5, extra_args=None)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings_none,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# Should work without error and include regular parameters
assert captured["temperature"] == 0.5
# Test with empty dict
captured.clear()
settings_empty = ModelSettings(temperature=0.3, extra_args={})
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings_empty,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# Should work without error and include regular parameters
assert captured["temperature"] == 0.3
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_reasoning_effort_falls_back_to_extra_args(monkeypatch):
"""
Ensure reasoning_effort from extra_args is promoted when reasoning settings are missing.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
settings = ModelSettings(
extra_args={"reasoning_effort": "none", "custom_param": "custom_value"}
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "none"
assert captured["custom_param"] == "custom_value"
assert settings.extra_args == {"reasoning_effort": "none", "custom_param": "custom_value"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_litellm_retry_settings_do_not_leak_and_disable_provider_retries_on_runner_retry(
monkeypatch,
):
"""Runner retries should disable LiteLLM's own retries without forwarding SDK retry config."""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="test response")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
retry=ModelRetrySettings(
max_retries=2,
backoff={"initial_delay": 0.25, "jitter": False},
),
extra_args={"max_retries": 7, "num_retries": 6, "custom_param": "custom_value"},
)
model = LitellmModel(model="test-model")
with provider_managed_retries_disabled(True):
await model.get_response(
system_instructions=None,
input="test input",
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
)
assert settings.retry is not None
assert settings.retry.backoff is not None
assert captured["custom_param"] == "custom_value"
assert captured["max_retries"] == 0
assert captured["num_retries"] == 0
assert "retry" not in captured
def test_litellm_get_retry_advice_uses_response_headers() -> None:
"""LiteLLM retry advice should expose OpenAI-compatible retry headers."""
model = LitellmModel(model="test-model")
error = RateLimitError(
message="rate limited",
llm_provider="openai",
model="gpt-4o-mini",
response=Response(
status_code=429,
headers=Headers({"x-should-retry": "true", "retry-after-ms": "250"}),
),
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
)
)
assert advice is not None
assert advice.suggested is True
assert advice.retry_after == 0.25
def test_litellm_get_retry_advice_keeps_stateful_transport_failures_ambiguous() -> None:
model = LitellmModel(model="test-model")
error = APIConnectionError(
message="connection error",
request=httpx.Request("POST", "https://api.openai.com/v1/responses"),
)
advice = model.get_retry_advice(
ModelRetryAdviceRequest(
error=error,
attempt=1,
stream=False,
previous_response_id="resp_prev",
)
)
assert advice is not None
assert advice.suggested is True
assert advice.replay_safety is None
@@ -0,0 +1,697 @@
from collections.abc import AsyncIterator
import pytest
from openai.types.chat.chat_completion_chunk import (
ChatCompletionChunk,
Choice,
ChoiceDelta,
ChoiceDeltaToolCall,
ChoiceDeltaToolCallFunction,
)
from openai.types.completion_usage import (
CompletionTokensDetails,
CompletionUsage,
PromptTokensDetails,
)
from openai.types.responses import (
Response,
ResponseCompletedEvent,
ResponseContentPartAddedEvent,
ResponseFunctionToolCall,
ResponseOutputMessage,
ResponseOutputRefusal,
ResponseOutputText,
ResponseReasoningItem,
ResponseRefusalDeltaEvent,
)
from agents.extensions.models.litellm_model import LitellmModel
from agents.extensions.models.litellm_provider import LitellmProvider
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_text_content(monkeypatch) -> None:
"""
Validate that `stream_response` emits the correct sequence of events when
streaming a simple assistant message consisting of plain text content.
We simulate two chunks of text returned from the chat completion stream.
"""
# Create two chunks that will be emitted by the fake stream.
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content="He"))],
)
# Mark last chunk with usage so stream_response knows this is final.
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content="llo"))],
usage=CompletionUsage(
completion_tokens=5,
prompt_tokens=7,
total_tokens=12,
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
prompt_tokens_details=PromptTokensDetails(cached_tokens=6),
),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
# Patch _fetch_response to inject our fake stream
async def patched_fetch_response(self, *args, **kwargs):
# `_fetch_response` is expected to return a Response skeleton and the async stream
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# We expect a response.created, then a response.output_item.added, content part added,
# two content delta events (for "He" and "llo"), a content part done, the assistant message
# output_item.done, and finally response.completed.
# There should be 8 events in total.
assert len(output_events) == 8
# First event indicates creation.
assert output_events[0].type == "response.created"
# The output item added and content part added events should mark the assistant message.
assert output_events[1].type == "response.output_item.added"
assert output_events[2].type == "response.content_part.added"
# Two text delta events.
assert output_events[3].type == "response.output_text.delta"
assert output_events[3].delta == "He"
assert output_events[4].type == "response.output_text.delta"
assert output_events[4].delta == "llo"
# After streaming, the content part and item should be marked done.
assert output_events[5].type == "response.content_part.done"
assert output_events[6].type == "response.output_item.done"
# Last event indicates completion of the stream.
assert output_events[7].type == "response.completed"
# The completed response should have one output message with full text.
completed_resp = output_events[7].response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
assert completed_resp.output[0].content[0].text == "Hello"
assert completed_resp.usage, "usage should not be None"
assert completed_resp.usage.input_tokens == 7
assert completed_resp.usage.output_tokens == 5
assert completed_resp.usage.total_tokens == 12
assert completed_resp.usage.input_tokens_details.cached_tokens == 6
assert completed_resp.usage.output_tokens_details.reasoning_tokens == 2
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_refusal_content(monkeypatch) -> None:
"""
Validate that when the model streams a refusal string instead of normal content,
`stream_response` emits the appropriate sequence of events including
`response.refusal.delta` events for each chunk of the refusal message and
constructs a completed assistant message with a `ResponseOutputRefusal` part.
"""
# Simulate refusal text coming in two pieces, like content but using the `refusal`
# field on the delta rather than `content`.
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(refusal="No"))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(refusal="Thanks"))],
usage=CompletionUsage(completion_tokens=2, prompt_tokens=2, total_tokens=4),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Expect sequence similar to text: created, output_item.added, content part added,
# two refusal delta events, content part done, output_item.done, completed.
assert len(output_events) == 8
assert output_events[0].type == "response.created"
assert output_events[1].type == "response.output_item.added"
assert output_events[2].type == "response.content_part.added"
assert output_events[3].type == "response.refusal.delta"
assert output_events[3].delta == "No"
assert output_events[4].type == "response.refusal.delta"
assert output_events[4].delta == "Thanks"
assert output_events[5].type == "response.content_part.done"
assert output_events[6].type == "response.output_item.done"
assert output_events[7].type == "response.completed"
completed_resp = output_events[7].response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
refusal_part = completed_resp.output[0].content[0]
assert isinstance(refusal_part, ResponseOutputRefusal)
assert refusal_part.refusal == "NoThanks"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_tool_call(monkeypatch) -> None:
"""
Validate that `stream_response` emits the correct sequence of events when
the model is streaming a function/tool call instead of plain text.
The function call will be split across two chunks.
"""
# Simulate a single tool call with complete function name in first chunk
# and arguments split across chunks (reflecting real API behavior)
tool_call_delta1 = ChoiceDeltaToolCall(
index=0,
id="tool-id",
function=ChoiceDeltaToolCallFunction(name="my_func", arguments="arg1"),
type="function",
)
tool_call_delta2 = ChoiceDeltaToolCall(
index=0,
id="tool-id",
function=ChoiceDeltaToolCallFunction(name=None, arguments="arg2"),
type="function",
)
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Sequence should be: response.created, then after loop we expect function call-related events:
# one response.output_item.added for function call, a response.function_call_arguments.delta,
# a response.output_item.done, and finally response.completed.
assert output_events[0].type == "response.created"
# The next three events are about the tool call.
assert output_events[1].type == "response.output_item.added"
# The added item should be a ResponseFunctionToolCall.
added_fn = output_events[1].item
assert isinstance(added_fn, ResponseFunctionToolCall)
assert added_fn.name == "my_func" # Name should be complete from first chunk
assert added_fn.arguments == "" # Arguments start empty
assert output_events[2].type == "response.function_call_arguments.delta"
assert output_events[2].delta == "arg1" # First argument chunk
assert output_events[3].type == "response.function_call_arguments.delta"
assert output_events[3].delta == "arg2" # Second argument chunk
assert output_events[4].type == "response.output_item.done"
assert output_events[5].type == "response.completed"
# Final function call should have complete arguments
final_fn = output_events[4].item
assert isinstance(final_fn, ResponseFunctionToolCall)
assert final_fn.name == "my_func"
assert final_fn.arguments == "arg1arg2"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_real_time_function_call_arguments(monkeypatch) -> None:
"""
Validate that LiteLLM `stream_response` also emits function call arguments in real-time
as they are received, ensuring consistent behavior across model providers.
"""
# Simulate realistic chunks: name first, then arguments incrementally
tool_call_delta1 = ChoiceDeltaToolCall(
index=0,
id="litellm-call-456",
function=ChoiceDeltaToolCallFunction(name="generate_code", arguments=""),
type="function",
)
tool_call_delta2 = ChoiceDeltaToolCall(
index=0,
function=ChoiceDeltaToolCallFunction(arguments='{"language": "'),
type="function",
)
tool_call_delta3 = ChoiceDeltaToolCall(
index=0,
function=ChoiceDeltaToolCallFunction(arguments='python", "task": "'),
type="function",
)
tool_call_delta4 = ChoiceDeltaToolCall(
index=0,
function=ChoiceDeltaToolCallFunction(arguments='hello world"}'),
type="function",
)
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
)
chunk3 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta3]))],
)
chunk4 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta4]))],
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2, chunk3, chunk4):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Extract events by type
function_args_delta_events = [
e for e in output_events if e.type == "response.function_call_arguments.delta"
]
output_item_added_events = [e for e in output_events if e.type == "response.output_item.added"]
# Verify we got real-time streaming (3 argument delta events)
assert len(function_args_delta_events) == 3
assert len(output_item_added_events) == 1
# Verify the deltas were streamed correctly
expected_deltas = ['{"language": "', 'python", "task": "', 'hello world"}']
for i, delta_event in enumerate(function_args_delta_events):
assert delta_event.delta == expected_deltas[i]
# Verify function call metadata
added_event = output_item_added_events[0]
assert isinstance(added_event.item, ResponseFunctionToolCall)
assert added_event.item.name == "generate_code"
assert added_event.item.call_id == "litellm-call-456"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_synthesizes_refusal_on_content_filter(monkeypatch) -> None:
"""A stream that terminates with finish_reason == "content_filter" and no
emitted content (as Anthropic-on-Bedrock does via LiteLLM) must synthesize a
ResponseOutputRefusal so the completed response carries an explicit refusal
rather than an empty assistant turn.
Mirrors the real Bedrock chunk shape: an empty-string content delta followed
by a terminal content_filter chunk with no content. The empty "" delta must
not open a text content part; the synthesized refusal must be the only
content part, at the same index in the stream and in response.completed.
"""
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(role="assistant", content=""))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
usage=CompletionUsage(
completion_tokens=0,
prompt_tokens=7,
total_tokens=7,
),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
types = [e.type for e in output_events]
# Coherent refusal sequence: the message + refusal part are opened, a refusal
# delta is emitted, and the parts/message are closed before completion.
assert "response.output_item.added" in types
assert "response.content_part.added" in types
assert "response.refusal.delta" in types
assert types[-1] == "response.completed"
assert "response.output_item.done" in types
# The refusal delta carries a non-empty message.
refusal_deltas = [e for e in output_events if e.type == "response.refusal.delta"]
assert refusal_deltas and refusal_deltas[0].delta
# Event coherence: the assistant message is announced exactly once, and every
# content part that is opened is also closed.
assert types.count("response.output_item.added") == 1
assert types.count("response.content_part.added") == types.count("response.content_part.done")
# The empty "" content delta must NOT open a text content part: no text part
# events and no output_text.delta are emitted at all.
assert "response.output_text.delta" not in types
added_parts = [e for e in output_events if e.type == "response.content_part.added"]
assert len(added_parts) == 1
assert isinstance(added_parts[0].part, ResponseOutputRefusal)
# The completed response contains exactly one content part: the refusal.
completed_event = output_events[-1]
assert isinstance(completed_event, ResponseCompletedEvent)
completed_resp = completed_event.response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
assert len(completed_resp.output[0].content) == 1
refusal_part = completed_resp.output[0].content[0]
assert isinstance(refusal_part, ResponseOutputRefusal)
assert refusal_part.refusal
# The refusal's streamed content_index matches its position in the completed
# response (0), so raw-event replay and the final response stay aligned.
assert added_parts[0].content_index == 0
assert refusal_deltas[0].content_index == 0
done_parts = [e for e in output_events if e.type == "response.content_part.done"]
assert len(done_parts) == 1
assert done_parts[0].content_index == 0
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_content_filter_does_not_clobber_text(monkeypatch) -> None:
"""A content_filter finish_reason that arrives AFTER real text was streamed
must not synthesize a refusal (the text stands)."""
chunk1 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content="answer"))],
)
chunk2 = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
usage=CompletionUsage(completion_tokens=1, prompt_tokens=7, total_tokens=8),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk1, chunk2):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = [
event
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
]
assert "response.refusal.delta" not in [e.type for e in output_events]
completed_event = output_events[-1]
assert isinstance(completed_event, ResponseCompletedEvent)
completed_resp = completed_event.response
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
assert completed_resp.output[0].content[0].text == "answer"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_content_filter_refusal_after_reasoning(monkeypatch) -> None:
"""A content_filter turn preceded by reasoning must still place the
synthesized refusal at content_index 0 of the assistant message. Reasoning
is a *separate* output item (it shifts the message's output_index, not its
content_index), so the refusal the sole content part stays at
content_index 0 in both the stream and response.completed."""
reasoning_delta = ChoiceDelta(role="assistant", content=None)
# reasoning_content is a provider extra field the handler reads via hasattr.
reasoning_delta.reasoning_content = "thinking..." # type: ignore[attr-defined]
chunk_reasoning = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=reasoning_delta)],
)
chunk_empty = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(content=""))],
)
chunk_filter = ChatCompletionChunk(
id="chunk-id",
created=1,
model="fake",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
usage=CompletionUsage(completion_tokens=0, prompt_tokens=7, total_tokens=7),
)
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
for c in (chunk_reasoning, chunk_empty, chunk_filter):
yield c
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, fake_stream()
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
model = LitellmProvider().get_model("gpt-4")
output_events = [
event
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
]
# A reasoning item was produced as a separate output item.
completed_event = output_events[-1]
assert isinstance(completed_event, ResponseCompletedEvent)
completed_resp = completed_event.response
assert isinstance(completed_resp.output[0], ResponseReasoningItem)
assistant_msg = completed_resp.output[1]
assert isinstance(assistant_msg, ResponseOutputMessage)
# The refusal is the sole content part of the assistant message, at index 0.
assert len(assistant_msg.content) == 1
assert isinstance(assistant_msg.content[0], ResponseOutputRefusal)
# The assistant message's output_index is 1 (after the reasoning item), and
# every refusal event uses that output_index and content_index 0 — matching
# the refusal's position in response.completed.
added = [
e
for e in output_events
if isinstance(e, ResponseContentPartAddedEvent)
and isinstance(e.part, ResponseOutputRefusal)
]
deltas = [e for e in output_events if isinstance(e, ResponseRefusalDeltaEvent)]
assert len(added) == 1
assert added[0].content_index == 0
assert added[0].output_index == 1
assert deltas and all(d.content_index == 0 and d.output_index == 1 for d in deltas)
# The empty "" delta still opens no text part.
assert "response.output_text.delta" not in [e.type for e in output_events]
@@ -0,0 +1,89 @@
import litellm
import pytest
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
async def _get_response(monkeypatch, *, finish_reason, content):
"""Drive get_response against a mocked litellm completion and return the items."""
async def fake_acompletion(model, messages=None, **kwargs):
msg = Message(role="assistant", content=content)
choice = Choices(index=0, finish_reason=finish_reason, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(model="test-model")
return await model.get_response(
system_instructions=None,
input=[],
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_content_filter_finish_reason_surfaces_refusal(monkeypatch):
"""A content-filter block (empty message, finish_reason=content_filter) must
become an explicit ResponseOutputRefusal, not zero output items.
Some providers (e.g. Anthropic on Amazon Bedrock) signal a safety block only
via ``finish_reason == "content_filter"`` with an empty message and no
``refusal`` field; without this the turn is indistinguishable from an empty
response and drives agent loops into fruitless retries.
"""
resp = await _get_response(monkeypatch, finish_reason="content_filter", content="")
refusals = [
content
for item in resp.output
if isinstance(item, ResponseOutputMessage)
for content in item.content
if isinstance(content, ResponseOutputRefusal)
]
assert refusals, f"expected a refusal item, got: {resp.output}"
assert refusals[0].refusal # non-empty message
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_content_filter_does_not_clobber_real_content(monkeypatch):
"""A content_filter finish_reason that still carries text is left alone — we
only synthesize a refusal when the message is genuinely empty."""
resp = await _get_response(
monkeypatch, finish_reason="content_filter", content="here is the answer"
)
refusals = [
content
for item in resp.output
if isinstance(item, ResponseOutputMessage)
for content in item.content
if isinstance(content, ResponseOutputRefusal)
]
assert not refusals, "should not synthesize a refusal when content is present"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_normal_stop_is_unaffected(monkeypatch):
"""A normal completion is unchanged — no spurious refusal."""
resp = await _get_response(monkeypatch, finish_reason="stop", content="all good")
refusals = [
content
for item in resp.output
if isinstance(item, ResponseOutputMessage)
for content in item.content
if isinstance(content, ResponseOutputRefusal)
]
assert not refusals
+265
View File
@@ -0,0 +1,265 @@
import logging
import litellm
import pytest
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from agents.extensions.models.litellm_model import LitellmModel
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_is_forwarded(monkeypatch):
"""
Forward `extra_body` via LiteLLM's dedicated kwarg.
This ensures that provider-specific request fields stay nested under `extra_body`
so LiteLLM can merge them into the upstream request body itself.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
temperature=0.1, extra_body={"cached_content": "some_cache", "foo": 123}
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["extra_body"] == {"cached_content": "some_cache", "foo": 123}
assert "cached_content" not in captured
assert "foo" not in captured
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_reasoning_effort_is_promoted(monkeypatch):
"""
Ensure reasoning_effort from extra_body is promoted to the top-level parameter.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
settings = ModelSettings(
extra_body={"reasoning_effort": "none", "cached_content": "some_cache"}
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "none"
assert captured["extra_body"] == {"cached_content": "some_cache"}
assert settings.extra_body == {"reasoning_effort": "none", "cached_content": "some_cache"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_reasoning_effort_prefers_model_settings(monkeypatch):
"""
Verify explicit ModelSettings.reasoning takes precedence over extra_body entries.
"""
from openai.types.shared import Reasoning
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
reasoning=Reasoning(effort="low"),
extra_body={"reasoning_effort": "high"},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
# reasoning_effort is string when no summary is provided (backward compatible)
assert captured["reasoning_effort"] == "low"
assert "extra_body" not in captured
assert settings.extra_body == {"reasoning_effort": "high"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_reasoning_effort_overrides_extra_args(monkeypatch):
"""
Ensure extra_body reasoning_effort wins over extra_args when both are provided.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
settings = ModelSettings(
extra_body={"reasoning_effort": "none"},
extra_args={"reasoning_effort": "low", "custom_param": "custom"},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "none"
assert captured["custom_param"] == "custom"
assert "extra_body" not in captured
assert settings.extra_args == {"reasoning_effort": "low", "custom_param": "custom"}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_extra_body_metadata_stays_nested(monkeypatch):
"""
Keep extra_body metadata nested even when top-level metadata is also set.
LiteLLM resolves top-level metadata and extra_body separately. Flattening the nested
metadata dict loses the caller's intended request shape for OpenAI-compatible proxies.
"""
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
metadata={"sdk": "agents"},
extra_body={
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
"cached_content": "some_cache",
},
)
model = LitellmModel(model="test-model")
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["metadata"] == {"sdk": "agents"}
assert captured["extra_body"] == {
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
"cached_content": "some_cache",
}
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name",
[
"openai/gpt-5-mini",
"anthropic/claude-sonnet-4-5",
"gemini/gemini-2.5-pro",
],
)
async def test_reasoning_summary_uses_scalar_effort_and_warns(
monkeypatch, caplog: pytest.LogCaptureFixture, model_name: str
):
"""
Ensure reasoning.summary does not change the LiteLLM chat-completions argument shape.
LitellmModel should continue to pass a scalar reasoning_effort value and warn that summary
is ignored on this path, regardless of the provider encoded in the model string.
"""
from openai.types.shared import Reasoning
captured: dict[str, object] = {}
async def fake_acompletion(model, messages=None, **kwargs):
captured.update(kwargs)
msg = Message(role="assistant", content="ok")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
settings = ModelSettings(
reasoning=Reasoning(effort="medium", summary="auto"),
)
model = LitellmModel(model=model_name)
with caplog.at_level(logging.WARNING, logger="openai.agents"):
await model.get_response(
system_instructions=None,
input=[],
model_settings=settings,
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assert captured["reasoning_effort"] == "medium"
warning_messages = [
record.message
for record in caplog.records
if "does not forward Reasoning.summary" in record.message
]
assert len(warning_messages) == 1
@@ -0,0 +1,29 @@
from __future__ import annotations
import importlib
import pytest
pytest.importorskip("litellm")
def test_litellm_logging_patch_env_var_controls_application(monkeypatch):
"""Assert the serializer patch only applies when the env var is enabled."""
litellm_logging = importlib.import_module("litellm.litellm_core_utils.litellm_logging")
litellm_model = importlib.import_module("agents.extensions.models.litellm_model")
monkeypatch.delenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", raising=False)
litellm_logging = importlib.reload(litellm_logging)
importlib.reload(litellm_model)
assert hasattr(
litellm_logging,
"_extract_response_obj_and_hidden_params",
), "LiteLLM removed _extract_response_obj_and_hidden_params; revisit warning patch."
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is False
monkeypatch.setenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", "true")
litellm_logging = importlib.reload(litellm_logging)
importlib.reload(litellm_model)
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is True
+89
View File
@@ -0,0 +1,89 @@
from __future__ import annotations
from typing import Any
import pytest
from agents import ModelSettings, ModelTracing, __version__
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
async def test_user_agent_header_litellm(override_ua: str | None, monkeypatch):
called_kwargs: dict[str, Any] = {}
expected_ua = override_ua or f"Agents/Python {__version__}"
import importlib
import sys
import types as pytypes
litellm_fake: Any = pytypes.ModuleType("litellm")
class DummyMessage:
role = "assistant"
content = "Hello"
tool_calls: list[Any] | None = None
def get(self, _key, _default=None):
return None
def model_dump(self):
return {"role": self.role, "content": self.content}
class Choices: # noqa: N801 - mimic litellm naming
def __init__(self):
self.message = DummyMessage()
class DummyModelResponse:
def __init__(self):
self.choices = [Choices()]
async def acompletion(**kwargs):
nonlocal called_kwargs
called_kwargs = kwargs
return DummyModelResponse()
utils_ns = pytypes.SimpleNamespace()
utils_ns.Choices = Choices
utils_ns.ModelResponse = DummyModelResponse
litellm_types = pytypes.SimpleNamespace(
utils=utils_ns,
llms=pytypes.SimpleNamespace(openai=pytypes.SimpleNamespace(ChatCompletionAnnotation=dict)),
)
litellm_fake.acompletion = acompletion
litellm_fake.types = litellm_types
monkeypatch.setitem(sys.modules, "litellm", litellm_fake)
litellm_mod = importlib.import_module("agents.extensions.models.litellm_model")
monkeypatch.setattr(litellm_mod, "litellm", litellm_fake, raising=True)
LitellmModel = litellm_mod.LitellmModel
model = LitellmModel(model="gpt-4")
if override_ua is not None:
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
else:
token = None
try:
await model.get_response(
system_instructions=None,
input="hi",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
finally:
if token is not None:
HEADERS_OVERRIDE.reset(token)
assert "extra_headers" in called_kwargs
assert called_kwargs["extra_headers"]["User-Agent"] == expected_ua
+209
View File
@@ -0,0 +1,209 @@
from typing import Any, cast
import pytest
from agents import (
Agent,
MultiProvider,
OpenAIResponsesModel,
OpenAIResponsesWSModel,
RunConfig,
UserError,
)
from agents.extensions.models.litellm_model import LitellmModel
from agents.models.multi_provider import MultiProviderMap
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.run_internal.run_loop import get_model
def test_no_prefix_is_openai():
agent = Agent(model="gpt-4o", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, OpenAIResponsesModel)
def test_openai_prefix_is_openai():
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, OpenAIResponsesModel)
def test_litellm_prefix_is_litellm():
agent = Agent(model="litellm/foo/bar", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, LitellmModel)
def test_any_llm_prefix_uses_any_llm_provider(monkeypatch):
import sys
import types as pytypes
captured_model: dict[str, Any] = {}
class FakeAnyLLMModel:
pass
class FakeAnyLLMProvider:
def get_model(self, model_name):
captured_model["value"] = model_name
return FakeAnyLLMModel()
fake_module: Any = pytypes.ModuleType("agents.extensions.models.any_llm_provider")
fake_module.AnyLLMProvider = FakeAnyLLMProvider
monkeypatch.setitem(sys.modules, "agents.extensions.models.any_llm_provider", fake_module)
agent = Agent(model="any-llm/openrouter/openai/gpt-5.4-mini", instructions="", name="test")
model = get_model(agent, RunConfig())
assert isinstance(model, FakeAnyLLMModel)
assert captured_model["value"] == "openrouter/openai/gpt-5.4-mini"
def test_no_prefix_can_use_openai_responses_websocket():
agent = Agent(model="gpt-4o", instructions="", name="test")
model = get_model(
agent,
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
)
assert isinstance(model, OpenAIResponsesWSModel)
def test_openai_prefix_can_use_openai_responses_websocket():
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
model = get_model(
agent,
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
)
assert isinstance(model, OpenAIResponsesWSModel)
def test_multi_provider_passes_websocket_base_url_to_openai_provider(monkeypatch):
captured_kwargs = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
captured_kwargs.update(kwargs)
def get_model(self, model_name):
raise AssertionError("This test only verifies constructor passthrough.")
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
MultiProvider(openai_websocket_base_url="wss://proxy.example.test/v1")
assert captured_kwargs["websocket_base_url"] == "wss://proxy.example.test/v1"
def test_multi_provider_forwards_openai_buffer_streamed_tool_calls_to_chat_model():
provider = MultiProvider(
openai_client=cast(Any, object()),
openai_use_responses=False,
openai_buffer_streamed_tool_calls=True,
)
model = provider.get_model("gpt-4o")
assert isinstance(model, OpenAIChatCompletionsModel)
assert model._buffer_streamed_tool_calls is True
def test_openai_prefix_defaults_to_alias_mode(monkeypatch):
captured_model: dict[str, Any] = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
captured_model["value"] = model_name
return object()
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider = MultiProvider()
provider.get_model("openai/gpt-4o")
assert captured_model["value"] == "gpt-4o"
def test_openai_prefix_can_be_preserved_as_literal_model_id(monkeypatch):
captured_model: dict[str, Any] = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
captured_model["value"] = model_name
return object()
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider = MultiProvider(openai_prefix_mode="model_id")
provider.get_model("openai/gpt-4o")
assert captured_model["value"] == "openai/gpt-4o"
def test_unknown_prefix_defaults_to_error():
provider = MultiProvider()
with pytest.raises(UserError, match="Unknown prefix: openrouter"):
provider.get_model("openrouter/openai/gpt-4o")
def test_unknown_prefix_can_be_preserved_for_openai_compatible_model_ids(monkeypatch):
captured_model: dict[str, Any] = {}
captured_result: dict[str, Any] = {}
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
captured_model["value"] = model_name
fake_model = object()
captured_result["value"] = fake_model
return fake_model
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider = MultiProvider(unknown_prefix_mode="model_id")
result = provider.get_model("openrouter/openai/gpt-4o")
assert result is captured_result["value"]
assert captured_model["value"] == "openrouter/openai/gpt-4o"
def test_provider_map_entries_override_openai_prefix_mode(monkeypatch):
captured_model: dict[str, Any] = {}
class FakeCustomProvider:
def get_model(self, model_name):
captured_model["value"] = model_name
return object()
class FakeOpenAIProvider:
def __init__(self, **kwargs):
pass
def get_model(self, model_name):
raise AssertionError("Expected the explicit provider_map entry to win.")
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
provider_map = MultiProviderMap()
provider_map.add_provider("openai", cast(Any, FakeCustomProvider()))
provider = MultiProvider(
provider_map=provider_map,
openai_prefix_mode="model_id",
)
provider.get_model("openai/gpt-4o")
assert captured_model["value"] == "gpt-4o"
def test_multi_provider_rejects_invalid_prefix_modes():
bad_openai_prefix_mode: Any = "invalid"
bad_unknown_prefix_mode: Any = "invalid"
with pytest.raises(UserError, match="openai_prefix_mode"):
MultiProvider(openai_prefix_mode=bad_openai_prefix_mode)
with pytest.raises(UserError, match="unknown_prefix_mode"):
MultiProvider(unknown_prefix_mode=bad_unknown_prefix_mode)
@@ -0,0 +1,186 @@
from __future__ import annotations
from collections.abc import Iterable, Iterator
from typing import Any, cast
import httpx
import pytest
from openai import omit
from openai.types.chat.chat_completion import ChatCompletion
from agents import (
ModelSettings,
ModelTracing,
OpenAIChatCompletionsModel,
OpenAIResponsesModel,
generation_span,
)
from agents.models import (
openai_chatcompletions as chat_module,
openai_responses as responses_module,
)
class _SingleUseIterable:
"""Helper iterable that raises if iterated more than once."""
def __init__(self, values: list[object]) -> None:
self._values = list(values)
self.iterations = 0
def __iter__(self) -> Iterator[object]:
if self.iterations:
raise RuntimeError("Iterable should have been materialized exactly once.")
self.iterations += 1
yield from self._values
def _force_materialization(value: object) -> None:
if isinstance(value, dict):
for nested in value.values():
_force_materialization(nested)
elif isinstance(value, list):
for nested in value:
_force_materialization(nested)
elif isinstance(value, Iterable) and not isinstance(value, str | bytes | bytearray):
list(value)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_chat_completions_materializes_iterator_payload(
monkeypatch: pytest.MonkeyPatch,
) -> None:
message_iter = _SingleUseIterable([{"type": "text", "text": "hi"}])
tool_iter = _SingleUseIterable([{"type": "string"}])
chat_converter = cast(Any, chat_module).Converter
monkeypatch.setattr(
chat_converter,
"items_to_messages",
classmethod(lambda _cls, _input, **kwargs: [{"role": "user", "content": message_iter}]),
)
monkeypatch.setattr(
chat_converter,
"tool_to_openai",
classmethod(
lambda _cls, _tool: {
"type": "function",
"function": {
"name": "dummy",
"parameters": {"properties": tool_iter},
},
}
),
)
captured_kwargs: dict[str, Any] = {}
class DummyCompletions:
async def create(self, **kwargs):
captured_kwargs.update(kwargs)
_force_materialization(kwargs["messages"])
if kwargs["tools"] is not omit:
_force_materialization(kwargs["tools"])
return ChatCompletion(
id="dummy-id",
created=0,
model="gpt-4",
object="chat.completion",
choices=[],
usage=None,
)
class DummyClient:
def __init__(self) -> None:
self.chat = type("_Chat", (), {"completions": DummyCompletions()})()
self.base_url = httpx.URL("http://example.test")
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=DummyClient()) # type: ignore[arg-type]
with generation_span(disabled=True) as span:
await cast(Any, model)._fetch_response(
system_instructions=None,
input="ignored",
model_settings=ModelSettings(),
tools=[object()],
output_schema=None,
handoffs=[],
span=span,
tracing=ModelTracing.DISABLED,
stream=False,
)
assert message_iter.iterations == 1
assert tool_iter.iterations == 1
assert isinstance(captured_kwargs["messages"][0]["content"], list)
assert isinstance(captured_kwargs["tools"][0]["function"]["parameters"]["properties"], list)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_responses_materializes_iterator_payload(monkeypatch: pytest.MonkeyPatch) -> None:
input_iter = _SingleUseIterable([{"type": "input_text", "text": "hello"}])
tool_iter = _SingleUseIterable([{"type": "string"}])
responses_item_helpers = cast(Any, responses_module).ItemHelpers
responses_converter = cast(Any, responses_module).Converter
monkeypatch.setattr(
responses_item_helpers,
"input_to_new_input_list",
classmethod(lambda _cls, _input: [{"role": "user", "content": input_iter}]),
)
converted_tools = responses_module.ConvertedTools(
tools=[
cast(
Any,
{
"type": "function",
"name": "dummy",
"parameters": {"properties": tool_iter},
},
)
],
includes=[],
)
monkeypatch.setattr(
responses_converter,
"convert_tools",
classmethod(lambda _cls, _tools, _handoffs, **_kwargs: converted_tools),
)
captured_kwargs: dict[str, Any] = {}
class DummyResponses:
async def create(self, **kwargs):
captured_kwargs.update(kwargs)
_force_materialization(kwargs["input"])
_force_materialization(kwargs["tools"])
return object()
class DummyClient:
def __init__(self) -> None:
self.responses = DummyResponses()
model = OpenAIResponsesModel(model="gpt-4.1", openai_client=DummyClient()) # type: ignore[arg-type]
await cast(Any, model)._fetch_response(
system_instructions=None,
input="ignored",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
previous_response_id=None,
conversation_id=None,
stream=False,
prompt=None,
)
assert input_iter.iterations == 1
assert tool_iter.iterations == 1
assert isinstance(captured_kwargs["input"][0]["content"], list)
assert isinstance(captured_kwargs["tools"][0]["parameters"]["properties"], list)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,756 @@
# Copyright (c) OpenAI
#
# Licensed under the MIT License.
# See LICENSE file in the project root for full license information.
"""
Unit tests for the internal `Converter` class defined in
`agents.models.openai_chatcompletions`. The converter is responsible for
translating between internal "item" structures (e.g., `ResponseOutputMessage`
and related types from `openai.types.responses`) and the ChatCompletion message
structures defined by the OpenAI client library.
These tests exercise both conversion directions:
- `Converter.message_to_output_items` turns a `ChatCompletionMessage` (as
returned by the OpenAI API) into a list of `ResponseOutputItem` instances.
- `Converter.items_to_messages` takes in either a simple string prompt, or a
list of input/output items such as `ResponseOutputMessage` and
`ResponseFunctionToolCallParam` dicts, and constructs a list of
`ChatCompletionMessageParam` dicts suitable for sending back to the API.
"""
from __future__ import annotations
import logging
from typing import Any, Literal, cast
import pytest
from openai import omit
from openai.types.chat import ChatCompletionMessage, ChatCompletionMessageFunctionToolCall
from openai.types.chat.chat_completion_message_custom_tool_call import (
ChatCompletionMessageCustomToolCall,
Custom,
)
from openai.types.chat.chat_completion_message_tool_call import Function
from openai.types.responses import (
ResponseFunctionToolCall,
ResponseFunctionToolCallParam,
ResponseInputAudioParam,
ResponseInputTextParam,
ResponseOutputMessage,
ResponseOutputRefusal,
ResponseOutputText,
)
from openai.types.responses.response_input_item_param import FunctionCallOutput
from agents.agent_output import AgentOutputSchema
from agents.exceptions import UserError
from agents.items import TResponseInputItem
from agents.models.chatcmpl_converter import Converter
from agents.models.fake_id import FAKE_RESPONSES_ID
def test_message_to_output_items_with_text_only():
"""
Make sure a simple ChatCompletionMessage with string content is converted
into a single ResponseOutputMessage containing one ResponseOutputText.
"""
msg = ChatCompletionMessage(role="assistant", content="Hello")
items = Converter.message_to_output_items(msg)
# Expect exactly one output item (the message)
assert len(items) == 1
message_item = cast(ResponseOutputMessage, items[0])
assert message_item.id == FAKE_RESPONSES_ID
assert message_item.role == "assistant"
assert message_item.type == "message"
assert message_item.status == "completed"
# Message content should have exactly one text part with the same text.
assert len(message_item.content) == 1
text_part = cast(ResponseOutputText, message_item.content[0])
assert text_part.type == "output_text"
assert text_part.text == "Hello"
def test_message_to_output_items_with_refusal():
"""
Make sure a message with a refusal string produces a ResponseOutputMessage
with a ResponseOutputRefusal content part.
"""
msg = ChatCompletionMessage(role="assistant", refusal="I'm sorry")
items = Converter.message_to_output_items(msg)
assert len(items) == 1
message_item = cast(ResponseOutputMessage, items[0])
assert len(message_item.content) == 1
refusal_part = cast(ResponseOutputRefusal, message_item.content[0])
assert refusal_part.type == "refusal"
assert refusal_part.refusal == "I'm sorry"
def test_message_to_output_items_with_tool_call():
"""
If the ChatCompletionMessage contains one or more tool_calls, they should
be reflected as separate `ResponseFunctionToolCall` items appended after
the message item.
"""
tool_call = ChatCompletionMessageFunctionToolCall(
id="tool1",
type="function",
function=Function(name="myfn", arguments='{"x":1}'),
)
msg = ChatCompletionMessage(role="assistant", content="Hi", tool_calls=[tool_call])
items = Converter.message_to_output_items(msg)
# Should produce a message item followed by one function tool call item
assert len(items) == 2
message_item = cast(ResponseOutputMessage, items[0])
assert isinstance(message_item, ResponseOutputMessage)
fn_call_item = cast(ResponseFunctionToolCall, items[1])
assert fn_call_item.id == FAKE_RESPONSES_ID
assert fn_call_item.call_id == tool_call.id
assert fn_call_item.name == tool_call.function.name
assert fn_call_item.arguments == tool_call.function.arguments
assert fn_call_item.type == "function_call"
def test_message_to_output_items_with_custom_tool_call_keeps_default_compatibility():
"""Custom tool calls should keep the default Chat Completions behavior."""
tool_call = ChatCompletionMessageCustomToolCall(
id="tool1",
type="custom",
custom=Custom(name="raw_tool", input="payload"),
)
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
assert Converter.message_to_output_items(msg) == []
def test_message_to_output_items_with_custom_tool_call_raises_in_strict_mode():
"""Strict validation should fail explicitly instead of dropping custom tool calls."""
tool_call = ChatCompletionMessageCustomToolCall(
id="tool1",
type="custom",
custom=Custom(name="raw_tool", input="payload"),
)
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
with pytest.raises(UserError, match="Custom tool calls are not supported"):
Converter.message_to_output_items(msg, strict_feature_validation=True)
def test_message_to_output_items_with_mixed_custom_tool_call_raises_in_strict_mode():
"""Strict validation should not partially hide an unsupported custom tool call."""
function_tool_call = ChatCompletionMessageFunctionToolCall(
id="function-tool",
type="function",
function=Function(name="myfn", arguments='{"x":1}'),
)
custom_tool_call = ChatCompletionMessageCustomToolCall(
id="custom-tool",
type="custom",
custom=Custom(name="raw_tool", input="payload"),
)
msg = ChatCompletionMessage(
role="assistant",
tool_calls=[function_tool_call, custom_tool_call],
)
with pytest.raises(UserError, match="Custom tool calls are not supported"):
Converter.message_to_output_items(msg, strict_feature_validation=True)
def test_items_to_messages_with_string_user_content():
"""
A simple string as the items argument should be converted into a user
message param dict with the same content.
"""
result = Converter.items_to_messages("Ask me anything")
assert isinstance(result, list)
assert len(result) == 1
msg = result[0]
assert msg["role"] == "user"
assert msg["content"] == "Ask me anything"
def test_items_to_messages_with_easy_input_message():
"""
Given an easy input message dict (just role/content), the converter should
produce the appropriate ChatCompletionMessageParam with the same content.
"""
items: list[TResponseInputItem] = [
{
"role": "user",
"content": "How are you?",
}
]
messages = Converter.items_to_messages(items)
assert len(messages) == 1
out = messages[0]
assert out["role"] == "user"
# For simple string inputs, the converter returns the content as a bare string
assert out["content"] == "How are you?"
def test_items_to_messages_accepts_raw_chat_completions_user_content_parts():
"""
Raw Chat Completions content parts should be accepted as aliases for the SDK's
canonical input content shapes.
"""
items: list[TResponseInputItem] = [
# Cast the fixture because mypy cannot infer this raw chat-style dict as a specific
# member of the TResponseInputItem TypedDict union on its own.
cast(
TResponseInputItem,
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {
"url": "https://example.com/image.png",
"detail": "high",
},
},
],
},
)
]
messages = Converter.items_to_messages(items)
assert len(messages) == 1
message = messages[0]
assert message["role"] == "user"
assert message["content"] == [
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {
"url": "https://example.com/image.png",
"detail": "high",
},
},
]
def test_items_to_messages_with_output_message_and_function_call():
"""
Given a sequence of one ResponseOutputMessageParam followed by a
ResponseFunctionToolCallParam, the converter should produce a single
ChatCompletionAssistantMessageParam that includes both the assistant's
textual content and a populated `tool_calls` reflecting the function call.
"""
# Construct output message param dict with two content parts.
output_text: ResponseOutputText = ResponseOutputText(
text="Part 1",
type="output_text",
annotations=[],
logprobs=[],
)
refusal: ResponseOutputRefusal = ResponseOutputRefusal(
refusal="won't do that",
type="refusal",
)
resp_msg: ResponseOutputMessage = ResponseOutputMessage(
id="42",
type="message",
role="assistant",
status="completed",
content=[output_text, refusal],
)
# Construct a function call item dict (as if returned from model)
func_item: ResponseFunctionToolCallParam = {
"id": "99",
"call_id": "abc",
"name": "math",
"arguments": "{}",
"type": "function_call",
}
items: list[TResponseInputItem] = [
resp_msg.model_dump(), # type:ignore
func_item,
]
messages = Converter.items_to_messages(items)
# Should return a single assistant message
assert len(messages) == 1
assistant = messages[0]
assert assistant["role"] == "assistant"
# Content combines text portions of the output message
assert "content" in assistant
assert assistant["content"] == "Part 1"
# Refusal in output message should be represented in assistant message
assert "refusal" in assistant
assert assistant["refusal"] == refusal.refusal
# Tool calls list should contain one ChatCompletionMessageFunctionToolCall dict
tool_calls = assistant.get("tool_calls")
assert isinstance(tool_calls, list)
assert len(tool_calls) == 1
tool_call = tool_calls[0]
assert tool_call["type"] == "function"
assert tool_call["function"]["name"] == "math"
assert tool_call["function"]["arguments"] == "{}"
def test_convert_tool_choice_handles_standard_and_named_options() -> None:
"""
The `Converter.convert_tool_choice` method should return the omit sentinel
if no choice is provided, pass through values like "auto", "required",
or "none" unchanged, and translate any other string into a function
selection dict.
"""
assert Converter.convert_tool_choice(None) is omit
assert Converter.convert_tool_choice("auto") == "auto"
assert Converter.convert_tool_choice("required") == "required"
assert Converter.convert_tool_choice("none") == "none"
tool_choice_dict = Converter.convert_tool_choice("mytool")
assert isinstance(tool_choice_dict, dict)
assert tool_choice_dict["type"] == "function"
assert tool_choice_dict["function"]["name"] == "mytool"
def test_convert_tool_choice_allows_tool_search_as_named_function_for_chat_models() -> None:
tool_choice_dict = Converter.convert_tool_choice("tool_search")
assert isinstance(tool_choice_dict, dict)
assert tool_choice_dict["type"] == "function"
assert tool_choice_dict["function"]["name"] == "tool_search"
def test_convert_response_format_returns_not_given_for_plain_text_and_dict_for_schemas() -> None:
"""
The `Converter.convert_response_format` method should return the omit sentinel
when no output schema is provided or if the output schema indicates
plain text. For structured output schemas, it should return a dict
with type `json_schema` and include the generated JSON schema and
strict flag from the provided `AgentOutputSchema`.
"""
# when output is plain text (schema None or output_type str), do not include response_format
assert Converter.convert_response_format(None) is omit
assert Converter.convert_response_format(AgentOutputSchema(str)) is omit
# For e.g. integer output, we expect a response_format dict
schema = AgentOutputSchema(int)
resp_format = Converter.convert_response_format(schema)
assert isinstance(resp_format, dict)
assert resp_format["type"] == "json_schema"
assert resp_format["json_schema"]["name"] == "final_output"
assert "strict" in resp_format["json_schema"]
assert resp_format["json_schema"]["strict"] == schema.is_strict_json_schema()
assert "schema" in resp_format["json_schema"]
assert resp_format["json_schema"]["schema"] == schema.json_schema()
def test_items_to_messages_with_function_output_item():
"""
A function call output item should be converted into a tool role message
dict with the appropriate tool_call_id and content.
"""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": '{"foo": "bar"}',
}
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == func_output_item["output"]
def test_items_to_messages_with_non_text_only_function_output_uses_placeholder_by_default(
caplog: pytest.LogCaptureFixture,
):
"""Default conversion should keep running without sending an empty tool message."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/image.png",
}
],
}
with caplog.at_level(logging.WARNING, logger="openai.agents"):
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == "[tool output omitted]"
assert "Replacing the tool output with a placeholder" in caplog.text
def test_items_to_messages_with_non_text_only_function_output_raises_in_strict_mode():
"""Strict validation should fail explicitly instead of silently losing the output."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/image.png",
}
],
}
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
def test_items_to_messages_with_empty_function_output_uses_placeholder_by_default(
caplog: pytest.LogCaptureFixture,
):
"""Default conversion should not send an empty tool message."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [],
}
with caplog.at_level(logging.WARNING, logger="openai.agents"):
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == "[tool output omitted]"
assert "Replacing the tool output with a placeholder" in caplog.text
def test_items_to_messages_with_empty_function_output_raises_in_strict_mode():
"""Strict validation should fail explicitly instead of sending empty output."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [],
}
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
def test_items_to_messages_with_mixed_function_output_keeps_text_by_default(
caplog: pytest.LogCaptureFixture,
):
"""Default conversion should preserve text parts and omit unsupported non-text parts."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{"type": "input_text", "text": "visible text"},
{
"type": "input_image",
"image_url": "https://example.com/image.png",
},
],
}
with caplog.at_level(logging.WARNING, logger="openai.agents"):
messages = Converter.items_to_messages([func_output_item])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == [{"type": "text", "text": "visible text"}]
assert "tool output omitted" not in caplog.text
def test_items_to_messages_can_preserve_non_text_function_output() -> None:
"""Compatible providers can opt in to preserving non-text tool output."""
func_output_item: FunctionCallOutput = {
"type": "function_call_output",
"call_id": "somecall",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/image.png",
}
],
}
messages = Converter.items_to_messages(
[func_output_item],
preserve_tool_output_all_content=True,
)
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "tool"
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
assert tool_msg["content"] == [
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.png", "detail": "auto"},
}
]
def test_extract_all_and_text_content_for_strings_and_lists():
"""
The converter provides helpers for extracting user-supplied message content
either as a simple string or as a list of `input_text` dictionaries.
When passed a bare string, both `extract_all_content` and
`extract_text_content` should return the string unchanged.
When passed a list of input dictionaries, `extract_all_content` should
produce a list of `ChatCompletionContentPart` dicts, and `extract_text_content`
should filter to only the textual parts.
"""
prompt = "just text"
assert Converter.extract_all_content(prompt) == prompt
assert Converter.extract_text_content(prompt) == prompt
text1: ResponseInputTextParam = {"type": "input_text", "text": "one"}
text2: ResponseInputTextParam = {"type": "input_text", "text": "two"}
all_parts = Converter.extract_all_content([text1, text2])
assert isinstance(all_parts, list)
assert len(all_parts) == 2
assert all_parts[0]["type"] == "text" and all_parts[0]["text"] == "one"
assert all_parts[1]["type"] == "text" and all_parts[1]["text"] == "two"
text_parts = Converter.extract_text_content([text1, text2])
assert isinstance(text_parts, list)
assert all(p["type"] == "text" for p in text_parts)
assert [p["text"] for p in text_parts] == ["one", "two"]
def test_extract_all_content_handles_input_audio():
"""
input_audio entries should translate into ChatCompletion input_audio parts.
"""
audio: ResponseInputAudioParam = {
"type": "input_audio",
"input_audio": {"data": "AAA=", "format": "wav"},
}
parts = Converter.extract_all_content([audio])
assert isinstance(parts, list)
assert parts == [
{
"type": "input_audio",
"input_audio": {"data": "AAA=", "format": "wav"},
}
]
def test_extract_all_content_preserves_prompt_cache_breakpoints() -> None:
breakpoint = {"mode": "explicit"}
content: list[dict[str, Any]] = [
{
"type": "input_text",
"text": "one",
"prompt_cache_breakpoint": breakpoint,
},
{
"type": "input_image",
"image_url": "https://example.com/image.png",
"prompt_cache_breakpoint": breakpoint,
},
{
"type": "input_audio",
"input_audio": {"data": "AAA=", "format": "wav"},
"prompt_cache_breakpoint": breakpoint,
},
{
"type": "input_file",
"file_data": "data:text/plain;base64,SGVsbG8=",
"filename": "hello.txt",
"prompt_cache_breakpoint": breakpoint,
},
]
parts = Converter.extract_all_content(content)
assert isinstance(parts, list)
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint] * 4
def test_raw_chat_content_aliases_preserve_prompt_cache_breakpoints() -> None:
breakpoint = {"mode": "explicit"}
parts = Converter.extract_all_content(
cast(
list[dict[str, Any]],
[
{"type": "text", "text": "one", "prompt_cache_breakpoint": breakpoint},
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.png"},
"prompt_cache_breakpoint": breakpoint,
},
],
)
)
assert isinstance(parts, list)
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint, breakpoint]
def test_extract_all_content_rejects_invalid_input_audio():
"""
input_audio requires both data and format fields to be present.
"""
audio_missing_data = cast(
ResponseInputAudioParam,
{
"type": "input_audio",
"input_audio": {"format": "wav"},
},
)
with pytest.raises(UserError):
Converter.extract_all_content([audio_missing_data])
def test_items_to_messages_handles_system_and_developer_roles():
"""
Roles other than `user` (e.g. `system` and `developer`) need to be
converted appropriately whether provided as simple dicts or as full
`message` typed dicts.
"""
sys_items: list[TResponseInputItem] = [{"role": "system", "content": "setup"}]
sys_msgs = Converter.items_to_messages(sys_items)
assert len(sys_msgs) == 1
assert sys_msgs[0]["role"] == "system"
assert sys_msgs[0]["content"] == "setup"
dev_items: list[TResponseInputItem] = [{"role": "developer", "content": "debug"}]
dev_msgs = Converter.items_to_messages(dev_items)
assert len(dev_msgs) == 1
assert dev_msgs[0]["role"] == "developer"
assert dev_msgs[0]["content"] == "debug"
def test_maybe_input_message_allows_message_typed_dict():
"""
The `Converter.maybe_input_message` should recognize a dict with
"type": "message" and a supported role as an input message. Ensure
that such dicts are passed through by `items_to_messages`.
"""
# Construct a dict with the proper required keys for a ResponseInputParam.Message
message_dict: TResponseInputItem = {
"type": "message",
"role": "user",
"content": "hi",
}
assert Converter.maybe_input_message(message_dict) is not None
# items_to_messages should process this correctly
msgs = Converter.items_to_messages([message_dict])
assert len(msgs) == 1
assert msgs[0]["role"] == "user"
assert msgs[0]["content"] == "hi"
def test_tool_call_conversion():
"""
Test that tool calls are converted correctly.
"""
function_call = ResponseFunctionToolCallParam(
id="tool1",
call_id="abc",
name="math",
arguments="{}",
type="function_call",
)
messages = Converter.items_to_messages([function_call])
assert len(messages) == 1
tool_msg = messages[0]
assert tool_msg["role"] == "assistant"
assert tool_msg.get("content") is None
# Verify the content key exists in the message even when it is None.
# This is for Chat Completions API compatibility.
assert "content" in tool_msg, "content key should be present in assistant message"
tool_calls = list(tool_msg.get("tool_calls", []))
assert len(tool_calls) == 1
tool_call = tool_calls[0]
assert tool_call["id"] == function_call["call_id"]
assert tool_call["function"]["name"] == function_call["name"] # type: ignore
assert tool_call["function"]["arguments"] == function_call["arguments"] # type: ignore
@pytest.mark.parametrize("role", ["user", "system", "developer"])
def test_input_message_with_all_roles(role: str):
"""
The `Converter.maybe_input_message` should recognize a dict with
"type": "message" and a supported role as an input message. Ensure
that such dicts are passed through by `items_to_messages`.
"""
# Construct a dict with the proper required keys for a ResponseInputParam.Message
casted_role = cast(Literal["user", "system", "developer"], role)
message_dict: TResponseInputItem = {
"type": "message",
"role": casted_role,
"content": "hi",
}
assert Converter.maybe_input_message(message_dict) is not None
# items_to_messages should process this correctly
msgs = Converter.items_to_messages([message_dict])
assert len(msgs) == 1
assert msgs[0]["role"] == casted_role
assert msgs[0]["content"] == "hi"
def test_item_reference_errors():
"""
Test that item references are converted correctly.
"""
with pytest.raises(UserError):
Converter.items_to_messages(
[
{
"type": "item_reference",
"id": "item1",
}
]
)
class TestObject:
pass
def test_unknown_object_errors():
"""
Test that unknown objects are converted correctly.
"""
with pytest.raises(UserError, match="Unhandled item type or structure"):
# Purposely ignore the type error
Converter.items_to_messages([TestObject()]) # type: ignore
def test_assistant_messages_in_history():
"""
Test that assistant messages are added to the history.
"""
messages = Converter.items_to_messages(
[
{
"role": "user",
"content": "Hello",
},
{
"role": "assistant",
"content": "Hello?",
},
{
"role": "user",
"content": "What was my Name?",
},
]
)
assert messages == [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hello?"},
{"role": "user", "content": "What was my Name?"},
]
assert len(messages) == 3
assert messages[0]["role"] == "user"
assert messages[0]["content"] == "Hello"
assert messages[1]["role"] == "assistant"
assert messages[1]["content"] == "Hello?"
assert messages[2]["role"] == "user"
assert messages[2]["content"] == "What was my Name?"
File diff suppressed because it is too large Load Diff
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
import pytest
from agents.models.openai_client_utils import (
is_official_openai_base_url,
is_official_openai_client,
)
@pytest.mark.parametrize(
"base_url",
[
"https://api.openai.com",
"https://api.openai.com/v1/",
],
)
def test_official_openai_base_url_matches_exact_host(base_url: str) -> None:
assert is_official_openai_base_url(base_url) is True
@pytest.mark.parametrize(
"base_url",
[
"https://api.openai.com.evil/v1/",
"https://api.openai.com.proxy.local/v1/",
"http://api.openai.com/v1/",
"https://custom.example.test/v1/",
],
)
def test_official_openai_base_url_rejects_non_openai_hosts(base_url: str) -> None:
assert is_official_openai_base_url(base_url) is False
def test_official_openai_websocket_base_url_matches_exact_host() -> None:
assert is_official_openai_base_url("wss://api.openai.com/v1/", websocket=True) is True
assert (
is_official_openai_base_url("wss://api.openai.com.proxy.local/v1/", websocket=True) is False
)
def test_official_openai_client_rejects_client_without_base_url() -> None:
assert is_official_openai_client(object()) is False # type: ignore[arg-type]
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+178
View File
@@ -0,0 +1,178 @@
"""Unit tests for the low-level helpers in :mod:`agents.models._openai_retry`.
These exercise the header-parsing, status-extraction, and error-code helpers
directly, plus a few public ``get_openai_retry_advice`` branches that the broader
behavioral suite in ``test_model_retry.py`` does not reach.
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from email.utils import format_datetime
import httpx
from agents.models._openai_retry import get_openai_retry_advice
from agents.models._retry_runtime import (
get_error_code as _get_error_code,
get_error_header as _get_header_value,
get_retry_after,
get_status_code as _get_status_code,
header_lookup as _header_lookup,
parse_retry_after_ms as _parse_retry_after_ms,
parse_retry_after_value as _parse_retry_after,
)
from agents.retry import ModelRetryAdviceRequest
from agents.run_internal.model_retry import _normalize_retry_error
class _HeaderError(Exception):
"""Error that exposes headers through a plain attribute rather than a response."""
def __init__(self, message: str, *, headers: dict[str, str] | None = None) -> None:
super().__init__(message)
if headers is not None:
self.headers = headers
def _make_request(error: Exception, **kwargs: object) -> ModelRetryAdviceRequest:
return ModelRetryAdviceRequest(error=error, attempt=1, stream=False, **kwargs) # type: ignore[arg-type]
def test_header_lookup_plain_mapping_matches_case_insensitively() -> None:
headers = {"Retry-After": "5", "X-Other": "ignored"}
assert _header_lookup(headers, "retry-after") == "5"
assert _header_lookup(headers, "missing") is None
def test_header_lookup_httpx_headers() -> None:
headers = httpx.Headers({"retry-after": "7"})
assert _header_lookup(headers, "retry-after") == "7"
assert _header_lookup(None, "retry-after") is None
def test_get_header_value_reads_response_headers_attr() -> None:
class _Err(Exception):
response_headers = {"retry-after": "3"}
assert _get_header_value(_Err("boom"), "retry-after") == "3"
def test_parse_retry_after_ms_invalid_returns_none() -> None:
assert _parse_retry_after_ms(None) is None
assert _parse_retry_after_ms("not-a-number") is None
assert _parse_retry_after_ms("-100") is None
assert _parse_retry_after_ms("1500") == 1.5
def test_parse_retry_after_numeric_and_http_date() -> None:
assert _parse_retry_after(None) is None
assert _parse_retry_after("2") == 2.0
assert _parse_retry_after("-1") is None
future = datetime.now(timezone.utc) + timedelta(seconds=120)
parsed = _parse_retry_after(format_datetime(future))
assert parsed is not None and parsed > 0
assert _parse_retry_after("definitely not a date") is None
def test_get_retry_after_preserves_outer_exception_precedence() -> None:
outer = _HeaderError("wrapped", headers={"retry-after": "2"})
outer.__cause__ = _HeaderError("provider", headers={"retry-after-ms": "1500"})
assert get_retry_after(outer) == 2.0
def test_get_status_code_from_status_code_and_status_attrs() -> None:
class _StatusCode(Exception):
status_code = 503
class _Status(Exception):
status = 504
assert _get_status_code(_StatusCode("a")) == 503
assert _get_status_code(_Status("b")) == 504
assert _get_status_code(Exception("none")) is None
def test_get_error_code_from_body_mapping() -> None:
class _NestedBody(Exception):
body = {"error": {"code": "rate_limit_exceeded"}}
class _TopLevelBody(Exception):
body = {"code": "server_error"}
assert _get_error_code(_NestedBody("a")) == "rate_limit_exceeded"
assert _get_error_code(_TopLevelBody("b")) == "server_error"
assert _get_error_code(Exception("none")) is None
def test_provider_and_runner_retry_normalization_share_metadata() -> None:
class _RetryableError(Exception):
status_code = 429
request_id = "req_test"
body = {"error": {"code": "rate_limit_exceeded"}}
headers = {"retry-after-ms": "1500"}
class _WrapperError(Exception):
headers = {"x-other": "ignored"}
error = _WrapperError("wrapped")
error.__cause__ = _RetryableError("slow down")
advice = get_openai_retry_advice(_make_request(error))
runner_normalized = _normalize_retry_error(error, None)
assert advice is not None
assert advice.normalized is not None
assert advice.normalized.status_code == runner_normalized.status_code
assert advice.normalized.error_code == runner_normalized.error_code
assert advice.normalized.request_id == runner_normalized.request_id
assert advice.normalized.retry_after == runner_normalized.retry_after
assert runner_normalized.retry_after == 1.5
def test_advice_unsafe_to_replay() -> None:
error = Exception("cannot replay")
error.unsafe_to_replay = True # type: ignore[attr-defined]
advice = get_openai_retry_advice(_make_request(error))
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
def test_advice_websocket_request_is_unsafe() -> None:
message = (
"The request may have been accepted, so the SDK will not automatically "
"retry this websocket request."
)
advice = get_openai_retry_advice(_make_request(Exception(message)))
assert advice is not None
assert advice.suggested is False
assert advice.replay_safety == "unsafe"
def test_advice_respects_x_should_retry_false() -> None:
error = _HeaderError("nope", headers={"x-should-retry": "false"})
advice = get_openai_retry_advice(_make_request(error))
assert advice is not None
assert advice.suggested is False
def test_advice_returns_retry_after_only_when_no_other_signal() -> None:
# A 400 with no x-should-retry header and no network/timeout signal would not
# normally retry, but a retry-after header still yields advice carrying the delay.
error = _HeaderError("slow down", headers={"retry-after": "2"})
advice = get_openai_retry_advice(_make_request(error))
assert advice is not None
assert advice.retry_after == 2.0
# This branch only conveys the server-provided delay; it does not assert a
# retry decision, so ``suggested`` keeps its unset default.
assert advice.suggested is None
+454
View File
@@ -0,0 +1,454 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from typing import Any, cast
import pytest
from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta
from openai.types.completion_usage import (
CompletionTokensDetails,
CompletionUsage,
PromptTokensDetails,
)
from openai.types.responses import (
Response,
ResponseOutputMessage,
ResponseOutputText,
ResponseReasoningItem,
)
from agents.model_settings import ModelSettings
from agents.models.interface import ModelTracing
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.models.openai_provider import OpenAIProvider
# Helper functions to create test objects consistently
def create_content_delta(content: str) -> dict[str, Any]:
"""Create a delta dictionary with regular content"""
return {"content": content, "role": None, "function_call": None, "tool_calls": None}
def create_reasoning_delta(content: str) -> dict[str, Any]:
"""Create a delta dictionary with reasoning content. The Only difference is reasoning_content"""
return {
"content": None,
"role": None,
"function_call": None,
"tool_calls": None,
"reasoning_content": content,
}
def create_chunk(delta: dict[str, Any], include_usage: bool = False) -> ChatCompletionChunk:
"""Create a ChatCompletionChunk with the given delta"""
# Create a ChoiceDelta object from the dictionary
delta_obj = ChoiceDelta(
content=delta.get("content"),
role=delta.get("role"),
function_call=delta.get("function_call"),
tool_calls=delta.get("tool_calls"),
)
# Add reasoning_content attribute dynamically if present in the delta
if "reasoning_content" in delta:
# Use direct assignment for the reasoning_content attribute
delta_obj_any = cast(Any, delta_obj)
delta_obj_any.reasoning_content = delta["reasoning_content"]
# Create the chunk
chunk = ChatCompletionChunk(
id="chunk-id",
created=1,
model="deepseek is usually expected",
object="chat.completion.chunk",
choices=[Choice(index=0, delta=delta_obj)],
)
if include_usage:
chunk.usage = CompletionUsage(
completion_tokens=4,
prompt_tokens=2,
total_tokens=6,
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
)
return chunk
async def create_fake_stream(
chunks: list[ChatCompletionChunk],
) -> AsyncIterator[ChatCompletionChunk]:
for chunk in chunks:
yield chunk
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_yields_events_for_reasoning_content(monkeypatch) -> None:
"""
Validate that when a model streams reasoning content,
`stream_response` emits the appropriate sequence of events including
`response.reasoning_summary_text.delta` events for each chunk of the reasoning content and
constructs a completed response with a `ResponseReasoningItem` part.
"""
# Create test chunks
chunks = [
# Reasoning content chunks
create_chunk(create_reasoning_delta("Let me think")),
create_chunk(create_reasoning_delta(" about this")),
# Regular content chunks
create_chunk(create_content_delta("The answer")),
create_chunk(create_content_delta(" is 42"), include_usage=True),
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# verify reasoning content events were emitted
reasoning_delta_events = [
e for e in output_events if e.type == "response.reasoning_summary_text.delta"
]
assert len(reasoning_delta_events) == 2
assert reasoning_delta_events[0].delta == "Let me think"
assert reasoning_delta_events[1].delta == " about this"
reasoning_done_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.reasoning_summary_part.done"
)
first_text_delta_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.output_text.delta"
)
assert reasoning_done_index < first_text_delta_index
# verify regular content events were emitted
content_delta_events = [e for e in output_events if e.type == "response.output_text.delta"]
assert len(content_delta_events) == 2
assert content_delta_events[0].delta == "The answer"
assert content_delta_events[1].delta == " is 42"
assistant_message_index_events = []
for event in output_events:
event_any = cast(Any, event)
if event.type in {"response.output_item.added", "response.output_item.done"}:
if event_any.item.type == "message":
assistant_message_index_events.append(event_any)
elif event.type in {
"response.content_part.added",
"response.output_text.delta",
"response.content_part.done",
}:
assistant_message_index_events.append(event_any)
assert assistant_message_index_events
for event in assistant_message_index_events:
assert event.output_index == 1
assert type(event.output_index) is int
# verify the final response contains both types of content
response_event = output_events[-1]
assert response_event.type == "response.completed"
assert len(response_event.response.output) == 2
# first item should be reasoning
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
assert response_event.response.output[0].summary[0].text == "Let me think about this"
# second item should be message with text
assert isinstance(response_event.response.output[1], ResponseOutputMessage)
assert isinstance(response_event.response.output[1].content[0], ResponseOutputText)
assert response_event.response.output[1].content[0].text == "The answer is 42"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_keeps_reasoning_item_open_across_interleaved_text(
monkeypatch,
) -> None:
chunks = [
create_chunk(create_reasoning_delta("Let me think")),
create_chunk(create_content_delta("The answer")),
create_chunk(create_reasoning_delta(" more carefully")),
create_chunk(create_content_delta(" is 42"), include_usage=True),
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
reasoning_part_added_events = [
event for event in output_events if event.type == "response.reasoning_summary_part.added"
]
assert [event.summary_index for event in reasoning_part_added_events] == [0, 1]
reasoning_part_done_events = [
event for event in output_events if event.type == "response.reasoning_summary_part.done"
]
assert [event.summary_index for event in reasoning_part_done_events] == [0, 1]
first_reasoning_done_index = output_events.index(reasoning_part_done_events[0])
first_text_delta_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.output_text.delta"
)
second_reasoning_delta_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.reasoning_summary_text.delta" and event.summary_index == 1
)
reasoning_item_done_index = next(
index
for index, event in enumerate(output_events)
if event.type == "response.output_item.done" and event.item.type == "reasoning"
)
assert first_reasoning_done_index < first_text_delta_index
assert second_reasoning_delta_index > first_text_delta_index
assert reasoning_item_done_index > second_reasoning_delta_index
response_event = output_events[-1]
assert response_event.type == "response.completed"
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
assert [summary.text for summary in response_event.response.output[0].summary] == [
"Let me think",
" more carefully",
]
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_get_response_with_reasoning_content(monkeypatch) -> None:
"""
Test that when a model returns reasoning content in addition to regular content,
`get_response` properly includes both in the response output.
"""
# create a message with reasoning content
msg = ChatCompletionMessage(
role="assistant",
content="The answer is 42",
)
# Use dynamic attribute for reasoning_content
# We need to cast to Any to avoid mypy errors since reasoning_content is not a defined attribute
msg_with_reasoning = cast(Any, msg)
msg_with_reasoning.reasoning_content = "Let me think about this question carefully"
# create a choice with the message
mock_choice = {
"index": 0,
"finish_reason": "stop",
"message": msg_with_reasoning,
"delta": None,
}
chat = ChatCompletion(
id="resp-id",
created=0,
model="deepseek is expected",
object="chat.completion",
choices=[mock_choice], # type: ignore[list-item]
usage=CompletionUsage(
completion_tokens=10,
prompt_tokens=5,
total_tokens=15,
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=6),
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
),
)
async def patched_fetch_response(self, *args, **kwargs):
return chat
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
resp = await model.get_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
)
# should have produced a reasoning item and a message with text content
assert len(resp.output) == 2
# first output should be the reasoning item
assert isinstance(resp.output[0], ResponseReasoningItem)
assert resp.output[0].summary[0].text == "Let me think about this question carefully"
# second output should be the message with text content
assert isinstance(resp.output[1], ResponseOutputMessage)
assert isinstance(resp.output[1].content[0], ResponseOutputText)
assert resp.output[1].content[0].text == "The answer is 42"
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_preserves_usage_from_earlier_chunk(monkeypatch) -> None:
"""
Test that when an earlier chunk has usage data and later chunks don't,
the usage from the earlier chunk is preserved in the final response.
This handles cases where some providers (e.g., LiteLLM) may not include
usage in every chunk.
"""
# Create test chunks where first chunk has usage, last chunk doesn't
chunks = [
create_chunk(create_content_delta("Hello"), include_usage=True), # Has usage
create_chunk(create_content_delta("")), # No usage (usage=None)
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# Verify the final response preserves usage from the first chunk
response_event = output_events[-1]
assert response_event.type == "response.completed"
assert response_event.response.usage is not None
assert response_event.response.usage.input_tokens == 2
assert response_event.response.usage.output_tokens == 4
assert response_event.response.usage.total_tokens == 6
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_stream_response_with_empty_reasoning_content(monkeypatch) -> None:
"""
Test that when a model streams empty reasoning content,
the response still processes correctly without errors.
"""
# create test chunks with empty reasoning content
chunks = [
create_chunk(create_reasoning_delta("")),
create_chunk(create_content_delta("The answer is 42"), include_usage=True),
]
async def patched_fetch_response(self, *args, **kwargs):
resp = Response(
id="resp-id",
created_at=0,
model="fake-model",
object="response",
output=[],
tool_choice="none",
tools=[],
parallel_tool_calls=False,
)
return resp, create_fake_stream(chunks)
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
output_events = []
async for event in model.stream_response(
system_instructions=None,
input="",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
conversation_id=None,
prompt=None,
):
output_events.append(event)
# verify the final response contains the content
response_event = output_events[-1]
assert response_event.type == "response.completed"
# should only have the message, not an empty reasoning item
assert len(response_event.response.output) == 1
assert isinstance(response_event.response.output[0], ResponseOutputMessage)
assert isinstance(response_event.response.output[0].content[0], ResponseOutputText)
assert response_event.response.output[0].content[0].text == "The answer is 42"
@@ -0,0 +1,403 @@
from __future__ import annotations
from typing import Any, cast
import httpx
import litellm
import pytest
from litellm.types.utils import Choices, Message, ModelResponse, Usage
from openai.types.chat.chat_completion import ChatCompletion, Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.completion_usage import CompletionUsage
from agents.extensions.models.litellm_model import LitellmModel
from agents.items import TResponseInputItem
from agents.model_settings import ModelSettings
from agents.models.chatcmpl_converter import Converter
from agents.models.interface import ModelTracing
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
from agents.models.reasoning_content_replay import ReasoningContentReplayContext
REASONING_CONTENT_MODEL_A = "reasoning-content-model-a"
REASONING_CONTENT_MODEL_B = "reasoning-content-model-b"
# The converter currently keys Anthropic thinking-block reconstruction off the model name,
# so this test model keeps the "anthropic" substring while staying otherwise generic.
REASONING_CONTENT_MODEL_C = "reasoning-content-model-c-anthropic"
def _second_turn_input_items(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _second_turn_input_items_with_message(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"id": "__fake_id__",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [
{
"type": "output_text",
"text": "I'll call the weather tool now.",
"annotations": [],
"logprobs": [],
}
],
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _second_turn_input_items_with_file_search(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "Find notes about Tokyo weather."},
{
"id": "__fake_id__",
"summary": [
{"text": "I should search the knowledge base first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"id": "__fake_file_search_id__",
"queries": ["Tokyo weather"],
"status": "completed",
"type": "file_search_call",
},
],
)
def _second_turn_input_items_with_message_then_reasoning(
model_name: str,
) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [
{
"type": "output_text",
"text": "I'll call the weather tool now.",
"annotations": [],
"logprobs": [],
}
],
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": None,
"encrypted_content": None,
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _second_turn_input_items_with_thinking_blocks(model_name: str) -> list[TResponseInputItem]:
return cast(
list[TResponseInputItem],
[
{"role": "user", "content": "What's the weather in Tokyo?"},
{
"id": "__fake_id__",
"summary": [
{"text": "I should call the weather tool first.", "type": "summary_text"}
],
"type": "reasoning",
"content": [
{
"type": "reasoning_text",
"text": "First, I need to inspect the request.",
}
],
"encrypted_content": "test-signature",
"status": None,
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
},
{
"arguments": '{"city": "Tokyo"}',
"call_id": "call_weather_123",
"name": "get_weather",
"type": "function_call",
"id": "__fake_id__",
"status": None,
"provider_data": {"model": model_name},
},
{
"type": "function_call_output",
"call_id": "call_weather_123",
"output": "The weather in Tokyo is sunny and 22°C.",
},
],
)
def _assistant_with_tool_calls(messages: list[Any]) -> dict[str, Any]:
for msg in messages:
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
return msg
raise AssertionError("Expected an assistant message with tool_calls.")
def test_converter_keeps_default_reasoning_replay_behavior_for_non_default_model() -> None:
messages = Converter.items_to_messages(
_second_turn_input_items(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
)
assistant = _assistant_with_tool_calls(messages)
assert "reasoning_content" not in assistant
def test_converter_preserves_reasoning_content_across_output_message_with_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_message(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["content"] == "I'll call the weather tool now."
assert assistant["reasoning_content"] == "I should call the weather tool first."
def test_converter_replays_reasoning_content_when_reasoning_follows_message_with_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_message_then_reasoning(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["content"] == "I'll call the weather tool now."
assert assistant["reasoning_content"] == "I should call the weather tool first."
def test_converter_replays_reasoning_content_for_file_search_call_with_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_file_search(REASONING_CONTENT_MODEL_A),
model=REASONING_CONTENT_MODEL_A,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["reasoning_content"] == "I should search the knowledge base first."
assert assistant["tool_calls"][0]["function"]["name"] == "file_search_call"
def test_converter_replays_reasoning_content_with_thinking_blocks_and_hook() -> None:
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
return True
messages = Converter.items_to_messages(
_second_turn_input_items_with_thinking_blocks(REASONING_CONTENT_MODEL_C),
model=REASONING_CONTENT_MODEL_C,
preserve_thinking_blocks=True,
should_replay_reasoning_content=should_replay_reasoning_content,
)
assistant = _assistant_with_tool_calls(messages)
assert assistant["reasoning_content"] == "I should call the weather tool first."
assert assistant["content"][0]["type"] == "thinking"
assert assistant["content"][0]["thinking"] == "First, I need to inspect the request."
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_openai_chatcompletions_hook_can_enable_reasoning_content_replay() -> None:
captured: dict[str, Any] = {}
contexts: list[ReasoningContentReplayContext] = []
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
contexts.append(context)
return context.model == REASONING_CONTENT_MODEL_B
class MockChatCompletions:
async def create(self, **kwargs):
captured.update(kwargs)
msg = ChatCompletionMessage(role="assistant", content="done")
choice = Choice(index=0, message=msg, finish_reason="stop")
return ChatCompletion(
id="test-id",
created=0,
model=REASONING_CONTENT_MODEL_B,
object="chat.completion",
choices=[choice],
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
)
class MockChat:
def __init__(self):
self.completions = MockChatCompletions()
class MockClient:
def __init__(self):
self.chat = MockChat()
self.base_url = httpx.URL("https://example.com/v1/")
model = OpenAIChatCompletionsModel(
model=REASONING_CONTENT_MODEL_B,
openai_client=cast(Any, MockClient()),
should_replay_reasoning_content=should_replay_reasoning_content,
)
await model.get_response(
system_instructions=None,
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
assert assistant["reasoning_content"] == "I should call the weather tool first."
assert len(contexts) == 1
assert contexts[0].model == REASONING_CONTENT_MODEL_B
assert contexts[0].base_url == "https://example.com/v1"
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_litellm_hook_can_enable_reasoning_content_replay(monkeypatch) -> None:
captured: dict[str, Any] = {}
contexts: list[ReasoningContentReplayContext] = []
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
contexts.append(context)
return context.model == REASONING_CONTENT_MODEL_B
async def fake_acompletion(model, messages=None, **kwargs):
captured["messages"] = messages
msg = Message(role="assistant", content="done")
choice = Choices(index=0, message=msg)
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
model = LitellmModel(
model=REASONING_CONTENT_MODEL_B,
should_replay_reasoning_content=should_replay_reasoning_content,
)
await model.get_response(
system_instructions=None,
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
previous_response_id=None,
)
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
assert assistant["reasoning_content"] == "I should call the weather tool first."
assert len(contexts) == 1
assert contexts[0].model == REASONING_CONTENT_MODEL_B
assert contexts[0].base_url is None
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
@@ -0,0 +1,162 @@
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
import pytest
from agents.models.fake_id import FAKE_RESPONSES_ID
from agents.models.openai_responses import OpenAIResponsesModel
@pytest.fixture
def model() -> OpenAIResponsesModel:
"""Create a model instance for testing."""
mock_client = MagicMock()
return OpenAIResponsesModel(model="gpt-5", openai_client=mock_client)
class TestRemoveOpenAIResponsesAPIIncompatibleFields:
"""Tests for _remove_openai_responses_api_incompatible_fields method."""
def test_returns_unchanged_when_no_provider_data(self, model: OpenAIResponsesModel):
"""When no items have provider_data, the input should be returned unchanged."""
list_input = [
{"type": "message", "content": "hello"},
{"type": "function_call", "call_id": "call_123", "name": "test"},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert result is list_input # Same object reference.
def test_removes_reasoning_items_with_provider_data(self, model: OpenAIResponsesModel):
"""Reasoning items with provider_data should be completely removed."""
list_input = [
{"type": "message", "content": "hello"},
{"type": "reasoning", "provider_data": {"model": "gemini/gemini-3"}},
{"type": "function_call", "call_id": "call_123"},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 2
assert result[0] == {"type": "message", "content": "hello"}
assert result[1] == {"type": "function_call", "call_id": "call_123"}
def test_keeps_reasoning_items_without_provider_data(self, model: OpenAIResponsesModel):
"""Reasoning items without provider_data should be kept."""
list_input = [
{"type": "reasoning", "summary": []},
{"type": "message", "content": "hello", "provider_data": {"foo": "bar"}},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 2
assert result[0] == {"type": "reasoning", "summary": []}
assert result[1] == {"type": "message", "content": "hello"}
def test_removes_provider_data_from_all_items(self, model: OpenAIResponsesModel):
"""provider_data field should be removed from all dict items."""
list_input = [
{"type": "message", "content": "hello", "provider_data": {"model": "gemini/gemini-3"}},
{
"type": "function_call",
"call_id": "call_123",
"provider_data": {"model": "gemini/gemini-3"},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 2
assert "provider_data" not in result[0]
assert "provider_data" not in result[1]
def test_removes_fake_responses_id(self, model: OpenAIResponsesModel):
"""Items with id equal to FAKE_RESPONSES_ID should have their id removed."""
list_input = [
{
"type": "message",
"id": FAKE_RESPONSES_ID,
"content": "hello",
"provider_data": {"model": "gemini/gemini-3"},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert len(result) == 1
assert "id" not in result[0]
assert result[0]["content"] == "hello"
def test_preserves_real_ids(self, model: OpenAIResponsesModel):
"""Real IDs (not FAKE_RESPONSES_ID) should be preserved."""
list_input = [
{
"type": "message",
"id": "msg_real123",
"content": "hello",
"provider_data": {},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert result[0]["id"] == "msg_real123"
def test_handles_empty_list(self, model: OpenAIResponsesModel):
"""Empty list should be returned unchanged."""
list_input: list[dict[str, Any]] = []
result = model._remove_openai_responses_api_incompatible_fields(list_input)
assert result == []
def test_combined_scenario(self, model: OpenAIResponsesModel):
"""Test a realistic scenario with multiple items needing different processing."""
list_input = [
{"type": "message", "content": "user input"},
{"type": "reasoning", "summary": [], "provider_data": {"model": "gemini/gemini-3"}},
{
"type": "function_call",
"call_id": "call_abc_123",
"name": "get_weather",
"provider_data": {"model": "gemini/gemini-3"},
},
{
"type": "function_call_output",
"call_id": "call_abc_123",
"output": '{"temp": 72}',
},
{
"type": "message",
"id": FAKE_RESPONSES_ID,
"content": "The weather is 72F",
"provider_data": {"model": "gemini/gemini-3"},
},
]
result = model._remove_openai_responses_api_incompatible_fields(list_input)
# Should have 4 items (reasoning with provider_data removed).
assert len(result) == 4
# First item unchanged (no provider_data).
assert result[0] == {"type": "message", "content": "user input"}
# Function call: __thought__ suffix removed, provider_data removed.
assert result[1]["type"] == "function_call"
assert result[1]["call_id"] == "call_abc_123"
assert "provider_data" not in result[1]
# Function call output: __thought__ suffix removed, provider_data removed.
assert result[2]["type"] == "function_call_output"
assert result[2]["call_id"] == "call_abc_123"
# Last message: fake id removed, provider_data removed.
assert result[3]["type"] == "message"
assert result[3]["content"] == "The weather is 72F"
assert "id" not in result[3]
assert "provider_data" not in result[3]
@@ -0,0 +1,149 @@
import importlib
import pytest
from agents import Agent, responses_websocket_session
from agents.models.multi_provider import MultiProvider
from agents.models.openai_provider import OpenAIProvider
@pytest.mark.asyncio
async def test_responses_websocket_session_builds_shared_run_config():
async with responses_websocket_session() as ws:
assert isinstance(ws.provider, OpenAIProvider)
assert ws.provider._use_responses is True
assert ws.provider._use_responses_websocket is True
assert isinstance(ws.run_config.model_provider, MultiProvider)
assert ws.run_config.model_provider.openai_provider is ws.provider
@pytest.mark.asyncio
async def test_responses_websocket_session_preserves_openai_prefix_routing(monkeypatch):
captured: dict[str, object] = {}
sentinel = object()
def fake_get_model(model_name):
captured["model_name"] = model_name
return sentinel
async with responses_websocket_session() as ws:
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
assert result is sentinel
assert captured["model_name"] == "gpt-4.1"
@pytest.mark.asyncio
async def test_responses_websocket_session_can_preserve_openai_prefix_model_ids(monkeypatch):
captured: dict[str, object] = {}
sentinel = object()
def fake_get_model(model_name):
captured["model_name"] = model_name
return sentinel
async with responses_websocket_session(openai_prefix_mode="model_id") as ws:
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
assert result is sentinel
assert captured["model_name"] == "openai/gpt-4.1"
@pytest.mark.asyncio
async def test_responses_websocket_session_can_preserve_unknown_prefix_model_ids(monkeypatch):
captured: dict[str, object] = {}
sentinel = object()
def fake_get_model(model_name):
captured["model_name"] = model_name
return sentinel
async with responses_websocket_session(unknown_prefix_mode="model_id") as ws:
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
result = ws.run_config.model_provider.get_model("openrouter/openai/gpt-4.1")
assert result is sentinel
assert captured["model_name"] == "openrouter/openai/gpt-4.1"
@pytest.mark.asyncio
async def test_responses_websocket_session_run_streamed_injects_run_config(monkeypatch):
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
captured = {}
sentinel = object()
def fake_run_streamed(starting_agent, input, **kwargs):
captured["starting_agent"] = starting_agent
captured["input"] = input
captured["kwargs"] = kwargs
return sentinel
ws_module = importlib.import_module("agents.responses_websocket_session")
monkeypatch.setattr(ws_module.Runner, "run_streamed", fake_run_streamed)
async with responses_websocket_session() as ws:
result = ws.run_streamed(agent, "hello")
assert result is sentinel
assert captured["starting_agent"] is agent
assert captured["input"] == "hello"
assert captured["kwargs"]["run_config"] is ws.run_config
@pytest.mark.asyncio
async def test_responses_websocket_session_run_injects_run_config(monkeypatch):
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
captured = {}
sentinel = object()
async def fake_run(starting_agent, input, **kwargs):
captured["starting_agent"] = starting_agent
captured["input"] = input
captured["kwargs"] = kwargs
return sentinel
ws_module = importlib.import_module("agents.responses_websocket_session")
monkeypatch.setattr(ws_module.Runner, "run", fake_run)
async with responses_websocket_session() as ws:
result = await ws.run(agent, "hello")
assert result is sentinel
assert captured["starting_agent"] is agent
assert captured["input"] == "hello"
assert captured["kwargs"]["run_config"] is ws.run_config
@pytest.mark.asyncio
async def test_responses_websocket_session_rejects_run_config_override():
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
async with responses_websocket_session() as ws:
with pytest.raises(ValueError, match="run_config"):
ws.run_streamed(agent, "hello", run_config=object())
@pytest.mark.asyncio
async def test_responses_websocket_session_context_manager_closes_provider(monkeypatch):
close_calls: list[OpenAIProvider] = []
async def fake_aclose(self):
close_calls.append(self)
monkeypatch.setattr(OpenAIProvider, "aclose", fake_aclose)
async with responses_websocket_session() as ws:
provider = ws.provider
assert close_calls == [provider]
@pytest.mark.asyncio
async def test_responses_websocket_session_does_not_expose_run_sync():
async with responses_websocket_session() as ws:
assert not hasattr(ws, "run_sync")
+32
View File
@@ -0,0 +1,32 @@
from agents.model_settings import ModelSettings
from agents.models._trace import model_config_for_trace, sanitize_url_for_trace
def test_sanitize_url_for_trace_strips_auth_query_and_fragment() -> None:
assert (
sanitize_url_for_trace("https://user:pass@example.com/v1?api-key=secret#fragment")
== "https://example.com/v1"
)
assert sanitize_url_for_trace("https://example.com/v1?token=secret") == "https://example.com/v1"
def test_model_config_for_trace_sanitizes_base_url_and_omits_request_extras() -> None:
config = model_config_for_trace(
ModelSettings(
temperature=0.5,
extra_headers={"Authorization": "Bearer provider-token"},
extra_query={"api-key": "query-token"},
extra_body={"secret": "body-token"},
extra_args={"api_key": "arg-token"},
),
base_url="https://user:pass@example.com/v1?api-key=secret#fragment",
extra_config={"model_impl": "test-model"},
)
assert config["temperature"] == 0.5
assert config["base_url"] == "https://example.com/v1"
assert config["model_impl"] == "test-model"
assert "extra_headers" not in config
assert "extra_query" not in config
assert "extra_body" not in config
assert "extra_args" not in config
View File
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
from typing import Any
import pytest
from agents import RunContextWrapper
from agents.realtime.agent import RealtimeAgent
def test_can_initialize_realtime_agent():
agent = RealtimeAgent(name="test", instructions="Hello")
assert agent.name == "test"
assert agent.instructions == "Hello"
@pytest.mark.asyncio
async def test_dynamic_instructions():
agent = RealtimeAgent(name="test")
assert agent.instructions is None
def _instructions(ctx, agt) -> str:
assert ctx.context is None
assert agt == agent
return "Dynamic"
agent = RealtimeAgent(name="test", instructions=_instructions)
instructions = await agent.get_system_prompt(RunContextWrapper(context=None))
assert instructions == "Dynamic"
def test_post_init_rejects_invalid_field_types() -> None:
with pytest.raises(TypeError, match="RealtimeAgent name must be a string"):
RealtimeAgent(name=1) # type: ignore[arg-type]
with pytest.raises(TypeError, match="RealtimeAgent tools must be a list"):
RealtimeAgent(name="x", tools="nope") # type: ignore[arg-type]
with pytest.raises(TypeError, match="RealtimeAgent handoffs must be a list"):
RealtimeAgent(name="x", handoffs="nope") # type: ignore[arg-type]
with pytest.raises(TypeError, match="RealtimeAgent instructions must be"):
RealtimeAgent(name="x", instructions=123) # type: ignore[arg-type]
def test_clone_does_not_mutate_original_lists() -> None:
"""Cloning with a new list must not affect the original agent's lists."""
original = RealtimeAgent(name="orig", tools=[], handoffs=[])
new_tools: list[Any] = ["t1"]
cloned = original.clone(tools=new_tools)
assert original.tools == []
assert len(cloned.tools) == 1
assert cloned.tools is not original.tools
# Shared reference when not overridden (documented shallow-copy behavior).
assert cloned.handoffs is original.handoffs
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
import importlib
import logging
from pathlib import Path
from types import ModuleType
import pytest
from agents.realtime.events import RealtimeEventInfo, RealtimeRawModelEvent
from agents.realtime.items import InputText, UserMessageItem
from agents.realtime.model_events import RealtimeModelItemUpdatedEvent
from agents.run_context import RunContextWrapper
@pytest.fixture
def app_server(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
app_dir = Path(__file__).parents[2] / "examples" / "realtime" / "app"
monkeypatch.chdir(app_dir)
monkeypatch.setenv("LOG_LEVEL", "DEBUG")
module = importlib.import_module("examples.realtime.app.server")
return importlib.reload(module)
def test_item_updated_debug_summary_uses_concrete_event_type(
app_server: ModuleType,
caplog: pytest.LogCaptureFixture,
) -> None:
item = UserMessageItem(
item_id="item-1",
content=[InputText(text="sensitive transcript")],
)
event = RealtimeRawModelEvent(
data=RealtimeModelItemUpdatedEvent(item=item),
info=RealtimeEventInfo(context=RunContextWrapper(None)),
)
with caplog.at_level(logging.DEBUG, logger=app_server.__name__):
app_server.manager._log_debug_event("session-1", event)
assert "item_updated" in caplog.text
assert "item-1" in caplog.text
assert "input_text" in caplog.text
assert "sensitive transcript" not in caplog.text
+53
View File
@@ -0,0 +1,53 @@
from openai.types.realtime.realtime_audio_formats import AudioPCM, AudioPCMA, AudioPCMU
from agents.realtime.audio_formats import to_realtime_audio_format
def test_to_realtime_audio_format_from_strings():
assert to_realtime_audio_format("pcm").type == "audio/pcm" # type: ignore[union-attr]
assert to_realtime_audio_format("pcm16").type == "audio/pcm" # type: ignore[union-attr]
assert to_realtime_audio_format("audio/pcm").type == "audio/pcm" # type: ignore[union-attr]
assert to_realtime_audio_format("pcmu").type == "audio/pcmu" # type: ignore[union-attr]
assert to_realtime_audio_format("audio/pcmu").type == "audio/pcmu" # type: ignore[union-attr]
assert to_realtime_audio_format("g711_ulaw").type == "audio/pcmu" # type: ignore[union-attr]
assert to_realtime_audio_format("pcma").type == "audio/pcma" # type: ignore[union-attr]
assert to_realtime_audio_format("audio/pcma").type == "audio/pcma" # type: ignore[union-attr]
assert to_realtime_audio_format("g711_alaw").type == "audio/pcma" # type: ignore[union-attr]
def test_to_realtime_audio_format_passthrough_and_unknown_logs():
fmt = AudioPCM(type="audio/pcm", rate=24000)
# Passing a RealtimeAudioFormats should return the same instance
assert to_realtime_audio_format(fmt) is fmt
# Unknown string returns None (and logs at debug level internally)
assert to_realtime_audio_format("something_else") is None
def test_to_realtime_audio_format_none():
assert to_realtime_audio_format(None) is None
def test_to_realtime_audio_format_from_mapping():
pcm_exact_rate = to_realtime_audio_format({"type": "audio/pcm", "rate": 24000})
assert isinstance(pcm_exact_rate, AudioPCM)
assert pcm_exact_rate.rate == 24000
pcm = to_realtime_audio_format({"type": "audio/pcm", "rate": 16000})
assert isinstance(pcm, AudioPCM)
assert pcm.type == "audio/pcm"
assert pcm.rate == 24000
pcm_default_rate = to_realtime_audio_format({"type": "audio/pcm"})
assert isinstance(pcm_default_rate, AudioPCM)
assert pcm_default_rate.rate == 24000
ulaw = to_realtime_audio_format({"type": "audio/pcmu"})
assert isinstance(ulaw, AudioPCMU)
assert ulaw.type == "audio/pcmu"
alaw = to_realtime_audio_format({"type": "audio/pcma"})
assert isinstance(alaw, AudioPCMA)
assert alaw.type == "audio/pcma"
assert to_realtime_audio_format({"type": "audio/unknown", "rate": 8000}) is None
+380
View File
@@ -0,0 +1,380 @@
from __future__ import annotations
import base64
from unittest.mock import Mock
import pytest
from openai.types.realtime.conversation_item_create_event import ConversationItemCreateEvent
from openai.types.realtime.conversation_item_truncate_event import ConversationItemTruncateEvent
from openai.types.realtime.input_audio_buffer_append_event import InputAudioBufferAppendEvent
from openai.types.realtime.realtime_conversation_item_function_call_output import (
RealtimeConversationItemFunctionCallOutput,
)
from pydantic import ValidationError
from agents.realtime.config import RealtimeModelTracingConfig
from agents.realtime.model_inputs import (
RealtimeModelSendAudio,
RealtimeModelSendRawMessage,
RealtimeModelSendToolOutput,
RealtimeModelSendUserInput,
RealtimeModelUserInputMessage,
)
from agents.realtime.openai_realtime import _ConversionHelper
class TestConversionHelperTryConvertRawMessage:
"""Test suite for _ConversionHelper.try_convert_raw_message method."""
def test_try_convert_raw_message_valid_session_update(self):
"""Test converting a valid session.update raw message."""
raw_message = RealtimeModelSendRawMessage(
message={
"type": "session.update",
"other_data": {
"session": {
"model": "gpt-realtime-2.1",
"type": "realtime",
"modalities": ["text", "audio"],
"voice": "ash",
}
},
}
)
result = _ConversionHelper.try_convert_raw_message(raw_message)
assert result is not None
assert result.type == "session.update"
def test_try_convert_raw_message_valid_response_create(self):
"""Test converting a valid response.create raw message."""
raw_message = RealtimeModelSendRawMessage(
message={
"type": "response.create",
"other_data": {},
}
)
result = _ConversionHelper.try_convert_raw_message(raw_message)
assert result is not None
assert result.type == "response.create"
def test_try_convert_raw_message_invalid_type(self):
"""Test converting an invalid message type returns None."""
raw_message = RealtimeModelSendRawMessage(
message={
"type": "invalid.message.type",
"other_data": {},
}
)
result = _ConversionHelper.try_convert_raw_message(raw_message)
assert result is None
def test_try_convert_raw_message_malformed_data(self):
"""Test converting malformed message data returns None."""
raw_message = RealtimeModelSendRawMessage(
message={
"type": "session.update",
"other_data": {
"session": "invalid_session_data" # Should be dict
},
}
)
result = _ConversionHelper.try_convert_raw_message(raw_message)
assert result is None
def test_try_convert_raw_message_missing_type(self):
"""Test converting message without type returns None."""
raw_message = RealtimeModelSendRawMessage(
message={
"type": "missing.type.test",
"other_data": {"some": "data"},
}
)
result = _ConversionHelper.try_convert_raw_message(raw_message)
assert result is None
class TestConversionHelperTracingConfig:
"""Test suite for _ConversionHelper.convert_tracing_config method."""
def test_convert_tracing_config_none(self):
"""Test converting None tracing config."""
result = _ConversionHelper.convert_tracing_config(None)
assert result is None
def test_convert_tracing_config_auto(self):
"""Test converting 'auto' tracing config."""
result = _ConversionHelper.convert_tracing_config("auto")
assert result == "auto"
def test_convert_tracing_config_dict_full(self):
"""Test converting full tracing config dict."""
tracing_config: RealtimeModelTracingConfig = {
"group_id": "test-group",
"metadata": {"env": "test"},
"workflow_name": "test-workflow",
}
result = _ConversionHelper.convert_tracing_config(tracing_config)
assert result is not None
assert result != "auto"
assert result.group_id == "test-group"
assert result.metadata == {"env": "test"}
assert result.workflow_name == "test-workflow"
def test_convert_tracing_config_dict_partial(self):
"""Test converting partial tracing config dict."""
tracing_config: RealtimeModelTracingConfig = {
"group_id": "test-group",
}
result = _ConversionHelper.convert_tracing_config(tracing_config)
assert result is not None
assert result != "auto"
assert result.group_id == "test-group"
assert result.metadata is None
assert result.workflow_name is None
def test_convert_tracing_config_empty_dict(self):
"""Test converting empty tracing config dict."""
tracing_config: RealtimeModelTracingConfig = {}
result = _ConversionHelper.convert_tracing_config(tracing_config)
assert result is not None
assert result != "auto"
assert result.group_id is None
assert result.metadata is None
assert result.workflow_name is None
class TestConversionHelperUserInput:
"""Test suite for _ConversionHelper user input conversion methods."""
def test_convert_user_input_to_conversation_item_string(self):
"""Test converting string user input to conversation item."""
event = RealtimeModelSendUserInput(user_input="Hello, world!")
result = _ConversionHelper.convert_user_input_to_conversation_item(event)
assert result.type == "message"
assert result.role == "user"
assert result.content is not None
assert len(result.content) == 1
assert result.content[0].type == "input_text"
assert result.content[0].text == "Hello, world!"
def test_convert_user_input_to_conversation_item_dict(self):
"""Test converting dict user input to conversation item."""
user_input_dict: RealtimeModelUserInputMessage = {
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "Hello"},
{"type": "input_text", "text": "World"},
],
}
event = RealtimeModelSendUserInput(user_input=user_input_dict)
result = _ConversionHelper.convert_user_input_to_conversation_item(event)
assert result.type == "message"
assert result.role == "user"
assert result.content is not None
assert len(result.content) == 2
assert result.content[0].type == "input_text"
assert result.content[0].text == "Hello"
assert result.content[1].type == "input_text"
assert result.content[1].text == "World"
def test_convert_user_input_to_conversation_item_dict_empty_content(self):
"""Test converting dict user input with empty content."""
user_input_dict: RealtimeModelUserInputMessage = {
"type": "message",
"role": "user",
"content": [],
}
event = RealtimeModelSendUserInput(user_input=user_input_dict)
result = _ConversionHelper.convert_user_input_to_conversation_item(event)
assert result.type == "message"
assert result.role == "user"
assert result.content is not None
assert len(result.content) == 0
def test_convert_user_input_to_item_create(self):
"""Test converting user input to item create event."""
event = RealtimeModelSendUserInput(user_input="Test message")
result = _ConversionHelper.convert_user_input_to_item_create(event)
assert isinstance(result, ConversationItemCreateEvent)
assert result.type == "conversation.item.create"
assert result.item.type == "message"
assert result.item.role == "user"
class TestConversionHelperAudio:
"""Test suite for _ConversionHelper.convert_audio_to_input_audio_buffer_append."""
def test_convert_audio_to_input_audio_buffer_append(self):
"""Test converting audio data to input audio buffer append event."""
audio_data = b"test audio data"
event = RealtimeModelSendAudio(audio=audio_data, commit=False)
result = _ConversionHelper.convert_audio_to_input_audio_buffer_append(event)
assert isinstance(result, InputAudioBufferAppendEvent)
assert result.type == "input_audio_buffer.append"
# Verify base64 encoding
expected_b64 = base64.b64encode(audio_data).decode("utf-8")
assert result.audio == expected_b64
def test_convert_audio_to_input_audio_buffer_append_empty(self):
"""Test converting empty audio data."""
audio_data = b""
event = RealtimeModelSendAudio(audio=audio_data, commit=True)
result = _ConversionHelper.convert_audio_to_input_audio_buffer_append(event)
assert isinstance(result, InputAudioBufferAppendEvent)
assert result.type == "input_audio_buffer.append"
assert result.audio == ""
def test_convert_audio_to_input_audio_buffer_append_large_data(self):
"""Test converting large audio data."""
audio_data = b"x" * 10000 # Large audio buffer
event = RealtimeModelSendAudio(audio=audio_data, commit=False)
result = _ConversionHelper.convert_audio_to_input_audio_buffer_append(event)
assert isinstance(result, InputAudioBufferAppendEvent)
assert result.type == "input_audio_buffer.append"
# Verify it can be decoded back
decoded = base64.b64decode(result.audio)
assert decoded == audio_data
class TestConversionHelperToolOutput:
"""Test suite for _ConversionHelper.convert_tool_output method."""
def test_convert_tool_output(self):
"""Test converting tool output to conversation item create event."""
mock_tool_call = Mock()
mock_tool_call.call_id = "call_123"
event = RealtimeModelSendToolOutput(
tool_call=mock_tool_call,
output="Function executed successfully",
start_response=False,
)
result = _ConversionHelper.convert_tool_output(event)
assert isinstance(result, ConversationItemCreateEvent)
assert result.type == "conversation.item.create"
assert result.item.type == "function_call_output"
assert isinstance(result.item, RealtimeConversationItemFunctionCallOutput)
tool_output_item = result.item
assert tool_output_item.output == "Function executed successfully"
assert tool_output_item.call_id == "call_123"
def test_convert_tool_output_no_call_id(self):
"""Test converting tool output with None call_id."""
mock_tool_call = Mock()
mock_tool_call.call_id = None
event = RealtimeModelSendToolOutput(
tool_call=mock_tool_call,
output="Output without call ID",
start_response=False,
)
with pytest.raises(
ValidationError,
match="1 validation error for RealtimeConversationItemFunctionCallOutput",
):
_ConversionHelper.convert_tool_output(event)
def test_convert_tool_output_empty_output(self):
"""Test converting tool output with empty output."""
mock_tool_call = Mock()
mock_tool_call.call_id = "call_456"
event = RealtimeModelSendToolOutput(
tool_call=mock_tool_call,
output="",
start_response=True,
)
result = _ConversionHelper.convert_tool_output(event)
assert isinstance(result, ConversationItemCreateEvent)
assert result.type == "conversation.item.create"
assert isinstance(result.item, RealtimeConversationItemFunctionCallOutput)
assert result.item.output == ""
assert result.item.call_id == "call_456"
class TestConversionHelperInterrupt:
"""Test suite for _ConversionHelper.convert_interrupt method."""
def test_convert_interrupt(self):
"""Test converting interrupt parameters to conversation item truncate event."""
current_item_id = "item_789"
current_audio_content_index = 2
elapsed_time_ms = 1500
result = _ConversionHelper.convert_interrupt(
current_item_id, current_audio_content_index, elapsed_time_ms
)
assert isinstance(result, ConversationItemTruncateEvent)
assert result.type == "conversation.item.truncate"
assert result.item_id == "item_789"
assert result.content_index == 2
assert result.audio_end_ms == 1500
def test_convert_interrupt_zero_time(self):
"""Test converting interrupt with zero elapsed time."""
result = _ConversionHelper.convert_interrupt("item_1", 0, 0)
assert isinstance(result, ConversationItemTruncateEvent)
assert result.type == "conversation.item.truncate"
assert result.item_id == "item_1"
assert result.content_index == 0
assert result.audio_end_ms == 0
def test_convert_interrupt_large_values(self):
"""Test converting interrupt with large values."""
result = _ConversionHelper.convert_interrupt("item_xyz", 99, 999999)
assert isinstance(result, ConversationItemTruncateEvent)
assert result.type == "conversation.item.truncate"
assert result.item_id == "item_xyz"
assert result.content_index == 99
assert result.audio_end_ms == 999999
def test_convert_interrupt_empty_item_id(self):
"""Test converting interrupt with empty item ID."""
result = _ConversionHelper.convert_interrupt("", 1, 100)
assert isinstance(result, ConversationItemTruncateEvent)
assert result.type == "conversation.item.truncate"
assert result.item_id == ""
assert result.content_index == 1
assert result.audio_end_ms == 100
@@ -0,0 +1,35 @@
from __future__ import annotations
from typing import Any, cast
import pytest
from websockets.asyncio.client import ClientConnection
from agents.realtime.openai_realtime import OpenAIRealtimeWebSocketModel
class _DummyWS:
def __init__(self) -> None:
self.sent: list[str] = []
async def send(self, data: str) -> None:
self.sent.append(data)
@pytest.mark.asyncio
async def test_no_auto_interrupt_on_vad_speech_started(monkeypatch: Any) -> None:
model = OpenAIRealtimeWebSocketModel()
called = {"interrupt": False}
async def _fake_interrupt(event: Any) -> None:
called["interrupt"] = True
# Prevent network use; _websocket only needed for other paths
model._websocket = cast(ClientConnection, _DummyWS())
monkeypatch.setattr(model, "_send_interrupt", _fake_interrupt)
# This event previously triggered an interrupt; now it should be ignored
await model._handle_ws_event({"type": "input_audio_buffer.speech_started"})
assert called["interrupt"] is False
+80
View File
@@ -0,0 +1,80 @@
from openai.types.realtime.realtime_conversation_item_assistant_message import (
Content as AssistantMessageContent,
RealtimeConversationItemAssistantMessage,
)
from openai.types.realtime.realtime_conversation_item_system_message import (
Content as SystemMessageContent,
RealtimeConversationItemSystemMessage,
)
from openai.types.realtime.realtime_conversation_item_user_message import (
Content as UserMessageContent,
RealtimeConversationItemUserMessage,
)
from agents.realtime.items import (
AssistantMessageItem,
RealtimeMessageItem,
SystemMessageItem,
UserMessageItem,
)
from agents.realtime.openai_realtime import _ConversionHelper
def test_user_message_conversion() -> None:
item = RealtimeConversationItemUserMessage(
id="123",
type="message",
role="user",
content=[
UserMessageContent(type="input_text", text=None),
],
)
converted: RealtimeMessageItem = _ConversionHelper.conversation_item_to_realtime_message_item(
item, None
)
assert isinstance(converted, UserMessageItem)
item = RealtimeConversationItemUserMessage(
id="123",
type="message",
role="user",
content=[
UserMessageContent(type="input_audio", audio=None),
],
)
converted = _ConversionHelper.conversation_item_to_realtime_message_item(item, None)
assert isinstance(converted, UserMessageItem)
def test_assistant_message_conversion() -> None:
item = RealtimeConversationItemAssistantMessage(
id="123",
type="message",
role="assistant",
content=[AssistantMessageContent(type="output_text", text=None)],
)
converted: RealtimeMessageItem = _ConversionHelper.conversation_item_to_realtime_message_item(
item, None
)
assert isinstance(converted, AssistantMessageItem)
def test_system_message_conversion() -> None:
item = RealtimeConversationItemSystemMessage(
id="123",
type="message",
role="system",
content=[SystemMessageContent(type="input_text", text=None)],
)
converted: RealtimeMessageItem = _ConversionHelper.conversation_item_to_realtime_message_item(
item, None
)
assert isinstance(converted, SystemMessageItem)
+51
View File
@@ -0,0 +1,51 @@
from typing import get_args
import agents.realtime as realtime
from agents.realtime.model_events import RealtimeModelEvent
from agents.usage import Usage
def test_all_events_have_type() -> None:
"""Test that all events have a type."""
events = get_args(RealtimeModelEvent)
assert len(events) > 0
for event in events:
assert event.type is not None
assert isinstance(event.type, str)
def test_usage_event_types_are_publicly_exported() -> None:
expected_exports = {
"RealtimeModelCachedTokensDetails",
"RealtimeModelInputTokensDetails",
"RealtimeModelOutputTokensDetails",
"RealtimeModelUsageEvent",
}
assert expected_exports <= set(realtime.__all__)
for name in expected_exports:
assert getattr(realtime, name) is not None
def test_custom_model_can_construct_typed_usage_without_openai_types() -> None:
event = realtime.RealtimeModelUsageEvent(
usage=Usage(requests=1, input_tokens=8, output_tokens=5, total_tokens=13),
input_tokens_details=realtime.RealtimeModelInputTokensDetails(
text_tokens=2,
audio_tokens=6,
cached_tokens=3,
cached_tokens_details=realtime.RealtimeModelCachedTokensDetails(
text_tokens=1,
audio_tokens=2,
),
),
output_tokens_details=realtime.RealtimeModelOutputTokensDetails(
text_tokens=1,
audio_tokens=4,
),
)
assert event.input_tokens_details is not None
assert event.input_tokens_details.audio_tokens == 6
assert event.output_tokens_details is not None
assert event.output_tokens_details.audio_tokens == 4
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,295 @@
from typing import cast
import pytest
from openai.types.realtime.realtime_conversation_item_user_message import (
RealtimeConversationItemUserMessage,
)
from openai.types.realtime.realtime_response_usage import RealtimeResponseUsage
from openai.types.realtime.realtime_tracing_config import (
TracingConfiguration,
)
from agents import Agent, function_tool, tool_namespace
from agents.exceptions import UserError
from agents.handoffs import handoff
from agents.realtime.config import RealtimeModelTracingConfig
from agents.realtime.model_inputs import (
RealtimeModelSendRawMessage,
RealtimeModelSendUserInput,
RealtimeModelUserInputMessage,
)
from agents.realtime.openai_realtime import (
OpenAIRealtimeWebSocketModel,
_ConversionHelper,
get_api_key,
)
from agents.tool import Tool
@pytest.mark.asyncio
async def test_get_api_key_from_env(monkeypatch):
monkeypatch.setenv("OPENAI_API_KEY", "env-key")
assert await get_api_key(None) == "env-key"
@pytest.mark.asyncio
async def test_get_api_key_from_callable_async():
async def f():
return "k"
assert await get_api_key(f) == "k"
def test_try_convert_raw_message_invalid_returns_none():
msg = RealtimeModelSendRawMessage(message={"type": "invalid.event", "other_data": {}})
assert _ConversionHelper.try_convert_raw_message(msg) is None
def test_convert_response_usage_preserves_modality_details():
event = _ConversionHelper.convert_response_usage(
RealtimeResponseUsage.model_validate(
{
"total_tokens": 253,
"input_tokens": 132,
"output_tokens": 121,
"input_token_details": {
"text_tokens": 119,
"audio_tokens": 13,
"image_tokens": 0,
"cached_tokens": 64,
"cached_tokens_details": {
"text_tokens": 60,
"audio_tokens": 4,
"image_tokens": 0,
},
},
"output_token_details": {"text_tokens": 30, "audio_tokens": 91},
}
)
)
assert event.usage.requests == 1
assert event.usage.input_tokens == 132
assert event.usage.output_tokens == 121
assert event.usage.total_tokens == 253
assert event.usage.input_tokens_details.cached_tokens == 64
assert event.input_tokens_details is not None
assert event.input_tokens_details.text_tokens == 119
assert event.input_tokens_details.audio_tokens == 13
assert event.input_tokens_details.image_tokens == 0
assert event.input_tokens_details.cached_tokens == 64
assert event.input_tokens_details.cached_tokens_details is not None
assert event.input_tokens_details.cached_tokens_details.text_tokens == 60
assert event.input_tokens_details.cached_tokens_details.audio_tokens == 4
assert event.input_tokens_details.cached_tokens_details.image_tokens == 0
assert event.output_tokens_details is not None
assert event.output_tokens_details.text_tokens == 30
assert event.output_tokens_details.audio_tokens == 91
def test_convert_response_usage_preserves_missing_details_and_derives_total():
event = _ConversionHelper.convert_response_usage(
RealtimeResponseUsage.model_validate(
{
"input_tokens": 12,
"output_tokens": 3,
"input_token_details": {"audio_tokens": 0},
}
)
)
assert event.usage.total_tokens == 15
assert event.input_tokens_details is not None
assert event.input_tokens_details.audio_tokens == 0
assert event.input_tokens_details.text_tokens is None
assert event.input_tokens_details.cached_tokens is None
assert event.input_tokens_details.cached_tokens_details is None
assert event.output_tokens_details is None
def test_convert_user_input_to_conversation_item_dict_and_str():
# Dict with mixed, including unknown parts (silently skipped)
dict_input_any = {
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "hello"},
{"type": "input_image", "image_url": "http://x/y.png", "detail": "auto"},
{"type": "bogus", "x": 1},
],
}
event = RealtimeModelSendUserInput(
user_input=cast(RealtimeModelUserInputMessage, dict_input_any)
)
item_any = _ConversionHelper.convert_user_input_to_conversation_item(event)
item = cast(RealtimeConversationItemUserMessage, item_any)
assert item.role == "user"
# String input becomes input_text
event2 = RealtimeModelSendUserInput(user_input="hi")
item2_any = _ConversionHelper.convert_user_input_to_conversation_item(event2)
item2 = cast(RealtimeConversationItemUserMessage, item2_any)
assert item2.content[0].type == "input_text"
def test_convert_user_input_dict_skips_invalid_input_text_parts():
"""input_text parts with missing/non-string text must be skipped, not
forwarded as Content(text=None) which the realtime API rejects."""
dict_input_any = {
"type": "message",
"role": "user",
"content": [
{"type": "input_text"}, # missing text
{"type": "input_text", "text": 123}, # non-string text
{"type": "input_text", "text": "ok"}, # valid
],
}
event = RealtimeModelSendUserInput(
user_input=cast(RealtimeModelUserInputMessage, dict_input_any)
)
item = cast(
RealtimeConversationItemUserMessage,
_ConversionHelper.convert_user_input_to_conversation_item(event),
)
assert item.content is not None
assert len(item.content) == 1
assert item.content[0].type == "input_text"
assert item.content[0].text == "ok"
def test_convert_tracing_config_variants():
from agents.realtime.openai_realtime import _ConversionHelper as CH
assert CH.convert_tracing_config(None) is None
assert CH.convert_tracing_config("auto") == "auto"
cfg: RealtimeModelTracingConfig = {
"group_id": "g",
"metadata": {"k": "v"},
"workflow_name": "wf",
}
oc_any = CH.convert_tracing_config(cfg)
oc = cast(TracingConfiguration, oc_any)
assert oc.group_id == "g"
assert oc.workflow_name == "wf"
def test_tools_to_session_tools_raises_on_non_function_tool():
class NotFunctionTool:
def __init__(self):
self.name = "x"
m = OpenAIRealtimeWebSocketModel()
with pytest.raises(UserError):
m._tools_to_session_tools(cast(list[Tool], [NotFunctionTool()]), [])
def test_tools_to_session_tools_includes_handoffs():
a = Agent(name="a")
h = handoff(a)
m = OpenAIRealtimeWebSocketModel()
out = m._tools_to_session_tools([], [h])
assert out[0].name is not None and out[0].name.startswith("transfer_to_")
def test_tools_to_session_tools_rejects_duplicate_function_tool_names():
tool_one = function_tool(lambda: "one", name_override="lookup_account")
tool_two = function_tool(lambda: "two", name_override="lookup_account")
m = OpenAIRealtimeWebSocketModel()
with pytest.raises(
UserError,
match=("Duplicate Realtime tool name found: 'lookup_account' \\(2 function tools\\)"),
):
m._tools_to_session_tools([tool_one, tool_two], [])
def test_tools_to_session_tools_rejects_function_handoff_name_conflict():
tool = function_tool(lambda: "ok", name_override="transfer_to_billing")
h = handoff(Agent(name="billing"), tool_name_override="transfer_to_billing")
m = OpenAIRealtimeWebSocketModel()
with pytest.raises(
UserError,
match=(
"Duplicate Realtime tool name found: "
"'transfer_to_billing' \\(function tool and handoff\\)"
),
):
m._tools_to_session_tools([tool], [h])
def test_tools_to_session_tools_ignores_disabled_function_tool_name_conflict():
tool = function_tool(
lambda: "ok",
name_override="transfer_to_billing",
is_enabled=False,
)
h = handoff(Agent(name="billing"), tool_name_override="transfer_to_billing")
m = OpenAIRealtimeWebSocketModel()
out = m._tools_to_session_tools([tool], [h])
assert [tool.name for tool in out] == ["transfer_to_billing"]
def test_tools_to_session_tools_omits_disabled_function_tool():
tool = function_tool(
lambda: "ok",
name_override="hidden_tool",
is_enabled=False,
)
m = OpenAIRealtimeWebSocketModel()
out = m._tools_to_session_tools([tool], [])
assert out == []
def test_tools_to_session_tools_ignores_disabled_handoff_name_conflict():
tool = function_tool(lambda: "ok", name_override="transfer_to_billing")
h = handoff(
Agent(name="billing"),
tool_name_override="transfer_to_billing",
is_enabled=False,
)
m = OpenAIRealtimeWebSocketModel()
out = m._tools_to_session_tools([tool], [h])
assert [tool.name for tool in out] == ["transfer_to_billing"]
def test_tools_to_session_tools_rejects_duplicate_handoff_names():
handoff_one = handoff(Agent(name="billing"), tool_name_override="transfer_to_support")
handoff_two = handoff(Agent(name="technical"), tool_name_override="transfer_to_support")
m = OpenAIRealtimeWebSocketModel()
with pytest.raises(
UserError,
match=("Duplicate Realtime tool name found: 'transfer_to_support' \\(2 handoffs\\)"),
):
m._tools_to_session_tools([], [handoff_one, handoff_two])
def test_tools_to_session_tools_rejects_namespaced_function_tools():
tool = tool_namespace(
name="crm",
description="CRM tools",
tools=[function_tool(lambda customer_id: customer_id, name_override="lookup_account")],
)[0]
m = OpenAIRealtimeWebSocketModel()
with pytest.raises(UserError, match="tool_namespace\\(\\)"):
m._tools_to_session_tools([tool], [])
def test_tools_to_session_tools_rejects_deferred_function_tools():
tool = function_tool(
lambda customer_id: customer_id,
name_override="lookup_account",
defer_loading=True,
)
m = OpenAIRealtimeWebSocketModel()
with pytest.raises(UserError, match="defer_loading=True"):
m._tools_to_session_tools([tool], [])
@@ -0,0 +1,56 @@
from __future__ import annotations
import asyncio
import pytest
from agents.exceptions import UserError
from agents.realtime.openai_realtime import OpenAIRealtimeSIPModel
class _DummyWebSocket:
def __init__(self) -> None:
self.sent_messages: list[str] = []
self.closed = False
def __aiter__(self):
return self
async def __anext__(self): # pragma: no cover - simple termination
raise StopAsyncIteration
async def send(self, data: str) -> None:
self.sent_messages.append(data)
async def close(self) -> None:
self.closed = True
@pytest.mark.asyncio
async def test_sip_model_uses_call_id_in_url(monkeypatch: pytest.MonkeyPatch) -> None:
dummy_ws = _DummyWebSocket()
captured: dict[str, object] = {}
async def fake_connect(url: str, **kwargs):
captured["url"] = url
captured["kwargs"] = kwargs
return dummy_ws
monkeypatch.setattr("agents.realtime.openai_realtime.websockets.connect", fake_connect)
model = OpenAIRealtimeSIPModel()
await model.connect({"api_key": "sk-test", "call_id": "call_789", "initial_model_settings": {}})
assert captured["url"] == "wss://api.openai.com/v1/realtime?call_id=call_789"
await asyncio.sleep(0) # allow listener task to start and finish
await model.close()
assert dummy_ws.closed
@pytest.mark.asyncio
async def test_sip_model_requires_call_id() -> None:
model = OpenAIRealtimeSIPModel()
with pytest.raises(UserError):
await model.connect({"api_key": "sk-test", "initial_model_settings": {}})
+193
View File
@@ -0,0 +1,193 @@
from unittest.mock import AsyncMock, patch
import pytest
from agents.realtime._default_tracker import ModelAudioTracker
from agents.realtime.model import RealtimePlaybackTracker
from agents.realtime.model_inputs import RealtimeModelSendInterrupt
from agents.realtime.openai_realtime import OpenAIRealtimeWebSocketModel
class TestPlaybackTracker:
"""Test playback tracker functionality for interrupt timing."""
@pytest.fixture
def model(self):
"""Create a fresh model instance for each test."""
return OpenAIRealtimeWebSocketModel()
@pytest.mark.asyncio
async def test_interrupt_timing_with_custom_playback_tracker(self, model):
"""Test interrupt uses custom playback tracker elapsed time instead of default timing."""
# Create custom tracker and set elapsed time
custom_tracker = RealtimePlaybackTracker()
custom_tracker.set_audio_format("pcm16")
custom_tracker.on_play_ms("item_1", 1, 500.0) # content_index 1, 500ms played
# Set up model with custom tracker directly
model._playback_tracker = custom_tracker
# Mock send_raw_message to capture interrupt
model._send_raw_message = AsyncMock()
# Send interrupt
await model._send_interrupt(RealtimeModelSendInterrupt())
# Should use custom tracker's 500ms elapsed time
truncate_events = [
call.args[0]
for call in model._send_raw_message.await_args_list
if getattr(call.args[0], "type", None) == "conversation.item.truncate"
]
assert truncate_events
assert truncate_events[0].audio_end_ms == 500
@pytest.mark.asyncio
async def test_interrupt_skipped_when_no_audio_playing(self, model):
"""Test interrupt returns early when no audio is currently playing."""
model._send_raw_message = AsyncMock()
# No audio playing (default state)
await model._send_interrupt(RealtimeModelSendInterrupt())
# Should not send any interrupt message
model._send_raw_message.assert_not_called()
@pytest.mark.asyncio
async def test_interrupt_skips_when_elapsed_exceeds_audio_length(self, model):
"""Test interrupt skips truncation when playback appears complete."""
model._send_raw_message = AsyncMock()
model._audio_state_tracker.set_audio_format("pcm16")
# 48_000 bytes of PCM16 at 24kHz equals ~1000ms of audio.
model._audio_state_tracker.on_audio_delta("item_1", 0, b"a" * 48_000)
model._playback_tracker = RealtimePlaybackTracker()
model._playback_tracker.on_play_ms("item_1", 0, 2000.0)
await model._send_interrupt(RealtimeModelSendInterrupt())
truncate_events = [
call.args[0]
for call in model._send_raw_message.await_args_list
if getattr(call.args[0], "type", None) == "conversation.item.truncate"
]
assert truncate_events == []
@pytest.mark.asyncio
async def test_interrupt_sends_truncate_when_ongoing_response(self, model):
"""Test interrupt still truncates while response is ongoing."""
model._ongoing_response = True
model._send_raw_message = AsyncMock()
model._audio_state_tracker.set_audio_format("pcm16")
# 48_000 bytes of PCM16 at 24kHz equals ~1000ms of audio.
model._audio_state_tracker.on_audio_delta("item_1", 0, b"a" * 48_000)
model._playback_tracker = RealtimePlaybackTracker()
model._playback_tracker.on_play_ms("item_1", 0, 2000.0)
await model._send_interrupt(RealtimeModelSendInterrupt())
truncate_events = [
call.args[0]
for call in model._send_raw_message.await_args_list
if getattr(call.args[0], "type", None) == "conversation.item.truncate"
]
assert truncate_events
assert truncate_events[0].audio_end_ms == 2000
def test_audio_delta_before_set_audio_format_does_not_raise(self):
"""ModelAudioTracker must tolerate audio deltas before a format is negotiated.
For transcription-only sessions or session payloads that omit an audio
format, ``set_audio_format`` is never called. Previously, the first
``on_audio_delta`` call raised ``AttributeError`` because ``self._format``
was unset. The length calculator already accepts ``None`` as the
unknown-format fallback, so the tracker should pass that through.
"""
tracker = ModelAudioTracker()
# Intentionally do NOT call set_audio_format here.
tracker.on_audio_delta("item_1", 0, b"test")
state = tracker.get_state("item_1", 0)
assert state is not None
# With no format, calculate_audio_length_ms falls back to PCM math.
expected_length = (4 / (24_000 * 2)) * 1000
assert state.audio_length_ms == pytest.approx(expected_length, rel=0, abs=1e-6)
assert tracker.get_last_audio_item() == ("item_1", 0)
def test_audio_state_accumulation_across_deltas(self):
"""Test ModelAudioTracker accumulates audio length across multiple deltas."""
tracker = ModelAudioTracker()
tracker.set_audio_format("pcm16")
# Send multiple deltas for same item
tracker.on_audio_delta("item_1", 0, b"test") # 4 bytes
tracker.on_audio_delta("item_1", 0, b"more") # 4 bytes
state = tracker.get_state("item_1", 0)
assert state is not None
# Should accumulate: 8 bytes -> 4 samples -> (4 / 24000) * 1000 ≈ 0.167ms
expected_length = (8 / (24_000 * 2)) * 1000
assert state.audio_length_ms == pytest.approx(expected_length, rel=0, abs=1e-6)
def test_default_playback_timing_uses_monotonic_clock(self, model):
model._audio_state_tracker.set_audio_format("pcm16")
with patch("agents.realtime._default_tracker.time.monotonic", return_value=42.0):
model._audio_state_tracker.on_audio_delta("item_1", 0, b"test")
with patch("agents.realtime.openai_realtime.time.monotonic", return_value=42.25):
state = model._get_playback_state()
assert state["current_item_id"] == "item_1"
assert state["current_item_content_index"] == 0
assert state["elapsed_ms"] == pytest.approx(250.0)
def test_state_cleanup_on_interruption(self):
"""Test both trackers properly reset state on interruption."""
# Test ModelAudioTracker cleanup
model_tracker = ModelAudioTracker()
model_tracker.set_audio_format("pcm16")
model_tracker.on_audio_delta("item_1", 0, b"test")
assert model_tracker.get_last_audio_item() == ("item_1", 0)
model_tracker.on_interrupted()
assert model_tracker.get_last_audio_item() is None
# Test RealtimePlaybackTracker cleanup
playback_tracker = RealtimePlaybackTracker()
playback_tracker.on_play_ms("item_1", 0, 100.0)
state = playback_tracker.get_state()
assert state["current_item_id"] == "item_1"
assert state["elapsed_ms"] == 100.0
playback_tracker.on_interrupted()
state = playback_tracker.get_state()
assert state["current_item_id"] is None
assert state["elapsed_ms"] is None
def test_audio_length_calculation_with_different_formats(self):
"""Test calculate_audio_length_ms handles g711 and PCM formats correctly."""
from agents.realtime._util import calculate_audio_length_ms
# Test g711 format (8kHz)
g711_bytes = b"12345678" # 8 bytes
g711_length = calculate_audio_length_ms("g711_ulaw", g711_bytes)
assert g711_length == 1 # (8 / 8000) * 1000
# Test PCM format (24kHz, default)
pcm_bytes = b"test" # 4 bytes
pcm_length = calculate_audio_length_ms("pcm16", pcm_bytes)
expected_pcm = (len(pcm_bytes) / (24_000 * 2)) * 1000
assert pcm_length == pytest.approx(expected_pcm, rel=0, abs=1e-6)
# Test None format (defaults to PCM)
none_length = calculate_audio_length_ms(None, pcm_bytes)
assert none_length == pytest.approx(expected_pcm, rel=0, abs=1e-6)
@@ -0,0 +1,23 @@
from agents.realtime.model import RealtimePlaybackTracker
def test_playback_tracker_on_play_bytes_and_state():
tr = RealtimePlaybackTracker()
tr.set_audio_format("pcm16") # PCM path
# 48k bytes -> (48000 / (24000 * 2)) * 1000 = 1_000ms
tr.on_play_bytes("item1", 0, b"x" * 48000)
st = tr.get_state()
assert st["current_item_id"] == "item1"
assert st["elapsed_ms"] and abs(st["elapsed_ms"] - 1_000.0) < 1e-6
# Subsequent play on same item accumulates
tr.on_play_ms("item1", 0, 500.0)
st2 = tr.get_state()
assert st2["elapsed_ms"] and abs(st2["elapsed_ms"] - 1_500.0) < 1e-6
# Interruption clears state
tr.on_interrupted()
st3 = tr.get_state()
assert st3["current_item_id"] is None
assert st3["elapsed_ms"] is None

Some files were not shown because too many files have changed in this diff Show More